From ebba4a749e13947559e6ba6e262fe8a58a07991b Mon Sep 17 00:00:00 2001 From: Jithin Date: Mon, 10 Aug 2026 15:21:46 +0530 Subject: [PATCH 1/2] feat(redteam): add Netra.redteam.runRedteam() to trigger an existing red-team config Co-Authored-By: Claude Sonnet 5 --- .env.sample | 8 + CHANGELOG.md | 6 + README.md | 3 + package-lock.json | 1378 ++++++++++++++++- package.json | 10 +- src/api/index.ts | 29 + src/api/redteam/__tests__/api.test.ts | 443 ++++++ src/api/redteam/__tests__/client.test.ts | 294 ++++ src/api/redteam/__tests__/e2e/helpers.ts | 35 + src/api/redteam/__tests__/e2e/mock-backend.ts | 534 +++++++ .../e2e/tc-01-04-run-creation.e2e.test.ts | 173 +++ .../e2e/tc-11-14-input-validation.e2e.test.ts | 63 + .../tc-15-23-callback-contract.e2e.test.ts | 218 +++ ...24-33h-client-driven-turn-loop.e2e.test.ts | 190 +++ .../tc-34-36-generation-gating.e2e.test.ts | 74 + .../e2e/tc-37-40-auth-tenancy.e2e.test.ts | 86 + ...tc-41-45-results-progress-risk.e2e.test.ts | 140 ++ .../e2e/tc-46-48-cancellation.e2e.test.ts | 171 ++ src/api/redteam/__tests__/task.test.ts | 72 + src/api/redteam/__tests__/utils.test.ts | 154 ++ src/api/redteam/api.ts | 328 ++++ src/api/redteam/client.ts | 322 ++++ src/api/redteam/index.ts | 48 + src/api/redteam/models.ts | 190 +++ src/api/redteam/task.ts | 81 + src/api/redteam/utils.ts | 159 ++ src/index.ts | 33 +- src/version.ts | 2 +- vitest.config.ts | 17 + 29 files changed, 5200 insertions(+), 61 deletions(-) create mode 100644 src/api/redteam/__tests__/api.test.ts create mode 100644 src/api/redteam/__tests__/client.test.ts create mode 100644 src/api/redteam/__tests__/e2e/helpers.ts create mode 100644 src/api/redteam/__tests__/e2e/mock-backend.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-01-04-run-creation.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-11-14-input-validation.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-15-23-callback-contract.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-24-33h-client-driven-turn-loop.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-34-36-generation-gating.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-37-40-auth-tenancy.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-41-45-results-progress-risk.e2e.test.ts create mode 100644 src/api/redteam/__tests__/e2e/tc-46-48-cancellation.e2e.test.ts create mode 100644 src/api/redteam/__tests__/task.test.ts create mode 100644 src/api/redteam/__tests__/utils.test.ts create mode 100644 src/api/redteam/api.ts create mode 100644 src/api/redteam/client.ts create mode 100644 src/api/redteam/index.ts create mode 100644 src/api/redteam/models.ts create mode 100644 src/api/redteam/task.ts create mode 100644 src/api/redteam/utils.ts create mode 100644 vitest.config.ts diff --git a/.env.sample b/.env.sample index 0130c64..3e2da92 100644 --- a/.env.sample +++ b/.env.sample @@ -34,3 +34,11 @@ NETRA_CONVERSATION_CONTENT_MAX_LEN= # ANTHROPIC_API_KEY= # GOOGLE_APPLICATION_CREDENTIALS= # GOOGLE_CLOUD_PROJECT= + +# Red-team SDK (Netra.redteam) — beta +# Ordinary REST timeout (seconds) for every red-team API call (default: 20) +NETRA_REDTEAM_TIMEOUT= +# Interval (seconds) between createRun retries while prompts are still generating (default: 2) +NETRA_REDTEAM_GENERATION_POLL_INTERVAL= +# Deadline (seconds) to wait for prompt generation before failing (default: 300) +NETRA_REDTEAM_GENERATION_TIMEOUT= diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ed104e..7754e0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.0-beta.1] - 2026-08-06 + +### Added + +- **Red-team SDK (`Netra.redteam`, beta)**: Trigger a red-team evaluation against a developer's local agent function via `Netra.redteam.runRedteam({ configId, handler, maxConcurrency? })`. `configId` identifies a red-team config created ahead of time (e.g. in the dashboard) — the config's agent, evaluators, and attack settings are decided there; the SDK only drives the run. `handler` is a plain callback `(prompt, sessionId, turnIndex) => Promise`, called once per turn — no class to extend. The client fetches the run's whole generated prompt list once, then drives every session's turns itself (bounded local concurrency via `maxConcurrency`, default/cap 5), submitting each turn's result directly. Returns a `RedteamResult` with `results`, `progress`, and `riskScore`. `Ctrl-C` (SIGINT/SIGTERM) cancels any in-flight run server-side before the process exits. + ## [1.8.0] - 2026-08-03 ### Added diff --git a/README.md b/README.md index c99b256..2814666 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,9 @@ You can configure the SDK using environment variables: | `NETRA_APP_NAME` | Name of your application | | `NETRA_ENV` | Environment (e.g., prod, dev) | | `NETRA_TRACE_CONTENT` | Capture prompt/completion content (default: true) | +| `NETRA_REDTEAM_TIMEOUT` | Red-team API request timeout in seconds — an ordinary, bounded REST timeout (default: `20`) | +| `NETRA_REDTEAM_GENERATION_POLL_INTERVAL` | Interval in seconds between `createRun` retries while prompts are still generating (default: `2`) | +| `NETRA_REDTEAM_GENERATION_TIMEOUT` | Deadline in seconds to wait for prompt generation to finish before failing (default: `300`) | ## 🤝 License diff --git a/package-lock.json b/package-lock.json index 9e92f2a..ebac98c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-beta.1", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.9.0", @@ -26,9 +26,11 @@ "devDependencies": { "@types/node": "^25.0.0", "@types/shimmer": "^1.2.0", + "@vitest/coverage-v8": "^4.1.10", "ts-node": "^10.9.2", "tsup": "^8.5.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^4.1.10" }, "engines": { "node": ">=18.0.0" @@ -88,6 +90,66 @@ } } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", @@ -2702,6 +2764,16 @@ "node": ">=14" } }, + "node_modules/@oxc-project/types": { + "version": "0.143.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.143.0.tgz", + "integrity": "sha512-u6JZdLBTLotrNC9Vd6vPssINdzcCzleKAH6EJKImQb7GtYvX5keN2dxkoK44stCc4tffE6QQRtZTXVSzsLUlWA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -2766,6 +2838,251 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.3.tgz", + "integrity": "sha512-zrJtHDcaZJ1Fp7xf4hNl+7seH9Cn/N5TwLYkhgXREtBwAd/jaqW3uqeHxpDugJLVICWg4eW44kOQEGJ1r6jCGw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.3.tgz", + "integrity": "sha512-ieIiibVCp0tX7TLu2cafoNPv8wJyYi01ekXpbf8q2j7F4rGAhhXb/eQh7ge9DRBY78GwmRQtvjZDux7EDbA8kA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.3.tgz", + "integrity": "sha512-Zh9tCon19eDXJoihx0rqKhMUlMYqzwj3aPsSuHmI4RWZh62dWUL+DJN4C5YQya5TcQBJU/Fe8+rY0jhXTQITqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.3.tgz", + "integrity": "sha512-nGbJWewA1wrXXZiQhjAT5rhibGfns5ZNkDVqxsO6zJ3f3YvpoDNNmGMSbbhLuXKjNScaBJVOAboztAWVespQMg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.3.tgz", + "integrity": "sha512-QNniJr5Kml0kDEB98jiDOJjXNroxIIi0IXIbdYzY26Xt1pVbeP62+KnoIZLwirOymX/0jDk/2gI/bNUv7A7OIw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.3.tgz", + "integrity": "sha512-TkqEAcmmvH3I/q4114NB4RVt6241Dao48pF45uLcFGrwAaIn0iITgTAKP/dLjbN0R4buJjGb91+UHSoFmpgIWw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.3.tgz", + "integrity": "sha512-NHqjnxpsndf4MPymxteFAWHHfkTL8HjWh1KB7z23ofZ6QO2euONuxDXjat69dKZRALnGypg8k8SsK8vZJoXv1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.3.tgz", + "integrity": "sha512-6tbrbwfz5GB9DQ4Jwo6hy9v+vR31xZlvzZ6n5Xut6Hhx5PvrA9q/HsK8KMaYQp063iqZGXwNvZtYNLD7EM/x0w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.3.tgz", + "integrity": "sha512-oyuXxXmoZHjXC917IAPFAAv4wWAa0cM9afk8nx1+9/jNNOX1uPf8yDA6p7G0RypOfw/X0PQt5IfoquY1um+zSg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.3.tgz", + "integrity": "sha512-TytMwF2KVGqP2tgd0I1OY0PAv78dZRAYcF5ssDzjM34SUXCED3uXvSd5+lHoC0bTD6eEdFz7LdQNCO1y0oVk9w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.3.tgz", + "integrity": "sha512-/E9m3qstrJFVPoULV25mVQblSNExY2+kBsYe4sy0Tn0yOOgJ8wZbZt3KnRbF/XeU2Gl1STKUQnDNTqhIE5MD4A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.3.tgz", + "integrity": "sha512-Kr0OcsoQI816i6HOl3vFHpd1K0eZyh76zgfj4c1nTyaTsd5r2Mj1lwM4R90y/qaCfmTn9eHy0SKwi98eitRxug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.3.tgz", + "integrity": "sha512-hOtMwTqnME+/gJcH/PCZ0wn0zPUjiWOgkHpxbSJpfGKMezHltx1S7/k1SitzVa7Ww2cqrDDaFbZEhcJZO8o+Jw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.3.tgz", + "integrity": "sha512-ekcqMMkI2PlhYnfzQnB/cEdYUVVJViWvoUyLrbzgDoi3Snfc1mVBwdnc306ufA5ejy8JSPjT2RlW1nQSjW7efg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.55.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.55.2.tgz", @@ -3116,6 +3433,13 @@ "win32" ] }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -3536,6 +3860,24 @@ "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", "license": "MIT" }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -3600,34 +3942,178 @@ "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", "license": "MIT" }, - "node_modules/a-sync-waterfall": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz", - "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==", - "license": "MIT" - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "node_modules/@vitest/coverage-v8": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.10.tgz", + "integrity": "sha512-IM49HmthevbgAO4anp1hwtoT9wYe59w0LR00gr+eagHE+ZJ5lK4sLPeO0ubgoJcwLk6dehU3R24N+FbEEKDc8g==", + "dev": true, "license": "MIT", "dependencies": { - "event-target-shim": "^5.0.0" + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.10", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" }, - "engines": { - "node": ">=6.5" + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.10", + "vitest": "4.1.10" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } } }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "node_modules/@vitest/expect": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", + "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "dev": true, "license": "MIT", - "bin": { - "acorn": "bin/acorn" + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" }, - "engines": { - "node": ">=0.4.0" + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", + "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.10", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", + "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", + "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.10", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", + "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "@vitest/utils": "4.1.10", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", + "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", + "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/a-sync-waterfall": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz", + "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==", + "license": "MIT" + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" } }, "node_modules/acorn-import-attributes": { @@ -3702,6 +4188,39 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", "license": "MIT" }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.5.tgz", + "integrity": "sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, + "node_modules/ast-v8-to-istanbul/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3842,6 +4361,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -3970,6 +4499,13 @@ "simple-wcswidth": "^1.1.2" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -4021,6 +4557,16 @@ "node": ">=0.4.0" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", @@ -4099,6 +4645,13 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "dev": true, + "license": "MIT" + }, "node_modules/es-object-atoms": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", @@ -4177,6 +4730,16 @@ "node": ">=6" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -4213,6 +4776,16 @@ "node": ">=18.0.0" } }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -4585,6 +5158,13 @@ "node": ">= 0.4" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -4691,6 +5271,58 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/joycon": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", @@ -4710,6 +5342,13 @@ "base64-js": "^1.5.1" } }, + "node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -4784,44 +5423,305 @@ } } }, - "node_modules/lilconfig": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", - "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", "dev": true, - "license": "MIT", + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, "engines": { - "node": ">=14" + "node": ">= 12.0.0" }, "funding": { - "url": "https://github.com/sponsors/antonk52" + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "MIT" + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } }, - "node_modules/load-tsconfig": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", - "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT", + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", - "license": "MIT" + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/load-tsconfig": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", + "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/lodash": { + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", + "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", "license": "MIT" @@ -4842,6 +5742,34 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.4.tgz", + "integrity": "sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -4929,6 +5857,25 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.17", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", + "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -4993,6 +5940,20 @@ "node": ">= 6" } }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/ollama": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.6.3.tgz", @@ -5107,9 +6068,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", "engines": { @@ -5141,6 +6102,35 @@ "pathe": "^2.0.1" } }, + "node_modules/postcss": { + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.17", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, "node_modules/postcss-load-config": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", @@ -5316,6 +6306,39 @@ "node": ">=14" } }, + "node_modules/rolldown": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.3.tgz", + "integrity": "sha512-rn9wpmxplLf7NLNyCk9FyWh3FM43DbY8jOzCdEPzH7uflhTftRbCEpqi6Ly2osgoU8OwObtmavMbWLaWy4LX7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.143.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.2.3", + "@rolldown/binding-darwin-arm64": "1.2.3", + "@rolldown/binding-darwin-x64": "1.2.3", + "@rolldown/binding-freebsd-x64": "1.2.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.3", + "@rolldown/binding-linux-arm64-gnu": "1.2.3", + "@rolldown/binding-linux-arm64-musl": "1.2.3", + "@rolldown/binding-linux-ppc64-gnu": "1.2.3", + "@rolldown/binding-linux-s390x-gnu": "1.2.3", + "@rolldown/binding-linux-x64-gnu": "1.2.3", + "@rolldown/binding-linux-x64-musl": "1.2.3", + "@rolldown/binding-openharmony-arm64": "1.2.3", + "@rolldown/binding-win32-arm64-msvc": "1.2.3", + "@rolldown/binding-win32-x64-msvc": "1.2.3" + } + }, "node_modules/rollup": { "version": "4.55.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.55.2.tgz", @@ -5399,6 +6422,13 @@ "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==", "license": "BSD-2-Clause" }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/simple-wcswidth": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", @@ -5415,6 +6445,30 @@ "node": ">= 12" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, "node_modules/stream-events": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", @@ -5605,6 +6659,13 @@ "node": ">=0.8" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", @@ -5613,14 +6674,14 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -5629,6 +6690,16 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinyrainbow": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -5838,6 +6909,184 @@ "dev": true, "license": "MIT" }, + "node_modules/vite": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.0.tgz", + "integrity": "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.23", + "rolldown": "~1.2.0", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", + "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.10", + "@vitest/mocker": "4.1.10", + "@vitest/pretty-format": "4.1.10", + "@vitest/runner": "4.1.10", + "@vitest/snapshot": "4.1.10", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.10", + "@vitest/browser-preview": "4.1.10", + "@vitest/browser-webdriverio": "4.1.10", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/ui": "4.1.10", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/vitest/node_modules/tinyexec": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -5861,6 +7110,23 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index f0ed327..97d228c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-beta.1", "description": "A comprehensive TypeScript/JavaScript SDK for AI application observability built on top of OpenTelemetry and Traceloop", "type": "module", "main": "./dist/index.cjs", @@ -20,7 +20,9 @@ "build": "tsup", "start:dev": "tsup --watch", "prepack": "npm run build", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage" }, "keywords": [ "netra", @@ -119,8 +121,10 @@ "devDependencies": { "@types/node": "^25.0.0", "@types/shimmer": "^1.2.0", + "@vitest/coverage-v8": "^4.1.10", "ts-node": "^10.9.2", "tsup": "^8.5.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^4.1.10" } } diff --git a/src/api/index.ts b/src/api/index.ts index 6dc1451..c2c859e 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -68,3 +68,32 @@ export type { // Prompts API export { Prompts } from "./prompts"; export type { GetPromptParams, PromptResponse } from "./prompts"; + +// Red-team API +export { + Redteam, + RedteamAuthError, + RedteamConfigError, + RedteamGenerationError, + RedteamGenerationTimeoutError, + RedteamHttpClient, + RedteamRunError, + executeHandler as executeRedteamHandler, +} from "./redteam"; +export type { + ConversationTurn as RedteamConversationTurn, + CreateRunResponse as RedteamCreateRunResponse, + RedteamAgentHandler, + RedteamAgentResponse, + RedteamResult, + RedteamRunOptions, + RedteamRunStatus, + RedteamTaskResult, + RedteamTurnType, + RiskScore as RedteamRiskScore, + RunProgress as RedteamRunProgress, + RunPromptItem as RedteamRunPromptItem, + RunPromptsResponse as RedteamRunPromptsResponse, + RunResultItem as RedteamRunResultItem, + RunResultsPage as RedteamRunResultsPage, +} from "./redteam"; diff --git a/src/api/redteam/__tests__/api.test.ts b/src/api/redteam/__tests__/api.test.ts new file mode 100644 index 0000000..5a0504a --- /dev/null +++ b/src/api/redteam/__tests__/api.test.ts @@ -0,0 +1,443 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mockClient = { + isInitialized: vi.fn(() => true), + createRun: vi.fn(), + getPrompts: vi.fn(), + submitTurn: vi.fn(), + getProgress: vi.fn(), + getResultsPage: vi.fn(), + getRiskScore: vi.fn(), + cancel: vi.fn(), +}; + +vi.mock("../client", () => { + return { + RedteamHttpClient: vi.fn().mockImplementation(function RedteamHttpClient() { + return mockClient; + }), + }; +}); + +import { Redteam } from "../api"; +import { RedteamRunOptions } from "../models"; + +const fakeConfig = {} as any; + +function resetMocks() { + for (const fn of Object.values(mockClient)) { + (fn as any).mockReset(); + } + mockClient.isInitialized.mockReturnValue(true); + mockClient.cancel.mockResolvedValue({ status: "cancelled" }); + mockClient.getProgress.mockResolvedValue({ completedSessions: 1 }); + mockClient.getRiskScore.mockResolvedValue({ latestSafetyScore: 95 }); +} + +describe("Redteam", () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + resetMocks(); + process.env.NETRA_REDTEAM_GENERATION_POLL_INTERVAL = "0"; + process.env.NETRA_REDTEAM_GENERATION_TIMEOUT = "5"; + }); + + afterEach(() => { + process.env = { ...originalEnv }; + vi.restoreAllMocks(); + }); + + it("happy path single-turn: create(running) -> fetch prompts once -> drive to done -> results + progress + risk score", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-1", configId: "cfg-1", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ + runId: "run-1", + status: "running", + turnType: "single", + multiTurnCount: 5, + prompts: [{ id: "p1", prompt: "attack", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }) + // Re-read at the end to determine final status. + .mockResolvedValueOnce({ runId: "run-1", status: "completed", turnType: "single", multiTurnCount: 5, prompts: [] }); + mockClient.submitTurn.mockResolvedValueOnce({ done: true }); + mockClient.getResultsPage.mockResolvedValueOnce({ + items: [{ evaluatorId: "ev-1", status: "pass", score: 1 }], + page: 1, + limit: 200, + total: 1, + }); + + const handler = vi.fn(async (prompt: string, _sessionId: string, _turnIndex: number) => `reply:${prompt}`); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-1", handler, maxConcurrency: 1 }; + + const result = await redteam.runRedteam(options); + + expect(handler).toHaveBeenCalledWith("attack", "p1", 1); + expect(mockClient.submitTurn).toHaveBeenCalledWith("run-1", { + promptId: "p1", + sessionId: "p1", + turnIndex: 1, + promptText: "attack", + output: "reply:attack", + }); + expect(result).not.toBeNull(); + expect(result!.success).toBe(true); + expect(result!.status).toBe("completed"); + expect(result!.results).toHaveLength(1); + expect(result!.progress).toEqual({ completedSessions: 1 }); + expect(result!.riskScore).toEqual({ latestSafetyScore: 95 }); + expect(mockClient.getPrompts).toHaveBeenCalledTimes(2); + }); + + it("still generating after create: retries createRun with {configId} on an interval until running", async () => { + mockClient.createRun + .mockResolvedValueOnce({ configId: "cfg-2", status: "generating" }) + .mockResolvedValueOnce({ configId: "cfg-2", status: "generating" }) + .mockResolvedValueOnce({ configId: "cfg-2", status: "generating" }) + .mockResolvedValueOnce({ runId: "run-2", configId: "cfg-2", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ runId: "run-2", status: "running", turnType: "multi", multiTurnCount: 5, prompts: [] }) + .mockResolvedValueOnce({ runId: "run-2", status: "completed", turnType: "multi", multiTurnCount: 5, prompts: [] }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + const handler = vi.fn(async () => "unused"); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-2", handler, maxConcurrency: 1 }; + + const result = await redteam.runRedteam(options); + + expect(mockClient.createRun).toHaveBeenNthCalledWith(1, { configId: "cfg-2" }); + expect(mockClient.createRun).toHaveBeenNthCalledWith(2, { configId: "cfg-2" }); + expect(mockClient.createRun).toHaveBeenNthCalledWith(3, { configId: "cfg-2" }); + expect(mockClient.createRun).toHaveBeenNthCalledWith(4, { configId: "cfg-2" }); + expect(mockClient.createRun).toHaveBeenCalledTimes(4); + expect(result).not.toBeNull(); + expect(result!.runId).toBe("run-2"); + expect(result!.configId).toBe("cfg-2"); + }); + + it("multi-turn: incrementing turnIndex across turns via nextPrompt/nextTurnIndex, done:false until the last submit", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-3", configId: "cfg-3", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ + runId: "run-3", + status: "running", + turnType: "multi", + multiTurnCount: 3, + prompts: [{ id: "p1", prompt: "p0", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }) + .mockResolvedValueOnce({ runId: "run-3", status: "completed", turnType: "multi", multiTurnCount: 3, prompts: [] }); + mockClient.submitTurn + .mockResolvedValueOnce({ done: false, nextPrompt: "p1", nextTurnIndex: 2 }) + .mockResolvedValueOnce({ done: false, nextPrompt: "p2", nextTurnIndex: 3 }) + .mockResolvedValueOnce({ done: true }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + const handler = vi.fn(async (prompt: string, _sessionId: string, turnIndex: number) => `r${turnIndex}:${prompt}`); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-3", handler, maxConcurrency: 1 }; + + const result = await redteam.runRedteam(options); + + expect(handler).toHaveBeenNthCalledWith(1, "p0", "p1", 1); + expect(handler).toHaveBeenNthCalledWith(2, "p1", "p1", 2); + expect(handler).toHaveBeenNthCalledWith(3, "p2", "p1", 3); + expect(mockClient.submitTurn).toHaveBeenNthCalledWith(1, "run-3", { + promptId: "p1", + sessionId: "p1", + turnIndex: 1, + promptText: "p0", + output: "r1:p0", + }); + expect(mockClient.submitTurn).toHaveBeenNthCalledWith(2, "run-3", { + promptId: "p1", + sessionId: "p1", + turnIndex: 2, + promptText: "p1", + output: "r2:p1", + }); + expect(mockClient.submitTurn).toHaveBeenNthCalledWith(3, "run-3", { + promptId: "p1", + sessionId: "p1", + turnIndex: 3, + promptText: "p2", + output: "r3:p2", + }); + expect(result!.success).toBe(true); + }); + + it("backend-reported run failure surfaces as success:false/status:\"failed\", not hard-coded completed", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-10", configId: "cfg-10", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ runId: "run-10", status: "running", turnType: "single", multiTurnCount: 5, prompts: [] }) + .mockResolvedValueOnce({ runId: "run-10", status: "failed", turnType: "single", multiTurnCount: 5, prompts: [] }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + const handler = vi.fn(async () => "unused"); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-10", handler, maxConcurrency: 1 }; + + const result = await redteam.runRedteam(options); + + expect(result).not.toBeNull(); + expect(result!.status).toBe("failed"); + expect(result!.success).toBe(false); + }); + + it("zero generated prompts: warns, still completes with empty results", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-11", configId: "cfg-11", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ runId: "run-11", status: "running", turnType: "single", multiTurnCount: 5, prompts: [] }) + .mockResolvedValueOnce({ runId: "run-11", status: "completed", turnType: "single", multiTurnCount: 5, prompts: [] }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + const handler = vi.fn(async () => "unused"); + const redteam = new Redteam(fakeConfig); + const result = await redteam.runRedteam({ configId: "cfg-11", handler, maxConcurrency: 1 }); + + expect(handler).not.toHaveBeenCalled(); + expect(mockClient.submitTurn).not.toHaveBeenCalled(); + expect(result!.results).toHaveLength(0); + }); + + it("fatal submitTurn error trips the shared stop signal (sibling stops instead of continuing) and propagates to the caller", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-13", configId: "cfg-13", status: "running" }); + mockClient.getPrompts.mockResolvedValueOnce({ + runId: "run-13", + status: "running", + turnType: "single", + multiTurnCount: 5, + prompts: [ + { id: "pA", prompt: "attack A", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }, + { id: "pB", prompt: "attack B", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }, + ], + }); + const fatal = new Error("503 exhausted"); + let submitCallsForB = 0; + mockClient.submitTurn.mockImplementation(async (_runId: string, body: any) => { + if (body.promptId === "pA") { + throw fatal; + } + // Poller B: would otherwise keep going turn after turn forever. A tiny + // real delay (unlike an instantly-resolved mock) lets the event loop + // actually yield between iterations, so the stop-signal check has a + // chance to interleave instead of the loop spinning unboundedly fast. + await new Promise((resolve) => setTimeout(resolve, 1)); + submitCallsForB++; + return { done: false, nextPrompt: "next", nextTurnIndex: body.turnIndex + 1 }; + }); + + const handler = vi.fn(async () => "unused"); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-13", handler, maxConcurrency: 2 }; + + await expect(redteam.runRedteam(options)).rejects.toThrow("503 exhausted"); + + const callsAtRejection = submitCallsForB; + await new Promise((resolve) => setTimeout(resolve, 15)); + // Session B must notice the tripped stop signal on its next loop check + // rather than continuing to submit turns indefinitely. + expect(submitCallsForB).toBeLessThanOrEqual(callsAtRejection + 1); + }); + + it("handler throws -> submits {error}, turn recorded error, run still finalizes", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-6", configId: "cfg-6", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ + runId: "run-6", + status: "running", + turnType: "single", + multiTurnCount: 5, + prompts: [{ id: "p1", prompt: "p0", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }) + .mockResolvedValueOnce({ runId: "run-6", status: "completed", turnType: "single", multiTurnCount: 5, prompts: [] }); + mockClient.submitTurn.mockResolvedValueOnce({ done: true }); + mockClient.getResultsPage.mockResolvedValueOnce({ + items: [{ evaluatorId: "ev-1", status: "error" }], + page: 1, + limit: 200, + total: 1, + }); + + const handler = vi.fn(async () => { + throw new Error("agent blew up"); + }); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-6", handler, maxConcurrency: 1 }; + + const result = await redteam.runRedteam(options); + + expect(mockClient.submitTurn).toHaveBeenCalledWith("run-6", { + promptId: "p1", + sessionId: "p1", + turnIndex: 1, + promptText: "p0", + error: "agent blew up", + }); + expect(result!.success).toBe(true); + expect(result!.results[0]).toMatchObject({ status: "error" }); + }); + + it("drives multiple sessions concurrently (bounded by maxConcurrency), each independently to done", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-7", configId: "cfg-7", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ + runId: "run-7", + status: "running", + turnType: "single", + multiTurnCount: 5, + prompts: [ + { id: "pA", prompt: "attack A", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }, + { id: "pB", prompt: "attack B", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }, + ], + }) + .mockResolvedValueOnce({ runId: "run-7", status: "completed", turnType: "single", multiTurnCount: 5, prompts: [] }); + mockClient.submitTurn.mockResolvedValue({ done: true }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + const seenSessions: string[] = []; + const handler = vi.fn(async (_prompt: string, sessionId: string) => { + seenSessions.push(sessionId); + return "ok"; + }); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-7", handler, maxConcurrency: 2 }; + + await redteam.runRedteam(options); + + expect(seenSessions.sort()).toEqual(["pA", "pB"]); + const submittedSessions = mockClient.submitTurn.mock.calls.map((call: any[]) => call[1].sessionId); + expect(submittedSessions.sort()).toEqual(["pA", "pB"]); + }); + + it("interrupt (SIGINT) -> single-fire cancel call, re-raises the signal so the process still terminates, run resolves as cancelled", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-8", configId: "cfg-8", status: "running" }); + // A session that never finishes on its own — the interrupt must cut it short. + mockClient.getPrompts.mockResolvedValueOnce({ + runId: "run-8", + status: "running", + turnType: "multi", + multiTurnCount: 1000, + prompts: [{ id: "p1", prompt: "attack", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }); + mockClient.submitTurn.mockImplementation(async (_runId: string, body: any) => { + // A tiny real delay lets the interrupt actually interleave instead of + // the loop spinning unboundedly fast against an instantly-resolved mock. + await new Promise((resolve) => setTimeout(resolve, 1)); + return { done: false, nextPrompt: "next", nextTurnIndex: body.turnIndex + 1 }; + }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + // Prevent the re-raised SIGINT from actually terminating the test + // process while still letting us assert it fired (LLD §11). + const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true as any); + + const handler = vi.fn(async () => "ok"); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-8", handler, maxConcurrency: 1 }; + + const runPromise = redteam.runRedteam(options); + + // Let a couple of turns happen, then interrupt. + await new Promise((resolve) => setTimeout(resolve, 10)); + process.emit("SIGINT" as any); + process.emit("SIGINT" as any); // second emission must NOT trigger a second cancel call + + const result = await runPromise; + + expect(mockClient.cancel).toHaveBeenCalledTimes(1); + expect(mockClient.cancel).toHaveBeenCalledWith("run-8"); + expect(result!.status).toBe("cancelled"); + expect(killSpy).toHaveBeenCalledTimes(1); + expect(killSpy).toHaveBeenCalledWith(process.pid, "SIGINT"); + + killSpy.mockRestore(); + }); + + it("does NOT install uncaughtException/unhandledRejection listeners, and an unrelated error never cancels the run", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-9", configId: "cfg-9", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ + runId: "run-9", + status: "running", + turnType: "multi", + multiTurnCount: 1000, + prompts: [{ id: "p1", prompt: "attack", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }) + .mockResolvedValueOnce({ runId: "run-9", status: "completed", turnType: "multi", multiTurnCount: 1000, prompts: [] }); + mockClient.submitTurn.mockImplementation(async (_runId: string, body: any) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + return { done: false, nextPrompt: "next", nextTurnIndex: body.turnIndex + 1 }; + }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + + const sigintBefore = process.listenerCount("SIGINT"); + const sigtermBefore = process.listenerCount("SIGTERM"); + const exceptionBefore = process.listenerCount("uncaughtException"); + const rejectionBefore = process.listenerCount("unhandledRejection"); + + const handler = vi.fn(async () => "ok"); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-9", handler, maxConcurrency: 1 }; + + const runPromise = redteam.runRedteam(options); + await new Promise((resolve) => setTimeout(resolve, 5)); + + // runRedteam must never add its own uncaughtException/unhandledRejection listeners — an + // unrelated error elsewhere in the host process must not be able to cancel this run. + expect(process.listenerCount("uncaughtException")).toBe(exceptionBefore); + expect(process.listenerCount("unhandledRejection")).toBe(rejectionBefore); + // SIGINT/SIGTERM listeners ARE expected while the run is in flight. + expect(process.listenerCount("SIGINT")).toBe(sigintBefore + 1); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore + 1); + + mockClient.submitTurn.mockResolvedValue({ done: true }); + const result = await runPromise; + + expect(mockClient.cancel).not.toHaveBeenCalled(); + expect(result!.status).toBe("completed"); + // Listeners must be removed once the run settles normally (no leak). + expect(process.listenerCount("SIGINT")).toBe(sigintBefore); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore); + }); + + it("getResults pages until a short page, concatenating all items", async () => { + mockClient.getResultsPage + .mockResolvedValueOnce({ + items: Array.from({ length: 200 }, (_, i) => ({ evaluatorId: `ev-${i}`, status: "pass" })), + page: 1, + limit: 200, + total: 401, + }) + .mockResolvedValueOnce({ + items: Array.from({ length: 200 }, (_, i) => ({ evaluatorId: `ev-${200 + i}`, status: "pass" })), + page: 2, + limit: 200, + total: 401, + }) + .mockResolvedValueOnce({ + items: [{ evaluatorId: "ev-400", status: "pass" }], + page: 3, + limit: 200, + total: 401, + }); + + const redteam = new Redteam(fakeConfig); + const results = await redteam.getResults("run-9"); + + expect(results).toHaveLength(401); + expect(mockClient.getResultsPage).toHaveBeenCalledTimes(3); + expect(mockClient.getResultsPage).toHaveBeenNthCalledWith(1, "run-9", { page: 1, limit: 200 }); + expect(mockClient.getResultsPage).toHaveBeenNthCalledWith(2, "run-9", { page: 2, limit: 200 }); + expect(mockClient.getResultsPage).toHaveBeenNthCalledWith(3, "run-9", { page: 3, limit: 200 }); + }); + + it("returns null for invalid input without any network call", async () => { + const redteam = new Redteam(fakeConfig); + const result = await redteam.runRedteam({ handler: "not-a-fn" } as any); + expect(result).toBeNull(); + expect(mockClient.createRun).not.toHaveBeenCalled(); + }); +}); diff --git a/src/api/redteam/__tests__/client.test.ts b/src/api/redteam/__tests__/client.test.ts new file mode 100644 index 0000000..f0d4366 --- /dev/null +++ b/src/api/redteam/__tests__/client.test.ts @@ -0,0 +1,294 @@ +import { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } from "@opentelemetry/core"; +import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; +import { + context, + propagation, + trace, + TraceFlags, +} from "@opentelemetry/api"; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const mockInstance = { + get: vi.fn(), + post: vi.fn(), + interceptors: { + request: { + use: vi.fn(), + }, + }, +}; + +vi.mock("axios", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { + ...actual.default, + create: vi.fn(() => mockInstance), + }, + }; +}); + +import axios from "axios"; +import { Config } from "../../../config"; +import { + RedteamAuthError, + RedteamConfigError, + RedteamGenerationError, + RedteamGenerationTimeoutError, + RedteamRunError, +} from "../models"; +import { RedteamHttpClient } from "../client"; + +function buildConfig(otlpEndpoint: string): Config { + process.env.NETRA_OTLP_ENDPOINT = otlpEndpoint; + process.env.NETRA_API_KEY = "test-api-key"; + return new Config({}); +} + +describe("RedteamHttpClient", () => { + beforeAll(() => { + propagation.setGlobalPropagator( + new CompositePropagator({ + propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()], + }), + ); + // A real (not Noop) context manager is required for context.with(...) to + // actually make the span context "active" for propagation.inject to see. + context.setGlobalContextManager(new AsyncHooksContextManager().enable()); + }); + + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.NETRA_OTLP_ENDPOINT; + delete process.env.NETRA_API_KEY; + }); + + afterEach(() => { + delete process.env.NETRA_OTLP_ENDPOINT; + delete process.env.NETRA_API_KEY; + }); + + it("strips a trailing /telemetry (and trailing slash) from the base URL", () => { + const cfg = buildConfig("https://api.getnetra.ai/telemetry/"); + new RedteamHttpClient(cfg); + expect(axios.create).toHaveBeenCalledWith( + expect.objectContaining({ baseURL: "https://api.getnetra.ai" }), + ); + }); + + it("injects the x-api-key header from config.apiKey", () => { + const cfg = buildConfig("https://api.getnetra.ai"); + new RedteamHttpClient(cfg); + expect(axios.create).toHaveBeenCalledWith( + expect.objectContaining({ headers: expect.objectContaining({ "x-api-key": "test-api-key" }) }), + ); + }); + + it("registers a request interceptor that injects traceparent headers", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + new RedteamHttpClient(cfg); + + expect(mockInstance.interceptors.request.use).toHaveBeenCalled(); + const [onFulfilled] = mockInstance.interceptors.request.use.mock.calls[0]; + + const spanContext = { + traceId: "0af7651916cd43dd8448eb211c80319c", + spanId: "b7ad6b7169203331", + traceFlags: TraceFlags.SAMPLED, + }; + const ctx = trace.setSpanContext(context.active(), spanContext); + + const fakeRequestConfig = { headers: {} as Record }; + const result = await context.with(ctx, () => onFulfilled(fakeRequestConfig)); + + expect(result.headers.traceparent).toBeDefined(); + expect(result.headers.traceparent).toContain(spanContext.traceId); + }); + + it("unwraps a single {data} envelope on createRun", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.post.mockResolvedValueOnce({ + data: { success: true, data: { runId: "run-1", configId: "cfg-1", status: "running" } }, + }); + const result = await client.createRun({ configId: "cfg-1" }); + expect(result).toEqual({ runId: "run-1", configId: "cfg-1", status: "running" }); + }); + + describe("getPrompts", () => { + it("fetches the run's whole prompt list in one call", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.get.mockResolvedValueOnce({ + data: { + success: true, + data: { + runId: "run-1", + status: "running", + turnType: "single", + multiTurnCount: 5, + prompts: [{ id: "p1", prompt: "attack", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }, + }, + }); + + const result = await client.getPrompts("run-1"); + + expect(mockInstance.get).toHaveBeenCalledWith("/redteam/sdk/runs/run-1/prompts"); + expect(result).toMatchObject({ + runId: "run-1", + status: "running", + prompts: [{ id: "p1", prompt: "attack", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }); + }); + }); + + describe("submitTurn", () => { + it("posts the turn body and returns {done, nextPrompt?, nextTurnIndex?}", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.post.mockResolvedValueOnce({ + data: { success: true, data: { done: false, nextPrompt: "turn 2", nextTurnIndex: 2 } }, + }); + const body = { promptId: "p1", sessionId: "s1", turnIndex: 1, promptText: "attack", output: "ok" }; + + const result = await client.submitTurn("run-1", body); + + expect(mockInstance.post).toHaveBeenCalledWith("/redteam/sdk/runs/run-1/turns", body); + expect(result).toEqual({ done: false, nextPrompt: "turn 2", nextTurnIndex: 2 }); + }); + + it("surfaces an exact-duplicate-turn (409) submission as {done: true} instead of throwing", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.post.mockRejectedValueOnce({ + isAxiosError: true, + response: { status: 409, data: { success: false, error: { message: "already submitted" } } }, + message: "Request failed with status code 409", + }); + const result = await client.submitTurn("run-1", { + promptId: "p1", + sessionId: "s1", + turnIndex: 1, + promptText: "attack", + output: "ok", + }); + expect(result).toEqual({ done: true }); + }); + }); + + describe("_extractErrorMessage / typed error mapping via createRun", () => { + const cases: Array<[number, any]> = [ + [400, RedteamConfigError], + [401, RedteamAuthError], + [403, RedteamAuthError], + [404, RedteamConfigError], + [409, RedteamRunError], + [422, RedteamConfigError], + [502, RedteamGenerationError], + [503, RedteamGenerationTimeoutError], + ]; + + for (const [status, ErrorClass] of cases) { + it(`maps HTTP ${status} to ${ErrorClass.name}`, async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + // Persistent (not "Once") since 502/503 are retried by the client's + // bounded-retry wrapper before finally throwing. + mockInstance.post.mockRejectedValue({ + isAxiosError: true, + response: { status, data: { success: false, error: { message: `error ${status}` } } }, + message: `Request failed with status code ${status}`, + }); + await expect(client.createRun({ configId: "cfg-1" })).rejects.toBeInstanceOf(ErrorClass); + }); + } + + it("uses the response envelope's error.message when present", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.post.mockRejectedValue({ + isAxiosError: true, + response: { status: 404, data: { success: false, error: { message: "config not found" } } }, + message: "Request failed with status code 404", + }); + await expect(client.createRun({ configId: "cfg-1" })).rejects.toThrow("config not found"); + }); + + it("passes through a non-axios Error unchanged", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + const boom = new Error("boom"); + mockInstance.post.mockRejectedValue(boom); + await expect(client.createRun({ configId: "cfg-1" })).rejects.toBe(boom); + }); + + it("retries a network error (no response) up to MAX_RETRIES before throwing", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + const networkError = { isAxiosError: true, message: "Network Error" }; + mockInstance.post.mockRejectedValue(networkError); + await expect(client.createRun({ configId: "cfg-1" })).rejects.toBeInstanceOf(Error); + // 1 initial attempt + 2 retries = 3 calls. + expect(mockInstance.post).toHaveBeenCalledTimes(3); + }); + + it("does not retry a 400 (non-transient) error", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.post.mockRejectedValue({ + isAxiosError: true, + response: { status: 400, data: { success: false, error: { message: "bad request" } } }, + message: "Request failed with status code 400", + }); + await expect(client.createRun({ configId: "cfg-1" })).rejects.toBeInstanceOf(RedteamConfigError); + expect(mockInstance.post).toHaveBeenCalledTimes(1); + }); + }); + + it("getProgress returns the unwrapped envelope contents", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.get.mockResolvedValueOnce({ data: { success: true, data: { completedSessions: 3 } } }); + const result = await client.getProgress("run-1"); + expect(result).toEqual({ completedSessions: 3 }); + }); + + it("getResultsPage sends page/limit/evaluatorId query params with defaults", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.get.mockResolvedValueOnce({ + data: { success: true, data: { data: [], page: 1, limit: 200, total: 0, hasNextPage: false } }, + }); + await client.getResultsPage("run-1"); + expect(mockInstance.get).toHaveBeenCalledWith("/redteam/sdk/runs/run-1/results", { + params: { page: 1, limit: 200, evaluatorId: undefined }, + }); + }); + + it("getRiskScore hits the config-scoped endpoint and returns the raw payload", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.get.mockResolvedValueOnce({ data: { success: true, data: { latestSafetyScore: 77 } } }); + const result = await client.getRiskScore("cfg-1"); + expect(result).toEqual({ latestSafetyScore: 77 }); + expect(mockInstance.get).toHaveBeenCalledWith("/redteam/sdk/configs/cfg-1/risk-score"); + }); + + it("cancel posts to the cancel endpoint and returns the unwrapped status", async () => { + const cfg = buildConfig("https://api.getnetra.ai"); + const client = new RedteamHttpClient(cfg); + mockInstance.post.mockResolvedValueOnce({ data: { success: true, data: { status: "cancelled" } } }); + const result = await client.cancel("run-1"); + expect(result).toEqual({ status: "cancelled" }); + expect(mockInstance.post).toHaveBeenCalledWith("/redteam/sdk/runs/run-1/cancel"); + }); + + it("throws RedteamAuthError when the client was never initialized (no endpoint)", async () => { + const cfg = new Config({}); + const client = new RedteamHttpClient(cfg); + expect(client.isInitialized()).toBe(false); + await expect(client.createRun({ configId: "cfg-1" })).rejects.toBeInstanceOf(RedteamAuthError); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/helpers.ts b/src/api/redteam/__tests__/e2e/helpers.ts new file mode 100644 index 0000000..08be594 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/helpers.ts @@ -0,0 +1,35 @@ +/** + * QA fixture — NOT product source. Shared setup helpers for the red-team SDK + * E2E suite. Constructs the REAL `Redteam` client (real `RedteamHttpClient`/ + * axios, no mocking of the client itself) wired at a real HTTP loopback + * connection to `MockRedteamBackend`. + */ +import { Config } from "../../../../config"; +import { Redteam } from "../../api"; +import { MockRedteamBackend } from "./mock-backend"; + +/** Build a real `Redteam` client pointed at `backend.url`, authenticated as `apiKey`. */ +export function newClient(backend: MockRedteamBackend, apiKey: string, extraEnv: Record = {}): Redteam { + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = apiKey; + for (const [k, v] of Object.entries(extraEnv)) { + process.env[k] = v; + } + const config = new Config({}); + return new Redteam(config); +} + +/** Reset the subset of env vars this suite touches, so tests don't leak config into each other. */ +export function resetRedteamEnv(): void { + delete process.env.NETRA_OTLP_ENDPOINT; + delete process.env.NETRA_API_KEY; + delete process.env.NETRA_REDTEAM_GENERATION_POLL_INTERVAL; + delete process.env.NETRA_REDTEAM_GENERATION_TIMEOUT; + delete process.env.NETRA_REDTEAM_TIMEOUT; +} + +/** Fast generation-gating poll, for tests that don't care about the "generating" gate's real timing. */ +export const FAST_POLL_ENV = { + NETRA_REDTEAM_GENERATION_POLL_INTERVAL: "0.05", + NETRA_REDTEAM_GENERATION_TIMEOUT: "5", +}; diff --git a/src/api/redteam/__tests__/e2e/mock-backend.ts b/src/api/redteam/__tests__/e2e/mock-backend.ts new file mode 100644 index 0000000..f8c395a --- /dev/null +++ b/src/api/redteam/__tests__/e2e/mock-backend.ts @@ -0,0 +1,534 @@ +/** + * QA fixture — NOT product source. + * + * A contract-faithful in-process mock of the `/redteam/sdk/*` API surface. + * It exists so the QA E2E suite can drive the REAL `netra-sdk-js` client + * (`RedteamHttpClient` + `Redteam`, unmocked) over a real HTTP loopback + * connection, exercising the actual wire contract, the client-driven turn + * loop, and error mapping end-to-end — without requiring the full backend + * NestJS app (Postgres/ClickHouse/Redis/LLM credentials), which is + * infeasible in this sandbox. + * + * This module implements the same *behavioral contract* the real backend + * does (envelope shape, whole-list prompt fetch, per-session vs per-run + * `done` scoping, optimistic concurrency on `turns` via duplicate-(runId, + * promptId, turnIndex) rejection, tenant scoping) so that black-box + * assertions made against it are meaningful re-checks of the contract, not + * tautologies. It does not implement real judge/attacker LLM calls (there + * are no credentials in this sandbox) — turn + * decisions are deterministic fakes. + */ +import http from "node:http"; +import { randomUUID } from "node:crypto"; +import type { AddressInfo } from "node:net"; + +export interface MockTenant { + apiKey: string; + orgId: string; + projectId: string; + featureFlagEnabled: boolean; +} + +export interface MockAgent { + id: string; + projectId: string; + hasSystemPrompt: boolean; +} + +export interface MockEvaluator { + id: string; + slug: string; + requiresSystemPrompt: boolean; + isJailbreak: boolean; +} + +export type GenerationMode = "immediate" | "delayed" | "failed" | "unavailable"; + +export interface MockConfigInput { + id?: string; + projectId: string; + orgId: string; + agentId: string; + evaluatorIds: string[]; + turnType?: "single" | "multi"; + sessionsPerEvaluator?: number; + multiTurnCount?: number; + generationMode?: GenerationMode; + /** For generationMode "delayed": number of "generating" answers before flipping to "running". */ + delayedFlipAfterPolls?: number; +} + +interface ConfigRecord extends Required> { + delayedFlipAfterPolls: number; + pollCount: number; + hasActiveRun: boolean; +} + +/** + * One generated prompt for a run — the unit of work the SDK client fetches + * once (via `GET .../prompts`) and drives to completion itself. No + * server-side claim/lease of any kind (revision 7 redesign): `submittedTurns` + * only exists here to detect an exact-duplicate resubmission (409), the same + * network-retry guard the real backend's `hasResultForTurn` check provides. + */ +interface PromptRecord { + runId: string; + promptId: string; + prompt: string; + evaluatorId: string; + evaluatorSlug: string; + status: "pending" | "done" | "failed"; + currentTurnIndex: number; + priorTurns: { role: string; content: string }[]; + submittedTurns: Set; +} + +interface RunRecord { + id: string; + configId: string; + projectId: string; + orgId: string; + status: "running" | "completed" | "failed" | "cancelled"; + triggeredBy: string | null; + promptIds: string[]; + results: any[]; +} + +export interface LoggedRequest { + method: string; + path: string; + query: Record; + body: any; + apiKey: string | null; +} + +const EARLY_STOP_SENTINEL = "STOP_EARLY"; +const JAILBREAK_CAP = 4; + +export class MockRedteamBackend { + tenants = new Map(); // keyed by apiKey + agents = new Map(); + evaluators = new Map(); + configs = new Map(); + runs = new Map(); + promptRecords = new Map(); // key `${runId}::${promptId}` + + requestLog: LoggedRequest[] = []; + + private server: http.Server; + private _url = ""; + + constructor() { + this.server = http.createServer((req, res) => this._handle(req, res)); + } + + async start(): Promise { + await new Promise((resolve) => this.server.listen(0, "127.0.0.1", resolve)); + const addr = this.server.address() as AddressInfo; + this._url = `http://127.0.0.1:${addr.port}`; + return this._url; + } + + async stop(): Promise { + await new Promise((resolve, reject) => + this.server.close((err) => (err ? reject(err) : resolve())), + ); + } + + get url(): string { + return this._url; + } + + // --------------------------------------------------------------------- + // Fixture setup helpers (used by tests to seed state) + // --------------------------------------------------------------------- + + addTenant(t: Partial = {}): MockTenant { + const tenant: MockTenant = { + apiKey: t.apiKey ?? `key-${randomUUID()}`, + orgId: t.orgId ?? `org-${randomUUID()}`, + projectId: t.projectId ?? `proj-${randomUUID()}`, + featureFlagEnabled: t.featureFlagEnabled ?? true, + }; + this.tenants.set(tenant.apiKey, tenant); + return tenant; + } + + addAgent(a: Partial & { projectId: string }): MockAgent { + const agent: MockAgent = { + id: a.id ?? `agent-${randomUUID()}`, + projectId: a.projectId, + hasSystemPrompt: a.hasSystemPrompt ?? true, + }; + this.agents.set(agent.id, agent); + return agent; + } + + addEvaluator(e: Partial = {}): MockEvaluator { + const ev: MockEvaluator = { + id: e.id ?? `eval-${randomUUID()}`, + slug: e.slug ?? "harmful-content", + requiresSystemPrompt: e.requiresSystemPrompt ?? false, + isJailbreak: e.isJailbreak ?? false, + }; + this.evaluators.set(ev.id, ev); + return ev; + } + + addConfig(c: MockConfigInput): ConfigRecord { + const record: ConfigRecord = { + id: c.id ?? `cfg-${randomUUID()}`, + projectId: c.projectId, + orgId: c.orgId, + agentId: c.agentId, + evaluatorIds: c.evaluatorIds, + turnType: c.turnType ?? "single", + sessionsPerEvaluator: c.sessionsPerEvaluator ?? 1, + multiTurnCount: c.multiTurnCount ?? 3, + generationMode: c.generationMode ?? "immediate", + delayedFlipAfterPolls: c.delayedFlipAfterPolls ?? 1, + pollCount: 0, + hasActiveRun: false, + }; + this.configs.set(record.id, record); + return record; + } + + getRun(runId: string): RunRecord | undefined { + return this.runs.get(runId); + } + + getConfig(configId: string): ConfigRecord | undefined { + return this.configs.get(configId); + } + + /** Directly seed a run's persisted result rows, bypassing the turn loop (QA-fixture-only; used for pagination-boundary testing, TC-42). */ + seedRunDone(runId: string, items: any[]): void { + const run = this.runs.get(runId); + if (!run) throw new Error(`seedRunDone: no such run ${runId}`); + run.results.push(...items); + run.status = "completed"; + for (const promptId of run.promptIds) { + const record = this.promptRecords.get(`${runId}::${promptId}`); + if (record) record.status = "done"; + } + } + + private _attackType(config: ConfigRecord): "single" | "multi" | "jailbreak" { + const hasJailbreak = config.evaluatorIds.some((id) => this.evaluators.get(id)?.isJailbreak); + if (hasJailbreak) return "jailbreak"; + return config.turnType; + } + + private _seedPromptsForRun(run: RunRecord, config: ConfigRecord): void { + const evaluatorId = config.evaluatorIds[0]; + const evaluatorSlug = this.evaluators.get(evaluatorId)?.slug ?? "harmful-content"; + for (let i = 0; i < config.sessionsPerEvaluator; i++) { + const promptId = `prompt-${randomUUID()}`; + run.promptIds.push(promptId); + this.promptRecords.set(`${run.id}::${promptId}`, { + runId: run.id, + promptId, + prompt: `adversarial prompt (${promptId})`, + evaluatorId, + evaluatorSlug, + status: "pending", + currentTurnIndex: 1, + priorTurns: [], + submittedTurns: new Set(), + }); + } + } + + // --------------------------------------------------------------------- + // HTTP handling + // --------------------------------------------------------------------- + + private async _handle(req: http.IncomingMessage, res: http.ServerResponse): Promise { + const url = new URL(req.url ?? "/", "http://localhost"); + const path = url.pathname; + const query: Record = {}; + url.searchParams.forEach((v, k) => (query[k] = v)); + + let body: any = undefined; + if (req.method === "POST") { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(chunk as Buffer); + const raw = Buffer.concat(chunks).toString("utf8"); + body = raw ? JSON.parse(raw) : {}; + } + + const apiKey = (req.headers["x-api-key"] as string) ?? null; + this.requestLog.push({ method: req.method ?? "", path, query, body, apiKey }); + + const send = (status: number, payload: any) => { + const json = JSON.stringify(payload); + res.writeHead(status, { "content-type": "application/json" }); + res.end(json); + }; + const ok = (data: any, meta: any = {}) => send(200, { success: true, data, error: null, meta }); + const accepted = (data: any) => send(202, { success: true, data, error: null, meta: {} }); + const errorEnv = (status: number, code: string, message: string) => + send(status, { success: false, data: null, error: { code, error: code, message }, meta: {} }); + + try { + if (!apiKey || !this.tenants.has(apiKey)) { + return errorEnv(401, "UNAUTHORIZED", "missing or invalid x-api-key"); + } + const tenant = this.tenants.get(apiKey)!; + if (!tenant.featureFlagEnabled) { + return errorEnv(403, "FORBIDDEN", "red-teaming not enabled for this org"); + } + + // POST /redteam/sdk/runs + if (req.method === "POST" && path === "/redteam/sdk/runs") { + return this._createRun(tenant, body, { ok, accepted, errorEnv }); + } + + const runMatch = path.match(/^\/redteam\/sdk\/runs\/([^/]+)\/(.+)$/); + if (runMatch) { + const [, runId, sub] = runMatch; + if (req.method === "GET" && sub === "prompts") { + return this._getPrompts(tenant, runId, { ok, errorEnv }); + } + if (req.method === "POST" && sub === "turns") { + return this._submitTurn(tenant, runId, body, { ok, errorEnv }); + } + if (req.method === "GET" && sub === "progress") { + return this._progress(tenant, runId, { ok, errorEnv }); + } + if (req.method === "GET" && sub === "results") { + return this._results(tenant, runId, query, { ok, errorEnv }); + } + if (req.method === "POST" && sub === "cancel") { + return this._cancel(tenant, runId, { ok, errorEnv }); + } + } + + const riskMatch = path.match(/^\/redteam\/sdk\/configs\/([^/]+)\/risk-score$/); + if (riskMatch && req.method === "GET") { + return this._riskScore(tenant, riskMatch[1], { ok, errorEnv }); + } + + send(404, { success: false, data: null, error: { code: "NOT_FOUND", error: "NOT_FOUND", message: "no such route" }, meta: {} }); + } catch (e) { + send(500, { success: false, data: null, error: { code: "INTERNAL", error: "INTERNAL", message: String(e) }, meta: {} }); + } + } + + private _createRun( + tenant: MockTenant, + body: any, + h: { ok: Function; accepted: Function; errorEnv: Function }, + ) { + const hasConfigId = body && typeof body.configId === "string"; + if (!hasConfigId) { + return h.errorEnv(400, "BAD_REQUEST", "configId is required"); + } + + const config = this.configs.get(body.configId); + if (!config || config.projectId !== tenant.projectId) { + return h.errorEnv(404, "NOT_FOUND", "config not found or not in this project"); + } + + // One active run per config, practically deduping repeated triggers. + if (config.hasActiveRun) { + return h.errorEnv(409, "CONFLICT", "a run is already active for this config"); + } + + if (config.generationMode === "unavailable") { + return h.errorEnv(503, "SERVICE_UNAVAILABLE", "generation did not complete (worker unavailable?)"); + } + if (config.generationMode === "failed") { + return h.errorEnv(502, "BAD_GATEWAY", "prompt generation failed"); + } + if (config.generationMode === "delayed") { + config.pollCount++; + if (config.pollCount <= config.delayedFlipAfterPolls) { + return h.accepted({ configId: config.id, status: "generating" }); + } + // fall through to running below (flip happens once threshold passed) + } + + // status === 'immediate', or 'delayed' past its flip threshold: create + start the run. + config.hasActiveRun = true; + const run: RunRecord = { + id: `run-${randomUUID()}`, + configId: config.id, + projectId: tenant.projectId, + orgId: tenant.orgId, + status: "running", + triggeredBy: null, + promptIds: [], + results: [], + }; + this.runs.set(run.id, run); + this._seedPromptsForRun(run, config); + if (run.promptIds.length === 0) { + // Empty run (zero prompts, TC-45): nothing to service — finalize immediately. + run.status = "completed"; + } + return h.accepted({ runId: run.id, configId: config.id, status: "running" }); + } + + private _runFullyDone(runId: string): boolean { + const records = [...this.promptRecords.values()].filter((r) => r.runId === runId); + if (records.length === 0) { + // No prompts were ever seeded for this run (TC-45, empty run) — done + // iff the run itself was already finalized at creation time. + const run = this.runs.get(runId); + return !!run && run.status !== "running"; + } + return records.every((r) => r.status === "done" || r.status === "failed"); + } + + /** + * `GET .../prompts` — the client fetches this ONCE per run, then drives + * every prompt's session to completion itself. No server-side claim of any + * kind (revision 7 redesign). + */ + private _getPrompts(tenant: MockTenant, runId: string, h: { ok: Function; errorEnv: Function }) { + const run = this.runs.get(runId); + if (!run || run.projectId !== tenant.projectId) return h.errorEnv(404, "NOT_FOUND", "run not found"); + const config = this.configs.get(run.configId)!; + const prompts = run.promptIds.map((promptId) => { + const record = this.promptRecords.get(`${runId}::${promptId}`)!; + return { + id: record.promptId, + prompt: record.prompt, + evaluatorId: record.evaluatorId, + evaluatorSlug: record.evaluatorSlug, + }; + }); + h.ok({ + runId, + status: run.status, + turnType: config.turnType, + multiTurnCount: config.multiTurnCount, + prompts, + }); + } + + /** + * `POST .../turns` — submits one turn's result for one prompt/session, + * identified directly by `promptId` (no server-issued invocation id, no + * persisted turn-state). Prior turns are reconstructed from this prompt's + * own accumulated `priorTurns`, mirroring how the real backend rebuilds + * them from `redteam_run_results` rows. + */ + private _submitTurn(tenant: MockTenant, runId: string, body: any, h: { ok: Function; errorEnv: Function }) { + const run = this.runs.get(runId); + if (!run || run.projectId !== tenant.projectId) return h.errorEnv(404, "NOT_FOUND", "run not found"); + if (!body || typeof body.promptId !== "string" || typeof body.turnIndex !== "number") { + return h.errorEnv(422, "UNPROCESSABLE", "promptId and turnIndex required"); + } + if (body.output === undefined && body.error === undefined) { + return h.errorEnv(422, "UNPROCESSABLE", "neither output nor error supplied"); + } + + const record = this.promptRecords.get(`${runId}::${body.promptId}`); + if (!record) { + return h.errorEnv(404, "NOT_FOUND", "promptId does not belong to this runId"); + } + if (record.submittedTurns.has(body.turnIndex)) { + // Exact-duplicate submission (network-retry guard) — 409, matching the + // real backend's `hasResultForTurn` check. + return h.errorEnv(409, "CONFLICT", "this turn has already been submitted"); + } + record.submittedTurns.add(body.turnIndex); + + const config = this.configs.get(run.configId)!; + const attackType = this._attackType(config); + + const output = body.error !== undefined ? `[error: ${body.error}]` : String(body.output); + run.results.push({ + evaluatorId: record.evaluatorId, + evaluatorSlug: record.evaluatorSlug, + status: body.error !== undefined ? "error" : "pass", + score: body.error !== undefined ? null : 1, + judgeOutput: body.error !== undefined ? `handler error: ${body.error}` : "no leak detected", + sessionId: body.sessionId, + turnIndex: body.turnIndex, + conversationHistory: [ + { role: "user", content: body.promptText }, + { role: "assistant", content: output }, + ], + }); + record.priorTurns.push({ role: "user", content: body.promptText }, { role: "assistant", content: output }); + + if (body.error !== undefined) { + record.status = "failed"; + if (this._runFullyDone(runId)) { + run.status = "completed"; + run.triggeredBy = "org-owner-fixture"; + } + return h.ok({ done: true }); + } + + const cap = attackType === "jailbreak" ? JAILBREAK_CAP : attackType === "multi" ? config.multiTurnCount : 1; + const earlyStop = typeof body.output === "string" && body.output.includes(EARLY_STOP_SENTINEL); + + let done: boolean; + let nextPrompt: string | undefined; + let nextTurnIndex: number | undefined; + if (earlyStop || body.turnIndex >= cap) { + record.status = "done"; + done = true; + } else { + record.currentTurnIndex = body.turnIndex + 1; + nextPrompt = `adversarial prompt turn ${record.currentTurnIndex} (${record.promptId})`; + nextTurnIndex = record.currentTurnIndex; + done = false; + } + + // Recompute run-level completion for observability/cancel semantics. + if (this._runFullyDone(runId)) { + run.status = "completed"; + run.triggeredBy = "org-owner-fixture"; + } + + return h.ok(done ? { done: true } : { done: false, nextPrompt, nextTurnIndex }); + } + + private _progress(tenant: MockTenant, runId: string, h: { ok: Function; errorEnv: Function }) { + const run = this.runs.get(runId); + if (!run || run.projectId !== tenant.projectId) return h.errorEnv(404, "NOT_FOUND", "run not found"); + const total = run.promptIds.length; + const doneCount = [...this.promptRecords.values()].filter((r) => r.runId === runId && r.status === "done").length; + h.ok({ runId, status: run.status, totalSessions: total, completedSessions: doneCount }); + } + + private _results(tenant: MockTenant, runId: string, query: Record, h: { ok: Function; errorEnv: Function }) { + const run = this.runs.get(runId); + if (!run || run.projectId !== tenant.projectId) return h.errorEnv(404, "NOT_FOUND", "run not found"); + const page = Number(query.page ?? "1"); + const limit = Number(query.limit ?? "200"); + const start = (page - 1) * limit; + const items = run.results.slice(start, start + limit); + // Field name is `data` (matching the real backend's paginated response DTO), not `items` — + // `items` is this SDK's own post-mapping `RunResultsPage` field name (see mapResultsPage). + h.ok({ data: items, page, limit, total: run.results.length, hasNextPage: start + limit < run.results.length }); + } + + private _riskScore(tenant: MockTenant, configId: string, h: { ok: Function; errorEnv: Function }) { + const config = this.configs.get(configId); + if (!config || config.projectId !== tenant.projectId) return h.errorEnv(404, "NOT_FOUND", "config not found"); + h.ok({ configId, latestSafetyScore: 92, change: -3, history: [{ at: new Date().toISOString(), score: 92 }] }); + } + + private _cancel(tenant: MockTenant, runId: string, h: { ok: Function; errorEnv: Function }) { + const run = this.runs.get(runId); + if (!run || run.projectId !== tenant.projectId) return h.errorEnv(404, "NOT_FOUND", "run not found"); + if (run.status !== "running") { + return h.errorEnv(409, "CONFLICT", "run is not currently RUNNING"); + } + run.status = "cancelled"; + for (const promptId of run.promptIds) { + const record = this.promptRecords.get(`${runId}::${promptId}`); + if (record) record.status = "done"; + } + const config = this.configs.get(run.configId); + if (config) config.hasActiveRun = false; + h.ok({ status: "cancelled" }); + } +} diff --git a/src/api/redteam/__tests__/e2e/tc-01-04-run-creation.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-01-04-run-creation.e2e.test.ts new file mode 100644 index 0000000..a65012c --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-01-04-run-creation.e2e.test.ts @@ -0,0 +1,173 @@ +/** + * E2E (real SDK client `Netra.redteam.runRedteam` + real HTTP loopback to a + * contract-faithful mock backend — see mock-backend.ts for the feasibility + * rationale): run creation, triggering an existing config. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import { RedteamConfigError, RedteamRunError } from "../../models"; + +describe("TC-01..TC-04 — Run creation", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + it("TC-01: trigger an existing config, single-turn, happy path", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator({ slug: "harmful-content" }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + turnType: "single", + }); + + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + const result = await client.runRedteam({ + configId: config.id, + handler: async () => "my agent's reply", + }); + + expect(result).not.toBeNull(); + expect(result!.success).toBe(true); + expect(result!.status).toBe("completed"); + expect(result!.results).toHaveLength(1); + expect(result!.riskScore).toBeDefined(); + }); + + it("multi-turn config: multiple turns with incrementing turnIndex", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator({ slug: "harmful-content" }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + turnType: "multi", + multiTurnCount: 3, + }); + + const seenTurnIndexes: number[] = []; + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + const result = await client.runRedteam({ + configId: config.id, + handler: async (_prompt, _sessionId, turnIndex) => { + seenTurnIndexes.push(turnIndex); + return "reply"; + }, + }); + + expect(result!.success).toBe(true); + expect(seenTurnIndexes).toEqual([1, 2, 3]); + }); + + it("iterative-jailbreak config: selected purely by evaluator slug, independent of turnType", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const jailbreakEvaluator = backend.addEvaluator({ slug: "system-prompt-jailbreak", isJailbreak: true }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [jailbreakEvaluator.id], + // turnType left at its "single" default — the server selects jailbreak + // behavior from the evaluator slug alone, independent of turnType. + }); + + const seenTurnIndexes: number[] = []; + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + const result = await client.runRedteam({ + configId: config.id, + handler: async (_prompt, _sessionId, turnIndex) => { + seenTurnIndexes.push(turnIndex); + return "reply"; + }, + }); + + expect(result!.success).toBe(true); + expect(seenTurnIndexes.length).toBeGreaterThan(1); + expect(seenTurnIndexes).toEqual([...seenTurnIndexes].sort((a, b) => a - b)); + }); + + it("TC-02: config still generating — SDK retries createRun transparently, no caller-visible error", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + generationMode: "delayed", + delayedFlipAfterPolls: 2, + }); + + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + const result = await client.runRedteam({ + configId: config.id, + handler: async () => "reply", + }); + + expect(result).not.toBeNull(); + expect(result!.success).toBe(true); + + const createRunCalls = backend.requestLog.filter( + (r) => r.method === "POST" && r.path === "/redteam/sdk/runs", + ); + // At least the initial call + the calls that saw "generating". + expect(createRunCalls.length).toBeGreaterThan(1); + for (const call of createRunCalls) { + expect(call.body).toEqual({ configId: config.id }); + } + }); + + it("TC-03: a config belonging to another tenant/project — 404, RedteamConfigError, no turn loop starts", async () => { + const tenantA = backend.addTenant(); + const tenantB = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenantB.projectId }); + const evaluator = backend.addEvaluator(); + const foreignConfig = backend.addConfig({ + projectId: tenantB.projectId, + orgId: tenantB.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + }); + + const client = newClient(backend, tenantA.apiKey, FAST_POLL_ENV); + await expect( + client.runRedteam({ configId: foreignConfig.id, handler: async () => "x" }), + ).rejects.toBeInstanceOf(RedteamConfigError); + + const promptsCalls = backend.requestLog.filter((r) => r.path.includes("/prompts")); + expect(promptsCalls).toHaveLength(0); + }); + + it("TC-04: a config with an already-active run — 409, RedteamRunError", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + }); + config.hasActiveRun = true; // simulate another RUNNING run for this config + + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + await expect( + client.runRedteam({ configId: config.id, handler: async () => "x" }), + ).rejects.toBeInstanceOf(RedteamRunError); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-11-14-input-validation.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-11-14-input-validation.e2e.test.ts new file mode 100644 index 0000000..955a490 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-11-14-input-validation.e2e.test.ts @@ -0,0 +1,63 @@ +/** + * E2E: input validation. These are rejected client-side before any network + * call reaches the (real, running) mock backend — asserted here by checking + * the request log stays empty, proving the real SDK's runtime guard actually + * fires. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import type { RedteamRunOptions } from "../../models"; + +describe("TC-11..TC-14 — Input validation", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + it("TC-12: configId missing — rejected client-side, no network call", async () => { + const tenant = backend.addTenant(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const options = { handler: async () => "reply" } as unknown as RedteamRunOptions; + + const result = await client.runRedteam(options); + expect(result).toBeNull(); + expect(backend.requestLog).toHaveLength(0); + }); + + it("TC-14: unknown extra fields on the options object are not forwarded to the wire", async () => { + // A caller who bypasses TypeScript (plain JS, or an `as any` cast) must + // not get any of these forwarded — `buildCreateRunBody` keys only on + // `configId`. + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const bypassed = { + configId: config.id, + unknownField: "should-not-be-forwarded", + multiTurnCount: 99, + handler: async () => "reply", + } as unknown as RedteamRunOptions; + + const result = await client.runRedteam(bypassed); + expect(result!.success).toBe(true); + + const createRunCall = backend.requestLog.find((r) => r.method === "POST" && r.path === "/redteam/sdk/runs"); + expect(createRunCall!.body).toEqual({ configId: config.id }); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-15-23-callback-contract.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-15-23-callback-contract.e2e.test.ts new file mode 100644 index 0000000..80c63a7 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-15-23-callback-contract.e2e.test.ts @@ -0,0 +1,218 @@ +/** + * E2E: the local-agent callback contract. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import type { RedteamRunOptions } from "../../models"; + +describe("TC-15..TC-23 — Callback contract", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + function seedSingleTurnConfig(sessionsPerEvaluator = 1) { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator({ slug: "harmful-content" }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + sessionsPerEvaluator, + }); + return { tenant, config }; + } + + it("TC-15: handler is a plain arrow function (no class) — accepted, called successfully", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + let called = false; + + const result = await client.runRedteam({ + configId: config.id, + handler: async (prompt, sessionId, turnIndex) => { + called = true; + expect(typeof prompt).toBe("string"); + expect(typeof sessionId).toBe("string"); + expect(typeof turnIndex).toBe("number"); + return "text"; + }, + }); + + expect(called).toBe(true); + expect(result!.success).toBe(true); + }); + + it("TC-16: handler is not a function — rejected client-side before any network call", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const withObject = { configId: config.id, handler: {} } as unknown as RedteamRunOptions; + expect(await client.runRedteam(withObject)).toBeNull(); + + const withUndefined = { configId: config.id, handler: undefined } as unknown as RedteamRunOptions; + expect(await client.runRedteam(withUndefined)).toBeNull(); + + expect(backend.requestLog).toHaveLength(0); + }); + + it("TC-17: handler returns a bare string — treated as the agent's message with no sessionId override", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const result = await client.runRedteam({ + configId: config.id, + handler: async () => "my reply", + }); + + expect(result!.success).toBe(true); + const submitCall = backend.requestLog.find((r) => r.path.includes("turns")); + expect(submitCall!.body.output).toBe("my reply"); + expect(submitCall!.body.sessionId).toBeDefined(); // the session it was polled for, not an override + }); + + it("TC-18: handler returns {message, sessionId} — overriding sessionId forwarded on turns", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const result = await client.runRedteam({ + configId: config.id, + handler: async () => ({ message: "x", sessionId: "custom-session-override" }), + }); + + expect(result!.success).toBe(true); + const submitCall = backend.requestLog.find((r) => r.path.includes("turns")); + expect(submitCall!.body.output).toBe("x"); + expect(submitCall!.body.sessionId).toBe("custom-session-override"); + }); + + it("TC-19: handler returns an unsupported shape — treated as a handler error, submitted as {error}, run continues to finalize", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const result = await client.runRedteam({ + configId: config.id, + handler: async () => 42 as unknown as string, + }); + + expect(result!.success).toBe(true); // run still finalizes + const submitCall = backend.requestLog.find((r) => r.path.includes("turns")); + expect(submitCall!.body.error).toBeDefined(); + expect(submitCall!.body.output).toBeUndefined(); + expect(result!.results[0].status).toBe("error"); + }); + + it("TC-20: handler throws synchronously or rejects — caught by SDK, submitted as {error}, run continues with partial results", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const result = await client.runRedteam({ + configId: config.id, + handler: async () => { + throw new Error("boom"); + }, + }); + + expect(result!.success).toBe(true); + const submitCall = backend.requestLog.find((r) => r.path.includes("turns")); + expect(submitCall!.body.error).toContain("boom"); + expect(result!.results[0].status).toBe("error"); + }); + + it("TC-21: handler receives correct turnIndex sequence — single-turn (turnIndex===1)", async () => { + const { tenant, config } = seedSingleTurnConfig(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const seen: number[] = []; + await client.runRedteam({ + configId: config.id, + handler: async (_p, _s, turnIndex) => { + seen.push(turnIndex); + return "reply"; + }, + }); + + expect(seen).toEqual([1]); + }); + + it("TC-22: handler receives correct turnIndex sequence — multi-turn 1,2,3 in order for a given session", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator({ slug: "harmful-content" }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + turnType: "multi", + multiTurnCount: 3, + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const seen: number[] = []; + await client.runRedteam({ + configId: config.id, + handler: async (_p, _s, turnIndex) => { + seen.push(turnIndex); + return "reply"; + }, + }); + + expect(seen).toEqual([1, 2, 3]); + }); + + it("TC-23: handler receives correct turnIndex sequence — iterative jailbreak, increasing up to the cap or an early stop", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const jailbreakEvaluator = backend.addEvaluator({ slug: "jailbreak-eval", isJailbreak: true }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [jailbreakEvaluator.id], + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const seen: number[] = []; + const result = await client.runRedteam({ + configId: config.id, + handler: async (_p, _s, turnIndex) => { + seen.push(turnIndex); + return "reply"; + }, + }); + + expect(result!.success).toBe(true); + expect(seen.length).toBeGreaterThan(1); + expect(seen).toEqual(seen.map((_v, i) => i + 1)); // strictly increasing from 1 + + // Early-stop path: the mock's early-stop sentinel makes the run finish + // before the iteration cap. + const seenEarly: number[] = []; + const config2 = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [jailbreakEvaluator.id], + }); + const result2 = await client.runRedteam({ + configId: config2.id, + handler: async (_p, _s, turnIndex) => { + seenEarly.push(turnIndex); + return turnIndex === 2 ? "STOP_EARLY now" : "reply"; + }, + }); + expect(result2!.success).toBe(true); + expect(seenEarly).toEqual([1, 2]); + expect(seenEarly.length).toBeLessThan(seen.length); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-24-33h-client-driven-turn-loop.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-24-33h-client-driven-turn-loop.e2e.test.ts new file mode 100644 index 0000000..5331d28 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-24-33h-client-driven-turn-loop.e2e.test.ts @@ -0,0 +1,190 @@ +/** + * E2E: the client-driven turn loop. The client fetches a run's whole prompt + * list ONCE (`GET .../prompts`), then drives every session's turns itself, + * holding all "what's next" state in memory between `POST .../turns` calls — + * there is no persisted turn-state, no server-side claim of any kind, no + * polling for "not-ready". All requests go over a real HTTP loopback to the + * mock backend, driven by the real SDK client (`Redteam`/`RedteamHttpClient`, + * unmocked). + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import { Config } from "../../../../config"; +import { RedteamHttpClient } from "../../client"; + +function newRawClient(backend: MockRedteamBackend, apiKey: string): RedteamHttpClient { + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = apiKey; + return new RedteamHttpClient(new Config({})); +} + +describe("TC-24..TC-33h — Client-driven turn loop (revision 7 architecture)", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + function seedRun(opts: { sessionsPerEvaluator?: number; turnType?: "single" | "multi"; multiTurnCount?: number } = {}) { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + sessionsPerEvaluator: opts.sessionsPerEvaluator ?? 1, + turnType: opts.turnType ?? "single", + multiTurnCount: opts.multiTurnCount, + }); + return { tenant, config }; + } + + it("TC-24: GET .../prompts returns the whole list in one call, immediately — no polling of any kind", async () => { + const { tenant, config } = seedRun({ sessionsPerEvaluator: 3 }); + const raw = newRawClient(backend, tenant.apiKey); + const created = await raw.createRun({ configId: config.id }); + if (created.status !== "running") throw new Error("expected running"); + + const start = Date.now(); + const resp = await raw.getPrompts(created.runId); + const elapsed = Date.now() - start; + + expect(resp.prompts).toHaveLength(3); + expect(elapsed).toBeLessThan(500); + // A second call is idempotent — same list, no side effects, no claiming. + const second = await raw.getPrompts(created.runId); + expect(second.prompts.map((p) => p.id).sort()).toEqual(resp.prompts.map((p) => p.id).sort()); + }); + + it("TC-27/TC-28: submitTurn reflects done correctly mid-run and at run end", async () => { + const { tenant, config } = seedRun({ turnType: "multi", multiTurnCount: 3 }); + const raw = newRawClient(backend, tenant.apiKey); + const created = await raw.createRun({ configId: config.id }); + if (created.status !== "running") throw new Error("expected running"); + + const { prompts } = await raw.getPrompts(created.runId); + const [prompt] = prompts; + + const turn1 = await raw.submitTurn(created.runId, { + promptId: prompt.id, + sessionId: prompt.id, + turnIndex: 1, + promptText: prompt.prompt, + output: "reply 1", + }); + expect(turn1.done).toBe(false); + expect(turn1.nextPrompt).toBeTruthy(); + expect(turn1.nextTurnIndex).toBe(2); + + const turn2 = await raw.submitTurn(created.runId, { + promptId: prompt.id, + sessionId: prompt.id, + turnIndex: 2, + promptText: turn1.nextPrompt as string, + output: "reply 2", + }); + expect(turn2.done).toBe(false); + + const turn3 = await raw.submitTurn(created.runId, { + promptId: prompt.id, + sessionId: prompt.id, + turnIndex: 3, + promptText: turn2.nextPrompt as string, + output: "reply 3", + }); + expect(turn3.done).toBe(true); + + const afterDone = await raw.getPrompts(created.runId); + expect(afterDone.status).toBe("completed"); + }); + + it("TC-30/TC-31: duplicate (run, promptId, turnIndex) submission — 409, not a silent overwrite or double-count", async () => { + const { tenant, config } = seedRun(); + const raw = newRawClient(backend, tenant.apiKey); + const created = await raw.createRun({ configId: config.id }); + if (created.status !== "running") throw new Error("expected running"); + + const { prompts } = await raw.getPrompts(created.runId); + const [prompt] = prompts; + const body = { promptId: prompt.id, sessionId: prompt.id, turnIndex: 1, promptText: prompt.prompt, output: "reply" }; + + const first = await raw.submitTurn(created.runId, body); + expect(first.done).toBe(true); + + // The client normalizes the backend's 409 to {done: true} rather than + // throwing (same treatment as an already-accepted turn), so a retry + // doesn't crash the caller — but it must not be double-counted server-side. + const duplicate = await raw.submitTurn(created.runId, body); + expect(duplicate.done).toBe(true); + + const run = backend.getRun(created.runId)!; + expect(run.results).toHaveLength(1); // not double-counted + }); + + it("TC-32/TC-33a: multiple sessions on one run advance independently, no lost turns, no cross-talk", async () => { + const { tenant, config } = seedRun({ sessionsPerEvaluator: 3, turnType: "multi", multiTurnCount: 2 }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const seenPromptIds = new Set(); + const result = await client.runRedteam({ + configId: config.id, + maxConcurrency: 3, + handler: async (_prompt: string, sessionId: string) => { + seenPromptIds.add(sessionId); + return "reply"; + }, + }); + + expect(result!.success).toBe(true); + expect(seenPromptIds.size).toBe(3); // every session was driven, none skipped or merged + const run = backend.getRun(result!.runId)!; + expect(run.results).toHaveLength(6); // 3 sessions * 2 turns each + }); + + it("TC-33f: promptId from a different run is rejected — 404", async () => { + const { tenant, config } = seedRun(); + const raw = newRawClient(backend, tenant.apiKey); + const created1 = await raw.createRun({ configId: config.id }); + if (created1.status !== "running") throw new Error("expected running"); + const { prompts } = await raw.getPrompts(created1.runId); + const [promptFromRun1] = prompts; + + const config2 = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: config.agentId, + evaluatorIds: config.evaluatorIds, + }); + const created2 = await raw.createRun({ configId: config2.id }); + if (created2.status !== "running") throw new Error("expected running"); + + await expect( + raw.submitTurn(created2.runId, { + promptId: promptFromRun1.id, + sessionId: promptFromRun1.id, + turnIndex: 1, + promptText: promptFromRun1.prompt, + output: "reply", + }), + ).rejects.toThrow(); + }); + + it("TC-45: zero prompts (empty run) — completes immediately with no turns to drive", async () => { + const { tenant, config } = seedRun({ sessionsPerEvaluator: 0 }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const handler = async () => "unused"; + const result = await client.runRedteam({ configId: config.id, handler }); + + expect(result!.status).toBe("completed"); + expect(result!.results).toHaveLength(0); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-34-36-generation-gating.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-34-36-generation-gating.e2e.test.ts new file mode 100644 index 0000000..c6fc05d --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-34-36-generation-gating.e2e.test.ts @@ -0,0 +1,74 @@ +/** + * E2E: prompt generation gating. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import { RedteamGenerationError, RedteamGenerationTimeoutError } from "../../models"; + +describe("TC-34..TC-36 — Prompt generation gating", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + it("TC-34: generation completes within the poll budget — SDK transparently waits (retrying create-run) then proceeds", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + generationMode: "delayed", + delayedFlipAfterPolls: 3, + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const result = await client.runRedteam({ configId: config.id, handler: async () => "reply" }); + expect(result!.success).toBe(true); + }); + + it("TC-35: generation fails (promptsGenerationFailedAt) — 502, RedteamGenerationError", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + generationMode: "failed", + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + await expect( + client.runRedteam({ configId: config.id, handler: async () => "reply" }), + ).rejects.toBeInstanceOf(RedteamGenerationError); + }); + + it("TC-36: generation worker unavailable / poll budget exhausted with no progress — 503, RedteamGenerationTimeoutError", async () => { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + generationMode: "unavailable", + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + await expect( + client.runRedteam({ configId: config.id, handler: async () => "reply" }), + ).rejects.toBeInstanceOf(RedteamGenerationTimeoutError); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-37-40-auth-tenancy.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-37-40-auth-tenancy.e2e.test.ts new file mode 100644 index 0000000..89fc086 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-37-40-auth-tenancy.e2e.test.ts @@ -0,0 +1,86 @@ +/** + * E2E: auth, tenancy, and entitlement. + * + * TC-40 (SDK-triggered run attribution / `triggered_by`) is a backend/DB + * column not exposed on any SDK-facing response shape (`RunProgress`/ + * `RiskScore` are opaque, backend-defined shapes, and `triggered_by` is not + * part of the SDK's contract at all). It is therefore not observable through + * this suite's black-box SDK-driven surface; it is asserted at the + * backend-integration level in the backend repo's own test suite. Marked + * unautomatable-here explicitly rather than silently dropped. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import { RedteamAuthError } from "../../models"; + +describe("TC-37..TC-39 — Auth, tenancy, and entitlement", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + it("TC-37: missing or invalid API key — 401, RedteamAuthError", async () => { + backend.addTenant({ apiKey: "the-real-key" }); + const client = newClient(backend, "garbage-key-not-registered", FAST_POLL_ENV); + + await expect( + client.runRedteam({ configId: "cfg-x", handler: async () => "reply" }), + ).rejects.toBeInstanceOf(RedteamAuthError); + }); + + it("TC-38: feature flag disabled for the org — 403, RedteamAuthError", async () => { + const tenant = backend.addTenant({ featureFlagEnabled: false }); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + }); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + await expect( + client.runRedteam({ configId: config.id, handler: async () => "reply" }), + ).rejects.toBeInstanceOf(RedteamAuthError); + }); + + it("TC-39: cross-tenant run/result access — 404 on every read endpoint, no data leaks across tenants", async () => { + const tenantA = backend.addTenant(); + const tenantB = backend.addTenant(); + const agentA = backend.addAgent({ projectId: tenantA.projectId }); + const evaluatorA = backend.addEvaluator(); + const configA = backend.addConfig({ + projectId: tenantA.projectId, + orgId: tenantA.orgId, + agentId: agentA.id, + evaluatorIds: [evaluatorA.id], + }); + + const clientA = newClient(backend, tenantA.apiKey, FAST_POLL_ENV); + const result = await clientA.runRedteam({ configId: configA.id, handler: async () => "reply" }); + expect(result!.success).toBe(true); + + const clientB = newClient(backend, tenantB.apiKey, FAST_POLL_ENV); + // Every read surface must 404 for tenant B against tenant A's run/config. + await expect(clientB.getResults(result!.runId)).rejects.toThrow(); + await expect(clientB.cancel(result!.runId)).rejects.toThrow(); + + const rawRiskCheck = async () => { + const { RedteamHttpClient } = await import("../../client"); + const { Config } = await import("../../../../config"); + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = tenantB.apiKey; + const raw = new RedteamHttpClient(new Config({})); + return raw.getRiskScore(configA.id); + }; + await expect(rawRiskCheck()).rejects.toThrow(); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-41-45-results-progress-risk.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-41-45-results-progress-risk.e2e.test.ts new file mode 100644 index 0000000..f81fb25 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-41-45-results-progress-risk.e2e.test.ts @@ -0,0 +1,140 @@ +/** + * E2E: results, progress, and risk score. + * + * Parity with a dashboard-driven run cannot be fully verified here: there is + * no dashboard-driven code path in this sandbox's mock backend (it only + * implements the SDK contract surface), and true parity means "identical + * persisted result rows / identical dashboard rendering", which requires the + * real backend DB and both real code paths side by side — that exact parity + * assertion belongs in the backend repo's own integration/e2e suite. This + * suite's contribution is the contract-shape half of parity: the SDK-driven + * run's results/progress/risk-score have exactly the shape the backend + * contract promises (so nothing SDK-specific leaks into what the dashboard + * would render) — see the "shape parity" test below. Marked explicitly + * rather than silently skipped. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; + +describe("TC-41..TC-45 — Results, progress, and risk score", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + function seed() { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator({ slug: "harmful-content" }); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + }); + return { tenant, config }; + } + + it("TC-41 (partial / contract-shape parity — see file docstring for full-parity scope note): SDK-triggered results carry no SDK-specific labeling and match the documented RunResultItem shape", async () => { + const { tenant, config } = seed(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + const result = await client.runRedteam({ configId: config.id, handler: async () => "reply" }); + + expect(result!.results).toHaveLength(1); + const item = result!.results[0]; + // Every documented RunResultItem field, and nothing SDK-specific bolted on. + expect(Object.keys(item).sort()).toEqual( + ["conversationHistory", "evaluatorId", "evaluatorSlug", "judgeOutput", "score", "sessionId", "status", "turnIndex"].sort(), + ); + }); + + it("TC-42: results pagination — a run with >200 result rows pages via page/limit, SDK aggregates with no gaps/duplicates", async () => { + const { tenant, config } = seed(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + // Seed the run directly (bypassing the turn loop) with 205 synthetic rows + // to exercise the pagination boundary without driving 205 real turns. + const created = await (async () => { + const { RedteamHttpClient } = await import("../../client"); + const { Config } = await import("../../../../config"); + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = tenant.apiKey; + const raw = new RedteamHttpClient(new Config({})); + return raw.createRun({ configId: config.id }); + })(); + if (created.status !== "running") throw new Error("expected running"); + + const items = Array.from({ length: 205 }, (_, i) => ({ + evaluatorId: "eval-1", + evaluatorSlug: "harmful-content", + status: "pass", + score: 1, + judgeOutput: "ok", + sessionId: "sess-synthetic", + turnIndex: i + 1, + conversationHistory: [], + })); + backend.seedRunDone(created.runId, items); + + const results = await client.getResults(created.runId); + expect(results).toHaveLength(205); + const turnIndexes = results.map((r) => r.turnIndex).sort((a, b) => (a ?? 0) - (b ?? 0)); + expect(turnIndexes).toEqual(Array.from({ length: 205 }, (_, i) => i + 1)); // no gaps, no duplicates + + const resultsPageCalls = backend.requestLog.filter((r) => r.path.includes("/results")); + expect(resultsPageCalls.length).toBeGreaterThanOrEqual(2); // at least 2 pages (200 + 5) + }); + + it("TC-43: partial results after some errored turns — final result includes all turns with individual statuses; success reflects overall completion not per-turn pass rate", async () => { + const { tenant, config } = seed(); + config.sessionsPerEvaluator = 2; + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + let call = 0; + const result = await client.runRedteam({ + configId: config.id, + handler: async () => { + call++; + if (call === 1) throw new Error("handler failure on this turn"); + return "ok reply"; + }, + }); + + expect(result!.success).toBe(true); // overall completion, not per-turn pass rate + expect(result!.results).toHaveLength(2); + const statuses = result!.results.map((r) => r.status).sort(); + expect(statuses).toEqual(["error", "pass"]); + }); + + it("TC-44: risk score reflects the run's config — aggregate safety score/change/history", async () => { + const { tenant, config } = seed(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + const result = await client.runRedteam({ configId: config.id, handler: async () => "reply" }); + + expect(result!.riskScore).toBeDefined(); + expect(result!.riskScore).toHaveProperty("latestSafetyScore"); + expect(result!.riskScore).toHaveProperty("change"); + expect(result!.riskScore).toHaveProperty("history"); + }); + + it("TC-45: empty run (zero prompts generated) finalizes immediately as done with an empty results[]; SDK does not throw", async () => { + const { tenant, config } = seed(); + config.sessionsPerEvaluator = 0; // config that produces zero adversarial prompts/sessions + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + // Full end-to-end runRedteam(): create -> poll loop (immediately sees + // scope:"run" done, since zero sessions were seeded) -> results/risk-score. + const result = await client.runRedteam({ configId: config.id, handler: async () => "unused" }); + + expect(result).not.toBeNull(); // SDK does not throw + expect(result!.success).toBe(true); + expect(result!.results).toEqual([]); + }); +}); diff --git a/src/api/redteam/__tests__/e2e/tc-46-48-cancellation.e2e.test.ts b/src/api/redteam/__tests__/e2e/tc-46-48-cancellation.e2e.test.ts new file mode 100644 index 0000000..1f8ccc0 --- /dev/null +++ b/src/api/redteam/__tests__/e2e/tc-46-48-cancellation.e2e.test.ts @@ -0,0 +1,171 @@ +/** + * E2E: cancellation and interruption. + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { MockRedteamBackend } from "./mock-backend"; +import { newClient, resetRedteamEnv, FAST_POLL_ENV } from "./helpers"; +import { Config } from "../../../../config"; +import { RedteamHttpClient } from "../../client"; + +describe("TC-46..TC-48 — Cancellation and interruption", () => { + let backend: MockRedteamBackend; + + beforeEach(async () => { + backend = new MockRedteamBackend(); + await backend.start(); + }); + afterEach(async () => { + await backend.stop(); + resetRedteamEnv(); + }); + + function seed() { + const tenant = backend.addTenant(); + const agent = backend.addAgent({ projectId: tenant.projectId }); + const evaluator = backend.addEvaluator(); + const config = backend.addConfig({ + projectId: tenant.projectId, + orgId: tenant.orgId, + agentId: agent.id, + evaluatorIds: [evaluator.id], + turnType: "multi", + multiTurnCount: 5, + }); + return { tenant, config }; + } + + it("TC-46: explicit cancel mid-run — run transitions to cancelled, pollers stop, result reflects status:cancelled", async () => { + const { tenant, config } = seed(); + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = tenant.apiKey; + const raw = new RedteamHttpClient(new Config({})); + const created = await raw.createRun({ configId: config.id }); + if (created.status !== "running") throw new Error("expected running"); + + // Simulate "turns in progress": fetch the prompt list, submit one turn (not final, multiTurnCount=5). + const promptsResp = await raw.getPrompts(created.runId); + const [prompt] = promptsResp.prompts; + await raw.submitTurn(created.runId, { + promptId: prompt.id, + sessionId: prompt.id, + turnIndex: 1, + promptText: prompt.prompt, + output: "reply", + }); + + const cancelResult = await raw.cancel(created.runId); + expect(cancelResult.status).toBe("cancelled"); + + const afterCancel = await raw.getPrompts(created.runId); + expect(afterCancel.status).toBe("cancelled"); // no more turns will be accepted + expect(backend.getRun(created.runId)!.status).toBe("cancelled"); + }); + + it("TC-46b (public-API path): Netra.redteam.cancel(runId) reaches the real backend and marks the run cancelled", async () => { + const { tenant, config } = seed(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = tenant.apiKey; + const raw = new RedteamHttpClient(new Config({})); + const created = await raw.createRun({ configId: config.id }); + if (created.status !== "running") throw new Error("expected running"); + + const cancelResult = await client.cancel(created.runId); + expect(cancelResult.status).toBe("cancelled"); + expect(backend.getRun(created.runId)!.status).toBe("cancelled"); + }); + + it("TC-47: process interrupt (SIGINT) mid-run — SDK issues a single cancel call before exiting; no orphaned RUNNING run", async () => { + // The real SDK's interrupt handler re-delivers the signal via + // `process.kill(process.pid, signal)` once it's done cancelling, so that + // Ctrl-C still terminates the developer's process normally. Sending a + // REAL SIGINT to the process running this test suite + // would kill the test runner, so `process.kill` is stubbed for the + // duration of this test to observe that re-delivery attempt safely, + // without ever executing it. This stubs only a Node global for isolation + // — it does not touch, weaken, or mock any product code path. + const originalKill = process.kill.bind(process); + const killCalls: Array<{ pid: number; signal?: string | number }> = []; + process.kill = ((pid: number, signal?: string | number) => { + killCalls.push({ pid, signal }); + return true; + }) as typeof process.kill; + + try { + const { tenant, config } = seed(); + const client = newClient(backend, tenant.apiKey, FAST_POLL_ENV); + + const runPromise = client.runRedteam({ + configId: config.id, + maxConcurrency: 1, + // A tiny per-turn delay keeps the (multiTurnCount=5) session from + // completing naturally before the interrupt has a chance to land. + handler: async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return "reply"; + }, + }); + + // Wait until the SDK has genuinely started driving the session (proves + // its SIGINT listener — registered right after create-run resolves, and + // before the prompt-list fetch — is already attached) before firing the + // interrupt, so it isn't lost to a race. + const deadline = Date.now() + 3000; + while (backend.requestLog.filter((r) => r.path.includes("/prompts")).length === 0) { + if (Date.now() > deadline) throw new Error("timed out waiting for the session drive to start"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + + process.emit("SIGINT" as any); + + const result = await runPromise; + expect(result).not.toBeNull(); + expect(result!.status).toBe("cancelled"); + + // The interrupt handler's own cancel() call is fire-and-forget from + // runRedteam's perspective — poll briefly for it to actually land. + const cancelDeadline = Date.now() + 2000; + while (backend.requestLog.filter((r) => r.path.includes("/cancel")).length === 0) { + if (Date.now() > cancelDeadline) throw new Error("timed out waiting for the interrupt's cancel() call"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + + const cancelCalls = backend.requestLog.filter((r) => r.path.includes("/cancel")); + expect(cancelCalls.length).toBe(1); // exactly one cancel call, not zero, not duplicated + + const run = [...backend.runs.values()].find((r) => r.configId === config.id); + expect(run!.status).not.toBe("running"); // no orphaned RUNNING run left behind + + expect(killCalls.length).toBe(1); // signal re-delivery was attempted exactly once + expect(killCalls[0].signal).toBe("SIGINT"); + } finally { + process.kill = originalKill; + } + }); + + it("TC-48: cancel an already-finished run — 409, no state corruption", async () => { + const { tenant, config } = seed(); + config.turnType = "single"; + process.env.NETRA_OTLP_ENDPOINT = backend.url; + process.env.NETRA_API_KEY = tenant.apiKey; + const raw = new RedteamHttpClient(new Config({})); + const created = await raw.createRun({ configId: config.id }); + if (created.status !== "running") throw new Error("expected running"); + + const promptsResp = await raw.getPrompts(created.runId); + const [prompt] = promptsResp.prompts; + const submit = await raw.submitTurn(created.runId, { + promptId: prompt.id, + sessionId: prompt.id, + turnIndex: 1, + promptText: prompt.prompt, + output: "reply", + }); + expect(submit.done).toBe(true); // run already completed (single-turn) + + await expect(raw.cancel(created.runId)).rejects.toThrow(); + // State unchanged by the failed cancel attempt. + expect(backend.getRun(created.runId)!.status).toBe("completed"); + }); +}); diff --git a/src/api/redteam/__tests__/task.test.ts b/src/api/redteam/__tests__/task.test.ts new file mode 100644 index 0000000..8dccfd5 --- /dev/null +++ b/src/api/redteam/__tests__/task.test.ts @@ -0,0 +1,72 @@ +import { readFileSync } from "fs"; +import { join } from "path"; +import { describe, expect, it, vi } from "vitest"; +import { executeHandler, RedteamAgentHandler } from "../task"; + +describe("executeHandler", () => { + it("resolves a plain async function's string return to {output}", async () => { + const handler: RedteamAgentHandler = async (prompt) => `echo:${prompt}`; + const result = await executeHandler(handler, "hello", "sess-1", 0); + expect(result).toEqual({ output: "echo:hello" }); + }); + + it("tolerates a sync function's string return (await on non-Promise is a no-op)", async () => { + const handler: RedteamAgentHandler = (prompt) => `sync:${prompt}`; + const result = await executeHandler(handler, "hi", "sess-2", 1); + expect(result).toEqual({ output: "sync:hi" }); + }); + + it("extracts {message, sessionId} return shape", async () => { + const handler: RedteamAgentHandler = async () => ({ + message: "the reply", + sessionId: "override-session", + }); + const result = await executeHandler(handler, "prompt", "sess-3", 2); + expect(result).toEqual({ output: "the reply", sessionId: "override-session" }); + }); + + it("extracts {message} without sessionId override", async () => { + const handler: RedteamAgentHandler = async () => ({ message: "no override" }); + const result = await executeHandler(handler, "prompt", "sess-4", 0); + expect(result).toEqual({ output: "no override", sessionId: undefined }); + }); + + it("throws on a bad-shape return (number)", async () => { + const handler = (async () => 42) as unknown as RedteamAgentHandler; + await expect(executeHandler(handler, "p", "s", 0)).rejects.toThrow( + /must return string/, + ); + }); + + it("throws on a bad-shape return (null)", async () => { + const handler = (async () => null) as unknown as RedteamAgentHandler; + await expect(executeHandler(handler, "p", "s", 0)).rejects.toThrow( + /must return string/, + ); + }); + + it("throws on a bad-shape return (object missing message)", async () => { + const handler = (async () => ({ foo: "bar" })) as unknown as RedteamAgentHandler; + await expect(executeHandler(handler, "p", "s", 0)).rejects.toThrow( + /must return string/, + ); + }); + + it("passes turnIndex as the third positional argument to the handler", async () => { + const handler = vi.fn(async (_prompt: string, _sessionId: string, _turnIndex: number) => "ok"); + await executeHandler(handler as RedteamAgentHandler, "prompt", "sess-5", 7); + expect(handler).toHaveBeenCalledWith("prompt", "sess-5", 7); + }); + + it("does not define/require any instanceof or BaseTask gate in this file", () => { + const source = readFileSync(join(__dirname, "..", "task.ts"), "utf-8"); + // Strip comments (which may reference BaseTask/instanceof only to + // document the intentional divergence from src/simulation) before + // asserting no executable instanceof/BaseTask gate exists. + const codeOnly = source + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/\/\/.*$/gm, ""); + expect(codeOnly).not.toMatch(/instanceof/); + expect(codeOnly).not.toMatch(/BaseTask/); + }); +}); diff --git a/src/api/redteam/__tests__/utils.test.ts b/src/api/redteam/__tests__/utils.test.ts new file mode 100644 index 0000000..bf91381 --- /dev/null +++ b/src/api/redteam/__tests__/utils.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { RedteamRunOptions } from "../models"; +import { + buildCreateRunBody, + getGenerationPollIntervalMs, + getGenerationTimeoutMs, + getRedteamTimeoutMs, + mapResultsPage, + mapRiskScore, + unwrapEnvelope, + validateRedteamInputs, +} from "../utils"; + +const noop = async () => "ok"; + +describe("validateRedteamInputs", () => { + it("rejects a non-function handler", () => { + const options = { configId: "cfg-1", handler: "not-a-fn" } as unknown as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBeNull(); + }); + + it("accepts a plain arrow function handler", () => { + const options = { configId: "cfg-1", handler: noop } as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBe(true); + }); + + it("rejects a missing configId", () => { + const options = { handler: noop } as unknown as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBeNull(); + }); + + it("rejects an empty-string configId", () => { + const options = { configId: "", handler: noop } as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBeNull(); + }); + + it("accepts a valid options object", () => { + const options: RedteamRunOptions = { configId: "cfg-1", handler: noop }; + expect(validateRedteamInputs(options)).toBe(true); + }); + + it("rejects maxConcurrency: 0 (would silently produce zero pollers)", () => { + const options = { configId: "cfg-1", handler: noop, maxConcurrency: 0 } as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBeNull(); + }); + + it("rejects a negative maxConcurrency", () => { + const options = { configId: "cfg-1", handler: noop, maxConcurrency: -1 } as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBeNull(); + }); + + it("rejects a non-integer maxConcurrency", () => { + const options = { configId: "cfg-1", handler: noop, maxConcurrency: 2.5 } as RedteamRunOptions; + expect(validateRedteamInputs(options)).toBeNull(); + }); + + it("accepts a valid positive integer maxConcurrency", () => { + const options: RedteamRunOptions = { configId: "cfg-1", handler: noop, maxConcurrency: 3 }; + expect(validateRedteamInputs(options)).toBe(true); + }); + + it("accepts an unset maxConcurrency (defaults elsewhere)", () => { + const options: RedteamRunOptions = { configId: "cfg-1", handler: noop }; + expect(validateRedteamInputs(options)).toBe(true); + }); +}); + +describe("buildCreateRunBody", () => { + it("emits only {configId}", () => { + const options: RedteamRunOptions = { configId: "cfg-123", handler: noop }; + expect(buildCreateRunBody(options)).toEqual({ configId: "cfg-123" }); + }); +}); + +describe("env parsing", () => { + const ENV_VARS = [ + "NETRA_REDTEAM_TIMEOUT", + "NETRA_REDTEAM_GENERATION_POLL_INTERVAL", + "NETRA_REDTEAM_GENERATION_TIMEOUT", + ]; + const originalEnv: Record = {}; + + beforeEach(() => { + for (const key of ENV_VARS) { + originalEnv[key] = process.env[key]; + delete process.env[key]; + } + }); + + afterEach(() => { + for (const key of ENV_VARS) { + if (originalEnv[key] === undefined) delete process.env[key]; + else process.env[key] = originalEnv[key]; + } + vi.restoreAllMocks(); + }); + + it("getRedteamTimeoutMs: unset -> default (20s -> 20000ms)", () => { + expect(getRedteamTimeoutMs()).toBe(20000); + }); + + it("getRedteamTimeoutMs: valid value is honored (seconds -> ms)", () => { + process.env.NETRA_REDTEAM_TIMEOUT = "10"; + expect(getRedteamTimeoutMs()).toBe(10000); + }); + + it("getRedteamTimeoutMs: NaN -> default + warn", () => { + process.env.NETRA_REDTEAM_TIMEOUT = "not-a-number"; + expect(getRedteamTimeoutMs()).toBe(20000); + }); + + it("getGenerationPollIntervalMs: unset -> default (2s -> 2000ms)", () => { + expect(getGenerationPollIntervalMs()).toBe(2000); + }); + + it("getGenerationTimeoutMs: unset -> default (300s -> 300000ms)", () => { + expect(getGenerationTimeoutMs()).toBe(300000); + }); +}); + +describe("unwrapEnvelope", () => { + it("unwraps a single {data} envelope", () => { + expect(unwrapEnvelope({ success: true, data: { foo: "bar" } })).toEqual({ foo: "bar" }); + }); + + it("does not additionally unwrap a payload whose own field happens to be named data (e.g. a results page)", () => { + expect(unwrapEnvelope({ success: true, data: { data: [{ foo: "bar" }], total: 1 } })).toEqual({ + data: [{ foo: "bar" }], + total: 1, + }); + }); +}); + +describe("response mappers", () => { + it("maps a results page", () => { + const page = mapResultsPage({ + data: [{ evaluatorId: "ev-1", status: "pass", score: 0.9 }], + page: 1, + limit: 200, + total: 1, + }); + expect(page.items).toHaveLength(1); + expect(page.items[0]).toMatchObject({ evaluatorId: "ev-1", status: "pass", score: 0.9 }); + }); + + it("maps an empty results page", () => { + const page = mapResultsPage({ data: [], page: 1, limit: 200, total: 0 }); + expect(page.items).toEqual([]); + }); + + it("maps a risk score payload through unchanged", () => { + expect(mapRiskScore({ latestSafetyScore: 82 })).toEqual({ latestSafetyScore: 82 }); + }); +}); diff --git a/src/api/redteam/api.ts b/src/api/redteam/api.ts new file mode 100644 index 0000000..72182d9 --- /dev/null +++ b/src/api/redteam/api.ts @@ -0,0 +1,328 @@ +/** + * Public API for running a red-team evaluation against a developer's local + * agent function. Exposed as `Netra.redteam`. + */ + +import pLimit from "p-limit"; +import { Config } from "../../config"; +import { Logger } from "../../logger"; +import { RedteamHttpClient } from "./client"; +import { + CreateRunResponse, + RedteamGenerationTimeoutError, + RedteamResult, + RedteamRunOptions, + RedteamRunStatus, + RiskScore, + RunProgress, + RunPromptItem, + RunResultItem, +} from "./models"; +import { executeHandler, RedteamAgentHandler } from "./task"; +import { buildCreateRunBody, getGenerationPollIntervalMs, getGenerationTimeoutMs, validateRedteamInputs } from "./utils"; + +const LOG_PREFIX = "netra.redteam"; +const MAX_AGENT_RESPONSE_CHARS = 5000; +const RESULTS_PAGE_LIMIT = 200; + +/** Shared stop flag every in-flight session-drive checks between turns, so an interrupt or a fatal sibling error halts the whole run promptly. */ +interface StopSignal { + stopped: boolean; +} + +/** + * Public orchestration class for red-team runs. Owns create -> await-ready -> + * fetch-the-whole-prompt-list-once -> drive every session's turns itself + * (no per-turn polling, no server-side turn-state of any kind) -> + * results/progress/risk-score aggregation. + */ +export class Redteam { + private _config: Config; + private _client: RedteamHttpClient; + + constructor(config: Config) { + this._config = config; + this._client = new RedteamHttpClient(config); + } + + /** + * Run a full red-team evaluation against an existing config: fetch the + * run's entire generated prompt list once, drive every session to + * completion itself (own local concurrency via `maxConcurrency`), then + * fetch results + progress + risk score. + * + * @param options - `{configId, handler}` — `configId` identifies a + * red-team config already created (e.g. in the dashboard); + * `handler` is the developer's local agent callback. + * @returns The aggregated `RedteamResult`, or `null` on invalid input / + * uninitialized client (logged, not thrown). + */ + async runRedteam(options: RedteamRunOptions): Promise { + if (!validateRedteamInputs(options)) { + return null; + } + if (!this._client.isInitialized()) { + Logger.error(`${LOG_PREFIX}: client not initialized (NETRA_OTLP_ENDPOINT/apiKey required)`); + return null; + } + + const maxConcurrency = Math.min(5, options.maxConcurrency ?? 5); + + const createResp = await this._client.createRun(buildCreateRunBody(options)); + + let runId: string; + let configId: string; + if (createResp.status === "generating") { + const ready = await this._awaitRunReady(createResp.configId); + runId = ready.runId; + configId = ready.configId; + } else { + runId = createResp.runId; + configId = createResp.configId; + } + + const stopSignal: StopSignal = { stopped: false }; + let interrupted = false; + const proc = typeof process !== "undefined" ? process : undefined; + + const removeListeners = () => { + if (proc && typeof proc.removeListener === "function") { + proc.removeListener("SIGINT", handleSigint); + proc.removeListener("SIGTERM", handleSigterm); + } + }; + + /** + * Single-fire interrupt finalizer: cancel the run server-side, then + * re-deliver the signal so the process's default + * disposition still applies once our listener is gone (matching + * `Simulation.finalizeFailure`) — Ctrl-C still terminates the process. + * + * Deliberately scoped to SIGINT/SIGTERM only — NOT `uncaughtException`/ + * `unhandledRejection`. Those are process-wide events with no way to + * tell whether the error came from this run's own session-drive loop or + * from unrelated code elsewhere in the host process; a global listener + * here would let any unrelated error silently cancel this run server-side + * (and, since every concurrent `runRedteam()` call registers its own + * listener, cancel every other in-flight run too). This run's own fatal + * errors are already surfaced through the normal awaited chain in + * `_driveSession`/`_driveAllSessions` — no gap this would need to fill. + */ + const finalizeCancel = (signal: NodeJS.Signals) => { + if (interrupted) return; + interrupted = true; + stopSignal.stopped = true; + removeListeners(); + void this._client + .cancel(runId) + .catch((e) => { + Logger.error(`${LOG_PREFIX}: interrupt cancel failed:`, e instanceof Error ? e.message : e); + }) + .finally(() => { + if (proc && typeof proc.kill === "function" && proc.pid !== undefined) { + proc.kill(proc.pid, signal); + } + }); + }; + const handleSigint = () => finalizeCancel("SIGINT"); + const handleSigterm = () => finalizeCancel("SIGTERM"); + + if (proc && typeof proc.once === "function") { + proc.once("SIGINT", handleSigint); + proc.once("SIGTERM", handleSigterm); + } + + const promptsResp = await this._client.getPrompts(runId); + if (promptsResp.prompts.length === 0) { + Logger.warn(`${LOG_PREFIX}: run ${runId} has zero generated prompts — nothing to drive`); + } + + try { + await this._driveAllSessions(runId, options.handler, promptsResp.prompts, maxConcurrency, stopSignal); + } finally { + removeListeners(); + } + + const results = await this.getResults(runId); + if (results.length === 0) { + Logger.warn(`${LOG_PREFIX}: run ${runId} finalized with zero results`); + } + + let progress: RunProgress | undefined; + try { + progress = await this._client.getProgress(runId); + } catch (error) { + Logger.warn(`${LOG_PREFIX}: failed to fetch progress:`, error instanceof Error ? error.message : error); + } + + let riskScore: RiskScore | undefined; + try { + riskScore = await this._client.getRiskScore(configId); + } catch (error) { + Logger.warn(`${LOG_PREFIX}: failed to fetch risk score:`, error instanceof Error ? error.message : error); + } + + // An interrupt always wins; otherwise re-read the run's own final status + // (every session's last submitTurn call already finalized it server-side + // — this is just a fresh read, not a wait). + const status: RedteamRunStatus = interrupted ? "cancelled" : await this._finalStatus(runId); + + return { + success: status === "completed", + status, + runId, + configId, + results, + progress, + riskScore, + }; + } + + /** + * Fetch all paginated per-turn results for a run, looping pages until a + * short page (< limit items) is returned. + */ + async getResults(runId: string): Promise { + const items: RunResultItem[] = []; + let page = 1; + // eslint-disable-next-line no-constant-condition + while (true) { + const pageResult = await this._client.getResultsPage(runId, { page, limit: RESULTS_PAGE_LIMIT }); + items.push(...pageResult.items); + if (pageResult.items.length < RESULTS_PAGE_LIMIT) { + break; + } + page++; + } + return items; + } + + /** Cancel an in-progress run. */ + async cancel(runId: string): Promise<{ status: "cancelled" }> { + return this._client.cancel(runId); + } + + /** + * The "generating" gate: re-issue `createRun` with the returned `configId` + * on a fixed interval up to a deadline, until the run is ready. + */ + private async _awaitRunReady( + configId: string, + ): Promise<{ runId: string; configId: string }> { + const intervalMs = getGenerationPollIntervalMs(); + const deadlineMs = getGenerationTimeoutMs(); + const start = Date.now(); + + // eslint-disable-next-line no-constant-condition + while (true) { + if (Date.now() - start > deadlineMs) { + throw new RedteamGenerationTimeoutError(); + } + + Logger.info(`${LOG_PREFIX}: waiting on prompt generation for config ${configId}...`); + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + + const resp: CreateRunResponse = await this._client.createRun({ configId }); + if (resp.status === "running") { + return { runId: resp.runId, configId: resp.configId }; + } + } + } + + /** Re-reads the run's own status once every session has been driven to completion. */ + private async _finalStatus(runId: string): Promise { + const resp = await this._client.getPrompts(runId); + return resp.status === "generating" ? "completed" : resp.status; + } + + /** + * Drives every prompt's session to completion, `maxConcurrency` at a time + * (bounded by `pLimit`). There is no server-side claim of any kind to + * arbitrate — each prompt is driven by exactly one local worker, and no + * other client process is racing for the same items, so `pLimit`'s own + * queue is the only concurrency control needed. + */ + private async _driveAllSessions( + runId: string, + handler: RedteamAgentHandler, + prompts: RunPromptItem[], + maxConcurrency: number, + stopSignal: StopSignal, + ): Promise { + const limit = pLimit(maxConcurrency); + const drives = prompts.map((prompt) => limit(() => this._driveSession(runId, handler, prompt, stopSignal))); + await Promise.all(drives); + } + + /** + * Drives one prompt's session from turn 1 through to `done`. Holds all + * "what's next" state in memory (`promptText`/`turnIndex`) between + * `submitTurn` calls — nothing is persisted server-side between turns. + */ + private async _driveSession( + runId: string, + handler: RedteamAgentHandler, + prompt: RunPromptItem, + stopSignal: StopSignal, + ): Promise { + // The prompt IS the session, 1:1, in this design — reusing its id as the + // sessionId avoids inventing a second identifier for the same thing. + let sessionId = prompt.id; + let turnIndex = 1; + let promptText = prompt.prompt; + + // eslint-disable-next-line no-constant-condition + while (true) { + if (stopSignal.stopped) { + return; + } + + let submitBody: { + promptId: string; + sessionId: string; + turnIndex: number; + promptText: string; + output?: string; + error?: string; + }; + try { + const { output, sessionId: overrideSessionId } = await executeHandler(handler, promptText, sessionId, turnIndex); + const truncated = this._truncateOutput(output); + sessionId = overrideSessionId ?? sessionId; + submitBody = { promptId: prompt.id, sessionId, turnIndex, promptText, output: truncated }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + Logger.error(`${LOG_PREFIX}: handler failed for session ${sessionId}, turn ${turnIndex}:`, message); + submitBody = { promptId: prompt.id, sessionId, turnIndex, promptText, error: message }; + } + + let result; + try { + result = await this._client.submitTurn(runId, submitBody); + } catch (error) { + // Fatal (non-retryable/exhausted-retry) error: trip the stop signal so + // sibling session-drives exit on their next loop check instead of + // being orphaned, then propagate the failure to the caller. + stopSignal.stopped = true; + throw error; + } + + if (result.done) { + return; + } + + promptText = result.nextPrompt as string; + turnIndex = result.nextTurnIndex as number; + } + } + + private _truncateOutput(output: string): string { + if (output.length <= MAX_AGENT_RESPONSE_CHARS) { + return output; + } + Logger.warn( + `${LOG_PREFIX}: agent response truncated from ${output.length} to ${MAX_AGENT_RESPONSE_CHARS} chars`, + ); + return output.slice(0, MAX_AGENT_RESPONSE_CHARS); + } +} diff --git a/src/api/redteam/client.ts b/src/api/redteam/client.ts new file mode 100644 index 0000000..a438783 --- /dev/null +++ b/src/api/redteam/client.ts @@ -0,0 +1,322 @@ +import axios, { AxiosInstance, AxiosResponse, AxiosError } from "axios"; +import { Config } from "../../config"; +import { Logger } from "../../logger"; +import { injectTraceContextHeaders } from "../../utils/context-propagation"; +import { + CreateRunRequestBody, + CreateRunResponse, + RedteamAuthError, + RedteamConfigError, + RedteamGenerationError, + RedteamGenerationTimeoutError, + RedteamRunError, + RiskScore, + RunProgress, + RunPromptsResponse, + RunResultsPage, + SubmitTurnBody, + SubmitTurnResult, +} from "./models"; +import { + getRedteamTimeoutMs, + mapResultsPage, + mapRiskScore, + unwrapEnvelope, +} from "./utils"; + +const LOG_PREFIX = "netra.redteam"; +const BASE_PATH = "/redteam/sdk"; +const MAX_RETRIES = 2; +const RETRY_BASE_DELAY_MS = 200; + +/** + * Internal HTTP client for the red-team SDK API surface. + * + * Modeled on `SimulationHttpClient` (axios + request interceptor + envelope + * unwrap conventions), but with an ordinary, short REST timeout — every + * backend call is fast and bounded, there is no server-held-open wait to + * accommodate. + */ +export class RedteamHttpClient { + private client: AxiosInstance | null = null; + + constructor(config: Config) { + this.client = this._createClient(config); + } + + isInitialized(): boolean { + return this.client !== null; + } + + private _createClient(config: Config): AxiosInstance | null { + const endpoint = (config.otlpEndpoint || "").trim(); + if (!endpoint) { + Logger.error(`${LOG_PREFIX}: NETRA_OTLP_ENDPOINT is required`); + return null; + } + + const baseURL = this._resolveBaseUrl(endpoint); + const headers = this._buildHeaders(config); + const timeout = getRedteamTimeoutMs(); + + try { + const instance = axios.create({ baseURL, headers, timeout }); + + instance.interceptors.request.use( + (cfg) => { + const traceHeaders = injectTraceContextHeaders({}); + Object.assign(cfg.headers, traceHeaders); + return cfg; + }, + (error) => Promise.reject(error), + ); + + return instance; + } catch (error) { + Logger.error(`${LOG_PREFIX}: Failed to create HTTP client:`, error); + return null; + } + } + + /** Strip a trailing `/` and a trailing `/telemetry` from the configured endpoint. */ + private _resolveBaseUrl(endpoint: string): string { + let baseUrl = endpoint.endsWith("/") ? endpoint.slice(0, -1) : endpoint; + if (baseUrl.endsWith("/telemetry")) { + baseUrl = baseUrl.slice(0, -"/telemetry".length); + } + return baseUrl; + } + + private _buildHeaders(config: Config): Record { + const headers: Record = { ...config.headers }; + if (config.apiKey) { + headers["x-api-key"] = config.apiKey; + } + return headers; + } + + private _ensureClient(): AxiosInstance { + if (!this.client) { + throw new RedteamAuthError("Netra red-team client is not initialized (NETRA_OTLP_ENDPOINT required)"); + } + return this.client; + } + + // ------------------------------------------------------------------------- + // Endpoints + // ------------------------------------------------------------------------- + + /** `POST /redteam/sdk/runs` — create/start (or continue generating) a run. */ + async createRun(body: CreateRunRequestBody): Promise { + try { + return await this._withRetry(async () => { + const client = this._ensureClient(); + const response: AxiosResponse = await client.post(`${BASE_PATH}/runs`, body); + return unwrapEnvelope(response.data); + }); + } catch (error) { + throw this._toTypedError(error); + } + } + + /** + * `GET /redteam/sdk/runs/{runId}/prompts` + * + * Fetched ONCE per run — the entire generated prompt list, in one call. No + * per-turn polling, no server-side claim of any kind: the client drives + * every session's turns itself from this list. + */ + async getPrompts(runId: string): Promise { + try { + return await this._withRetry(async () => { + const client = this._ensureClient(); + const response: AxiosResponse = await client.get(`${BASE_PATH}/runs/${runId}/prompts`); + return unwrapEnvelope(response.data); + }); + } catch (error) { + throw this._toTypedError(error); + } + } + + /** + * `POST /redteam/sdk/runs/{runId}/turns` + * + * Not retried: an exact-duplicate (run, promptId, turnIndex) submission + * returns a clean `409` (a network-retry guard, since there is no + * persisted claim to check against) — this method surfaces that as + * `{done: true}` rather than throwing, since a duplicate of an already- + * accepted turn means this session's work here is already recorded. + */ + async submitTurn(runId: string, body: SubmitTurnBody): Promise { + try { + const client = this._ensureClient(); + const response: AxiosResponse = await client.post(`${BASE_PATH}/runs/${runId}/turns`, body); + return unwrapEnvelope(response.data); + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 409) { + Logger.debug( + `${LOG_PREFIX}: turn (promptId=${body.promptId}, turnIndex=${body.turnIndex}) already submitted; treating as done`, + ); + return { done: true }; + } + throw this._toTypedError(error); + } + } + + /** `GET /redteam/sdk/runs/{runId}/progress` */ + async getProgress(runId: string): Promise { + try { + return await this._withRetry(async () => { + const client = this._ensureClient(); + const response: AxiosResponse = await client.get(`${BASE_PATH}/runs/${runId}/progress`); + return unwrapEnvelope(response.data) ?? {}; + }); + } catch (error) { + throw this._toTypedError(error); + } + } + + /** + * `GET /redteam/sdk/runs/{runId}/results?page&limit&evaluatorId` — one page. + * Callers loop pages until `items.length < limit` (see `Redteam.getResults` + * in `api.ts`). + */ + async getResultsPage( + runId: string, + params: { page?: number; limit?: number; evaluatorId?: string } = {}, + ): Promise { + try { + return await this._withRetry(async () => { + const client = this._ensureClient(); + const response: AxiosResponse = await client.get(`${BASE_PATH}/runs/${runId}/results`, { + params: { + page: params.page ?? 1, + limit: params.limit ?? 200, + evaluatorId: params.evaluatorId, + }, + }); + return mapResultsPage(unwrapEnvelope(response.data)); + }); + } catch (error) { + throw this._toTypedError(error); + } + } + + /** `GET /redteam/sdk/configs/{configId}/risk-score` */ + async getRiskScore(configId: string): Promise { + try { + return await this._withRetry(async () => { + const client = this._ensureClient(); + const response: AxiosResponse = await client.get( + `${BASE_PATH}/configs/${configId}/risk-score`, + ); + return mapRiskScore(unwrapEnvelope(response.data)); + }); + } catch (error) { + throw this._toTypedError(error); + } + } + + /** `POST /redteam/sdk/runs/{runId}/cancel` */ + async cancel(runId: string): Promise<{ status: "cancelled" }> { + try { + const client = this._ensureClient(); + const response: AxiosResponse = await client.post(`${BASE_PATH}/runs/${runId}/cancel`); + return unwrapEnvelope(response.data); + } catch (error) { + throw this._toTypedError(error); + } + } + + // ------------------------------------------------------------------------- + // Retry + error mapping + // ------------------------------------------------------------------------- + + /** Bounded retry (max 2) with linear backoff on network errors / 5xx / timeout. Never retries 4xx. */ + private async _withRetry(fn: () => Promise): Promise { + let lastError: unknown; + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + if (!this._isRetryable(error) || attempt === MAX_RETRIES) { + throw error; + } + const delay = RETRY_BASE_DELAY_MS * (attempt + 1); + Logger.debug(`${LOG_PREFIX}: retrying after error (attempt ${attempt + 1}/${MAX_RETRIES}):`, error); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw lastError; + } + + private _isRetryable(error: unknown): boolean { + if (axios.isAxiosError(error)) { + if (!error.response) { + // Network error / timeout — no response received. + return true; + } + const status = error.response.status; + return status === 502 || status === 503 || status >= 500; + } + return false; + } + + /** + * Extract a human-readable message from an axios error or ErrorEnvelope, + * covering 400/401/403/404/409/422/502/503. + */ + private _extractErrorMessage(error: any): string { + if (axios.isAxiosError(error)) { + const axiosError = error as AxiosError; + if (axiosError.response?.data) { + const responseData = axiosError.response.data as any; + if ( + typeof responseData === "object" && + responseData.error && + typeof responseData.error === "object" + ) { + return responseData.error.message || responseData.error.error || error.message; + } + } + return axiosError.message; + } + return error?.message || String(error); + } + + /** Map a raw axios/HTTP error to a typed red-team error. */ + private _toTypedError(error: unknown): Error { + if (error instanceof Error && !axios.isAxiosError(error)) { + return error; + } + + const status = axios.isAxiosError(error) ? error.response?.status : undefined; + const message = this._extractErrorMessage(error); + + switch (status) { + case 400: + return new RedteamConfigError(message || "invalid request: configId is required"); + case 401: + return new RedteamAuthError(message || "check NETRA_API_KEY"); + case 403: + return new RedteamAuthError(message || "red-teaming not enabled for this org"); + case 404: + return new RedteamConfigError(message || "config not found or not in this project"); + case 409: + return new RedteamRunError(message || "a run is already active for this config"); + case 422: + return new RedteamConfigError( + message || "agent is missing application details (systemPrompt)", + ); + case 502: + return new RedteamGenerationError(message || "prompt generation failed"); + case 503: + return new RedteamGenerationTimeoutError( + message || "generation did not complete (worker unavailable?)", + ); + default: + return error instanceof Error ? error : new Error(message); + } + } +} diff --git a/src/api/redteam/index.ts b/src/api/redteam/index.ts new file mode 100644 index 0000000..42e1214 --- /dev/null +++ b/src/api/redteam/index.ts @@ -0,0 +1,48 @@ +/** + * Red-team SDK module exports. + */ + +export { Redteam } from "./api"; +export { RedteamHttpClient } from "./client"; +export { executeHandler } from "./task"; +export type { + RedteamAgentHandler, + RedteamAgentResponse, + RedteamTaskResult, +} from "./task"; +export { + RedteamAuthError, + RedteamConfigError, + RedteamGenerationError, + RedteamGenerationTimeoutError, + RedteamRunError, +} from "./models"; +export type { + ConversationTurn, + CreateRunGeneratingResponse, + CreateRunRequestBody, + CreateRunResponse, + CreateRunRunningResponse, + RedteamResult, + RedteamRunOptions, + RedteamRunStatus, + RedteamTurnType, + RiskScore, + RunProgress, + RunPromptItem, + RunPromptsResponse, + RunResultItem, + RunResultsPage, + SubmitTurnBody, + SubmitTurnResult, +} from "./models"; +export { + buildCreateRunBody, + getGenerationPollIntervalMs, + getGenerationTimeoutMs, + getRedteamTimeoutMs, + mapResultsPage, + mapRiskScore, + unwrapEnvelope, + validateRedteamInputs, +} from "./utils"; diff --git a/src/api/redteam/models.ts b/src/api/redteam/models.ts new file mode 100644 index 0000000..1c53aa4 --- /dev/null +++ b/src/api/redteam/models.ts @@ -0,0 +1,190 @@ +/** + * Public + wire types for the red-team SDK client. + */ + +import { RedteamAgentHandler } from "./task"; + +/** The attack style a run's underlying config was set up with. */ +export type RedteamTurnType = "single" | "multi"; + +/** Server-side run lifecycle status. */ +export type RedteamRunStatus = "running" | "completed" | "failed" | "cancelled"; + +// --------------------------------------------------------------------------- +// Public input +// --------------------------------------------------------------------------- + +/** + * Options for `Netra.redteam.runRedteam()` — triggers a red-team run against a + * config that already exists (created ahead of time in the dashboard: agent, + * evaluators, attack settings are all decided there). The SDK only drives the + * run; it never creates or edits the config. + */ +export interface RedteamRunOptions { + /** The id of an existing red-team config to run. */ + configId: string; + /** The developer's local agent callback — a PLAIN function. */ + handler: RedteamAgentHandler; + /** Client-side session concurrency; default 5, capped at 5. */ + maxConcurrency?: number; +} + +// --------------------------------------------------------------------------- +// Wire (request) types — exact backend field names +// --------------------------------------------------------------------------- + +/** Wire body for `POST .../runs`. */ +export interface CreateRunRequestBody { + configId: string; +} + +/** + * Wire body for `POST .../turns` — submits one turn's result for one session, identified + * directly by `promptId` (from `GET .../prompts`; there is no server-issued invocation id). + */ +export interface SubmitTurnBody { + promptId: string; + sessionId: string; + turnIndex: number; + /** What was actually sent to the agent THIS turn — the catalog prompt verbatim on turn 1, or the previous call's `nextPrompt` for turn > 1. */ + promptText: string; + output?: string; + error?: string; +} + +// --------------------------------------------------------------------------- +// Wire (response) types +// --------------------------------------------------------------------------- + +export interface CreateRunRunningResponse { + runId: string; + configId: string; + status: "running"; +} + +export interface CreateRunGeneratingResponse { + configId: string; + status: "generating"; +} + +export type CreateRunResponse = + | CreateRunRunningResponse + | CreateRunGeneratingResponse; + +export interface ConversationTurn { + role: string; + content: string; +} + +/** One generated attack prompt — the unit of work the client drives to completion itself. */ +export interface RunPromptItem { + id: string; + prompt: string; + evaluatorId: string; + evaluatorSlug: string; +} + +/** + * `GET .../prompts` response — the client fetches this ONCE per run, then drives every + * session's turns itself (its own local concurrency, its own in-memory "what's next" state). + * No per-session claim/lease exists server-side; there is no other client racing for the same + * items, so no server-side arbitration is needed. + */ +export interface RunPromptsResponse { + runId: string; + status: RedteamRunStatus | "generating"; + turnType: RedteamTurnType; + multiTurnCount: number; + prompts: RunPromptItem[]; +} + +export interface SubmitTurnResult { + /** True if this session is finished. */ + done: boolean; + /** Present when done=false — the next message to send to the agent. */ + nextPrompt?: string; + /** Present when done=false. */ + nextTurnIndex?: number; +} + +/** Reused `RunProgressResponse` shape from the existing UI-facing service. */ +export type RunProgress = Record; + +export interface RunResultItem { + evaluatorId: string; + evaluatorSlug?: string; + status: "pass" | "fail" | "error"; + score?: number | null; + judgeOutput?: string | null; + sessionId?: string | null; + turnIndex?: number | null; + conversationHistory?: ConversationTurn[]; +} + +export interface RunResultsPage { + items: RunResultItem[]; + page: number; + limit: number; + total: number; +} + +/** Reused `RiskScoreResponse` shape. */ +export type RiskScore = Record; + +// --------------------------------------------------------------------------- +// Public output — the developer-facing result +// --------------------------------------------------------------------------- + +export interface RedteamResult { + success: boolean; + status: RedteamRunStatus; + runId: string; + configId: string; + results: RunResultItem[]; + progress?: RunProgress; + riskScore?: RiskScore; +} + +// --------------------------------------------------------------------------- +// Typed errors +// --------------------------------------------------------------------------- + +/** 401 — missing/invalid `x-api-key`. */ +export class RedteamAuthError extends Error { + constructor(message = "check NETRA_API_KEY") { + super(message); + this.name = "RedteamAuthError"; + } +} + +/** 400/404/422 — bad/unknown config, agent, or evaluator ids. */ +export class RedteamConfigError extends Error { + constructor(message: string) { + super(message); + this.name = "RedteamConfigError"; + } +} + +/** 502 — prompt generation failed. */ +export class RedteamGenerationError extends Error { + constructor(message = "prompt generation failed") { + super(message); + this.name = "RedteamGenerationError"; + } +} + +/** Generation deadline exceeded / 503 poll-budget exhausted. */ +export class RedteamGenerationTimeoutError extends Error { + constructor(message = "generation did not complete (worker unavailable?)") { + super(message); + this.name = "RedteamGenerationTimeoutError"; + } +} + +/** 409 — active-run conflict, or other run-level failure. */ +export class RedteamRunError extends Error { + constructor(message: string) { + super(message); + this.name = "RedteamRunError"; + } +} diff --git a/src/api/redteam/task.ts b/src/api/redteam/task.ts new file mode 100644 index 0000000..9cc82e0 --- /dev/null +++ b/src/api/redteam/task.ts @@ -0,0 +1,81 @@ +/** + * The red-team local-agent callback contract. + * + * Unlike `src/simulation/task.ts`'s `BaseTask` abstract class, the red-team + * handler is a **plain function** — no class, no `instanceof` gate. This is + * an intentional divergence chosen to keep the developer's code short — no + * class to extend. The signature also carries a + * `turnIndex` (simulation's `BaseTask.run` has no turn number). + */ + +/** + * The developer's local agent response. Either a plain string message, or a + * richer result carrying an explicit session id override. + */ +export type RedteamAgentResponse = string | RedteamTaskResult; + +/** + * Richer handler return shape, when the developer needs to report a + * different/derived session id alongside the message. + */ +export interface RedteamTaskResult { + message: string; + sessionId?: string; +} + +/** + * The developer-authored local agent callback. + * + * @param prompt - The prompt/attack text generated by the red-team for this turn. + * @param sessionId - The session identifier this turn belongs to. + * @param turnIndex - The zero-based index of this turn within the session. + * @returns The agent's response — a plain string, or a `{message, sessionId}` + * object — synchronously or as a Promise. + */ +export type RedteamAgentHandler = ( + prompt: string, + sessionId: string, + turnIndex: number, +) => Promise | RedteamAgentResponse; + +/** + * Invoke the developer's handler and normalize its return value. + * + * Tolerates both sync and async handlers (awaiting a non-Promise value is a + * no-op). The *return value* is still duck-typed, mirroring + * `src/simulation/utils.ts:executeTask` — a plain string, or an object with a + * string `message` (and optional `sessionId`). Any other shape throws. + * + * @param handler - The developer's local agent callback. + * @param prompt - The prompt to pass as the handler's first argument. + * @param sessionId - The session id to pass as the handler's second argument. + * @param turnIndex - The turn index to pass as the handler's third argument. + * @returns `{ output, sessionId? }` extracted from the handler's return value. + * @throws Error if the handler's resolved return value is not a string or a + * `{message: string}`-shaped object. + */ +export async function executeHandler( + handler: RedteamAgentHandler, + prompt: string, + sessionId: string, + turnIndex: number, +): Promise<{ output: string; sessionId?: string }> { + const result = await handler(prompt, sessionId, turnIndex); + + if (typeof result === "string") { + return { output: result }; + } + + if ( + result && + typeof result === "object" && + typeof (result as RedteamTaskResult).message === "string" + ) { + const r = result as RedteamTaskResult; + return { output: r.message, sessionId: r.sessionId }; + } + + throw new Error( + `redteam handler must return string | { message: string }, got ${typeof result}`, + ); +} diff --git a/src/api/redteam/utils.ts b/src/api/redteam/utils.ts new file mode 100644 index 0000000..adf3b3a --- /dev/null +++ b/src/api/redteam/utils.ts @@ -0,0 +1,159 @@ +import { Logger } from "../../logger"; +import { + CreateRunRequestBody, + RedteamRunOptions, + RiskScore, + RunResultItem, + RunResultsPage, +} from "./models"; + +const LOG_PREFIX = "netra.redteam"; + +// --------------------------------------------------------------------------- +// Validation +// --------------------------------------------------------------------------- + +/** + * Validate a `RedteamRunOptions` object before any network call. + * + * Rules: + * 1. `typeof handler === "function"`. + * 2. `configId` must be present. + * 3. `maxConcurrency`, if given, must be a positive integer — 0/negative/non-integer would + * silently produce zero pollers (`_runSessionsAsync` resolves immediately with no work done), + * which `runRedteam` would otherwise report as a misleading `{success:true, results:[]}`. + * + * @returns `true` when valid; `null` when invalid (after logging the reason). + */ +export function validateRedteamInputs( + options: RedteamRunOptions | null | undefined, +): true | null { + if (!options || typeof options !== "object") { + Logger.error(`${LOG_PREFIX}: options object is required`); + return null; + } + + if (typeof options.handler !== "function") { + Logger.error( + `${LOG_PREFIX}: handler must be a function (prompt, sessionId, turnIndex) => Promise`, + ); + return null; + } + + if (!options.configId) { + Logger.error(`${LOG_PREFIX}: configId is required`); + return null; + } + + if ( + options.maxConcurrency !== undefined && + (!Number.isInteger(options.maxConcurrency) || options.maxConcurrency < 1) + ) { + Logger.error(`${LOG_PREFIX}: maxConcurrency must be a positive integer, got ${options.maxConcurrency}`); + return null; + } + + return true; +} + +// --------------------------------------------------------------------------- +// Wire body builder +// --------------------------------------------------------------------------- + +/** Build the create-run request body from public `RedteamRunOptions`. */ +export function buildCreateRunBody( + options: RedteamRunOptions, +): CreateRunRequestBody { + return { configId: options.configId }; +} + +// --------------------------------------------------------------------------- +// Env parsing +// --------------------------------------------------------------------------- + +const DEFAULT_REDTEAM_TIMEOUT_S = 20; // ordinary REST timeout +const DEFAULT_GENERATION_POLL_INTERVAL_S = 2; +const DEFAULT_GENERATION_TIMEOUT_S = 300; + +function parseNumericEnv( + envVar: string, + defaultValue: number, +): number { + const raw = process.env[envVar]; + if (!raw) { + return defaultValue; + } + const parsed = parseFloat(raw); + if (isNaN(parsed)) { + Logger.warn( + `${LOG_PREFIX}: Invalid ${envVar} value '${raw}', using default ${defaultValue}`, + ); + return defaultValue; + } + return parsed; +} + +/** `NETRA_REDTEAM_TIMEOUT` (seconds) -> ms. Ordinary REST timeout, not a long-poll wait. */ +export function getRedteamTimeoutMs(): number { + return parseNumericEnv("NETRA_REDTEAM_TIMEOUT", DEFAULT_REDTEAM_TIMEOUT_S) * 1000; +} + +/** `NETRA_REDTEAM_GENERATION_POLL_INTERVAL` (seconds) -> ms. */ +export function getGenerationPollIntervalMs(): number { + return parseNumericEnv("NETRA_REDTEAM_GENERATION_POLL_INTERVAL", DEFAULT_GENERATION_POLL_INTERVAL_S) * 1000; +} + +/** `NETRA_REDTEAM_GENERATION_TIMEOUT` (seconds) -> ms. */ +export function getGenerationTimeoutMs(): number { + return parseNumericEnv("NETRA_REDTEAM_GENERATION_TIMEOUT", DEFAULT_GENERATION_TIMEOUT_S) * 1000; +} + +// --------------------------------------------------------------------------- +// Envelope unwrap + response mappers +// --------------------------------------------------------------------------- + +/** + * Unwrap the backend's `{ data: ... }` response envelope. The backend's + * `ResponseTransformerInterceptor` wraps every response exactly once — never + * doubly-nested — so this must NOT also unwrap an inner `data` field: some + * payloads (e.g. the paginated results page) legitimately have their own + * `data` field (the page's items), which an extra unwrap would silently + * discard. + */ +export function unwrapEnvelope(raw: any): T { + if (raw && typeof raw === "object" && "data" in raw) { + return raw.data as T; + } + return raw as T; +} + +/** + * Map a raw paginated results payload. The backend's page DTO names the + * items field `data` (`{ data, total, page, limit, hasNextPage }`), not + * `items` — `items` is this SDK's own `RunResultsPage` field name. + */ +export function mapResultsPage(raw: any): RunResultsPage { + const items: RunResultItem[] = Array.isArray(raw?.data) + ? raw.data.map((item: any) => ({ + evaluatorId: item.evaluatorId, + evaluatorSlug: item.evaluatorSlug, + status: item.status, + score: item.score ?? null, + judgeOutput: item.judgeOutput ?? null, + sessionId: item.sessionId ?? null, + turnIndex: item.turnIndex ?? null, + conversationHistory: item.conversationHistory ?? [], + })) + : []; + return { + items, + page: raw?.page ?? 1, + limit: raw?.limit ?? items.length, + total: raw?.total ?? items.length, + }; +} + +/** Map a raw risk-score payload (backend shape is `additionalProperties: true`). */ +export function mapRiskScore(raw: any): RiskScore { + return (raw ?? {}) as RiskScore; +} diff --git a/src/index.ts b/src/index.ts index d9393f1..95f61d6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import { context, Span, SpanKind, trace } from "@opentelemetry/api"; import { createRequire } from "module"; -import { Prompts, Dashboard, Evaluation, Usage } from "./api"; +import { Prompts, Dashboard, Evaluation, Usage, Redteam } from "./api"; import { Config, NetraConfig } from "./config"; import { initInstrumentations, instrumentationsReady, uninstrumentAll } from "./instrumentation"; import { Logger } from "./logger"; @@ -68,6 +68,14 @@ export { Usage, // Prompts API Prompts, + // Red-team API + Redteam, + RedteamAuthError, + RedteamConfigError, + RedteamGenerationError, + RedteamGenerationTimeoutError, + RedteamHttpClient, + RedteamRunError, } from "./api"; export type { @@ -107,6 +115,22 @@ export type { TraceSummary, GetPromptParams, PromptResponse, + // Red-team API + RedteamAgentHandler, + RedteamAgentResponse, + RedteamConversationTurn, + RedteamCreateRunResponse, + RedteamResult, + RedteamRiskScore, + RedteamRunOptions, + RedteamRunProgress, + RedteamRunPromptItem, + RedteamRunPromptsResponse, + RedteamRunResultItem, + RedteamRunResultsPage, + RedteamRunStatus, + RedteamTaskResult, + RedteamTurnType, } from "./api"; // Export simulation types and classes @@ -148,6 +172,7 @@ export class Netra { static dashboard: Dashboard; static simulation: Simulation; static prompts: Prompts; + static redteam: Redteam; static getConfig(): Config { if (!this._config) { @@ -214,6 +239,12 @@ export class Netra { Logger.warn("Netra: failed to initialize prompts client:", e); } + try { + this.redteam = new Redteam(cfg); + } catch (e) { + Logger.warn("Netra: failed to initialize redteam client:", e); + } + this._initialized = true; Logger.info("Netra successfully initialized."); diff --git a/src/version.ts b/src/version.ts index f27afec..2dbf4fa 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const SDK_VERSION = "1.8.0"; +export const SDK_VERSION = "1.9.0-beta.1"; diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..4679828 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.test.ts"], + coverage: { + provider: "v8", + reporter: ["text", "lcov"], + include: ["src/api/redteam/**/*.ts"], + exclude: ["src/api/redteam/**/__tests__/**", "src/api/redteam/index.ts"], + thresholds: { + lines: 80, + }, + }, + }, +}); From 2c4dadad1fefac7a8ad00348b2eb62dc0e8ef0fc Mon Sep 17 00:00:00 2001 From: Jithin Date: Fri, 21 Aug 2026 12:09:24 +0530 Subject: [PATCH 2/2] fix(redteam): fix SIGINT/SIGTERM cancel race, add runNumber MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Netra.init() and runRedteam() each installed their own independent SIGINT/SIGTERM listener; Netra.init()'s process.exit(0) could win the race and kill the process before runRedteam()'s cancel POST got a response back from the backend, leaving runs stuck "running" server-side. Replaces both with a single shared, dependency-free shutdown-hook registry (src/utils/shutdown-hooks.ts) that installs the SDK's one real signal listener lazily, on first registration — so it works for a standalone `new Redteam(config)` instance with no Netra.init() call too. Also: - Adds runNumber to RedteamResult/RunProgress, passed through from the backend's new getRunNumber() (matches the dashboard's "Run #N"). - Wraps each redteam turn in its own span and stamps netra.trace.origin=redteam on the root span, mirroring the backend's existing filter for redteam-originated traces. Verified live: real SIGINT against a running backend, both via Netra.init() and via a standalone Redteam instance, each confirmed server-side (independent of client-side logs) to leave the run "cancelled" rather than stuck "running". Co-Authored-By: Claude Sonnet 5 --- src/api/redteam/__tests__/api.test.ts | 46 +++++++++++---- src/api/redteam/api.ts | 83 +++++++++++++-------------- src/api/redteam/client.ts | 3 + src/api/redteam/models.ts | 2 + src/index.ts | 20 ++++--- src/utils/shutdown-hooks.ts | 63 ++++++++++++++++++++ 6 files changed, 154 insertions(+), 63 deletions(-) create mode 100644 src/utils/shutdown-hooks.ts diff --git a/src/api/redteam/__tests__/api.test.ts b/src/api/redteam/__tests__/api.test.ts index 5a0504a..8c2ec2a 100644 --- a/src/api/redteam/__tests__/api.test.ts +++ b/src/api/redteam/__tests__/api.test.ts @@ -21,6 +21,7 @@ vi.mock("../client", () => { import { Redteam } from "../api"; import { RedteamRunOptions } from "../models"; +import { _hookCountForTests } from "../../../utils/shutdown-hooks"; const fakeConfig = {} as any; @@ -30,7 +31,7 @@ function resetMocks() { } mockClient.isInitialized.mockReturnValue(true); mockClient.cancel.mockResolvedValue({ status: "cancelled" }); - mockClient.getProgress.mockResolvedValue({ completedSessions: 1 }); + mockClient.getProgress.mockResolvedValue({ completedSessions: 1, runNumber: 3 }); mockClient.getRiskScore.mockResolvedValue({ latestSafetyScore: 95 }); } @@ -86,9 +87,36 @@ describe("Redteam", () => { expect(result!.success).toBe(true); expect(result!.status).toBe("completed"); expect(result!.results).toHaveLength(1); - expect(result!.progress).toEqual({ completedSessions: 1 }); + expect(result!.progress).toEqual({ completedSessions: 1, runNumber: 3 }); expect(result!.riskScore).toEqual({ latestSafetyScore: 95 }); expect(mockClient.getPrompts).toHaveBeenCalledTimes(2); + expect(result!.runNumber).toBe(3); + }); + + it("runNumber is undefined when the progress fetch fails, instead of throwing", async () => { + mockClient.createRun.mockResolvedValueOnce({ runId: "run-2", configId: "cfg-2", status: "running" }); + mockClient.getPrompts + .mockResolvedValueOnce({ + runId: "run-2", + status: "running", + turnType: "single", + multiTurnCount: 5, + prompts: [{ id: "p1", prompt: "attack", evaluatorId: "ev-1", evaluatorSlug: "harmful-hate" }], + }) + .mockResolvedValueOnce({ runId: "run-2", status: "completed", turnType: "single", multiTurnCount: 5, prompts: [] }); + mockClient.submitTurn.mockResolvedValueOnce({ done: true }); + mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); + mockClient.getProgress.mockRejectedValueOnce(new Error("progress endpoint down")); + + const handler = vi.fn(async () => "ok"); + const redteam = new Redteam(fakeConfig); + const options: RedteamRunOptions = { configId: "cfg-2", handler, maxConcurrency: 1 }; + + const result = await redteam.runRedteam(options); + + expect(result!.progress).toBeUndefined(); + expect(result!.runNumber).toBeUndefined(); + expect(result!.status).toBe("completed"); }); it("still generating after create: retries createRun with {configId} on an interval until running", async () => { @@ -373,10 +401,9 @@ describe("Redteam", () => { }); mockClient.getResultsPage.mockResolvedValueOnce({ items: [], page: 1, limit: 200, total: 0 }); - const sigintBefore = process.listenerCount("SIGINT"); - const sigtermBefore = process.listenerCount("SIGTERM"); const exceptionBefore = process.listenerCount("uncaughtException"); const rejectionBefore = process.listenerCount("unhandledRejection"); + const hooksBefore = _hookCountForTests(); const handler = vi.fn(async () => "ok"); const redteam = new Redteam(fakeConfig); @@ -389,18 +416,17 @@ describe("Redteam", () => { // unrelated error elsewhere in the host process must not be able to cancel this run. expect(process.listenerCount("uncaughtException")).toBe(exceptionBefore); expect(process.listenerCount("unhandledRejection")).toBe(rejectionBefore); - // SIGINT/SIGTERM listeners ARE expected while the run is in flight. - expect(process.listenerCount("SIGINT")).toBe(sigintBefore + 1); - expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore + 1); + // A shutdown hook IS registered while the run is in flight — checked via + // the hook count, not process.listenerCount() (see utils/shutdown-hooks.ts). + expect(_hookCountForTests()).toBe(hooksBefore + 1); mockClient.submitTurn.mockResolvedValue({ done: true }); const result = await runPromise; expect(mockClient.cancel).not.toHaveBeenCalled(); expect(result!.status).toBe("completed"); - // Listeners must be removed once the run settles normally (no leak). - expect(process.listenerCount("SIGINT")).toBe(sigintBefore); - expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore); + // The hook must be unregistered once the run settles normally (no leak). + expect(_hookCountForTests()).toBe(hooksBefore); }); it("getResults pages until a short page, concatenating all items", async () => { diff --git a/src/api/redteam/api.ts b/src/api/redteam/api.ts index 72182d9..c8d9b16 100644 --- a/src/api/redteam/api.ts +++ b/src/api/redteam/api.ts @@ -6,6 +6,8 @@ import pLimit from "p-limit"; import { Config } from "../../config"; import { Logger } from "../../logger"; +import { RootSpanProcessor } from "../../processors/root-span-processor"; +import { SpanWrapper } from "../../span-wrapper"; import { RedteamHttpClient } from "./client"; import { CreateRunResponse, @@ -20,9 +22,17 @@ import { } from "./models"; import { executeHandler, RedteamAgentHandler } from "./task"; import { buildCreateRunBody, getGenerationPollIntervalMs, getGenerationTimeoutMs, validateRedteamInputs } from "./utils"; +import { registerShutdownHook } from "../../utils/shutdown-hooks"; const LOG_PREFIX = "netra.redteam"; const MAX_AGENT_RESPONSE_CHARS = 5000; +const TURN_SPAN_NAME = "Netra.Redteam.Turn"; +// Same attribute/value the backend already sets on its own redteam-originated +// spans and already filters on for Insights/auto-eval — stamping it here too +// so traces produced by the developer's own instrumented handler get excluded +// the same way. +const TRACE_ORIGIN_ATTRIBUTE = "netra.trace.origin"; +const TRACE_ORIGIN_REDTEAM = "redteam"; const RESULTS_PAGE_LIMIT = 200; /** Shared stop flag every in-flight session-drive checks between turns, so an interrupt or a fatal sibling error halts the whole run promptly. */ @@ -83,54 +93,25 @@ export class Redteam { const stopSignal: StopSignal = { stopped: false }; let interrupted = false; - const proc = typeof process !== "undefined" ? process : undefined; - - const removeListeners = () => { - if (proc && typeof proc.removeListener === "function") { - proc.removeListener("SIGINT", handleSigint); - proc.removeListener("SIGTERM", handleSigterm); - } - }; /** - * Single-fire interrupt finalizer: cancel the run server-side, then - * re-deliver the signal so the process's default - * disposition still applies once our listener is gone (matching - * `Simulation.finalizeFailure`) — Ctrl-C still terminates the process. - * - * Deliberately scoped to SIGINT/SIGTERM only — NOT `uncaughtException`/ - * `unhandledRejection`. Those are process-wide events with no way to - * tell whether the error came from this run's own session-drive loop or - * from unrelated code elsewhere in the host process; a global listener - * here would let any unrelated error silently cancel this run server-side - * (and, since every concurrent `runRedteam()` call registers its own - * listener, cancel every other in-flight run too). This run's own fatal - * errors are already surfaced through the normal awaited chain in - * `_driveSession`/`_driveAllSessions` — no gap this would need to fill. + * Cancels the run server-side on shutdown — see ../../utils/shutdown-hooks.ts. + * Deliberately not hooked into uncaughtException/unhandledRejection: those + * are process-wide, so reacting to them here could cancel unrelated + * concurrent runs too. This run's own errors already surface through + * _driveSession/_driveAllSessions. */ - const finalizeCancel = (signal: NodeJS.Signals) => { + const unregisterShutdownHook = registerShutdownHook(async () => { if (interrupted) return; interrupted = true; stopSignal.stopped = true; - removeListeners(); - void this._client - .cancel(runId) - .catch((e) => { - Logger.error(`${LOG_PREFIX}: interrupt cancel failed:`, e instanceof Error ? e.message : e); - }) - .finally(() => { - if (proc && typeof proc.kill === "function" && proc.pid !== undefined) { - proc.kill(proc.pid, signal); - } - }); - }; - const handleSigint = () => finalizeCancel("SIGINT"); - const handleSigterm = () => finalizeCancel("SIGTERM"); - - if (proc && typeof proc.once === "function") { - proc.once("SIGINT", handleSigint); - proc.once("SIGTERM", handleSigterm); - } + try { + await this._client.cancel(runId); + Logger.debug(`${LOG_PREFIX}: run ${runId} cancelled server-side`); + } catch (e) { + Logger.error(`${LOG_PREFIX}: interrupt cancel failed:`, e instanceof Error ? e.message : e); + } + }); const promptsResp = await this._client.getPrompts(runId); if (promptsResp.prompts.length === 0) { @@ -140,7 +121,7 @@ export class Redteam { try { await this._driveAllSessions(runId, options.handler, promptsResp.prompts, maxConcurrency, stopSignal); } finally { - removeListeners(); + unregisterShutdownHook(); } const results = await this.getResults(runId); @@ -167,11 +148,14 @@ export class Redteam { // — this is just a fresh read, not a wait). const status: RedteamRunStatus = interrupted ? "cancelled" : await this._finalStatus(runId); + const runNumber = typeof progress?.runNumber === "number" ? progress.runNumber : undefined; + return { success: status === "completed", status, runId, configId, + runNumber, results, progress, riskScore, @@ -285,8 +269,17 @@ export class Redteam { output?: string; error?: string; }; + // Wrap the handler call in its own span (mirroring simulation's per-turn + // span) so there's always a root span to tag — the developer's own + // instrumentation may not start one on its own (e.g. a plain fetch to + // their agent with no outer span active). + const turnSpan = new SpanWrapper(TURN_SPAN_NAME, {}, LOG_PREFIX); + turnSpan.start(); try { - const { output, sessionId: overrideSessionId } = await executeHandler(handler, promptText, sessionId, turnIndex); + const { output, sessionId: overrideSessionId } = await turnSpan.withActive(() => { + RootSpanProcessor.setAttributeOnRootSpan(TRACE_ORIGIN_ATTRIBUTE, TRACE_ORIGIN_REDTEAM); + return executeHandler(handler, promptText, sessionId, turnIndex); + }); const truncated = this._truncateOutput(output); sessionId = overrideSessionId ?? sessionId; submitBody = { promptId: prompt.id, sessionId, turnIndex, promptText, output: truncated }; @@ -294,6 +287,8 @@ export class Redteam { const message = error instanceof Error ? error.message : String(error); Logger.error(`${LOG_PREFIX}: handler failed for session ${sessionId}, turn ${turnIndex}:`, message); submitBody = { promptId: prompt.id, sessionId, turnIndex, promptText, error: message }; + } finally { + turnSpan.end(); } let result; diff --git a/src/api/redteam/client.ts b/src/api/redteam/client.ts index a438783..26fa09e 100644 --- a/src/api/redteam/client.ts +++ b/src/api/redteam/client.ts @@ -219,11 +219,14 @@ export class RedteamHttpClient { /** `POST /redteam/sdk/runs/{runId}/cancel` */ async cancel(runId: string): Promise<{ status: "cancelled" }> { + Logger.debug(`${LOG_PREFIX}: POST ${BASE_PATH}/runs/${runId}/cancel`); try { const client = this._ensureClient(); const response: AxiosResponse = await client.post(`${BASE_PATH}/runs/${runId}/cancel`); + Logger.debug(`${LOG_PREFIX}: cancel response status=${response.status}`, response.data); return unwrapEnvelope(response.data); } catch (error) { + Logger.debug(`${LOG_PREFIX}: cancel request failed:`, error instanceof Error ? error.message : error); throw this._toTypedError(error); } } diff --git a/src/api/redteam/models.ts b/src/api/redteam/models.ts index 1c53aa4..1221067 100644 --- a/src/api/redteam/models.ts +++ b/src/api/redteam/models.ts @@ -140,6 +140,8 @@ export interface RedteamResult { status: RedteamRunStatus; runId: string; configId: string; + /** Matches the dashboard's "Run #N" (oldest = 1). Undefined if the progress fetch failed. */ + runNumber?: number; results: RunResultItem[]; progress?: RunProgress; riskScore?: RiskScore; diff --git a/src/index.ts b/src/index.ts index 95f61d6..753880f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ import { SpanWrapper } from "./span-wrapper"; import { Tracer } from "./tracer"; import { SpanType, SpanCallback, SpanOptions, SpanAttributes } from "./types"; import { wrapResponse } from "./utils/response-handler"; +import { registerShutdownHook, runShutdownHooks } from "./utils/shutdown-hooks"; export { Config, @@ -265,12 +266,6 @@ export class Netra { } // Graceful shutdown logic - const handleSignal = async (signal: string) => { - Logger.log(`\nReceived ${signal}. Shutting down Netra SDK...`); - await this.shutdown(); - process.exit(0); - }; - const handleUncaughtException = async (error: Error) => { Logger.error("Uncaught exception:", error); Logger.error("Shutting down Netra SDK due to crash..."); @@ -289,9 +284,9 @@ export class Netra { await this.shutdown(); }); - // Handle termination signals - process.once("SIGINT", () => handleSignal("SIGINT")); - process.once("SIGTERM", () => handleSignal("SIGTERM")); + // SIGINT/SIGTERM go through the shared shutdown-hook registry, not a + // listener here — see ./utils/shutdown-hooks.ts. + registerShutdownHook(() => this.shutdown()); // Handle crashes process.once("uncaughtException", handleUncaughtException); @@ -332,6 +327,10 @@ export class Netra { return; } + // Runs other registered hooks (e.g. an in-flight redteam run's cancel). + // No-op if we're already inside a signal-triggered pass (re-entrancy guard). + await runShutdownHooks(); + // Unpatch any monkey-patched instrumentations first try { await uninstrumentAll(); @@ -580,6 +579,9 @@ export class Netra { } static withBlockedSpansLocal = withBlockedSpansLocal; + + /** @internal Not a stable public API — see ./utils/shutdown-hooks.ts. */ + static registerShutdownHook = registerShutdownHook; } export default Netra; diff --git a/src/utils/shutdown-hooks.ts b/src/utils/shutdown-hooks.ts new file mode 100644 index 0000000..bccf22d --- /dev/null +++ b/src/utils/shutdown-hooks.ts @@ -0,0 +1,63 @@ +/** + * Shared shutdown-hook registry — avoids multiple independent + * process.once('SIGINT', ...) listeners racing each other to exit first. + * Installs the SDK's one real signal listener lazily, on first registration, + * so it works even for a standalone Redteam instance with no Netra.init(). + */ + +export const SHUTDOWN_HOOK_TIMEOUT_MS = 5000; + +type ShutdownHook = () => void | Promise; + +const hooks = new Set(); +let installed = false; +let fired = false; +let running = false; + +/** @internal Not a stable public API. Returns an unregister function. */ +export function registerShutdownHook(hook: ShutdownHook): () => void { + hooks.add(hook); + + if (!installed) { + installed = true; + const proc = typeof process !== "undefined" ? process : undefined; + const onSignal = (signal: NodeJS.Signals) => { + if (fired) return; + fired = true; + void runShutdownHooks().finally(() => { + if (proc && typeof proc.kill === "function" && proc.pid !== undefined) { + proc.kill(proc.pid, signal); + } + }); + }; + if (proc && typeof proc.once === "function") { + proc.once("SIGINT", () => onSignal("SIGINT")); + proc.once("SIGTERM", () => onSignal("SIGTERM")); + } + } + + return () => { + hooks.delete(hook); + }; +} + +/** + * @internal Runs every hook concurrently, bounded by SHUTDOWN_HOOK_TIMEOUT_MS. + * Guarded against re-entrancy: Netra.shutdown() is itself a registered hook + * and also calls this function, which would otherwise recurse forever. + */ +export async function runShutdownHooks(): Promise { + if (running) return; + running = true; + try { + const settle = Promise.allSettled([...hooks].map((hook) => hook())); + await Promise.race([settle, new Promise((resolve) => setTimeout(resolve, SHUTDOWN_HOOK_TIMEOUT_MS))]); + } finally { + running = false; + } +} + +/** @internal Test-only — the real SIGINT/SIGTERM listener is a singleton, so listenerCount() can't tell runs apart. */ +export function _hookCountForTests(): number { + return hooks.size; +}