diff --git a/.env.example b/.env.example index 880353b..5d81f7e 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,7 @@ GATEWAY_ADMIN_TOKEN=change-me-to-something-long OPENAI_API_KEY= ANTHROPIC_API_KEY= GROQ_API_KEY= +GEMINI_API_KEY= # Any other provider slug, e.g. for slug "together": # GATEWAY_UPSTREAM_TOGETHER_KEY= diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0595eaf..234a760 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,11 @@ updates: open-pull-requests-limit: 5 commit-message: prefix: "deps" + ignore: + # Spring Boot 4 is a breaking migration (Jackson 3, renamed starters); + # take it deliberately, not as an automatic PR. + - dependency-name: "org.springframework.boot:*" + update-types: ["version-update:semver-major"] groups: spring: patterns: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 29ada51..a3578ca 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,5 +7,5 @@ - [ ] `./mvnw verify` passes locally - [ ] `python3 tools/smoke_test.py` passes against `tools/mock-upstream.py` - [ ] New behaviour is covered by a test (unit or smoke) -- [ ] README updated if a user-facing endpoint, config key or field changed +- [ ] The matching file in `docs/` updated if an endpoint, config key or field changed - [ ] No credentials, `.env` files or `data/` contents included diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 0000000..bc6477c --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,85 @@ +name: Provider canary + +on: + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + canary: + name: One tiny call per provider + runs-on: ubuntu-26.04 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up JDK 21 + uses: actions/setup-java@d1374f8ac74474200d16ef8ecee22c7ae7a51880 # v6 + with: + distribution: temurin + java-version: "21" + cache: maven + + - name: Skip when no provider secrets are configured + id: gate + run: | + if [ -z "$OPENAI_API_KEY$ANTHROPIC_API_KEY$GROQ_API_KEY" ]; then + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "No provider secrets configured; skipping the canary." + else + echo "enabled=true" >> "$GITHUB_OUTPUT" + fi + + - name: Start the gateway + if: steps.gate.outputs.enabled == 'true' + run: | + ./mvnw -B -DskipTests package + GATEWAY_ADMIN_TOKEN=canary java -jar target/path-*.jar > /tmp/gateway.log 2>&1 & + for i in $(seq 1 60); do + curl -sf http://127.0.0.1:8080/actuator/health > /dev/null && break + sleep 1 + done + KEY=$(grep -o 'sk-gw-[A-Za-z0-9]*' /tmp/gateway.log | head -1) + echo "PATH_KEY=$KEY" >> "$GITHUB_ENV" + echo "KEY_OK=$([ -n "$KEY" ] && echo yes || echo no)" >> "$GITHUB_ENV" + + - name: Probe each provider + if: steps.gate.outputs.enabled == 'true' + run: | + set -e + failed=0 + check() { + model="$1" + code=$(curl -s -o /tmp/out.json -w '%{http_code}' http://127.0.0.1:8080/v1/chat/completions \ + -H "Authorization: Bearer $PATH_KEY" -H 'Content-Type: application/json' \ + -d "{\"model\":\"$model\",\"max_tokens\":8,\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}]}") + if [ "$code" != "200" ]; then + echo "::error::canary failed for $model (HTTP $code): $(head -c 200 /tmp/out.json)" + failed=1 + else + echo "canary ok: $model" + fi + } + [ -n "$OPENAI_API_KEY" ] && check gpt-4o-mini + [ -n "$ANTHROPIC_API_KEY" ] && check claude-3.5-haiku + [ -n "$GROQ_API_KEY" ] && check llama-3.3-70b + exit $failed + + - name: Open an issue when the canary fails + if: failure() && steps.gate.outputs.enabled == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh issue create --repo "${{ github.repository }}" \ + --title "Provider canary failed on $(date -u +%Y-%m-%d)" \ + --body "The nightly canary could not complete a minimal call. See ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. This usually means a provider changed its API or a key was revoked." + + - name: Show gateway log on failure + if: failure() + run: tail -60 /tmp/gateway.log || true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1833cbe..415aa23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [main] + branches: [main, Develop] pull_request: - branches: [main] + branches: [main, Develop] workflow_dispatch: permissions: @@ -17,37 +17,72 @@ concurrency: jobs: build: name: Build and unit tests - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v4 - + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@d1374f8ac74474200d16ef8ecee22c7ae7a51880 # v6 with: distribution: temurin java-version: "21" cache: maven + - name: Validate the changelog is parseable + run: python3 tools/changelog.py 0.1.0 > /dev/null + - name: Build and test run: ./mvnw -B verify - name: Upload test reports if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: surefire-reports path: target/surefire-reports/ retention-days: 7 + helm: + name: Helm chart + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 + with: + version: v3.16.3 + + - name: Lint + run: helm lint deploy/helm/path --set existingSecret=path-credentials + + - name: Render every toggle + run: | + helm template path deploy/helm/path --set existingSecret=x > /dev/null + helm template path deploy/helm/path --set existingSecret=x \ + --set autoscaling.enabled=true --set postgres.enabled=true \ + --set valkey.enabled=true --set serviceMonitor.enabled=true > /dev/null + + - name: No credentials are rendered inline + run: | + if helm template path deploy/helm/path --set existingSecret=x | grep -q "kind: Secret"; then + echo "::error::The chart rendered a Secret while existingSecret was set" + exit 1 + fi + + - name: The drain guard must fail closed + run: | + if helm template path deploy/helm/path --set existingSecret=x \ + --set drainTimeoutSeconds=60 --set terminationGracePeriodSeconds=30 > /dev/null 2>&1; then + echo "::error::The chart rendered with a grace period below the drain timeout" + exit 1 + fi + e2e: name: End-to-end smoke test - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 needs: build steps: - - uses: actions/checkout@v4 - + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@d1374f8ac74474200d16ef8ecee22c7ae7a51880 # v6 with: distribution: temurin java-version: "21" @@ -60,7 +95,11 @@ jobs: run: | mkdir -p /tmp/opencode python3 tools/mock-upstream.py 9099 > /tmp/opencode/mock.log 2>&1 & - GATEWAY_ADMIN_TOKEN=test-admin java -jar target/path-*.jar \ + GATEWAY_ADMIN_TOKEN=test-admin \ + GATEWAY_PROMPT_LOGGING=true \ + GATEWAY_GUARDRAILS_MODERATION=true \ + GATEWAY_SEMANTIC_MODEL=embed-test \ + java -jar target/path-*.jar \ > /tmp/opencode/gateway.log 2>&1 & for i in $(seq 1 60); do if curl -sf http://127.0.0.1:8080/actuator/health > /dev/null; then @@ -70,8 +109,25 @@ jobs: sleep 1 done curl -sf http://127.0.0.1:8080/actuator/health + curl -sf http://127.0.0.1:8080/v3/api-docs -o /tmp/opencode/openapi.json + test -s /tmp/opencode/openapi.json + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8080/v1/models | grep -q 'gpt-4o-mini'; then + echo "catalogue seeded after ${i}s" + break + fi + sleep 1 + done python3 tools/smoke_test.py + - name: Upload OpenAPI spec + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: openapi + path: /tmp/opencode/openapi.json + retention-days: 14 + - name: Show gateway log on failure if: failure() run: tail -60 /tmp/opencode/gateway.log diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2c6121f..5658e68 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -15,25 +15,23 @@ permissions: jobs: analyze: name: Analyze Java - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v4 - + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@d1374f8ac74474200d16ef8ecee22c7ae7a51880 # v6 with: distribution: temurin java-version: "21" cache: maven - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4 with: languages: java-kotlin build-mode: autobuild - name: Autobuild - uses: github/codeql-action/autobuild@v3 - + uses: github/codeql-action/autobuild@33d7b85da1a04171e211befe2c739303b9a4c066 # v4 - name: Analyze - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4 \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8972830..0096cd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,36 +12,61 @@ permissions: jobs: release: - name: Publish jar, container and release notes - runs-on: ubuntu-latest + name: Scan, sign off and publish + runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Verify the tag is on main + run: | + git fetch --no-tags origin main:refs/remotes/origin/main + if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then + echo "::error::Tag $GITHUB_REF_NAME points at a commit that is not on main." + exit 1 + fi + echo "Tag is on main: $GITHUB_SHA" - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@d1374f8ac74474200d16ef8ecee22c7ae7a51880 # v6 with: distribution: temurin java-version: "21" cache: maven - - name: Build jar + - name: Build jar and SBOM run: ./mvnw -B -DskipTests package - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + # --- supply chain: checksums ------------------------------------------- + - name: Collect SBOM + run: cp target/classes/META-INF/sbom/application.cdx.json target/bom.json - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 + - name: Write SHA-256 checksums + run: | + cd target + sha256sum path-*.jar bom.json | sort -k2 > checksums.txt + ls -l path-*.jar bom.json checksums.txt + cat checksums.txt + + - name: Verify the checksums we just wrote + run: cd target && sha256sum -c checksums.txt + # --- container: build, scan, push -------------------------------------- + - name: Set up QEMU + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 + - name: Set up Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push container image - uses: docker/build-push-action@v6 + id: image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: context: . push: true @@ -52,10 +77,64 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Scan the published image + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + env: + TRIVY_USERNAME: ${{ github.actor }} + TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + with: + image-ref: ghcr.io/${{ github.repository }}:${{ github.ref_name }} + # CRITICAL blocks the release; HIGH is visible in the scan output but the + # Debian base image rotates packages too often to gate on it. + severity: CRITICAL + exit-code: "1" + ignore-unfixed: true + + - name: Record the image digest + run: | + echo "### Container digest" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "ghcr.io/${{ github.repository }}@${{ steps.image.outputs.digest }}" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + + # --- Maven Central (optional) ------------------------------------------- + - name: Publish to Maven Central + if: vars.PUBLISH_TO_CENTRAL == 'true' + env: + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }} + run: | + mkdir -p ~/.gnupg && chmod 700 ~/.gnupg + printf '%s' "${{ secrets.MAVEN_GPG_PRIVATE_KEY }}" | gpg --batch --import + mkdir -p ~/.m2 + cat > ~/.m2/settings.xml < + + + central + ${CENTRAL_USERNAME} + ${CENTRAL_PASSWORD} + + + + XML + ./mvnw -B -P release -DskipTests deploy + + # --- publish ----------------------------------------------------------- + - name: Extract release notes from CHANGELOG.md + run: | + python3 tools/changelog.py "${GITHUB_REF_NAME}" > /tmp/release-notes.md + test -s /tmp/release-notes.md + cat /tmp/release-notes.md + - name: Create GitHub release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@8a0cd651e5b2365bd640854a3fa5dfc70e4875f3 # v3 with: name: ${{ github.ref_name }} - generate_release_notes: true - files: target/path-*.jar + body_path: /tmp/release-notes.md + files: | + target/path-*.jar + target/bom.json + target/checksums.txt fail_on_unmatched_files: true diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..b1fe8fc --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,62 @@ +name: Security + +on: + pull_request: + branches: [main, Develop] + push: + branches: [main, Develop] + schedule: + - cron: "0 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: security-${{ github.ref }} + cancel-in-progress: true + +jobs: + secrets: + name: Secret scan + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Scan history for committed secrets + uses: gitleaks/gitleaks-action@e0c47f4f8be36e29cdc102c57e68cb5cbf0e8d1e # v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + dependencies: + name: Dependency review + if: github.event_name == 'pull_request' + runs-on: ubuntu-26.04 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Reject new high-severity dependencies + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5 + with: + fail-on-severity: high + comment-summary-in-pr: always + + misconfig: + name: Config and file scan + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Trivy (secrets and misconfiguration) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + scan-type: fs + scan-ref: . + scanners: secret,misconfig + severity: HIGH,CRITICAL + exit-code: "1" + ignore-unfixed: true + skip-dirs: target,data diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..05566ca --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,33 @@ +# gitleaks configuration +# +# Test fixtures, documentation examples and generated files intentionally contain +# placeholder credentials (`sk-gw-...`, `mock-secret`, `sk-ant-...`). Real credentials +# are never committed; anything that looks real should still fail the scan. + +title = "Path" + +[extend] +useDefault = true + +[allowlist] +description = "Placeholders in tests, docs and example configuration" +paths = [ + '''src/test/.*''', + '''docs/.*''', + '''tools/.*''', + '''README\.md''', + '''AGENTS\.md''', + '''\.env\.example''', + '''src/main/resources/application\.yml''', +] + +regexes = [ + '''sk-gw\.\.\.''', + '''sk-gw-does-not-exist''', + '''sk-gw-[A-Za-z0-9]{0,4}$''', + '''mock-secret''', + '''test-admin''', + '''sk-ant-\.\.\.''', + '''gsk_\.\.\.''', + '''sk-\.\.\.''', +] diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 216df05..058db7b 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1,3 +1,4 @@ wrapperVersion=3.3.4 distributionType=only-script -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.16/apache-maven-3.9.16-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.10.1/apache-maven-3.10.1-bin.zip +distributionSha256Sum=9c1ef9174ef250a5009ed0dec97cbe8699acf7479add4e62bf7b0ea791c0adb5 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2a0a7a1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,274 @@ +# AGENTS.md — Path + +Working notes for agents and contributors. Read this before changing code; the +gotchas at the bottom each cost real debugging time. + +## What this is + +A provider-agnostic LLM gateway. One HTTP surface in front of OpenAI, Anthropic, +Groq and any OpenAI-compatible endpoint, with model routing, automatic fallback, +per-key credits, rate limits and cache-aware token metering. + +The client-facing contract is the OpenAI chat-completions shape because that is what +SDKs and tools already implement. That is a compatibility decision, **not** a vendor +dependency: the gateway translates to each provider's native protocol. + +## TL;DR + +```bash +# unit + integration tests (85) +./mvnw verify + +# full end-to-end suite against a mock upstream (104 checks, no real keys) +python3 tools/mock-upstream.py 9099 & +GATEWAY_ADMIN_TOKEN=test-admin java -jar target/path-*.jar & +python3 tools/smoke_test.py + +# run locally +export OPENAI_API_KEY=... ANTHROPIC_API_KEY=... GATEWAY_ADMIN_TOKEN=... +./mvnw spring-boot:run +``` + +The seeder prints a demo `sk-gw-...` key on the first boot only. + +## Layout + +``` +com.anindra.path +├── api/ ChatCompletionsController (public surface), errors, Durations +├── admin/ AdminController + services (providers, routes, keys, sync, stats) +├── config/ GatewayProperties, AuthFilter, CORS +├── domain/ JPA entities: Provider, ModelRoute, ClientKey, UsageRecord +├── gateway/ the engine: adapters, routing, streaming, metering +│ ├── ProviderAdapter wire-format contract +│ ├── OpenAiCompatibleAdapter OpenAI + Groq + vLLM + Together + ... +│ ├── AnthropicAdapter Messages API, tools, cache tokens +│ ├── ChatService non-streaming path with fallback +│ ├── StreamService SSE path; owns commit semantics +│ ├── ProviderCircuitBreaker per-provider OPEN/HALF_OPEN/CLOSED, Retry-After +│ ├── IdempotencyService Idempotency-Key claims and replay +│ ├── ConcurrencyLimiter per-key in-flight permits +│ ├── RateLimitStore in-memory or Valkey RPM limiter +│ ├── ResponseCacheService exact-match replay, opt-in per route +│ ├── DrainState shutdown drain + readiness +│ └── (config) RequestIdFilter X-Request-Id + MDC correlation +│ ├── RouteResolver public model id → ordered chain (aliases, weights) +│ ├── ModelAliasService cheap/fast/long-context/vision +│ ├── EmbeddingsService /v1/embeddings for OPENAI-type providers +│ └── (admin) AuditService, AlertService, PromptLogService, TenancyService +│ ├── CreditGuard pre-flight balance/worst-case check +│ ├── UsageService cost breakdown, aggregates, metering +│ └── TokenUsage normalised token accounting +├── repo/ Spring Data repositories +├── secret/ AES-GCM for stored provider credentials +└── seed/ first-boot bootstrap +``` + +## Request flow + +1. `AuthFilter` resolves `Authorization: Bearer sk-gw-...` → `ClientKey`, applies the + RPM limiter, or checks `X-Admin-Token` for `/admin/**`. +2. `ChatRequestSpec.parse` validates the body and keeps the original JSON for + passthrough. +3. `RouteResolver` turns the public model id (plus `models[]` and `provider.order`) + into an ordered chain of `(route, provider)`. +4. `CreditGuard` refuses the call if the balance cannot cover a worst-case estimate. +5. `ChatService` / `StreamService` walk the chain: one attempt per route, adapter + translates, `UpstreamClient` sends raw HTTP. +6. `UsageService` prices the call, writes a `UsageRecord`, charges the key. + +## Conventions + +- **No AI SDKs.** The gateway speaks HTTP + SSE with Jackson. SDKs own the request + model and drop unknown fields, which breaks passthrough; they also add retries that + corrupt fallback accounting. See docs/LIMITS.md. +- **`promptTokens` means total prompt**, including cache reads and cache writes: + `billableInput = prompt - cached - cacheWrite`. OpenAI reports cache as a subset; + the Anthropic adapter sums it. Keep this invariant when adding providers. +- **Cost is 8 decimals**, each component rounded before summing. Do not "fix" the + smoke test's `1e-8` tolerance by widening it further. +- **Admin JSON is snake_case**, with camelCase accepted via `@JsonAlias` on the input + records. Responses are always snake_case maps. +- **Errors use the OpenAI envelope**: `{"error":{"message","type","code","param"}}`. +- **Never log a credential.** Admin views show `credential_hint` (last 4) only. +- **Streaming commits on the first byte.** Fallback is possible only before that; + afterwards the client gets an SSE error object and `[DONE]`. + +## Where to add things + +| Task | Where | +|---|---| +| New provider (OpenAI-shaped or Anthropic-shaped) | No code: `POST /admin/providers` + routes, or `GATEWAY_UPSTREAM__KEY` | +| New wire format | Implement `ProviderAdapter` + `StreamTranslator`, add tests | +| New public endpoint | `api/`, reuse `ApiException` + the error envelope | +| New metered field | `TokenUsage` → both adapters → `UsageRecord` → `UsageService` → `/v1/usage` and `/admin/usage` | +| Admin action | `AdminService` + a thin `AdminController` method | + +## Schema + +`src/main/resources/db/migration/` is the only source of truth. Hibernate validates on +startup; a mismatch fails the boot. Tests run the same migrations against in-memory H2. + +## Documentation map + +`README.md` is the landing page. Details live in: + +| File | Contents | +|---|---| +| `docs/API.md` | endpoints, errors, idempotency, cache, embeddings | +| `docs/ARCHITECTURE.md` | request flow, routing, metering fields | +| `docs/CONFIGURATION.md` | every environment variable and profile | +| `docs/OPERATIONS.md` | admin API, metrics, migrations, alerts | +| `docs/LIMITS.md` | design notes and known gaps | +| `docs/TESTING.md` | test suites and the mock upstream | +| `docs/ROADMAP.md` | delivered work and the open stories | +| `changelog/` | one release-notes file per tag, published as the release body | + +Keep README short: a feature can be one bullet there and a section in the matching doc. + +## Testing + +- `GatewayApiTest` — boots the full app on in-memory H2: auth, catalogue, credits, + admin token, snake_case inputs. +- `AnthropicAdapterTest` — request translation (system, images, tools, tool results), + response translation, stream events, cache tokens. +- `UsageCostTest` — cost breakdown, cache pricing, missing prices, usage parsing. +- `ProviderCircuitBreakerTest` — threshold, half-open probe, rate-limit parking. +- `IdempotencyServiceTest` — replay, conflicts, concurrent claims, key scoping. +- `ConcurrencyLimiterTest` — per-key permits, no over-release. +- `TokenEstimateTest` — real tokenization with a heuristic fallback. +- `ResponseCacheEligibilityTest` / `ResponseCacheServiceTest` — cache rules, scoping, expiry. +- `CreditGuardTest` — budget windows and max_tokens clamping. +- `DrainStateTest` — in-flight accounting and refusal while draining. +- `tools/smoke_test.py` — 104 checks against `tools/mock-upstream.py`. The mock can + simulate failures by model id: `boom` (500), `ratelimit` (429), `unauthorized` (401), + `badrequest` (400), `slow`, `empty`. + +Any behaviour change needs a unit test **and**, if it touches routing/streaming/billing, +a smoke check. Never claim a fix without running both. + +## Branches and CI + +- `Develop` is the default branch; direct pushes are allowed. +- `main` is release-only: PR required, `Build and unit tests` + `End-to-end smoke test` + must pass, no bypassing (admins included), no force-push, no deletion. +- Maven Central coordinates are `io.github.an1ndra:path` (GitHub namespace); the Java + package stays `com.anindra.path`. Publishing lives behind `-P release` (plain jar + + sources + javadoc, signed); see `docs/PUBLISHING.md`. Never run `deploy` against a used + version — Central is immutable. After a `-P release` build the jar in `target/` is the + plain library, so rebuild before running the smoke suite. +- CI pins `ubuntu-26.04` (latest LTS) instead of `ubuntu-latest`; bump it deliberately. +- **Every action is pinned to a commit SHA**, with the tag in a trailing comment + (`actions/checkout@3d3c42e5... # v7.0.1`). A mutable tag can be repointed at any commit, + which is exactly the supply-chain hole this closes. Bump by merging the Dependabot + `github-actions` PR, or `pinact run` locally; never replace a SHA with `@vN` again. +- Release: copy `changelog/unreleased.md` to `changelog/vX.Y.Z.md`, commit, then push a + `v*.*.*` tag → jar + SBOM + checksums + multi-arch `ghcr.io/an1ndra/path`. The workflow + publishes `changelog/.md` verbatim as the release body and fails if the file is + missing or empty (`tools/changelog.py`). + +## Issue labels + +Every open story carries exactly one `area/` label and one `effort/` label, so the +queue is filterable by `label:area/routing` or by size. + +| Label | Meaning | +|---|---| +| `area/routing` | model selection, fallback chains, aliases | +| `area/billing` | credits, budgets, idempotent charging | +| `area/metering` | usage records, reporting, token counting | +| `area/providers` | wire formats, adapters, provider support | +| `area/security` | PII, guardrails, auth, secrets | +| `area/ops` | deployment, scaling, cluster behaviour | +| `area/api` | public and admin API surface | +| `effort/s` | roughly a day or less | +| `effort/m` | roughly two to five days | +| `effort/l` | a week or more, or needs a design decision | +| `blocked` | waiting on something outside this repository | + +A story may carry two `area/` labels when it genuinely spans two (for example a +billing change that also adds an endpoint). Never three, and never none. `effort/*` +is a guess, not a commitment — re-tag when reality disagrees. + +## Gotchas + +- **Flyway owns the schema; `ddl-auto` is `validate`.** Add `V__name.sql` for every + entity change and never edit an applied migration. `V1__baseline.sql` is written in + portable SQL: `bigint generated by default as identity`, `timestamp(6) with time zone`, + enums as `varchar` + check constraint (native enum types would not port between H2 and + Postgres). Existing pre-Flyway databases baseline at V1 automatically. +- **The circuit breaker is in-memory per node.** Like the rate limiter, state is not + shared between instances; a restart clears it. That is deliberate for v1. +- **Idempotency claims are written before the upstream call.** If the process dies + mid-request the row stays PENDING until `GATEWAY_IDEMPOTENCY_TTL`, so retries get + 409 `idempotency_in_progress`. The purge job only removes expired rows. +- **Spend windows use stored usage, not a counter.** `daily_limit_usd` and + `monthly_limit_usd` compare against `sum(cost_usd)` since UTC midnight / month start, + so they survive restarts and stay correct across instances. +- **Shutdown drains for `GATEWAY_DRAIN_TIMEOUT`.** `DrainState` flips readiness and + refuses new chat requests while streams finish; keep the value below your orchestrator's + termination grace period. +- **System-only conversations are rejected.** Anthropic needs a non-empty message list; + `ChatRequestSpec.parse` returns 400 instead of forwarding an empty array. The property + test found this; keep the guard. +- **`tools/load-test.js`** is a k6 script, not part of CI (it needs a running gateway and + a real key). +- **`allowed_models` is enforced in `CreditGuard`**, before any provider call. Globs end + with `*`; blank means every model. Aliases are checked by their alias name, so add the + alias to the allowlist too if a key should use `cheap`. +- **Prompt logging is a PII store.** Off by default; when enabled it redacts provider + keys and bearer tokens, but it still stores user prompts. Retention is enforced by a + scheduled purge. +- **Audit and alert writes must never break the operation.** Both swallow their own + failures and log a warning. +- **Aliases are computed from the route table on every resolve**, so editing a price can + change what `cheap` means. There is no alias table to drift out of sync. +- **Canary weights must stay deterministic per key.** `RouteResolver.weighted` hashes + `keyId` and the model; changing that hash reshuffles every client's assignment. +- **Tracing is off by default** (`GATEWAY_TRACE_SAMPLE=0`). Raising the probability + without setting `OTEL_EXPORTER_OTLP_ENDPOINT` produces export warnings, not traces. +- **The `prod` profile switches logs to ECS JSON.** `requestId` comes from + `RequestIdFilter`'s MDC entry; `traceId`/`spanId` are added by the tracing bridge. +- **RPM limiting defaults to memory; set `GATEWAY_RATE_LIMIT_BACKEND=valkey` in a + cluster.** Budget reservations are database-backed either way, so money is safe + without Valkey. +- **Admin tokens are scoped.** The bootstrap env token is read+write; DB tokens are + read-only unless `scopes` contains `write`. +- **Weighting is opt-in.** A chain of default-weight (100) routes keeps priority order; + canary shuffling only happens once a weight differs from 100. Otherwise fallback would + become a lottery. +- **Rate limiting is in-memory** by default, one node. Running two instances doubles the effective + RPM limit. +- **H2 rejects `AUTO_SERVER=TRUE` together with `DB_CLOSE_ON_EXIT`.** Don't re-add it. +- **Admin credentials are resolved DB → config → env.** A key stored via the admin API + silently wins over `OPENAI_API_KEY`. +- **Anthropic rejects parameters it has no equivalent for** with 422 + `unsupported_parameter` (`seed`, `logit_bias`, penalties, `logprobs`, `n > 1`, + non-text `response_format`). Do not "fix" a failing call by dropping the field; the + divergence is what the 422 exists to prevent. `user` maps to `metadata.user_id`. +- **The response cache only serves explicit `temperature: 0`, tool-free requests on a + route with `cache_enabled`.** Entries are keyed per client key so tenants cannot read + each other's completions. Do not relax the temperature rule: a cached sample at + temperature 1 is a correctness bug. +- **Token estimates use jtokkit** when the upstream model has a known encoding, and fall + back to `chars / 4`. The estimate only pre-authorises spend; billing always uses the + provider-reported usage. +- **`usage_estimated=true`** means the provider returned no `usage` block and the counts + were inferred from text length. Never present those as observed. +- **The demo key is unlimited** and is created when the key table is empty. Set + `GATEWAY_BOOTSTRAP_DEMO_KEY=false` anywhere real. +- **`tools/smoke_test.py` needs a clean DB** for the demo-key path; otherwise it creates + its own key through the admin API. + +## Security gates + +`main` requires the CI, CodeQL and Security checks. `.gitleaks.toml` allowlists the +placeholder credentials in tests and docs — do not broaden it to hide a real key. +Release artifacts carry `checksums.txt` and the wrapper pins the Maven distribution +SHA-256. + +## Secrets + +Never commit `.env`, `data/`, `*.mvdb`, master keys, or provider credentials. Stored +credentials are AES-GCM encrypted; `data/master.key` decrypts them and must be backed +up, never shared. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a17b956 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +Release notes live in [`changelog/`](changelog/), one file per tag, and each is published +verbatim as the body of the matching GitHub release. + +| Version | Date | Notes | +|---|---|---| +| v0.2.0 | 2026-11-17 | [changelog/v0.2.0.md](changelog/v0.2.0.md) | +| v0.1.0 | 2026-11-17 | [changelog/v0.1.0.md](changelog/v0.1.0.md) | + +Add new entries to [`changelog/unreleased.md`](changelog/unreleased.md) as you work; the +release steps are in [`changelog/README.md`](changelog/README.md). diff --git a/Dockerfile b/Dockerfile index ae26f6a..e1f2740 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,15 @@ RUN ./mvnw -B -q -DskipTests package FROM eclipse-temurin:21-jre WORKDIR /app COPY --from=build /app/target/path-*.jar app.jar -RUN mkdir -p /app/data + +# Run as an unprivileged user (Trivy DS-0002) and keep ./data writable for the H2 file +# and the generated master key. +RUN groupadd --system --gid 1001 path \ + && useradd --system --uid 1001 --gid path --home-dir /app --shell /usr/sbin/nologin path \ + && mkdir -p /app/data \ + && chown -R path:path /app +USER path + ENV PORT=8080 EXPOSE 8080 ENTRYPOINT ["java", "-jar", "/app/app.jar"] diff --git a/README.md b/README.md index a15f9e1..b1c9ccb 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,17 @@ [![Release](https://github.com/an1ndra/path/actions/workflows/release.yml/badge.svg)](https://github.com/an1ndra/path/actions/workflows/release.yml) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) -An OpenAI-compatible LLM gateway in the mould of OpenRouter: one API in front of many -providers, with model routing, automatic fallback, per-key credits, rate limits and -usage metering. Spring Boot 3.5, Java 21, H2 by default (Postgres ready). +**The LLM gateway for teams that run Java.** A provider-agnostic gateway — one HTTP +surface in front of OpenAI, Anthropic, Groq, Google Gemini and anything +OpenAI-compatible — with model routing, automatic fallback, per-key credits and +cache-aware token metering. Spring Boot 3.5, Java 21, one process to operate. + +New here? [Why Path](docs/WHY-PATH.md) compares it honestly with LiteLLM, Portkey and +OpenRouter, including where it loses. + +Clients speak the OpenAI chat-completions shape because that is what most SDKs already +implement. The gateway translates to each provider's native protocol, so adding or +swapping a provider never changes client code. ```mermaid flowchart LR @@ -21,58 +29,36 @@ flowchart LR X -.-> U ``` -Every request enters through one OpenAI-compatible surface, is authenticated and -authorised, then routed to the highest-priority provider that can serve it; the -response is metered on the way back. - ## What it does -- **One wire format.** Clients speak OpenAI chat-completions. Anthropic's Messages API - is translated in both directions, including streaming and tool calls. -- **Routing.** A public model id (e.g. `gpt-4o-mini`, `auto-fast`) maps to one or more - ordered routes. Lower `priority` wins. -- **Fallback.** 429 / 408 / 5xx / timeouts fall through to the next route. Streaming - falls back only before the first chunk; afterwards the client is committed. -- **Clients and credits.** Issue `sk-gw-...` keys, set a USD credit limit and an RPM - limit, watch `used_usd` grow as requests are metered by token price. -- **Metering.** Every call is recorded with model, provider, tokens, cost, latency and - status. Percentiles and per-provider/model breakdowns via `/admin/stats`. -- **Model catalogue.** `/v1/models` exposes every enabled route with providers and - pricing; `/admin/models/sync` imports each provider's real model list. - -## What is metered - -Every call records the full token picture both providers report, not just totals: - -| Field | Source (OpenAI / Anthropic) | -|---|---| -| `prompt_tokens` | `usage.prompt_tokens` / `input_tokens` + cache read + cache write | -| `completion_tokens` | `usage.completion_tokens` / `output_tokens` (includes thinking) | -| `cached_input_tokens` | `prompt_tokens_details.cached_tokens` / `cache_read_input_tokens` | -| `cache_write_tokens` | — / `cache_creation_input_tokens` | -| `reasoning_tokens` | `completion_tokens_details.reasoning_tokens` / in `output_tokens` | -| `audio_input/output_tokens` | `*_tokens_details.audio_tokens` / — | - -Cost is stored itemised — `input_cost_usd`, `cached_input_cost_usd`, -`cache_write_cost_usd`, `output_cost_usd`, `cost_usd` — plus -`cached_savings_usd` (what the cached tokens would have cost at the full input rate). -Cached tokens are billed at their own rate when the route has one, otherwise at the -plain input rate; Anthropic cache writes are billed at their premium rate. +- **One surface, many providers** — OpenAI, Anthropic, Groq, Google Gemini and any + OpenAI-compatible endpoint, including streaming and tool calls. +- **Routing and fallback** — a model id maps to an ordered chain; 429/5xx/timeouts fall + through, with capability aliases (`cheap`, `fast`, `long-context`) and weighted canary + splits. +- **Keys, budgets and limits** — `sk-gw-...` keys with lifetime/daily/monthly budgets, + per-key model allowlists, RPM and in-flight caps; projects and scoped admin tokens for + teams. +- **Cache-aware metering** — prompt, completion, cached and reasoning tokens, itemised + cost, TTFT and throughput per call. +- **Operable** — Flyway migrations, health/readiness, Prometheus metrics, tracing, + structured logs, an admin audit trail and an OpenAPI spec. + +## Install on Kubernetes -Also captured per call: `ttft_ms` and `tokens_per_second` (streaming), -`upstream_request_id`, `finish_reason`, `attempts`, `fallback_from`, and -`usage_estimated` — true when the provider returned no usage block and the counts were -inferred from text length rather than observed. +```bash +helm install path deploy/helm/path --namespace path --create-namespace \ + --set existingSecret=path-credentials +``` -Aggregates: `/v1/usage` adds cache hit ratio and savings for the calling key; -`/admin/stats` adds cache hit ratio and savings, p50/p95 TTFT, median throughput, -fallback count and estimated-usage count. +The chart refuses to render when `terminationGracePeriodSeconds` is not above +`GATEWAY_DRAIN_TIMEOUT`, because that combination truncates live SSE streams on a +rolling update. See [deploy/helm/path/README.md](deploy/helm/path/README.md) for the rest. ## Quick start ```bash -cd ai-gateway -export OPENAI_API_KEY=sk-... # any subset works +export OPENAI_API_KEY=sk-... # any subset works export ANTHROPIC_API_KEY=sk-ant-... export GROQ_API_KEY=gsk_... export GATEWAY_ADMIN_TOKEN=pick-something-long @@ -80,211 +66,52 @@ export GATEWAY_ADMIN_TOKEN=pick-something-long ./mvnw spring-boot:run ``` -On first boot the seeder creates three providers, a starter route table and one demo -client key, printed in the log: +First boot seeds four providers and a starter route table. To also mint a demo key +(off by default, never in the `prod` profile), start with +`GATEWAY_BOOTSTRAP_DEMO_KEY=true`; it is printed once in the log: ``` Created demo API key (no credit limit): sk-gw-... ``` -Send a request: - ```bash KEY=sk-gw-... -# non-streaming curl -s http://localhost:8080/v1/chat/completions \ -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ - -d '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"Say hi"}]}' - -# streaming -curl -N http://localhost:8080/v1/chat/completions \ - -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ - -d '{"model":"claude-3.5-haiku","stream":true, - "stream_options":{"include_usage":true}, - "messages":[{"role":"user","content":"Say hi"}]}' + -d '{"model":"auto-fast","messages":[{"role":"user","content":"Say hi"}]}' ``` -Non-streaming responses carry `X-Gateway-Provider`, `X-Gateway-Upstream-Model`, -`X-Gateway-Cost-Usd` and `X-Gateway-Latency-Ms`. - -Any OpenAI SDK works — set `base_url` to `http://localhost:8080/v1` and the key above: +Any OpenAI SDK works — point `base_url` at `http://localhost:8080/v1`: ```python from openai import OpenAI client = OpenAI(base_url="http://localhost:8080/v1", api_key="sk-gw-...") print(client.chat.completions.create( - model="auto-fast", + model="cheap", messages=[{"role": "user", "content": "hello"}], ).choices[0].message.content) ``` -## Routing - -Routes are rows of `(public_model_id, provider, upstream_model_id, priority, price)`. - -`auto-fast` in the seed data has two routes, which gives a cross-provider fallback: - -| public id | provider | upstream | priority | -|---|---|---|---| -| auto-fast | openai | gpt-4o-mini | 1 | -| auto-fast | groq | llama-3.3-70b-versatile | 2 | - -Request-level controls, OpenRouter style: - -```jsonc -{ - "model": "gpt-4o-mini", - "models": ["gpt-4o-mini", "claude-3.5-haiku"], // extra fallback models, in order - "provider": {"order": ["groq", "openai"], "allow_fallbacks": false}, - "messages": [...] -} -``` - -`provider.allow_fallbacks: false` pins the request to the first route, so an upstream -500 surfaces to the caller instead of being masked. - -## Admin API - -Every `/admin/**` call needs `X-Admin-Token: $GATEWAY_ADMIN_TOKEN`. If that variable is -empty the admin API is disabled (403). - -```bash -A='X-Admin-Token: pick-something-long' - -# providers -curl -s localhost:8080/admin/providers -H "$A" -curl -s -X PATCH localhost:8080/admin/providers/1 -H "$A" -H 'Content-Type: application/json' \ - -d '{"base_url":"https://api.openai.com/v1","api_key":"sk-..."}' -curl -s -X POST localhost:8080/admin/providers -H "$A" -H 'Content-Type: application/json' \ - -d '{"slug":"together","display_name":"Together","type":"OPENAI", - "base_url":"https://api.together.xyz/v1","api_key":"..."}' - -# routes -curl -s -X POST localhost:8080/admin/routes -H "$A" -H 'Content-Type: application/json' \ - -d '{"public_model_id":"gpt-4o-mini","provider_id":1,"upstream_model_id":"gpt-4o-mini", - "input_price_per_million":0.15,"output_price_per_million":0.60,"priority":10}' - -# client keys — the plaintext is returned exactly once -curl -s -X POST localhost:8080/admin/keys -H "$A" -H 'Content-Type: application/json' \ - -d '{"name":"team-a","credit_limit_usd":25,"rpm_limit":120}' -curl -s -X POST localhost:8080/admin/keys/1/topup -H "$A" -H 'Content-Type: application/json' \ - -d '{"amount_usd":10}' -curl -s -X POST localhost:8080/admin/keys/1/revoke -H "$A" - -# import each provider's model list (built-in prices for well-known ids, 0.00 otherwise) -curl -s -X POST localhost:8080/admin/models/sync -H "$A" - -# reporting -curl -s 'localhost:8080/admin/stats?window=24h' -H "$A" -curl -s 'localhost:8080/admin/usage?window=7d&limit=50' -H "$A" -``` +## Documentation -Admin JSON accepts snake_case (`credit_limit_usd`) and camelCase (`creditLimitUsd`). - -## Public endpoints - -| Method | Path | Auth | Purpose | -|---|---|---|---| -| POST | `/v1/chat/completions` | bearer key | chat, `stream: true` for SSE | -| GET | `/v1/models` | none if `gateway.public-models=true` | catalogue | -| GET | `/v1/models/{id}` | same | one model | -| GET | `/v1/usage?window=24h` | bearer key | this key's tokens, spend, recent calls | -| GET | `/v1/credits` | bearer key | limit, used, remaining | -| GET | `/actuator/health` | none | liveness | - -Errors use the OpenAI envelope: - -```json -{"error": {"message": "...", "type": "invalid_request_error", "code": "model_not_found"}} -``` - -Codes include `missing_api_key`, `invalid_api_key`, `key_disabled`, -`model_not_found`, `insufficient_credits`, `rate_limit_exceeded`, -`upstream_timeout`, `upstream_error`, `provider_auth_error`, `all_providers_failed`. - -## Configuration - -| Variable | Default | Meaning | -|---|---|---| -| `PORT` | `8080` | HTTP port | -| `GATEWAY_ADMIN_TOKEN` | empty (admin off) | admin shared secret | -| `GATEWAY_PUBLIC_MODELS` | `true` | allow keyless `/v1/models` | -| `GATEWAY_CORS_ORIGINS` | `*` | comma-separated origin patterns | -| `GATEWAY_MASTER_KEY` | generated | base64 32-byte AES key for stored credentials | -| `GATEWAY_MASTER_KEY_PATH` | `./data/master.key` | where the generated key is kept (0600) | -| `GATEWAY_BOOTSTRAP_DEMO_KEY` | `true` | create a demo key when the table is empty | -| `GATEWAY_DEFAULT_RPM` | `120` | default requests/minute for new keys | -| `GATEWAY_DEFAULT_CREDIT_USD` | unlimited | default credit limit for new keys | -| `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` / `GROQ_API_KEY` | — | upstream credentials | -| `GATEWAY_UPSTREAM__KEY` | — | credential for any other provider slug | -| `DB_URL`, `DB_USER`, `DB_PASSWORD` | H2 file | database | - -Timeouts live in `application.yml` under `gateway.*`: `connect-timeout` (10s), -`request-timeout` (180s), `stream-idle-timeout` (120s). - -Credentials stored through the admin API are AES-GCM encrypted; the environment -variables are never written to disk. - -## Postgres - -```bash -export SPRING_PROFILES_ACTIVE=postgres -export DB_URL=jdbc:postgresql://localhost:5432/aigateway -export DB_USER=aigateway DB_PASSWORD=aigateway -./mvnw spring-boot:run -``` - -or `docker compose up --build`. - -## Tests - -```bash -./mvnw test # 11 tests: API surface, auth, credits, Anthropic translation -``` - -End-to-end against a mock upstream (no real keys needed, exercises both wire formats, -streaming, fallback, 402/429, metering and sync): - -```bash -python3 tools/mock-upstream.py 9099 & -GATEWAY_ADMIN_TOKEN=test-admin java -jar target/path-0.1.0.jar & -python3 tools/smoke_test.py # 49 checks -``` - -## Design notes and limits - -- **Rate limiting is in-memory, one node.** It is exact on a single instance and - approximate across a cluster. Move the map in `RateLimiter` to Redis for multi-node. -- **Credits are charged after the call** from reported token usage, with a worst-case - estimate checked before it. An upstream that reports no usage is billed from an - estimate and flagged with `usage_estimated=true`, so inferred spend is never - presented as observed. -- **Cost precision is 8 decimals.** Each component is rounded before summing, so a - call's stored total can differ from the exact product by at most one unit in the - eighth decimal (USD 1e-8). -- **Streaming has no resume.** Once bytes are sent the route cannot change; failures - arrive as an SSE error object followed by `data: [DONE]`. -- **Chat only.** Like OpenRouter, this gateway does not proxy embeddings/images/audio; - the adapters are written around chat-completions. -- **Anthropic tool translation** covers text, images (base64 and URL), tools, - tool results, `tool_choice` and `parallel_tool_calls`. Anthropic-specific betas, - prompt caching and extended thinking blocks are not forwarded. -- **Model sync** filters out non-chat ids (embeddings, TTS, whisper, image models) - and prices only the ids listed in `ModelSyncService.PRICES`; everything else lands - at 0.00 until edited. +| Document | Contents | +|---|---| +| [API reference](docs/API.md) | Every endpoint, request/response shapes, error codes, idempotency, cache, embeddings | +| [Architecture](docs/ARCHITECTURE.md) | Request flow, providers, routing, what is metered | +| [Configuration](docs/CONFIGURATION.md) | Every environment variable, timeouts, credentials, Postgres | +| [Operations](docs/OPERATIONS.md) | Admin API, metrics, tracing, migrations, alerts, load testing | +| [Limits](docs/LIMITS.md) | Design decisions and known gaps | +| [Testing](docs/TESTING.md) | Unit tests, the 99-check smoke suite, the mock upstream | +| [Why Path](docs/WHY-PATH.md) | Honest comparison and a 60-second demo | +| [Publishing](docs/PUBLISHING.md) | Maven Central setup, signing and the release profile | +| [Roadmap](docs/ROADMAP.md) | Delivered work and the open stories | +| [Changelog](CHANGELOG.md) | Release notes, also used as the GitHub release body | +| [Security policy](SECURITY.md) | How to report a vulnerability and what CI checks | +| [AGENTS.md](AGENTS.md) | Contributor conventions and gotchas | +| [Postman](docs/path.postman_collection.json) | 26 requests over the public and admin surfaces | +| [Bruno](bruno/README.md) | the same 26 requests as git-friendly `.bru` files | ## License -Copyright (C) 2026 Anindra - -Path is free software: you can redistribute it and/or modify it under the terms of the -GNU General Public License as published by the Free Software Foundation, either -version 3 of the License, or (at your option) any later version. - -Path is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; -without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the [GNU General Public License](LICENSE) along with -this program. If not, see . +GNU General Public License v3.0 — see [LICENSE](LICENSE). Copyright (C) 2026 Anindra. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2d5a628 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,66 @@ +# Security policy + +## Reporting a vulnerability + +Please report vulnerabilities privately through +[GitHub security advisories](https://github.com/an1ndra/path/security/advisories/new) +rather than as a public issue. + +Include the version or commit, a reproduction, and the impact you believe it has. You can +expect an acknowledgement within a few days. Please do not open a public issue, and do not +test against infrastructure you do not own. + +## Scope + +In scope: + +- the gateway's HTTP surface (`/v1/**`, `/admin/**`) +- authentication and authorisation (client keys, admin tokens, scopes) +- credential handling (AES-GCM storage, redaction, logging) +- the container image and its dependencies + +Out of scope: + +- vulnerabilities in upstream model providers +- findings that require a compromised host or an already-leaked admin token +- denial of service from traffic you are authorised to send at unlimited volume + +## How this repository is checked + +Every pull request runs: + +| Check | Tool | +|---|---| +| Secret scanning (full history) | gitleaks | +| New dependency review (fails on high severity) | `actions/dependency-review-action` | +| Static analysis for Java | CodeQL | +| Secrets and misconfiguration in the tree | Trivy | +| Repointed action tags (every `uses:` is pinned to a commit SHA) | Dependabot + review | +| Admin auth brute force (10 failures / 5 min per address) | built-in limiter | +| Unit, integration and end-to-end tests | Maven + the smoke suite | + +Releases additionally scan the built container image with Trivy (fails on +HIGH/CRITICAL) and publish a CycloneDX SBOM plus SHA-256 checksums with the artifacts. + +## Verifying a release + +Each release ships `path-.jar`, `bom.json` and `checksums.txt`: + +```bash +sha256sum -c checksums.txt # both files must report OK +cosign verify ... # not used yet: artifacts are not signed +``` + +The container digest is printed in the workflow summary; pin it by digest: + +```bash +docker pull ghcr.io/an1ndra/path@sha256: +``` + +The Maven wrapper also verifies its own distribution: `distributionSha256Sum` in +`.mvn/wrapper/maven-wrapper.properties` is checked on every download. + +## Supported versions + +The latest tagged release receives fixes. `Develop` is the development branch and is not +supported for production use. diff --git "a/bruno/00 \302\267 Setup/Create API key.bru" "b/bruno/00 \302\267 Setup/Create API key.bru" new file mode 100644 index 0000000..cff0410 --- /dev/null +++ "b/bruno/00 \302\267 Setup/Create API key.bru" @@ -0,0 +1,33 @@ +meta { + name: Create API key + type: http + seq: 1 +} + +post { + url: {{baseUrl}}/admin/keys + body: json + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} + Content-Type: application/json +} + +body:json { + { + "name": "bruno-demo", + "credit_limit_usd": 5, + "rpm_limit": 120 + } +} + +docs { + Mints a client key and stores it in `apiKey`. The plaintext is returned once; every later chat request depends on it. +} + +script:post-response { + bru.setVar("apiKey", res.body.key); + bru.setVar("keyId", res.body.id); +} diff --git "a/bruno/00 \302\267 Setup/List providers.bru" "b/bruno/00 \302\267 Setup/List providers.bru" new file mode 100644 index 0000000..d953306 --- /dev/null +++ "b/bruno/00 \302\267 Setup/List providers.bru" @@ -0,0 +1,14 @@ +meta { + name: List providers + type: http + seq: 2 +} + +get { + url: {{baseUrl}}/admin/providers + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/01 \302\267 Health & catalogue/Get one model.bru" "b/bruno/01 \302\267 Health & catalogue/Get one model.bru" new file mode 100644 index 0000000..d15446b --- /dev/null +++ "b/bruno/01 \302\267 Health & catalogue/Get one model.bru" @@ -0,0 +1,14 @@ +meta { + name: Get one model + type: http + seq: 3 +} + +get { + url: {{baseUrl}}/v1/models/{{model}} + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} +} diff --git "a/bruno/01 \302\267 Health & catalogue/Health.bru" "b/bruno/01 \302\267 Health & catalogue/Health.bru" new file mode 100644 index 0000000..ea17a8c --- /dev/null +++ "b/bruno/01 \302\267 Health & catalogue/Health.bru" @@ -0,0 +1,10 @@ +meta { + name: Health + type: http + seq: 1 +} + +get { + url: {{baseUrl}}/actuator/health + auth: none +} diff --git "a/bruno/01 \302\267 Health & catalogue/List models.bru" "b/bruno/01 \302\267 Health & catalogue/List models.bru" new file mode 100644 index 0000000..a58c0b8 --- /dev/null +++ "b/bruno/01 \302\267 Health & catalogue/List models.bru" @@ -0,0 +1,14 @@ +meta { + name: List models + type: http + seq: 2 +} + +get { + url: {{baseUrl}}/v1/models + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} +} diff --git "a/bruno/02 \302\267 Chat/Chat \342\200\224 Anthropic route.bru" "b/bruno/02 \302\267 Chat/Chat \342\200\224 Anthropic route.bru" new file mode 100644 index 0000000..daeff92 --- /dev/null +++ "b/bruno/02 \302\267 Chat/Chat \342\200\224 Anthropic route.bru" @@ -0,0 +1,32 @@ +meta { + name: Chat — Anthropic route + type: http + seq: 5 +} + +post { + url: {{baseUrl}}/v1/chat/completions + body: json + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} + Content-Type: application/json +} + +body:json { + { + "model": "claude-3.5-haiku", + "messages": [ + { + "role": "system", + "content": "Be terse." + }, + { + "role": "user", + "content": "hello" + } + ] + } +} diff --git "a/bruno/02 \302\267 Chat/Chat \342\200\224 fallback chain.bru" "b/bruno/02 \302\267 Chat/Chat \342\200\224 fallback chain.bru" new file mode 100644 index 0000000..2ff9b78 --- /dev/null +++ "b/bruno/02 \302\267 Chat/Chat \342\200\224 fallback chain.bru" @@ -0,0 +1,39 @@ +meta { + name: Chat — fallback chain + type: http + seq: 3 +} + +post { + url: {{baseUrl}}/v1/chat/completions + body: json + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} + Content-Type: application/json +} + +body:json { + { + "model": "{{model}}", + "models": [ + "gpt-4o-mini", + "claude-3.5-haiku" + ], + "provider": { + "order": [ + "openai", + "anthropic" + ], + "allow_fallbacks": true + }, + "messages": [ + { + "role": "user", + "content": "hello" + } + ] + } +} diff --git "a/bruno/02 \302\267 Chat/Chat \342\200\224 non-streaming.bru" "b/bruno/02 \302\267 Chat/Chat \342\200\224 non-streaming.bru" new file mode 100644 index 0000000..4d57b23 --- /dev/null +++ "b/bruno/02 \302\267 Chat/Chat \342\200\224 non-streaming.bru" @@ -0,0 +1,28 @@ +meta { + name: Chat — non-streaming + type: http + seq: 1 +} + +post { + url: {{baseUrl}}/v1/chat/completions + body: json + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} + Content-Type: application/json +} + +body:json { + { + "model": "{{model}}", + "messages": [ + { + "role": "user", + "content": "Say hi in five words." + } + ] + } +} diff --git "a/bruno/02 \302\267 Chat/Chat \342\200\224 streaming.bru" "b/bruno/02 \302\267 Chat/Chat \342\200\224 streaming.bru" new file mode 100644 index 0000000..d61bf46 --- /dev/null +++ "b/bruno/02 \302\267 Chat/Chat \342\200\224 streaming.bru" @@ -0,0 +1,36 @@ +meta { + name: Chat — streaming + type: http + seq: 2 +} + +post { + url: {{baseUrl}}/v1/chat/completions + body: json + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} + Content-Type: application/json +} + +body:json { + { + "model": "{{model}}", + "stream": true, + "stream_options": { + "include_usage": true + }, + "messages": [ + { + "role": "user", + "content": "Count to five." + } + ] + } +} + +docs { + Open the Bruno's console to inspect the SSE frames. The response ends with `data: [DONE]`. +} diff --git "a/bruno/02 \302\267 Chat/Chat \342\200\224 tool calling.bru" "b/bruno/02 \302\267 Chat/Chat \342\200\224 tool calling.bru" new file mode 100644 index 0000000..8ab242b --- /dev/null +++ "b/bruno/02 \302\267 Chat/Chat \342\200\224 tool calling.bru" @@ -0,0 +1,49 @@ +meta { + name: Chat — tool calling + type: http + seq: 4 +} + +post { + url: {{baseUrl}}/v1/chat/completions + body: json + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} + Content-Type: application/json +} + +body:json { + { + "model": "{{model}}", + "messages": [ + { + "role": "user", + "content": "What is the weather in Pune?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto" + } +} diff --git "a/bruno/03 \302\267 Usage & credits/My credits.bru" "b/bruno/03 \302\267 Usage & credits/My credits.bru" new file mode 100644 index 0000000..cb5e581 --- /dev/null +++ "b/bruno/03 \302\267 Usage & credits/My credits.bru" @@ -0,0 +1,14 @@ +meta { + name: My credits + type: http + seq: 2 +} + +get { + url: {{baseUrl}}/v1/credits + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} +} diff --git "a/bruno/03 \302\267 Usage & credits/My usage.bru" "b/bruno/03 \302\267 Usage & credits/My usage.bru" new file mode 100644 index 0000000..fadd998 --- /dev/null +++ "b/bruno/03 \302\267 Usage & credits/My usage.bru" @@ -0,0 +1,14 @@ +meta { + name: My usage + type: http + seq: 1 +} + +get { + url: {{baseUrl}}/v1/usage?window=24h + auth: none +} + +headers { + Authorization: Bearer {{apiKey}} +} diff --git "a/bruno/04 \302\267 Admin \342\200\224 providers/Create provider.bru" "b/bruno/04 \302\267 Admin \342\200\224 providers/Create provider.bru" new file mode 100644 index 0000000..2a51f92 --- /dev/null +++ "b/bruno/04 \302\267 Admin \342\200\224 providers/Create provider.bru" @@ -0,0 +1,29 @@ +meta { + name: Create provider + type: http + seq: 2 +} + +post { + url: {{baseUrl}}/admin/providers + body: json + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} + Content-Type: application/json +} + +body:json { + { + "slug": "bruno-demo", + "display_name": "Bruno test", + "type": "OPENAI", + "base_url": "https://api.together.xyz/v1" + } +} + +script:post-response { + bru.setVar("createdProviderId", res.body.id); +} diff --git "a/bruno/04 \302\267 Admin \342\200\224 providers/Delete provider.bru" "b/bruno/04 \302\267 Admin \342\200\224 providers/Delete provider.bru" new file mode 100644 index 0000000..946e834 --- /dev/null +++ "b/bruno/04 \302\267 Admin \342\200\224 providers/Delete provider.bru" @@ -0,0 +1,18 @@ +meta { + name: Delete provider + type: http + seq: 4 +} + +delete { + url: {{baseUrl}}/admin/providers/{{createdProviderId}} + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} + +docs { + Removes the throwaway provider created above. +} diff --git "a/bruno/04 \302\267 Admin \342\200\224 providers/List providers.bru" "b/bruno/04 \302\267 Admin \342\200\224 providers/List providers.bru" new file mode 100644 index 0000000..b410663 --- /dev/null +++ "b/bruno/04 \302\267 Admin \342\200\224 providers/List providers.bru" @@ -0,0 +1,14 @@ +meta { + name: List providers + type: http + seq: 1 +} + +get { + url: {{baseUrl}}/admin/providers + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/04 \302\267 Admin \342\200\224 providers/Update provider.bru" "b/bruno/04 \302\267 Admin \342\200\224 providers/Update provider.bru" new file mode 100644 index 0000000..2857f6a --- /dev/null +++ "b/bruno/04 \302\267 Admin \342\200\224 providers/Update provider.bru" @@ -0,0 +1,23 @@ +meta { + name: Update provider + type: http + seq: 3 +} + +patch { + url: {{baseUrl}}/admin/providers/{{createdProviderId}} + body: json + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} + Content-Type: application/json +} + +body:json { + { + "display_name": "Bruno test (updated)", + "enabled": true + } +} diff --git "a/bruno/05 \302\267 Admin \342\200\224 routes/Create route.bru" "b/bruno/05 \302\267 Admin \342\200\224 routes/Create route.bru" new file mode 100644 index 0000000..3599e5f --- /dev/null +++ "b/bruno/05 \302\267 Admin \342\200\224 routes/Create route.bru" @@ -0,0 +1,31 @@ +meta { + name: Create route + type: http + seq: 2 +} + +post { + url: {{baseUrl}}/admin/routes + body: json + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} + Content-Type: application/json +} + +body:json { + { + "public_model_id": "postman-route-bruno-demo", + "provider_id": "{{providerId}}", + "upstream_model_id": "meta-llama/llama-4-scout-17b-16e-instruct", + "input_price_per_million": 0.11, + "output_price_per_million": 0.34, + "priority": 1 + } +} + +script:post-response { + bru.setVar("createdRouteId", res.body.id); +} diff --git "a/bruno/05 \302\267 Admin \342\200\224 routes/Delete route.bru" "b/bruno/05 \302\267 Admin \342\200\224 routes/Delete route.bru" new file mode 100644 index 0000000..715df55 --- /dev/null +++ "b/bruno/05 \302\267 Admin \342\200\224 routes/Delete route.bru" @@ -0,0 +1,14 @@ +meta { + name: Delete route + type: http + seq: 4 +} + +delete { + url: {{baseUrl}}/admin/routes/{{createdRouteId}} + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/05 \302\267 Admin \342\200\224 routes/List routes.bru" "b/bruno/05 \302\267 Admin \342\200\224 routes/List routes.bru" new file mode 100644 index 0000000..8d474cd --- /dev/null +++ "b/bruno/05 \302\267 Admin \342\200\224 routes/List routes.bru" @@ -0,0 +1,14 @@ +meta { + name: List routes + type: http + seq: 1 +} + +get { + url: {{baseUrl}}/admin/routes?model={{publicModel}} + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/05 \302\267 Admin \342\200\224 routes/Update route.bru" "b/bruno/05 \302\267 Admin \342\200\224 routes/Update route.bru" new file mode 100644 index 0000000..d5eaa31 --- /dev/null +++ "b/bruno/05 \302\267 Admin \342\200\224 routes/Update route.bru" @@ -0,0 +1,23 @@ +meta { + name: Update route + type: http + seq: 3 +} + +patch { + url: {{baseUrl}}/admin/routes/{{createdRouteId}} + body: json + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} + Content-Type: application/json +} + +body:json { + { + "priority": 2, + "enabled": true + } +} diff --git "a/bruno/06 \302\267 Admin \342\200\224 keys/List keys.bru" "b/bruno/06 \302\267 Admin \342\200\224 keys/List keys.bru" new file mode 100644 index 0000000..fc8fa09 --- /dev/null +++ "b/bruno/06 \302\267 Admin \342\200\224 keys/List keys.bru" @@ -0,0 +1,14 @@ +meta { + name: List keys + type: http + seq: 1 +} + +get { + url: {{baseUrl}}/admin/keys + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/06 \302\267 Admin \342\200\224 keys/Revoke key.bru" "b/bruno/06 \302\267 Admin \342\200\224 keys/Revoke key.bru" new file mode 100644 index 0000000..2d34e49 --- /dev/null +++ "b/bruno/06 \302\267 Admin \342\200\224 keys/Revoke key.bru" @@ -0,0 +1,18 @@ +meta { + name: Revoke key + type: http + seq: 3 +} + +post { + url: {{baseUrl}}/admin/keys/{{keyId}}/revoke + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} + +docs { + Run last: the setup key stops working after this. +} diff --git "a/bruno/06 \302\267 Admin \342\200\224 keys/Top up key.bru" "b/bruno/06 \302\267 Admin \342\200\224 keys/Top up key.bru" new file mode 100644 index 0000000..be11683 --- /dev/null +++ "b/bruno/06 \302\267 Admin \342\200\224 keys/Top up key.bru" @@ -0,0 +1,22 @@ +meta { + name: Top up key + type: http + seq: 2 +} + +post { + url: {{baseUrl}}/admin/keys/{{keyId}}/topup + body: json + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} + Content-Type: application/json +} + +body:json { + { + "amount_usd": 5 + } +} diff --git "a/bruno/07 \302\267 Admin \342\200\224 reporting/Stats.bru" "b/bruno/07 \302\267 Admin \342\200\224 reporting/Stats.bru" new file mode 100644 index 0000000..afbd24e --- /dev/null +++ "b/bruno/07 \302\267 Admin \342\200\224 reporting/Stats.bru" @@ -0,0 +1,14 @@ +meta { + name: Stats + type: http + seq: 1 +} + +get { + url: {{baseUrl}}/admin/stats?window=24h + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/07 \302\267 Admin \342\200\224 reporting/Sync models.bru" "b/bruno/07 \302\267 Admin \342\200\224 reporting/Sync models.bru" new file mode 100644 index 0000000..a26d10a --- /dev/null +++ "b/bruno/07 \302\267 Admin \342\200\224 reporting/Sync models.bru" @@ -0,0 +1,14 @@ +meta { + name: Sync models + type: http + seq: 3 +} + +post { + url: {{baseUrl}}/admin/models/sync + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git "a/bruno/07 \302\267 Admin \342\200\224 reporting/Usage rows.bru" "b/bruno/07 \302\267 Admin \342\200\224 reporting/Usage rows.bru" new file mode 100644 index 0000000..61f4ede --- /dev/null +++ "b/bruno/07 \302\267 Admin \342\200\224 reporting/Usage rows.bru" @@ -0,0 +1,14 @@ +meta { + name: Usage rows + type: http + seq: 2 +} + +get { + url: {{baseUrl}}/admin/usage?window=24h&limit=50 + auth: none +} + +headers { + X-Admin-Token: {{adminToken}} +} diff --git a/bruno/README.md b/bruno/README.md new file mode 100644 index 0000000..7e2026f --- /dev/null +++ b/bruno/README.md @@ -0,0 +1,51 @@ +# Bruno collection + +Same 26 requests as the Postman collection, in [Bruno](https://www.usebruno.com)'s +plain-text `.bru` format, so the calls live in git and diff like code. + +## Open it + +1. Bruno → **Open Collection** → select this `bruno/` folder. +2. Pick the **Local** environment (top right). +3. Set `adminToken` to your `GATEWAY_ADMIN_TOKEN` (the environment ships with + `test-admin`, matching the smoke suite). +4. Run **00 · Setup → Create API key** first. It mints a key and stores it in the + `apiKey` variable; every other request uses it. + +## Layout + +| Folder | What | +|---|---| +| `00 · Setup` | create a client key, list providers | +| `01 · Health & catalogue` | health, `/v1/models`, one model | +| `02 · Chat` | non-streaming, streaming, fallback chain, tools, Anthropic | +| `03 · Usage & credits` | the calling key's spend and balance | +| `04 · Admin — providers` | list, create, update, delete | +| `05 · Admin — routes` | list, create, update, delete | +| `06 · Admin — keys` | list, top up, revoke | +| `07 · Admin — reporting` | stats, usage rows, model sync | + +Variables: `baseUrl`, `adminToken`, `apiKey`, `model`, `publicModel`, `providerId`, +`keyId`. The create requests also set `createdProviderId` / `createdRouteId` for the +PATCH and DELETE calls. + +## Regenerating + +The collection mirrors `docs/path.postman_collection.json`. After changing the Postman +collection: + +```bash +python3 tools/postman_to_bruno.py +``` + +## CLI + +Run from inside the collection (the CLI treats the current directory as the root): + +```bash +cd bruno +npx @usebruno/cli run . -r --env Local +``` + +Verified against a local gateway: 26 requests, 26 passed. Useful for a smoke run; not +wired into CI because it needs a running instance and a key. diff --git a/bruno/bruno.json b/bruno/bruno.json new file mode 100644 index 0000000..83a8953 --- /dev/null +++ b/bruno/bruno.json @@ -0,0 +1,10 @@ +{ + "version": "1", + "name": "Path", + "type": "collection", + "ignore": [ + "node_modules", + ".git", + ".DS_Store" + ] +} diff --git a/bruno/environments/Local.bru b/bruno/environments/Local.bru new file mode 100644 index 0000000..8e051b2 --- /dev/null +++ b/bruno/environments/Local.bru @@ -0,0 +1,9 @@ +vars { + baseUrl: http://localhost:8080 + adminToken: test-admin + apiKey: + model: gpt-4o-mini + publicModel: cheap + providerId: 1 + keyId: 1 +} diff --git a/changelog/README.md b/changelog/README.md new file mode 100644 index 0000000..56a3616 --- /dev/null +++ b/changelog/README.md @@ -0,0 +1,35 @@ +# Changelog + +One file per release, named exactly like the git tag: + +``` +changelog/ +├── README.md this file +├── unreleased.md notes for the next release (never published on its own) +└── v0.1.0.md published as the body of the v0.1.0 GitHub release +``` + +## Adding changes + +Append to `changelog/unreleased.md` as you work. Keep the sections in the +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) order: Added, Changed, +Deprecated, Removed, Fixed, Security. + +## Cutting a release + +1. Copy `changelog/unreleased.md` to `changelog/vX.Y.Z.md` (the tag name, including `v`). +2. Rewrite the first heading to `# vX.Y.Z` and adjust the sections. +3. Reset `unreleased.md` to an empty template. +4. Commit on `Develop`, open the PR into `main`. +5. Tag `vX.Y.Z` on `main` and push. + +The release workflow runs `tools/changelog.py "$GITHUB_REF_NAME"`, which reads +`changelog/.md` and uses it verbatim as the release body. A tag without a file, or +with no real entries (only `Nothing yet.` placeholders), **fails the release** rather than +publishing blank notes. + +Preview locally before tagging: + +```bash +python3 tools/changelog.py v0.1.0 +``` diff --git a/changelog/unreleased.md b/changelog/unreleased.md new file mode 100644 index 0000000..fd65349 --- /dev/null +++ b/changelog/unreleased.md @@ -0,0 +1,16 @@ +# Unreleased + +Notes for the next release. Nothing is published from this file; copy it to +`vX.Y.Z.md` when cutting a release. See [README.md](README.md). + +## Added + +- Nothing yet. + +## Changed + +- Nothing yet. + +## Fixed + +- Nothing yet. diff --git a/changelog/v0.1.0.md b/changelog/v0.1.0.md new file mode 100644 index 0000000..ccde00a --- /dev/null +++ b/changelog/v0.1.0.md @@ -0,0 +1,48 @@ +# v0.1.0 + +First public release: one OpenAI-compatible surface in front of OpenAI, Anthropic, +Groq and any OpenAI-compatible endpoint. + +## Added + +- **Routing** — priority chains with cross-provider fallback, `models[]` and + `provider.order`/`allow_fallbacks` request controls, capability aliases (`cheap`, + `fast`, `long-context`, `vision`) resolved from the live price table, and weighted + canary splits that stay stable per client key. +- **Providers** — native OpenAI chat-completions passthrough and a full Anthropic + Messages adapter (system messages, images, tools, tool results, streaming), with + per-provider base URL, credential, timeouts and circuit state. +- **Streaming** — SSE relay with `[DONE]`, optional usage chunks, time-to-first-token + and throughput recorded per call; fallback only before the first byte. +- **Keys and money** — `sk-gw-...` client keys stored as SHA-256, lifetime/daily/monthly + USD budgets with pre-flight estimation and atomic reservation, RPM and in-flight + limits, per-key model allowlists, revocation. +- **Metering** — prompt/completion/cached/cache-write/reasoning tokens, itemised cost, + cache savings, request id, finish reason, attempts and fallback source per call; + `/v1/usage` and `/admin/stats` aggregates with TTFT and throughput percentiles. +- **Cache and idempotency** — opt-in exact-match response cache (streams included) and + `Idempotency-Key` replay with conflict detection. +- **Catalogues and sync** — `/v1/models` with pricing and routing detail, + `POST /admin/models/sync` importing each provider's model list. +- **Operations** — Flyway migrations, readiness gating, shutdown drain, provider probe + and health, admin audit trail, webhook alerts, Prometheus metrics, OpenTelemetry + tracing, ECS JSON logs and an OpenAPI document. +- **Features** — `/v1/embeddings` for OpenAI-compatible providers, opt-in redacted + prompt logging, projects and scoped admin tokens. + +## Security + +- Upstream credentials encrypted with AES-GCM at rest; the master key is generated to a + 0600 file when not supplied. +- Admin tokens are hashed, scoped (`read` / `read,write`) and revocable. +- CI runs gitleaks, dependency review, CodeQL and Trivy; releases scan the container + image and publish a CycloneDX SBOM with SHA-256 checksums. +- The Maven wrapper verifies its distribution against a pinned SHA-256. + +## Fixed + +- Anthropic prompt-cache reads were billed at the full input rate rather than 0.1×. +- Requests could reach providers before first-boot seeding finished, because health + reported UP before the seeder ran. +- System-only conversations produced an invalid empty Anthropic message list. +- Weighted routing applied to ordinary fallback chains, turning routing into a lottery. diff --git a/changelog/v0.2.0.md b/changelog/v0.2.0.md new file mode 100644 index 0000000..788f5b7 --- /dev/null +++ b/changelog/v0.2.0.md @@ -0,0 +1,91 @@ +# v0.2.0 + +Everything since the `v0.1.0` tag: the gateway gains a second routing strategy, guardrails, +an end-user billing axis, a semantic cache, Azure OpenAI and a Helm chart — plus the +fixes from a twelve-point security and billing audit. + +## Added + +### Routing and model selection + +- **Adaptive routing.** A route can declare `strategy`: `priority` (the default, unchanged), + `fastest` (ordered by an exponentially weighted latency) or `cheapest-viable` (cheapest + route under a `latency_slo_ms`, the rest demoted to fallback). Route views expose + `ewma_latency_ms` and `ewma_error_rate` so the decision is auditable, and `provider.order` + still overrides any strategy. +- **Operator-defined aliases.** `GET/POST /admin/aliases` stores an ordered chain + (`prod-chat` → `claude-sonnet-4.5`, then `gpt-4o`). Chains are validated at write time — + an unknown model, a self-reference or more than ten hops is a 400, so a broken alias + cannot become a 500 on a customer request. The four computed aliases (`cheap`, `fast`, + `long-context`, `vision`) are unchanged. + +### Billing and limits + +- **End-user attribution.** `end_user` (or OpenAI's `user`) is stored on every usage row + and indexed, `/v1/usage` returns a per-end-user breakdown, `/admin/usage` and the CSV + export filter by it, and a key can cap each customer with `end_user_daily_limit_usd` — + reserved with an atomic conditional update, so concurrency cannot overshoot. +- **Project rate limits.** Projects carry `rpm_limit` and `max_concurrent_requests`, + enforced in addition to each key's caps, and shared across replicas through the Valkey + limiter. +- **Project budget alerts.** `project.budget_low` and `project.budget_exhausted` fire once + per project, window and period. +- **`POST /v1/tokens/count`.** Free but authenticated: reports prompt tokens and whether the + prompt fits, counting tool definitions, which is where estimates are usually wrong. +- **Idempotent embeddings.** `/v1/embeddings` honours `Idempotency-Key`; a retried batch is + no longer billed twice. + +### Safety and privacy + +- **Guardrails.** PII redaction for prompt logs (on by default) and optionally before send, + plus opt-in moderation passthrough that refuses a flagged prompt with 400 + `content_flagged`. `GET /admin/settings` reports what is active. +- **Semantic cache tier.** With `semantic_cache_enabled` on a route and + `GATEWAY_SEMANTIC_MODEL` set, a near-duplicate prompt can reuse a stored completion, + under the same `temperature: 0`, tool-free, per-key rules as the exact cache. + +### Providers and deployment + +- **Azure OpenAI.** Setting `api_version` on a provider puts the deployment in the path, the + version in `?api-version` and the credential in the `api-key` header, for chat, + embeddings, moderation and the probe. +- **Helm chart** in `deploy/helm/path`: drain-aware grace period enforced at render time, + HPA with a custom-metric hook, PDB, optional Postgres and Valkey, ServiceMonitor. +- **Per-provider health** at `/actuator/health/providers`; overall health only reports + OUT_OF_SERVICE when every enabled provider is circuit-open. + +## Fixed + +The post-0.1.0 audit found twelve issues; all are fixed: + +- `models/sync` imported unpriced routes enabled, serving inference for free. +- 500 responses returned internal exception messages; they now return a reference id and + log the detail server-side. +- Chunked request bodies bypassed the size limit; reading is now counted and 413s. +- The demo API key was created by default, including in the `prod` profile. +- A large `max_tokens` could reserve credit the call would never use. +- Reservation now uses the expected cost while still refusing when it does not fit. +- `/v3/api-docs` was served in production; off in the `prod` profile. +- The in-memory rate-limit windows were never evicted. +- Release tags were not verified to be on `main`. +- `UpstreamClient` allocated an `ObjectMapper` per error. +- Project filters were missing from `/admin/usage`, `/admin/stats` and the CSV export. +- Admin auth had no lockout; ten failures in five minutes now return 429. + +Also fixed: the end-user cap was skipped for keys without a credit limit, and `api_version` +was ignored on provider `PATCH`. + +## Security + +- Every GitHub Action is pinned to a commit SHA; a mutable tag can be repointed at any + commit and would run with this repository's token. +- Master key rotation (`POST /admin/rotate-master-key`) re-encrypts stored credentials + before switching the active key. + +## Notes + +- Maven Central publishing is configured (`-P release`, `io.github.an1ndra:path`) and + verified locally against a throwaway key; the upload itself is pending a namespace and a + Portal token. See `docs/PUBLISHING.md`. +- Five stories remain open: Spring AI module, Vertex/Gemini native, Valkey-shared breaker, + usage partitioning, and the Central publication. diff --git a/deploy/helm/path/Chart.yaml b/deploy/helm/path/Chart.yaml new file mode 100644 index 0000000..62d1dd1 --- /dev/null +++ b/deploy/helm/path/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: path +description: Path — a provider-agnostic LLM gateway with routing, fallback, credits and cache-aware token metering +type: application +version: 0.2.0 +appVersion: "0.2.0" +kubeVersion: ">=1.27.0-0" +home: https://github.com/an1ndra/path +sources: + - https://github.com/an1ndra/path +keywords: + - llm + - gateway + - openai + - anthropic + - java diff --git a/deploy/helm/path/README.md b/deploy/helm/path/README.md new file mode 100644 index 0000000..27ef8cf --- /dev/null +++ b/deploy/helm/path/README.md @@ -0,0 +1,41 @@ +# Path Helm chart + +Path is an I/O-bound HTTP service with one JVM per pod, a database for keys and usage, +and — in a cluster — Valkey for shared rate limiting. + +```bash +helm install path deploy/helm/path \ + --namespace path --create-namespace \ + --set existingSecret=path-credentials + +kubectl -n path port-forward svc/path 8080:8080 +curl -s localhost:8080/actuator/health +curl -s -H "X-Admin-Token: $GATEWAY_ADMIN_TOKEN" localhost:8080/actuator/health/providers +``` + +## What the chart makes hard to get wrong + +| Decision | Why it is in the chart, not in a comment | +|---|---| +| `terminationGracePeriodSeconds` > `GATEWAY_DRAIN_TIMEOUT` | Path drains in-flight SSE streams on SIGTERM. If the grace period is shorter, a rolling update truncates them. The template **fails to render** when the relation is violated. | +| `maxUnavailable: 0`, `maxSurge: 1` | A chat gateway should not lose capacity during a deploy while it is also draining. | +| Readiness on `/actuator/health` | A single parked provider must not remove the pod from the Service: fallback still works. Only a total outage reports `OUT_OF_SERVICE`. | +| `emptyDir` for `/app/data` | That directory holds the H2 database and `master.key`. With Postgres those are not needed, and an `emptyDir` makes it obvious rather than silently losing a key on reschedule. | +| No inline Secret by default | `existingSecret` is required unless `secrets` is set, because a chart-managed Secret is a bad place for `GATEWAY_MASTER_KEY`. | + +## Values worth changing first + +- `replicaCount` (default 2) — below 2 there is no disruption budget to honour. +- `config.dbUrl` — point it at a managed Postgres; `postgres.enabled` exists for a demo. +- `valkey.enabled` — without it, rate limits are per pod and the effective limit is + `replicaCount × rpm`. Budget reservations are database-backed either way, so money is + safe without it. +- `autoscaling` — CPU is a poor signal for a gateway. If you have the Prometheus adapter, + scale on in-flight requests instead. + +## What this chart does not do + +- No ingress or TLS: terminate at your own gateway, and keep `GATEWAY_API_DOCS=false`. +- No migrations Job: Flyway runs at startup. With `replicaCount > 1` and a fresh database, + Flyway takes a lock, so the pods serialise; that is slow but correct. For a large + schema, run `./mvnw flyway:migrate` as a pre-install Job instead. diff --git a/deploy/helm/path/templates/NOTES.txt b/deploy/helm/path/templates/NOTES.txt new file mode 100644 index 0000000..b08b767 --- /dev/null +++ b/deploy/helm/path/templates/NOTES.txt @@ -0,0 +1,33 @@ +Path is running in the {{ .Release.Namespace }} namespace. + + In-cluster URL: http://{{ include "path.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.service.port }} + +Check it once it is up: + + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "path.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl -s localhost:{{ .Values.service.port }}/actuator/health + curl -s -H "X-Admin-Token: $GATEWAY_ADMIN_TOKEN" localhost:{{ .Values.service.port }}/actuator/health/providers + +The provider health group is the one to watch: overall health stays UP while fallback +exists, and only a total outage reports OUT_OF_SERVICE. + +Resources: +{{- if .Values.postgres.enabled }} + * Postgres: a StatefulSet with a {{ .Values.postgres.persistence.size }} volume. For + anything real, point config.dbUrl at a managed database instead. +{{- end }} +{{- if .Values.valkey.enabled }} + * Valkey: shared rate limiting and a shared view of rate-limit state. +{{- end }} +{{- if not .Values.valkey.enabled }} + * Valkey is off, so rate limiting is per pod and the effective limit is + {{ .Values.replicaCount }}x the configured rpm. Enable it before trusting a cluster. +{{- end }} + +Before exposing this to clients: + + * set GATEWAY_ADMIN_TOKEN and GATEWAY_MASTER_KEY in {{ include "path.secretName" . }} + * keep GATEWAY_API_DOCS=false in production + * terminationGracePeriodSeconds ({{ .Values.terminationGracePeriodSeconds }}s) must stay + above GATEWAY_DRAIN_TIMEOUT ({{ .Values.drainTimeoutSeconds }}s) or a rolling update + truncates live SSE streams; the chart refuses to render if that is violated diff --git a/deploy/helm/path/templates/_helpers.tpl b/deploy/helm/path/templates/_helpers.tpl new file mode 100644 index 0000000..a7b42a0 --- /dev/null +++ b/deploy/helm/path/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* Chart name, overridable. */}} +{{- define "path.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Fully qualified name. */}} +{{- define "path.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "path.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "path.labels" -}} +helm.sh/chart: {{ include "path.chart" . }} +{{ include "path.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "path.selectorLabels" -}} +app.kubernetes.io/name: {{ include "path.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* The Secret holding credentials: either the operator's or one we create. */}} +{{- define "path.secretName" -}} +{{- if .Values.existingSecret -}} +{{- .Values.existingSecret -}} +{{- else if .Values.secrets -}} +{{- printf "%s-credentials" (include "path.fullname" .) -}} +{{- else -}} +{{- fail "set existingSecret (recommended) or secrets; Path will not start without an admin token" -}} +{{- end -}} +{{- end -}} + +{{- define "path.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) -}} +{{- end -}} diff --git a/deploy/helm/path/templates/deployment.yaml b/deploy/helm/path/templates/deployment.yaml new file mode 100644 index 0000000..a6bb0e6 --- /dev/null +++ b/deploy/helm/path/templates/deployment.yaml @@ -0,0 +1,118 @@ +{{- if le (int .Values.terminationGracePeriodSeconds) (int .Values.drainTimeoutSeconds) -}} +{{- fail (printf "terminationGracePeriodSeconds (%d) must exceed drainTimeoutSeconds (%d), or a rolling update truncates live streams" (int .Values.terminationGracePeriodSeconds) (int .Values.drainTimeoutSeconds)) -}} +{{- end -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "path.fullname" . }} + labels: + {{- include "path.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + {{- include "path.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/values: {{ toJson .Values.env | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "path.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: path + image: {{ include "path.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + env: + - name: SPRING_PROFILES_ACTIVE + value: {{ .Values.config.profile | quote }} + - name: DB_URL + value: {{ .Values.config.dbUrl | quote }} + - name: DB_USER + value: {{ .Values.config.dbUser | quote }} + - name: GATEWAY_DRAIN_TIMEOUT + value: {{ printf "%ds" (int .Values.drainTimeoutSeconds) | quote }} + {{- if .Values.valkey.enabled }} + - name: GATEWAY_RATE_LIMIT_BACKEND + value: valkey + - name: VALKEY_HOST + value: {{ printf "%s-valkey" (include "path.fullname" .) | quote }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "path.secretName" . }} + key: {{ .Values.config.dbPasswordSecretKey }} + envFrom: + - secretRef: + name: {{ include "path.secretName" . }} + # A failing upstream parks the provider; readiness intentionally stays up so + # the gateway keeps serving through fallback. Only a total outage goes DOWN. + startupProbe: + httpGet: + path: /actuator/health + port: http + periodSeconds: {{ .Values.probes.startup.periodSeconds }} + failureThreshold: {{ .Values.probes.startup.failureThreshold }} + readinessProbe: + httpGet: + path: /actuator/health + port: http + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + livenessProbe: + httpGet: + path: /actuator/health + port: http + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /app/data + volumes: + - name: data + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/helm/path/templates/hpa.yaml b/deploy/helm/path/templates/hpa.yaml new file mode 100644 index 0000000..c21875a --- /dev/null +++ b/deploy/helm/path/templates/hpa.yaml @@ -0,0 +1,26 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "path.fullname" . }} + labels: + {{- include "path.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "path.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.metrics }} + {{- toYaml .Values.autoscaling.metrics | nindent 4 }} + {{- else }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/deploy/helm/path/templates/pdb.yaml b/deploy/helm/path/templates/pdb.yaml new file mode 100644 index 0000000..14f3935 --- /dev/null +++ b/deploy/helm/path/templates/pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "path.fullname" . }} + labels: + {{- include "path.labels" . | nindent 4 }} +spec: + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "path.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/path/templates/postgres.yaml b/deploy/helm/path/templates/postgres.yaml new file mode 100644 index 0000000..f88f972 --- /dev/null +++ b/deploy/helm/path/templates/postgres.yaml @@ -0,0 +1,68 @@ +{{- if .Values.postgres.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "path.fullname" . }}-postgres + labels: + {{- include "path.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + serviceName: {{ include "path.fullname" . }}-postgres + replicas: 1 + selector: + matchLabels: + {{- include "path.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: postgres + template: + metadata: + labels: + {{- include "path.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: postgres + spec: + containers: + - name: postgres + image: {{ .Values.postgres.image }} + ports: + - containerPort: 5432 + name: postgres + env: + - name: POSTGRES_DB + value: {{ .Values.postgres.database | quote }} + - name: POSTGRES_USER + value: {{ .Values.postgres.username | quote }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "path.secretName" . }} + key: {{ .Values.config.dbPasswordSecretKey }} + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.postgres.username | quote }}] + periodSeconds: 5 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.persistence.size }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "path.fullname" . }}-postgres + labels: + {{- include "path.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + ports: + - port: 5432 + targetPort: postgres + selector: + {{- include "path.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: postgres +{{- end }} diff --git a/deploy/helm/path/templates/secret.yaml b/deploy/helm/path/templates/secret.yaml new file mode 100644 index 0000000..86bf620 --- /dev/null +++ b/deploy/helm/path/templates/secret.yaml @@ -0,0 +1,13 @@ +{{- if and (not .Values.existingSecret) .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "path.secretName" . }} + labels: + {{- include "path.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- range $key, $value := .Values.secrets }} + {{ $key }}: {{ $value | quote }} + {{- end }} +{{- end }} diff --git a/deploy/helm/path/templates/service.yaml b/deploy/helm/path/templates/service.yaml new file mode 100644 index 0000000..b8f550b --- /dev/null +++ b/deploy/helm/path/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "path.fullname" . }} + labels: + {{- include "path.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "path.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/path/templates/serviceMonitor.yaml b/deploy/helm/path/templates/serviceMonitor.yaml new file mode 100644 index 0000000..dd46ef5 --- /dev/null +++ b/deploy/helm/path/templates/serviceMonitor.yaml @@ -0,0 +1,19 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "path.fullname" . }} + labels: + {{- include "path.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "path.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + path: /actuator/prometheus + interval: {{ .Values.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/helm/path/templates/valkey.yaml b/deploy/helm/path/templates/valkey.yaml new file mode 100644 index 0000000..66dd10f --- /dev/null +++ b/deploy/helm/path/templates/valkey.yaml @@ -0,0 +1,44 @@ +{{- if .Values.valkey.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "path.fullname" . }}-valkey + labels: + {{- include "path.labels" . | nindent 4 }} + app.kubernetes.io/component: valkey +spec: + replicas: 1 + selector: + matchLabels: + {{- include "path.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: valkey + template: + metadata: + labels: + {{- include "path.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: valkey + spec: + containers: + - name: valkey + image: {{ .Values.valkey.image }} + ports: + - containerPort: 6379 + name: redis + resources: + {{- toYaml .Values.valkey.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "path.fullname" . }}-valkey + labels: + {{- include "path.labels" . | nindent 4 }} + app.kubernetes.io/component: valkey +spec: + ports: + - port: 6379 + targetPort: redis + selector: + {{- include "path.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: valkey +{{- end }} diff --git a/deploy/helm/path/values.yaml b/deploy/helm/path/values.yaml new file mode 100644 index 0000000..fe8de34 --- /dev/null +++ b/deploy/helm/path/values.yaml @@ -0,0 +1,117 @@ +# Defaults for Path. Every knob that changes behaviour is commented with the trade-off. + +image: + repository: ghcr.io/an1ndra/path + # Defaults to the chart's appVersion when left empty. + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +replicaCount: 2 + +# Path drains in-flight streams on SIGTERM instead of truncating them, so the grace +# period MUST be longer than drainTimeoutSeconds. The template refuses to render +# otherwise; that check is the point of this pair. +terminationGracePeriodSeconds: 45 +drainTimeoutSeconds: 25 + +service: + type: ClusterIP + port: 8080 + annotations: {} + +# Secrets are never rendered inline by default: point existingSecret at a Secret you +# manage (External Secrets, Sealed Secrets, Vault) with these keys. +# +# GATEWAY_ADMIN_TOKEN admin API token +# GATEWAY_MASTER_KEY base64 32-byte key encrypting provider credentials +# OPENAI_API_KEY optional provider credentials +# ANTHROPIC_API_KEY +# GROQ_API_KEY +# GEMINI_API_KEY +# GATEWAY_UPASTASH_TOKEN ... +existingSecret: "" + +# Only for a throwaway install. Prefer existingSecret. +secrets: {} + +env: + # Any extra environment variable, e.g. GATEWAY_API_DOCS: "false" + GATEWAY_API_DOCS: "false" + GATEWAY_RATE_LIMIT_BACKEND: "memory" + +config: + # Path stores its own data in H2 by default; production wants Postgres. + profile: postgres + dbUrl: jdbc:postgresql://postgres:5432/aigateway + dbUser: aigateway + dbPasswordSecretKey: DB_PASSWORD + +# Optional in-chart Postgres. Off by default: an operator-managed database is better. +postgres: + enabled: false + image: postgres:16-alpine + database: aigateway + username: aigateway + password: aigateway + persistence: + size: 10Gi + +# Optional in-chart Valkey (Redis protocol), needed for shared rate limiting across +# replicas and for a shared circuit breaker. +valkey: + enabled: false + image: valkey/valkey:8-alpine + resources: {} + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + memory: 1Gi + +probes: + startup: + failureThreshold: 30 + periodSeconds: 5 + readiness: + periodSeconds: 10 + failureThreshold: 3 + liveness: + periodSeconds: 20 + failureThreshold: 3 + +# Path is I/O bound, so CPU is a poor scaling signal. The HPA defaults to memory +# without a custom metric; point metrics.custom at the Prometheus adapter when +# available (path_requests_total or in-flight requests). +autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + metrics: [] + # - type: Pods + # pods: + # metric: { name: path_in_flight_requests } + # target: { type: AverageValue, averageValue: "20" } + +podDisruptionBudget: + enabled: true + minAvailable: 1 + +podAnnotations: {} +podLabels: {} + +nodeSelector: {} +tolerations: [] +affinity: {} +topologySpreadConstraints: [] + +serviceMonitor: + enabled: false + interval: 30s + labels: {} diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..1e51dee --- /dev/null +++ b/docs/API.md @@ -0,0 +1,398 @@ +# API reference + +Ready-made clients: [Bruno](../bruno/README.md) (26 requests, `.bru` files) and the +[Postman collection](path.postman_collection.json). A machine-readable OpenAPI 3 +document is served at `/v3/api-docs` (Swagger UI at +`/swagger-ui.html` when `GATEWAY_SWAGGER_UI=true`). Everything below is the same +contract in prose. + +Base URL: `http://localhost:8080` (override with `PORT`). + +Two auth schemes: + +| Surface | Header | Secret | +|---|---|---| +| `/v1/**` | `Authorization: Bearer sk-gw-...` | client key from `/admin/keys` | +| `/admin/**` | `X-Admin-Token: ...` | `GATEWAY_ADMIN_TOKEN` | + +All bodies are JSON. Streaming responses are `text/event-stream`. + +A ready-to-import Postman collection lives in +[`path.postman_collection.json`](path.postman_collection.json) with +[`path.postman_environment.json`](path.postman_environment.json). + +--- + +## Errors + +Every failure uses the OpenAI envelope: + +```json +{"error": {"message": "The model 'nope' does not exist.", "type": "invalid_request_error", "code": "model_not_found"}} +``` + +| Code | HTTP | Meaning | +|---|---|---| +| `missing_api_key` | 401 | no `Authorization` header | +| `invalid_api_key` | 401 | unknown or malformed key | +| `key_disabled` | 401 | key revoked or disabled | +| `invalid_admin_token` | 401 | bad `X-Admin-Token` | +| `forbidden` | 403 | admin API disabled (no token configured) | +| `model_not_found` | 404 | no enabled route for that model id | +| `route_not_found` / `provider_not_found` / `key_not_found` | 404 | unknown admin target | +| `invalid_request` / `invalid_json` | 400 | body validation | +| `insufficient_credits` | 402 | balance cannot cover the call | +| `rate_limit_exceeded` | 429 | per-key RPM limit (has `Retry-After`) | +| `provider_not_configured` | 502 | provider row has no credential | +| `provider_auth_error` | 502 | upstream rejected the gateway's key | +| `upstream_invalid_request` | 400 | upstream rejected the client's body | +| `upstream_timeout` | 504 | connect/read/idle timeout | +| `upstream_error` | 502 | upstream 5xx or unmapped failure | +| `all_providers_failed` | 502 | every route in the chain failed | +| `circuit_open` | 503 | provider parked by the circuit breaker | +| `concurrency_limit_exceeded` | 429 | too many in-flight requests for this key | +| `payload_too_large` | 413 | request body above `GATEWAY_MAX_REQUEST_BYTES` | +| `unsupported_parameter` | 422 | parameter has no equivalent for this provider | +| `model_not_allowed` | 403 | key's `allowed_models` does not cover the model | +| `idempotency_conflict` | 409 | same `Idempotency-Key`, different body | +| `idempotency_in_progress` | 409 | duplicate while the first attempt is running | +| `shutting_down` | 503 | instance is draining; retry elsewhere | +| `unsupported_endpoint` | 501 | endpoint not available for this provider type | +| `route_exists` / `provider_exists` | 409 | duplicate admin create | + +--- + +## Public API + +### `POST /v1/chat/completions` + +OpenAI chat-completions body. The gateway adds three optional routing fields: + +| Field | Type | Meaning | +|---|---|---| +| `models` | string[] | fallback model ids, tried after `model` | +| `provider.order` | string[] | provider slug preference, e.g. `["groq","openai"]` | +| `provider.allow_fallbacks` | bool | `false` pins the first route; upstream errors surface | + +Everything the provider understands is forwarded untouched; `model` is rewritten to +the route's upstream id. Gateway-only fields are stripped before forwarding. + +**Non-streaming** + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -d '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"Say hi"}]}' +``` + +```json +{ + "id": "chatcmpl-...", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18, + "prompt_tokens_details": {"cached_tokens": 5}} +} +``` + +Response headers: + +| Header | Example | Notes | +|---|---|---| +| `X-Gateway-Provider` | `groq` | provider that actually served it | +| `X-Gateway-Upstream-Model` | `llama-3.3-70b-versatile` | upstream id used | +| `X-Gateway-Cost-Usd` | `0.00000548` | 8-decimal itemised total | +| `X-Gateway-Latency-Ms` | `412` | end-to-end | +| `X-Idempotent-Replay` | `true` | present only when replaying a stored response | +| `X-Path-Cache` | `hit` / `miss` / `off` | response cache outcome for this call | + +**Response cache** + +Opt-in per route (`cache_enabled`, `cache_ttl_seconds`). A request qualifies when it +sets `temperature: 0` explicitly and uses no tools. A hit replays the stored body +(streams replay as the same SSE sequence), costs nothing, and counts as savings in +`/v1/usage`. Cache keys are scoped to the client key. + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -d '{"model":"gpt-4o-mini","temperature":0,"messages":[{"role":"user","content":"Hi"}]}' \ + -D - | grep -i x-path-cache +``` + +**Idempotency** + +Send an `Idempotency-Key` header (up to 255 characters) to make a request safe to retry: + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -H "Idempotency-Key: 7f2c-orders-2026-11-17-001" \ + -d '{...}' +``` + +| Situation | Result | +|---|---| +| First request | runs normally, response stored for `GATEWAY_IDEMPOTENCY_TTL` | +| Repeat, same body | stored response replayed, `X-Idempotent-Replay: true`, no upstream call, no charge | +| Repeat, different body | `409 idempotency_conflict` | +| Repeat while first is still running | `409 idempotency_in_progress` | + +Streaming responses are captured and replayed as the same SSE sequence. A response +larger than `GATEWAY_IDEMPOTENCY_MAX_BYTES` is not stored; a retry will call the +provider again. + +**Streaming** + +```bash +curl -N http://localhost:8080/v1/chat/completions \ + -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -d '{"model":"claude-3.5-haiku","stream":true, + "stream_options":{"include_usage":true}, + "messages":[{"role":"user","content":"Say hi"}]}' +``` + +``` +: open + +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":""}}]} +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Hi"}}]} +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":19,"completion_tokens":7,"total_tokens":26}} +data: [DONE] +``` + +- `stream_options.include_usage` adds the usage-only chunk before `[DONE]`. +- A failure after the first chunk arrives as an error object then `[DONE]`; the HTTP + status is already 200, so clients must inspect the payload. +- Fallback happens only before the first chunk. + +**Tool calls** are passed through for OpenAI-shaped providers and translated to +`tool_use` / `tool_result` blocks for Anthropic. + +**End-user attribution** + +```json +{"model":"gpt-4o-mini","end_user":"customer-8821", + "messages":[{"role":"user","content":"..."}]} +``` + +`end_user` is stored on the usage row and indexed with the key and timestamp, so a key +with `end_user_daily_limit_usd` can cap each of its customers separately and `/v1/usage` +can report `end_users[]` back to the caller. `user` is accepted as an alias because that +is the field OpenAI clients already send; `end_user` wins when both are present. +At most 128 characters. + +### `POST /v1/tokens/count` + +Free, authenticated, and bounded exactly like chat: the model allowlist, drain state, +concurrency limiter and RPM limiter all apply. Returns `prompt_tokens`, `estimated` +(true when the chars/4 heuristic was used rather than a real encoding) and a `fits` block +with the route's context window and your `max_output_tokens`. Tool definitions and +tool-result messages are counted, since that is where estimates are usually wrong. + +### `POST /v1/embeddings` + +OpenAI-shaped embeddings. Only `OPENAI`-type providers are supported; an Anthropic route +returns `501 unsupported_endpoint`. + +```bash +curl -s http://localhost:8080/v1/embeddings \ + -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -d '{"model":"text-embedding-3-small","input":"hello"}' +``` + +Metered on input tokens only, with the same headers as chat. + +### `GET /v1/models` + +Public when `gateway.public-models=true` (default). + +```json +{ + "object": "list", + "data": [{ + "id": "auto-fast", "object": "model", "created": 1700000000, "owned_by": "openai", + "context_length": 128000, + "providers": ["openai", "groq"], + "pricing": {"prompt_per_million": 0.15, "completion_per_million": 0.60}, + "routing": [ + {"provider": "openai", "upstream_model": "gpt-4o-mini", "priority": 1, + "input_price_per_million": 0.15, "output_price_per_million": 0.60} + ] + }] +} +``` + +### `GET /v1/models/{id}` + +One catalogue entry. Dots and slashes in the id are supported. 404 `model_not_found` +when unknown. + +### `GET /v1/usage?window=24h` + +Per-key totals for the window (`30s`, `15m`, `24h`, `7d`, `4w` or ISO-8601). + +```json +{ + "object": "usage", "window": "PT24H", "requests": 18, + "prompt_tokens": 213, "completion_tokens": 117, "total_tokens": 330, + "cached_input_tokens": 69, "cache_hit_ratio": 0.3239, + "cached_savings_usd": 0.0000195, "reasoning_tokens": 0, + "cost_usd": 0.000378, + "cost_breakdown": {"input": 0.0000858, "cached_input": 0.0000192, + "cache_write": 0.000012, "output": 0.0002622}, + "credit_limit_usd": null, "used_usd": 0.000378, "remaining_usd": null, + "recent": [{"id": 42, "at": "...", "model": "gpt-4o-mini", "provider": "openai", + "cached_input_tokens": 5, "ttft_ms": 12, "tokens_per_second": 61.5, + "attempts": 1, "usage_estimated": false}] +} +``` + +### `GET /v1/credits` + +```json +{"object":"credits","key_id":1,"key_name":"demo-local","currency":"USD", + "credit_limit_usd":25.00,"used_usd":0.000378,"remaining_usd":24.999622,"unlimited":false} +``` + +--- + +## Admin API + +All admin calls need `X-Admin-Token`. When `GATEWAY_ADMIN_TOKEN` is empty the whole +surface returns 403. + +### Providers + +| Method | Path | Purpose | +|---|---|---| +| GET | `/admin/providers` | list with credential source/hint | +| POST | `/admin/providers` | create | +| PATCH | `/admin/providers/{id}` | update | +| DELETE | `/admin/providers/{id}` | delete (cascades routes) | + +```json +{ + "slug": "together", + "display_name": "Together", + "type": "OPENAI", + "base_url": "https://api.together.xyz/v1", + "api_key": "optional; omit to use GATEWAY_UPSTREAM_TOGETHER_KEY", + "enabled": true +} +``` + +`type` is `OPENAI` (chat-completions) or `ANTHROPIC` (Messages API). Response adds +`credential_source` (`database` / `config` / `env`), `credential_configured`, +`credential_hint` (last 4 only), `route_count`. + +### Routes + +| Method | Path | Purpose | +|---|---|---| +| GET | `/admin/routes?model=` | list (all enabled, or one public id) | +| POST | `/admin/routes` | create | +| PATCH | `/admin/routes/{id}` | update | +| DELETE | `/admin/routes/{id}` | delete | + +```json +{ + "public_model_id": "fast", + "provider_id": 1, + "upstream_model_id": "gpt-4o-mini", + "input_price_per_million": 0.15, + "cached_input_price_per_million": 0.075, + "output_price_per_million": 0.60, + "cache_write_price_per_million": null, + "priority": 1, + "context_length": 128000, + "supports_streaming": true, + "supports_tools": true, + "enabled": true, + "description": "cheap default" +} +``` + +Lower `priority` runs first. Cached/write prices fall back to the input price when +omitted. + +### Client keys + +| Method | Path | Purpose | +|---|---|---| +| GET | `/admin/keys` | list (masked) | +| POST | `/admin/keys` | create, returns the plaintext key **once** | +| PATCH | `/admin/keys/{id}` | rename, change limits, enable/disable | +| POST | `/admin/keys/{id}/revoke` | disable permanently | +| POST | `/admin/keys/{id}/topup` | `{"amount_usd": 10}` | + +```json +{"name": "team-a", "credit_limit_usd": 25, "rpm_limit": 120} +``` + +```json +{"id": 4, "name": "team-a", "masked_key": "sk-gw-AbCd...Wxyz", + "key": "sk-gw-...", "warning": "Store this key now. It is not shown again.", + "credit_limit_usd": 25, "used_usd": 0, "rpm_limit": 120, "enabled": true} +``` + +### Model sync + +`POST /admin/models/sync` — calls each provider's `/models`, filters non-chat ids, +upserts routes at priority 100. Known ids get prices from a built-in table; unknown +ids land at `0.00` and are flagged in `description`. + +```json +{"created": 4, "updated": 0, "skipped_providers": ["groq: no credential"], "errors": []} +``` + +### Reporting + +| Method | Path | Purpose | +|---|---|---| +| GET | `/admin/stats?window=24h` | totals, percentiles, cache, providers/models | +| GET | `/admin/usage?window=24h&limit=100&keyId=` | raw usage rows | + +```json +{ + "requests": 21, "total_tokens": 384, "cost_usd": 0.000393, + "latency_ms": {"p50": 2, "p95": 44, "p99": 44}, + "ttft_ms": {"p50": 1, "p95": 5}, + "tokens_per_second": {"p50": 7000.0}, + "cache": {"prompt_tokens": 246, "cached_input_tokens": 84, + "hit_ratio": 0.3415, "savings_usd": 0.00002061}, + "fallback_requests": 3, + "estimated_usage_requests": 0, + "by_provider": {"openai": {"requests": 12, "cost_usd": 0.0002}}, + "by_model": [{"model": "gpt-4o-mini", "requests": 12, "tokens": 180, "cost_usd": 0.0002}] +} +``` + +--- + +## Metering fields + +Each usage row records: + +| Field | Meaning | +|---|---| +| `prompt_tokens` | total prompt, including cache read and cache write | +| `completion_tokens` | output tokens, including reasoning | +| `cached_input_tokens` | subset of prompt served from cache | +| `cache_write_tokens` | Anthropic cache writes (subset of prompt) | +| `reasoning_tokens` | reasoning/thinking tokens | +| `input_cost_usd`, `cached_input_cost_usd`, `cache_write_cost_usd`, `output_cost_usd` | itemised | +| `cost_usd` | total, 8 decimals | +| `cached_savings_usd` | what cached tokens would have cost at the input rate | +| `ttft_ms` | time to first streamed token | +| `tokens_per_second` | generation throughput (post-TTFT window) | +| `upstream_request_id` | provider request id, for support | +| `finish_reason` | `stop`, `length`, `tool_calls`, ... | +| `attempts` | 1 = first route worked; >1 = fallback happened | +| `fallback_from` | provider that failed first | +| `usage_estimated` | counts were inferred, not reported | + +Billing rule: `billableInput = prompt_tokens − cached_input_tokens − cache_write_tokens`. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..786a447 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,106 @@ +# Architecture + +## Request flow + +1. `RequestIdFilter` assigns `X-Request-Id` and puts it in the MDC. +2. `AuthFilter` resolves `Authorization: Bearer sk-gw-...` to a `ClientKey`, applies the + RPM limiter, or checks `X-Admin-Token` for `/admin/**`. +3. `DrainState` refuses new work while shutting down; `ConcurrencyLimiter` takes a + per-key in-flight permit. +4. `ChatRequestSpec.parse` validates the body and keeps the original JSON for passthrough. +5. `RouteResolver` turns the public model id (plus `models[]`, `provider.order`, alias and + weights) into an ordered chain of `(route, provider)`. +6. `CreditGuard` checks allowlist, budgets and the worst-case cost, and clamps `max_tokens`. +7. `ChatService` / `StreamService` walk the chain: one attempt per route, the adapter + translates, `UpstreamClient` sends raw HTTP. `ProviderCircuitBreaker` skips parked + providers. +8. `UsageService` prices the call, writes a `UsageRecord`, charges the key and feeds the + metrics. + +## Providers + +Two wire formats cover every provider worth proxying: + +| Type | Speaks | Works with | +|---|---|---| +| `OPENAI` | chat-completions | OpenAI, Groq, Google Gemini (compatible endpoint), Together, Fireworks, DeepSeek, vLLM, LM Studio, Ollama's `/v1`, any compatible endpoint | +| `ANTHROPIC` | Messages API | Anthropic (native, with prompt caching and tool calls) | + +Any other provider is a configuration change, not a code change: register it with +`POST /admin/providers`, give it a base URL and a wire type, then add routes. A new wire +format means implementing `ProviderAdapter` (~200 lines for an OpenAI-shaped API, ~400 for +Anthropic). + +## Routing + +Routes are rows of `(public_model_id, provider, upstream_model_id, priority, price)`. + +`auto-fast` in the seed data has two routes, which gives a cross-provider fallback: + +| public id | provider | upstream | priority | +|---|---|---|---| +| auto-fast | openai | gpt-4o-mini | 1 | +| auto-fast | groq | llama-3.3-70b-versatile | 2 | + +Request-level controls, OpenRouter style: + +```jsonc +{ + "model": "gpt-4o-mini", + "models": ["gpt-4o-mini", "claude-3.5-haiku"], // extra fallback models, in order + "provider": {"order": ["groq", "openai"], "allow_fallbacks": false}, + "messages": [...] +} +``` + +`provider.allow_fallbacks: false` pins the request to the first route, so an upstream +500 surfaces to the caller instead of being masked. + +**Capability aliases** are resolved from the live route table: + +| Alias | Resolves to | +|---|---| +| `cheap` | enabled route with the lowest input price | +| `fast` | lowest output price | +| `long-context` | largest declared context window | +| `vision` | first route whose description mentions vision | + +They appear in `/v1/models` with a `resolves_to` field. + +**Canary / A-B** — each route has a `weight` (default 100). The starting route is chosen +by a stable hash of `(client key, model)` over the weights, so a key keeps landing on the +same variant and the rest of the chain stays as fallback. Set it with +`PATCH /admin/routes/{id} {"weight": 20}`. + +## What is metered + +Every call records the full token picture both providers report: + +| Field | Source (OpenAI / Anthropic) | +|---|---| +| `prompt_tokens` | `usage.prompt_tokens` / `input_tokens` + cache read + cache write | +| `completion_tokens` | `usage.completion_tokens` / `output_tokens` (includes thinking) | +| `cached_input_tokens` | `prompt_tokens_details.cached_tokens` / `cache_read_input_tokens` | +| `cache_write_tokens` | — / `cache_creation_input_tokens` | +| `reasoning_tokens` | `completion_tokens_details.reasoning_tokens` / in `output_tokens` | +| `audio_input/output_tokens` | `*_tokens_details.audio_tokens` / — | + +Cost is stored itemised — `input_cost_usd`, `cached_input_cost_usd`, +`cache_write_cost_usd`, `output_cost_usd`, `cost_usd` — plus `cached_savings_usd`. +Cached tokens bill at their own rate when the route has one, otherwise at the plain input +rate; Anthropic cache writes bill at their premium rate. + +Also captured: `ttft_ms`, `tokens_per_second`, `upstream_request_id`, `finish_reason`, +`attempts`, `fallback_from`, and `usage_estimated` (true when the provider returned no +usage block and the counts were inferred). + +Aggregates: `/v1/usage` adds cache hit ratio and savings for the calling key; +`/admin/stats` adds cache hit ratio, p50/p95 TTFT, median throughput, fallback count and +estimated-usage count. + +## Tool calls + +OpenAI-shaped providers pass tool calls through untouched. The Anthropic adapter +translates `tool_calls` to `tool_use` blocks and `tool` messages to `tool_result` +blocks, including `tool_choice` and `parallel_tool_calls`. Images are forwarded as +base64 or URL sources. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 0000000..0c272e1 --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,155 @@ +# Configuration + +Every setting is an environment variable with a working default; nothing is required +except the provider credentials you actually use. + +## Server + +| Variable | Default | Meaning | +|---|---|---| +| `PORT` | `8080` | HTTP port | +| `GATEWAY_ADMIN_TOKEN` | empty (admin off) | admin shared secret | +| `GATEWAY_PUBLIC_MODELS` | `true` | allow keyless `/v1/models` | +| `GATEWAY_CORS_ORIGINS` | `*` | comma-separated origin patterns | +| `GATEWAY_MAX_REQUEST_BYTES` | `1048576` | request bodies larger than this get 413 | +| `GATEWAY_DRAIN_TIMEOUT` | `30s` | how long shutdown waits for in-flight streams | + +## Keys and budgets + +| Variable | Default | Meaning | +|---|---|---| +| `GATEWAY_BOOTSTRAP_DEMO_KEY` | `false` | create a demo key when the table is empty (ignored in `prod`) | +| `GATEWAY_DEFAULT_RPM` | `120` | default requests/minute for new keys | +| `GATEWAY_DEFAULT_CREDIT_USD` | unlimited | default credit limit for new keys | +| `GATEWAY_DEFAULT_MAX_CONCURRENT` | `8` | in-flight requests per key (0 = unlimited) | +| `GATEWAY_MAX_OUTPUT_TOKENS` | `0` (off) | hard cap on `max_tokens` for every request | +| `GATEWAY_USAGE_RETENTION` | `180d` | usage rows older than this are deleted nightly (0 keeps them) | + +## Upstream behaviour + +| Variable | Default | Meaning | +|---|---|---| +| `GATEWAY_BREAKER_FAILURES` | `5` | consecutive failures before a provider is parked | +| `GATEWAY_BREAKER_OPEN` | `30s` | how long a parked provider is skipped | +| `GATEWAY_MAX_ATTEMPTS` | `3` | routes tried per request, including the first | +| `GATEWAY_IDEMPOTENCY_TTL` | `24h` | how long a completed response stays replayable | +| `GATEWAY_IDEMPOTENCY_MAX_BYTES` | `2097152` | larger responses are not stored for replay | +| `GATEWAY_IDEMPOTENCY_PENDING` | `2m` | abandoned PENDING claims are reclaimed after this | +| `GATEWAY_CACHE_MAX_BYTES` | `2097152` | larger responses are not cached | + +Timeouts live in `application.yml` under `gateway.*`: `connect-timeout` (10s), +`request-timeout` (180s), `stream-idle-timeout` (120s). A provider row can override the +last two with `request_timeout_ms` and `stream_idle_timeout_ms`. + +## Observability + +| Variable | Default | Meaning | +|---|---|---| +| `GATEWAY_TRACE_SAMPLE` | `0` | trace sampling probability (0 disables export) | +| `GATEWAY_SWAGGER_UI` | `false` | serve Swagger UI at `/swagger-ui.html` | +| `GATEWAY_ALERT_WEBHOOK` | empty | Slack-compatible webhook for alerts | +| `GATEWAY_ALERT_COOLDOWN` | `1h` | minimum gap between alerts of the same event | +| `GATEWAY_ALERT_BUDGET_THRESHOLD` | `0.8` | alert when a key has less than this fraction left | +| `GATEWAY_SEMANTIC_MODEL` | blank (off) | public model used to embed prompts for the semantic cache | +| `GATEWAY_SEMANTIC_MAX_ENTRIES` | `200` | candidates compared per key and model | +| `GATEWAY_GUARDRAILS_LOG_REDACTION` | `true` | redact PII before a prompt is written to the prompt log | +| `GATEWAY_GUARDRAILS_REDACT` | `false` | redact the prompt the provider receives | +| `GATEWAY_GUARDRAILS_MODERATION` | `false` | pre-flight moderation call (OpenAI-shaped providers) | +| `GATEWAY_GUARDRAILS_MODERATION_MODEL` | `omni-moderation-latest` | model used for that call | +| `GATEWAY_ADMIN_AUTH_FAILURES` | `10` | failed admin auth attempts before 429s | +| `GATEWAY_ADMIN_AUTH_WINDOW` | `5m` | window for those failures | +| `GATEWAY_API_DOCS` | `true` (false in `prod`) | serve `/v3/api-docs` | +| `GATEWAY_PROMPT_LOGGING` | `false` | store redacted request/response bodies | +| `GATEWAY_PROMPT_LOG_RETENTION` | `7d` | how long prompt logs are kept | +| `GATEWAY_PROMPT_LOG_MAX_BYTES` | `65536` | per-body cap for prompt logs | + +## Rate limiting + +| Variable | Default | Meaning | +|---|---|---| +| `GATEWAY_RATE_LIMIT_BACKEND` | `memory` | `memory` (one node) or `valkey` (shared) | +| `GATEWAY_RATE_LIMIT_PREFIX` | `path` | key prefix for Valkey counters | +| `SPRING_DATA_REDIS_HOST` / `_PORT` | `localhost` / `6379` | Valkey endpoint | + +Valkey speaks the Redis protocol, so the Spring Data Redis client works unchanged. +Point it at a Valkey instance and set `GATEWAY_RATE_LIMIT_BACKEND=valkey` to make the RPM +limit exact across instances. Budget reservations are enforced in the database either +way, so they are correct in a cluster regardless of this setting. + +## Provider credentials + +| Variable | Meaning | +|---|---| +| `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` / `GROQ_API_KEY` / `GEMINI_API_KEY` | credentials for the seeded providers | +| `GATEWAY_UPSTREAM__KEY` | credential for any other provider slug (e.g. `GATEWAY_UPSTREAM_TOGETHER_KEY`) | + +Resolution order: **database → `application.yml` → environment**. A key stored through +the admin API silently wins over the environment, which makes rotation possible without +a restart. Credentials in the database are AES-GCM encrypted with a key from +`GATEWAY_MASTER_KEY` or the generated `./data/master.key` (0600). + +A provider row must exist before its environment variable is read; `openai`, `anthropic`, +`groq` and `gemini` are seeded. + +### Gemini + +Gemini is seeded as an `OPENAI`-type provider pointing at Google's +OpenAI-compatible endpoint: + +``` +base_url = https://generativelanguage.googleapis.com/v1beta/openai +``` + +Set `GEMINI_API_KEY` and add a route per model you want to expose; the upstream id is +whatever Gemini calls it (`gemini-2.5-flash`, `gemini-3-flash`, …): + +```bash +A='X-Admin-Token: ...' +curl -s -X POST localhost:8080/admin/routes -H "$A" -H 'Content-Type: application/json' \ + -d '{"public_model_id":"gemini-2.5-flash","provider_id":4, + "upstream_model_id":"gemini-2.5-flash", + "input_price_per_million":0.30,"output_price_per_million":2.50, + "priority":10}' +``` + +`POST /admin/models/sync` imports Gemini's list (image and embedding ids are filtered +out) but leaves unknown prices at 0.00, so set them from Google's pricing page before +sending traffic. The native `generateContent` protocol is not used; the compatible layer +covers chat, streaming, tools, images and embeddings. + +## Database + +| Variable | Default | Meaning | +|---|---|---| +| `DB_URL` | `jdbc:h2:file:./data/gateway;MODE=PostgreSQL;AUTO_SERVER=TRUE` | JDBC URL | +| `DB_USER`, `DB_PASSWORD` | `sa`, empty | credentials | + +### Postgres + +```bash +export SPRING_PROFILES_ACTIVE=postgres +export DB_URL=jdbc:postgresql://localhost:5432/aigateway +export DB_USER=aigateway DB_PASSWORD=aigateway +./mvnw spring-boot:run +``` + +or `docker compose up --build`. + +### Profiles + +| Profile | Effect | +|---|---| +| default | H2 file database, human-readable logs | +| `postgres` | Postgres dialect and driver | +| `prod` | ECS JSON logs with `requestId`/`traceId`/`spanId`, conservative tracing | + +## Ports and endpoints + +| Path | Notes | +|---|---| +| `/v1/**` | public API, bearer key | +| `/admin/**` | `X-Admin-Token` | +| `/actuator/health` | liveness + readiness | +| `/actuator/health/providers` | per-provider circuit state and retry-after | +| `/actuator/prometheus` | metrics | +| `/v3/api-docs` | OpenAPI 3 | diff --git a/docs/LIMITS.md b/docs/LIMITS.md new file mode 100644 index 0000000..6779ba8 --- /dev/null +++ b/docs/LIMITS.md @@ -0,0 +1,167 @@ +# Design notes and limits + +## Behaviour worth knowing + +- **Idempotency is opt-in per request.** Send `Idempotency-Key` and a repeat of the same + request replays the stored response (streams included) with `X-Idempotent-Replay: true`; + a different body under the same key gets 409 `idempotency_conflict`, a concurrent + duplicate gets 409 `idempotency_in_progress`. Completed responses live for + `GATEWAY_IDEMPOTENCY_TTL`; responses over `GATEWAY_IDEMPOTENCY_MAX_BYTES` are not + stored, so a retry calls the provider again. +- **The response cache is opt-in per route and exact-match.** With `cache_enabled` and an + explicit `temperature: 0`, a tool-free request with an identical body replays the stored + response (streams included) with `X-Path-Cache: hit`, billed at zero and reported as + savings. Entries last `cache_ttl_seconds` (default 3600) and are scoped to the client + key. +- **Operator aliases are stable; capability aliases are not.** `POST /admin/aliases` + stores a chain (`prod-chat` → `claude-sonnet-4.5`, then `gpt-4o`) that resolves in that + order. The four built-in aliases (`cheap`, `fast`, `long-context`, `vision`) are still + recomputed from prices on every request, so they can move when a price changes. An + operator alias may shadow a built-in one, and the API flags it with + `shadows_computed`. +- **Alias chains are validated at write time.** An unknown model, a self-reference or a + chain deeper than 10 hops is rejected with 400, so a broken alias cannot turn into a + 500 on a customer request. +- **Keys can be restricted to models.** `allowed_models` accepts ids and globs + (`gpt-4o-mini,claude-*`); anything else returns 403 `model_not_allowed`. +- **Budgets have three windows.** A key carries a lifetime credit limit plus + `daily_limit_usd` and `monthly_limit_usd`, both checked against stored spend before the + call. +- **Requests can carry an end user.** `end_user` (Path-specific) or `user` (the OpenAI + field) is stored on the usage row and indexed with the key and timestamp. A key may set + `end_user_daily_limit_usd`, which reserves against that end user's own daily budget the + same way key credit is reserved — a reservation row per (key, end user), taken with an + atomic conditional update, so two concurrent calls cannot both take the last of it. The + cap applies even to a key with no credit limit of its own. +- **`/v1/usage` breaks spend down per end user**, so a product can bill its own customers + without a second system of record. `/admin/usage?endUser=` and the CSV export filter by + the same field. +- **Project limits sit above key limits.** A project can carry `rpm_limit` and + `max_concurrent_requests`, enforced in addition to the key's own caps: a request must + pass both. A project limit of `0` means unlimited. The 429 names which cap was hit + ("Project 'acme' rate limit exceeded"). +- **Limits are enforced before the call.** `max_concurrent_requests` in flight per key + (default 8) and bodies above `max-request-bytes` rejected with 413 before Jackson reads + them. +- **Unsupported parameters are rejected, not dropped.** Anthropic has no equivalent for + `seed`, `logit_bias`, penalties, `logprobs`, `n > 1` or a non-text `response_format`; + those return 422 `unsupported_parameter`. +- **System-only conversations are rejected.** Anthropic needs a non-empty message list; + the gateway returns 400 instead of forwarding an empty array. +- **Semantic tier sits above the exact cache and is off by default.** With + `semantic_cache_enabled` on a route and `GATEWAY_SEMANTIC_MODEL` set, a request that + misses the exact lookup is embedded and matched against stored entries by cosine + similarity (default threshold 0.95, per route). Every exact-cache rule still applies: + the route opted into caching, the request is `temperature: 0`, tool-free, and entries + are scoped per client key. It is off by default because a hit here answers a *different* + prompt. +- **The semantic tier costs an embedding call per lookup.** That is why it is opt-in: the + gateway embeds the prompt before it can know whether anything is similar. Candidates + are bounded by `GATEWAY_SEMANTIC_MAX_ENTRIES` per key and model, so the comparison is a + small scan rather than a table scan; a large deployment wants an ANN index instead. +- **The semantic tier serves non-streaming requests only.** Streaming still gets the exact + tier, which is byte-identical and cheaper to reason about. +- **Azure OpenAI is a provider shape, not a provider type.** Setting `api_version` on a + provider switches three things: the deployment name goes in the path + (`/openai/deployments//chat/completions`), the version goes in + `?api-version`, and auth uses the `api-key` header instead of a bearer token. The route + table therefore maps a public model to an Azure *deployment*, which is what Azure + actually requires. The provider probe uses the deployment listing, because Azure has no + `/models`, and an unsupported version surfaces Azure's own message rather than a + generic 502. +- **Routing strategies are opt-in per route.** `strategy` defaults to `priority`, which + keeps the declared order. `fastest` orders the chain by exponentially weighted latency + (~5-sample half-life); `cheapest-viable` prefers the cheapest route whose EWMA latency + is under the route's `latency_slo_ms`, and demotes the rest to fallback. A route with no + measurements yet counts as viable, so a cold start cannot deadlock routing. The EWMA is + per JVM, like the circuit breaker. +- **Circuit breaker per provider.** Five consecutive failures park a provider for 30s; + routes on a parked provider are skipped in microseconds. An upstream 429 with + `Retry-After` parks it for exactly that long. State appears as `circuit_state` on + `/admin/providers`. +- **Timeouts are per provider.** `request_timeout_ms` and `stream_idle_timeout_ms` + override the global defaults. +- **Shutdown drains instead of truncating.** Readiness goes DOWN, new requests get 503, + in-flight streams finish within `GATEWAY_DRAIN_TIMEOUT`. +- **Credits are charged after the call** from reported usage, with a worst-case estimate + checked first. A provider that reports nothing is billed from an estimate and flagged + `usage_estimated=true`. +- **Cost precision is 8 decimals.** Each component is rounded before summing, so a stored + total can differ from the exact product by at most 1e-8 USD. +- **Streaming has no resume.** Once bytes are sent the route cannot change; failures + arrive as an SSE error object followed by `data: [DONE]`. +- **Unpriced synced models are disabled.** `/admin/models/sync` imports unknown ids with + `enabled=false`, so a model cannot serve traffic for free before someone sets its price. +- **Reservations use the expected cost, not the cap.** A call reserves what a typical + completion costs (512 output tokens) while still refusing when that estimate exceeds the + balance, so a big `max_tokens` no longer blocks a small balance. +- **Admin auth is rate limited.** Ten failed attempts from one address in five minutes get + 429s; a successful auth clears the counter. +- **Usage history is pruned.** Rows older than `GATEWAY_USAGE_RETENTION` (default 180 + days) are deleted nightly; set it to `0` to keep history forever and plan for + partitioning at that point. +- **Abandoned idempotency claims are reclaimed.** A PENDING claim older than + `GATEWAY_IDEMPOTENCY_PENDING` (default 2 min) is treated as crashed and the retry runs, + so one dead instance cannot block a key for the full TTL. +- **Traces continue upstream.** The W3C `traceparent` header is injected into every + provider call, so provider-side logs join the same trace. +- **Prompt logging covers streaming too.** The SSE payload is captured and stored the + same way as a non-streaming body, capped and redacted. +- **Anthropic prompt caching is honoured.** A `cache_control` marker on a text or image + part is forwarded unchanged, so clients can cache long prefixes. +- **Projects have their own budgets.** A project can carry a lifetime, daily and monthly + limit; they are checked against the spend of every key in it, and + `GET /admin/projects/{id}/usage` rolls up today, this month and all time. +- **Prompt logs are redacted twice.** Credentials first (`sk-`, `gsk_`, bearer tokens), + then personal identifiers: emails, phones, card numbers that pass the Luhn checksum, + Aadhaar and PAN. Redaction is on by default for logs (`GATEWAY_GUARDRAILS_LOG_REDACTION`) + and is a pure function, so the same prompt always redacts the same way. Tokens are + numbered per call (`[EMAIL_1]`), which keeps a stored conversation readable. +- **Redaction before send is off by default.** `GATEWAY_GUARDRAILS_REDACT` makes the + provider see redacted text too. It changes what the model can answer, so it is an + operator decision, not a default. When on, redaction happens before the request is + hashed, stored or replayed, so a retry cannot resurrect the original text. +- **Moderation is off by default and never silent.** `GATEWAY_GUARDRAILS_MODERATION` adds a + pre-flight call to the provider's moderation endpoint (OpenAI-shaped providers only, + since it needs a compatible `/moderations`). A flagged request is refused with 400 + `content_flagged` and the categories; a moderation outage is logged and the request + proceeds, because a moderation dependency must not become a gateway outage. +- **Phone detection is deliberately narrow.** Contiguous digits are only treated as a + phone when they look like a mobile number (10-13 digits starting 6-9), so a 10-digit + epoch timestamp is not redacted into `[PHONE_1]`. Numbers with separators or a leading + `+` are treated as phones; ISO dates and dotted versions are not. +- **`GET /admin/settings` reports what is active**, so an operator (or the smoke suite) + can see whether logging, redaction and moderation are on without reading the env. +- **Prompt logging is off by default.** When enabled it redacts `sk-`/`gsk_`/`Bearer` + tokens, caps each body, and purges after the retention window — but it still stores user + prompts. Treat retention as a compliance decision. + +## Known gaps + +**Per-node by design** + +- **Circuit state and the in-flight limiter live in the JVM.** Breaking load across + instances makes each breaker see only its own failures, and the concurrency cap is per + instance. The RPM limiter and budget reservations *are* shared (Valkey and the + database), so money and rate limits stay correct in a cluster. +- **Fuzzing is time-boxed.** `ChatRequestFuzzTest` runs Jazzer for 10s per build; deeper + fuzzing is a manual `jazzer` run rather than a long CI job. + +**Provider-inherent** + +- **Embeddings are OpenAI-compatible only.** Anthropic publishes no embeddings endpoint, + so such a route returns 501 `unsupported_endpoint`. +- **Audio is not proxied.** Chat images are supported (base64 and URL); audio inputs + would need a per-provider upload flow. +- **Anthropic beta features beyond prompt caching are not forwarded** — extended + thinking and server-side tools pass through only as ordinary content blocks. + +**Operational** + +- **`/v3/api-docs` is off in the prod profile** unless `GATEWAY_API_DOCS=true`. +- **Idempotency applies to chat only.** `/v1/embeddings` honours drain, concurrency and + budgets, but not `Idempotency-Key` or the response cache. +- **Model sync pricing uses a built-in table.** Ids in `ModelSyncService.PRICES` get a + price; anything else lands at 0.00 until edited, so a synced model can look free. +- **Prompt logs are per-instance settings.** They apply to the whole gateway, not to a + single key. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..a44b5b0 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,166 @@ +# Operations + +## Admin API + +Every `/admin/**` call needs `X-Admin-Token: $GATEWAY_ADMIN_TOKEN`. If that variable is +empty the admin API returns 403. Admin JSON accepts snake_case (`credit_limit_usd`) and +camelCase (`creditLimitUsd`); responses are snake_case. + +```bash +A='X-Admin-Token: pick-something-long' + +# providers +curl -s localhost:8080/admin/providers -H "$A" +curl -s -X PATCH localhost:8080/admin/providers/1 -H "$A" -H 'Content-Type: application/json' \ + -d '{"base_url":"https://api.openai.com/v1","api_key":"sk-..."}' +curl -s -X POST localhost:8080/admin/providers -H "$A" -H 'Content-Type: application/json' \ + -d '{"slug":"together","display_name":"Together","type":"OPENAI", + "base_url":"https://api.together.xyz/v1","api_key":"..."}' + +# routes +curl -s -X POST localhost:8080/admin/routes -H "$A" -H 'Content-Type: application/json' \ + -d '{"public_model_id":"gpt-4o-mini","provider_id":1,"upstream_model_id":"gpt-4o-mini", + "input_price_per_million":0.15,"output_price_per_million":0.60,"priority":10}' + +# client keys — the plaintext is returned exactly once +curl -s -X POST localhost:8080/admin/keys -H "$A" -H 'Content-Type: application/json' \ + -d '{"name":"team-a","credit_limit_usd":25,"rpm_limit":120}' +curl -s -X POST localhost:8080/admin/keys/1/topup -H "$A" -H 'Content-Type: application/json' \ + -d '{"amount_usd":10}' +curl -s -X POST localhost:8080/admin/keys/1/revoke -H "$A" + +# import each provider's model list (built-in prices for well-known ids, 0.00 otherwise) +curl -s -X POST localhost:8080/admin/models/sync -H "$A" +``` + +## Projects and admin tokens + +Projects group keys and usage. Admin tokens carry `read` or `read,write`; the bootstrap +`GATEWAY_ADMIN_TOKEN` always has full access. + +```bash +curl -s -X POST localhost:8080/admin/projects -H "$A" -H 'Content-Type: application/json' \ + -d '{"slug":"team-a","name":"Team A"}' + +# issue a read-only token for a dashboard +curl -s -X POST localhost:8080/admin/tokens -H "$A" -H 'Content-Type: application/json' \ + -d '{"name":"dashboard","scopes":"read"}' + +curl -s 'localhost:8080/admin/keys?projectId=1' -H "$A" +curl -s -X POST localhost:8080/admin/tokens/1/revoke -H "$A" +``` + +A read-only token can `GET` any admin endpoint and gets 403 on anything mutating. + +## Provider operations + +```bash +# reachability check: one /models call per provider, reports latency and model count +curl -s -X POST localhost:8080/admin/providers/1/probe -H "$A" + +# fleet view: circuit state and credential status +curl -s localhost:8080/admin/providers/health -H "$A" +``` + +## Reporting + +```bash +curl -s 'localhost:8080/admin/stats?window=24h' -H "$A" +curl -s 'localhost:8080/admin/usage?window=7d&limit=50' -H "$A" +curl -s 'localhost:8080/admin/usage/export?window=30d&limit=5000' -H "$A" -o usage.csv +curl -s 'localhost:8080/admin/audit?action=route.update&limit=50' -H "$A" +curl -s 'localhost:8080/admin/prompt-logs?limit=20' -H "$A" +``` + +## CI runners + +All workflows run on `ubuntu-26.04`, the current Ubuntu LTS, pinned rather than +`ubuntu-latest` so an image migration cannot change the build under us. Bump the label +deliberately when the next LTS ships. + +## Security gates + +`main` only accepts a PR that passes all of these: + +| Check | Workflow | Fails on | +|---|---|---| +| `Build and unit tests` | CI | compile or test failure | +| `End-to-end smoke test` | CI | any of the 104 checks | +| `Analyze Java` | CodeQL | new code-scanning alerts | +| `Secret scan` | Security | a secret in the git history | +| `Dependency review` | Security | a new dependency at high severity | +| `Config and file scan` | Security | secrets or misconfiguration in the tree | + +A release tag additionally builds the image, scans it with Trivy (fails on +HIGH/CRITICAL), and publishes the jar, a CycloneDX SBOM and `checksums.txt`. The image +digest is written to the workflow summary so it can be pinned. + +## Observability + +- **Provider health** at `/actuator/health/providers`: circuit state and retry-after per + provider. Overall health stays UP while fallback exists; only a total outage returns + OUT_OF_SERVICE. +- **Metrics** at `/actuator/prometheus`: `path_requests_total` (model, provider, status, + streamed), `path_latency_seconds`, `path_tokens_total` (prompt/completion/cached), + `path_cost_usd_total`, `path_cache_hits_total`. +- **Traces** via Micrometer + OpenTelemetry. Off by default (`GATEWAY_TRACE_SAMPLE=0`); + set a sampling probability and `OTEL_EXPORTER_OTLP_ENDPOINT` to export. +- **Correlation** — every response carries `X-Request-Id`, echoed into the MDC and into + structured logs as `requestId`, alongside `traceId` and `spanId`. +- **Structured logs** — the `prod` profile emits ECS JSON; the default profile stays + human-readable. +- **OpenAPI** — springdoc publishes `/v3/api-docs`; CI uploads the generated spec as a + build artifact so drift is visible. + +## Schema migrations + +Flyway owns the schema; Hibernate runs with `ddl-auto: validate` and fails fast when an +entity and the database disagree. + +- `src/main/resources/db/migration/V1__baseline.sql` is the baseline, written in portable + SQL (`identity` columns, `timestamp with time zone`, enums as varchar). +- **Never edit an applied migration.** Add `V__name.sql`. +- An existing pre-Flyway database is baselined at V1 automatically, so upgrading a + running install does not drop data. +- Tests run the same migrations against in-memory H2, so a broken migration fails CI. + +## Shutdown + +On SIGTERM the readiness probe goes DOWN, new chat requests get `503 shutting_down`, and +in-flight streams drain for up to `GATEWAY_DRAIN_TIMEOUT`. Keep that value below your +orchestrator's termination grace period. + +## Release checklist + +1. Copy `changelog/unreleased.md` to `changelog/vX.Y.Z.md` and rewrite it for the release. +2. Reset `changelog/unreleased.md` to the empty template. +3. Bump the version in `pom.xml` if it changed. +4. Commit on `Develop`, then open a PR into `main` (all security gates must pass). +5. Tag on `main`: `git tag -a vX.Y.Z -m "..." && git push origin vX.Y.Z`. +6. The release workflow publishes `changelog/.md` verbatim as the release body, + scans the image, and attaches the jar, SBOM and `checksums.txt`. A missing or empty + file fails the release before anything is published. + +Preview the notes locally with `python3 tools/changelog.py vX.Y.Z`. + +## Alerts + +Set `GATEWAY_ALERT_WEBHOOK` to a Slack-compatible URL to receive +`provider.circuit_open`, `provider.rate_limited` and `key.budget_low`, at most once per +event per cooldown. Payloads are JSON with `source`, `event`, `at` and event-specific +fields. + +## Audit + +Every provider, route and key mutation is recorded with actor, action, target and +payload in `admin_audit`, readable at `GET /admin/audit`. Auditing never blocks the +operation it describes. + +## Load testing + +```bash +k6 run -e BASE=http://127.0.0.1:8080 -e KEY=sk-gw-... tools/load-test.js +``` + +Two scenarios (steady chat and a streaming arrival rate) with thresholds on error rate and +p95 latency, so a regression fails the run. Not part of CI. diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md new file mode 100644 index 0000000..d43f54f --- /dev/null +++ b/docs/PUBLISHING.md @@ -0,0 +1,134 @@ +# Publishing to Maven Central + +The `release` profile is wired and verified: it builds a plain library jar plus sources +and javadoc, signs all four artifacts (jar, sources, javadoc, pom) and uploads them to +the Sonatype Central Portal. + +The published coordinates are **`io.github.an1ndra:path`**, verified through GitHub. The +Java package stays `com.anindra.path`; only the Maven coordinates use the namespace. + +Three things cannot be automated from this repository, because they need your account: + +1. **A verified namespace**, `io.github.an1ndra`, proved by a public GitHub repository of + that exact name. +2. **A GPG key** whose public half is on a keyserver. +3. **A Portal token**, stored in `~/.m2/settings.xml` (local) or as repository secrets + (CI). + +Until those exist, `mvn deploy` will fail at authentication or signature validation — +and a Central release is **immutable**, so do not retry with a version that was already +published. + +## One-time setup + +### 1. Namespace + +`io.github.an1ndra` is a **GitHub namespace**: Sonatype proves you own it by finding a +public repository with that exact name, so no domain is needed. + +1. Create a **public** GitHub repository named exactly `io.github.an1ndra` + (dots included — that is the required convention). +2. `central.sonatype.com` → **Namespaces** → **Add Namespace** → `io.github.an1ndra`. +3. Press **Verify**. It reads the repository through your GitHub account and turns + green immediately. + +Until that badge is green, every upload fails with **403**. + +### 2. Signing key + +```bash +gpg --full-generate-key # RSA 4096 signing key, 0 = never expires +gpg --keyserver keys.openpgp.org --send-keys +gpg --list-secret-keys --keyid-format LONG +``` + +`keys.openpgp.org` then emails a confirmation link to the key's UID address. The key +stays invisible to consumers (404 on lookup) until that link is clicked, and Central +cannot validate uploads signed by an unpublished key. + +Export the private key for CI: + +```bash +gpg --armor --export-secret-keys > path-signing-key.asc +``` + +### 3. Portal credentials + +`central.sonatype.com → Account → Generate User Token`, then in `~/.m2/settings.xml`: + +```xml + + + + central + TOKEN_USERNAME + TOKEN_PASSWORD + + + +``` + +## Publish + +```bash +# 1. Confirm what will be uploaded (nothing leaves the machine). +./mvnw -P release -DskipTests verify +ls target/path-*.jar target/*.asc + +# 2. Upload to the Portal. autoPublish=false, so you review before release. +MAVEN_GPG_PASSPHRASE='...' ./mvnw -P release -DskipTests deploy + +# 3. In central.sonatype.com, open the deployment and press Publish. +``` + +Expected upload set: + +``` +path-0.1.0.jar the library jar (not the Spring Boot fat jar) +path-0.1.0-sources.jar +path-0.1.0-javadoc.jar +path-0.1.0.pom +*.asc one signature per file +``` + +The runnable fat jar is **not** published; the GitHub release and container image carry +it. `-P release` skips `spring-boot:repackage` for that reason — which also means +`target/path-0.1.0.jar` is the **plain library** after a release build and `java -jar` on +it fails with "no main manifest attribute". Run a normal `./mvnw package` to get the +runnable jar back. + +## From CI + +Add three repository secrets and one variable: + +| Kind | Name | Value | +|---|---|---| +| secret | `MAVEN_GPG_PRIVATE_KEY` | the armored private key | +| secret | `MAVEN_GPG_PASSPHRASE` | its passphrase | +| secret | `CENTRAL_USERNAME` / `CENTRAL_PASSWORD` | the Portal token | +| variable | `PUBLISH_TO_CENTRAL` | `true` to enable the step | + +The release workflow then publishes on each `v*` tag. With the variable unset the step is +skipped, so a fork never tries to publish. + +## Consumers + +```xml + + io.github.an1ndra + path + 0.1.0 + +``` + +```kotlin +implementation("io.github.an1ndra:path:0.1.0") +``` + +Resolvable path once published: + +``` +https://repo.maven.apache.org/maven2/io/github/an1ndra/path/0.1.0/path-0.1.0.pom +``` + +Only after the first successful release. Until then the coordinates do not resolve. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..32feebd --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,40 @@ +# Roadmap + +Everything below is tracked as a GitHub issue; this page is the short version for +anyone reading the repo rather than the issue tracker. + +## Open stories + +| # | Story | Blocked on | +|---|---|---| +| [#62](https://github.com/an1ndra/path/issues/62) | Spring AI `ChatModel` adapter module | a decision on the module name and whether it publishes as a second artifact | +| [#75](https://github.com/an1ndra/path/issues/75) | Vertex AI / Gemini native | a decision: Vertex (service-account auth) or AI Studio only | +| [#74](https://github.com/an1ndra/path/issues/74) | Valkey-shared breaker and limiter | a Valkey instance to verify the cross-replica behaviour honestly | +| [#61](https://github.com/an1ndra/path/issues/61) | Usage retention and partitioning | Postgres only; retention shipped, partitioning unverified on H2 | +| [#59](https://github.com/an1ndra/path/issues/59) | Publish to Maven Central | a verified namespace, a GPG key and a Portal token | + +## Delivered + +| Area | What shipped | +|---|---| +| Routing | priority chains, cross-provider fallback, capability aliases (`cheap`, `fast`, `long-context`, `vision`), weighted canary splits | +| Money | cache-aware token accounting, itemised cost, idempotent retries, lifetime/daily/monthly budgets, `max_tokens` clamping, model allowlists, CSV statements | +| Safety | per-provider circuit breaker with `Retry-After`, request size limits, per-key concurrency caps, parameter-fidelity 422s | +| Operations | Flyway migrations, readiness gating, shutdown drain, provider probe/health, admin audit log, webhook alerts, Prometheus metrics, OpenTelemetry tracing, ECS JSON logs, OpenAPI | +| Features | exact-match response cache (streaming included), embeddings, opt-in redacted prompt logging, nightly provider canary | +| Scale and tenancy | Valkey-backed shared RPM limiter, database budget reservations, projects, scoped admin tokens | +| Assurance | Jazzer fuzzing of the request parser, property tests for translation invariants | +| Post-0.1.0 audit | all twelve fixes (#38–#51): unpriced routes disabled, generic 500s with a reference id, chunked-body limits, admin auth limiter, master-key rotation, `projectId` filters, tag-on-main guard | +| Runnability | `tools/demo.sh` (60-second tour against the mock), `docs/WHY-PATH.md` | +| Routing and billing (0.2) | adaptive routing strategies (#63), operator-defined aliases (#68), project rate and concurrency limits (#65), project budget alerts (#69), `POST /v1/tokens/count` (#66), idempotent embeddings (#67) | +| Product (0.3) | semantic cache tier (#71), guardrails: PII redaction and moderation (#70), end-user attribution and budgets (#64), token counting (#66), idempotent embeddings (#67) | +| Reach | Azure OpenAI (#73) | +| Kubernetes | `deploy/helm/path` (#72): drain-aware grace period enforced at render time, HPA, PDB, optional Postgres and Valkey | + +## How to pick one up + +1. Branch from `Develop` (direct pushes allowed; `main` is release-only behind a PR). +2. Read `AGENTS.md` for conventions and gotchas. +3. Add a unit test and, if it touches routing, streaming or billing, a smoke check in + `tools/smoke_test.py`. +4. `./mvnw verify` and the smoke suite must pass before the PR. diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..5a20ddf --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,65 @@ +# Testing + +## Unit and integration tests + +```bash +./mvnw verify # 72 tests +``` + +| Class | Covers | +|---|---| +| `GatewayApiTest` | boots the full app on H2: auth, catalogue, credits, admin token, snake_case inputs, 413, OpenAPI, request ids | +| `AnthropicAdapterTest` | request translation (system, images, tools, tool results), response translation, stream events, cache tokens, rejected parameters | +| `AnthropicTranslationPropertyTest` | 200 generated conversations against translation invariants | +| `UsageCostTest` | cost breakdown, cache pricing, missing prices, usage parsing | +| `ProviderCircuitBreakerTest` | threshold, half-open probe, rate-limit parking | +| `IdempotencyServiceTest` | replay, conflicts, concurrent claims, key scoping | +| `ResponseCacheEligibilityTest` / `ResponseCacheServiceTest` | cache rules, scoping, expiry | +| `CreditGuardTest` | budget windows, max_tokens clamping, allowlist | +| `DrainStateTest` | in-flight accounting, refusal while draining | +| `RoutingTest` | aliases, weighted assignment, fallback order | +| `TokenEstimateTest` | real tokenization with a heuristic fallback | +| `ConcurrencyLimiterTest` | per-key permits, no over-release | +| `AuditServiceTest` / `AlertServiceTest` / `PromptLogServiceTest` | audit trail, webhook cooldown, redaction | + +## End-to-end suite + +99 checks against a mock upstream — no real keys needed. It exercises both wire formats, +streaming, fallback, credits, rate limits, idempotency, caching, aliases, embeddings, the +admin surface and metering. + +```bash +python3 tools/mock-upstream.py 9099 & +GATEWAY_ADMIN_TOKEN=test-admin java -jar target/path-0.1.0.jar & +python3 tools/smoke_test.py +``` + +The mock switches behaviour by model id, which is how failure paths are tested: + +| Model id contains | Behaviour | +|---|---| +| `boom` | HTTP 500 | +| `ratelimit` | HTTP 429 | +| `unauthorized` | HTTP 401 | +| `badrequest` | HTTP 400 | +| `slow` | stream sleeps between chunks | +| `empty` | stream closes without data | + +The suite patches providers to point at the mock through the admin API, so it also +verifies the admin surface end to end. + +## Load testing + +```bash +k6 run -e BASE=http://127.0.0.1:8080 -e KEY=sk-gw-... tools/load-test.js +``` + +Steady chat plus a streaming arrival rate, with thresholds on error rate and p95 latency. +It is not part of CI. + +## CI + +`.github/workflows/ci.yml` runs `./mvnw verify`, then starts the gateway against the mock +and runs the full smoke suite, uploading the OpenAPI spec as an artifact. +`.github/workflows/canary.yml` sends one 8-token call per configured provider nightly and +opens an issue on failure. diff --git a/docs/WHY-PATH.md b/docs/WHY-PATH.md new file mode 100644 index 0000000..739f578 --- /dev/null +++ b/docs/WHY-PATH.md @@ -0,0 +1,58 @@ +# Why Path + +Path is for teams that run Java and want their LLM traffic to stay inside their own +JVM, database and process — not behind a Python-only sidecar or a vendor's SaaS. + +## Where Path is the right tool + +- **One process.** A Spring Boot jar, Flyway migrations and an embedded H2 (or your + Postgres). No Python runtime, no Node gateway, no sidecar to operate. +- **Deterministic routing you can defend.** Price and priority decide where a request + goes; a customer dispute can be answered with "OpenAI returned 500, Groq was next". +- **Billing that accounts for caches.** Prompt, completion, cached and cache-write tokens + are priced separately, so a cache-heavy workload is not overbilled 10×. +- **Self-hosted by construction.** Provider keys are AES-GCM encrypted at rest, admin + tokens are scoped and audited, and nothing leaves your network except the upstream call. +- **Boring operations.** Readiness gates on seeding, shutdown drains long streams, + per-provider circuit breakers, Prometheus, OpenTelemetry and an OpenAPI spec. + +## Where Path is weaker + +Be honest about this before choosing it. + +| | Path | LiteLLM | Portkey | OpenRouter | +|---|---|---|---|---| +| Runtime | Java 21 / Spring Boot | Python | Node/Go, hosted | hosted SaaS | +| Model coverage | 2 wire formats, 4 seeded providers | 100+ providers | 250+ models | 300+ models | +| Hosted option | none (self-host only) | community | yes | yes | +| SDKs | any OpenAI SDK | Python SDK | JS/Python SDKs | JS/Python SDKs | +| Guardrails / PII | not built | plugins | built-in | built-in | +| Semantic cache | no (exact match only) | plugins | yes | no | +| Dashboard | JSON APIs + Postman/Bruno | UI | full UI | full UI | +| Licence | GPL-3.0 | MIT | Apache-2.0 core | proprietary | + +Model count is the biggest gap. If you need Gemini's native protocol, Bedrock signing, +or a hundred long-tail providers on day one, a larger project covers more ground. + +Path's answer is not "more providers". It is "providers you already use, routed and +billed precisely, in the runtime your team already operates". + +## A 60-second demo + +No API keys needed — the demo talks to a mock upstream, so it shows the mechanics +(routing, streaming, cache, metering) without spending anything. + +```bash +bash tools/demo.sh +``` + +What it prints: + +1. the seeded catalogue (`/v1/models`), including capability aliases +2. a non-streaming chat through `auto-fast`, with provider and cost headers +3. a streaming chat, showing SSE chunks and `[DONE]` +4. the same deterministic request twice, showing `X-Path-Cache: hit` and zero cost +5. `/v1/usage` for the key: tokens, cache savings, recent calls + +To point it at real providers instead, see the Quick start in the [README](../README.md) +and set `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GROQ_API_KEY` or `GEMINI_API_KEY`. diff --git a/docs/path.postman_collection.json b/docs/path.postman_collection.json new file mode 100644 index 0000000..0088342 --- /dev/null +++ b/docs/path.postman_collection.json @@ -0,0 +1,1158 @@ +{ + "info": { + "name": "Path \u2014 LLM gateway", + "description": "Public OpenAI-compatible surface plus the admin API.\n\n1. Set `baseUrl` and `adminToken` in the environment.\n2. Run **00 \u00b7 Setup** first: it mints an API key into `apiKey`.\n3. Then any Chat request.\n\nChat needs either a real provider credential or the mock upstream (`python3 tools/mock-upstream.py 9099`, with providers pointed at it).", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "apiKey", + "value": "" + }, + { + "key": "model", + "value": "gpt-4o-mini" + }, + { + "key": "publicModel", + "value": "fast" + }, + { + "key": "providerId", + "value": "1" + }, + { + "key": "createdProviderId", + "value": "" + }, + { + "key": "createdRouteId", + "value": "" + }, + { + "key": "keyId", + "value": "1" + } + ], + "item": [ + { + "name": "00 \u00b7 Setup", + "item": [ + { + "name": "Create API key", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/keys", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "keys" + ] + }, + "description": "Mints a client key and stores it in `apiKey`. The plaintext is returned once; every later chat request depends on it.", + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"postman-{{$timestamp}}\",\n \"credit_limit_usd\": 5,\n \"rpm_limit\": 120\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.collectionVariables.set(\"apiKey\", json.key);", + "pm.collectionVariables.set(\"keyId\", json.id);", + "pm.test(\"key returned once\", () => pm.expect(json.key).to.match(/^sk-gw-/));" + ] + } + } + ] + }, + { + "name": "List providers", + "request": { + "method": "GET", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/providers", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "providers" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.test(\"at least one provider\", () => pm.expect(json.length).to.be.above(0));", + "if (json.length) pm.collectionVariables.set(\"providerId\", json[0].id);" + ] + } + } + ] + } + ] + }, + { + "name": "01 \u00b7 Health & catalogue", + "item": [ + { + "name": "Health", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/actuator/health", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "actuator", + "health" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"UP\", () => pm.expect(pm.response.json().status).to.eql(\"UP\"));" + ] + } + } + ] + }, + { + "name": "List models", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/models", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "models" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.test(\"catalogue is a list\", () => pm.expect(json.data).to.be.an(\"array\"));", + "pm.test(\"has at least one model\", () => pm.expect(json.data.length).to.be.above(0));" + ] + } + } + ] + }, + { + "name": "Get one model", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/models/{{model}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "models", + "{{model}}" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.test(\"id matches\", () => pm.expect(json.id).to.eql(pm.collectionVariables.get(\"model\")));" + ] + } + } + ] + } + ] + }, + { + "name": "02 \u00b7 Chat", + "item": [ + { + "name": "Chat \u2014 non-streaming", + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hi in five words.\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.test(\"has content\", () => pm.expect(json.choices[0].message.content).to.be.a(\"string\"));", + "pm.test(\"model echoed as public id\", () => pm.expect(json.model).to.eql(pm.collectionVariables.get(\"model\")));", + "pm.test(\"provider header present\", () => pm.expect(pm.response.headers.get(\"X-Gateway-Provider\")).to.be.a(\"string\"));", + "pm.test(\"cost header is numeric\", () => pm.expect(Number(pm.response.headers.get(\"X-Gateway-Cost-Usd\"))).to.be.at.least(0));" + ] + } + } + ] + }, + { + "name": "Chat \u2014 streaming", + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "description": "Open the Postman console to inspect the SSE frames. The response ends with `data: [DONE]`.", + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"{{model}}\",\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n },\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Count to five.\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"SSE content type\", () => pm.expect(pm.response.headers.get(\"Content-Type\")).to.include(\"text/event-stream\"));", + "pm.test(\"stream ends with [DONE]\", () => pm.expect(pm.response.text()).to.include(\"[DONE]\"));" + ] + } + } + ] + }, + { + "name": "Chat \u2014 fallback chain", + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"{{model}}\",\n \"models\": [\n \"gpt-4o-mini\",\n \"claude-3.5-haiku\"\n ],\n \"provider\": {\n \"order\": [\n \"openai\",\n \"anthropic\"\n ],\n \"allow_fallbacks\": true\n },\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"hello\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"provider header tells you who served it\", () => pm.expect(pm.response.headers.get(\"X-Gateway-Provider\")).to.be.a(\"string\"));" + ] + } + } + ] + }, + { + "name": "Chat \u2014 tool calling", + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"{{model}}\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"What is the weather in Pune?\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n }\n ],\n \"tool_choice\": \"auto\"\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + }, + { + "name": "Chat \u2014 Anthropic route", + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-3.5-haiku\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"Be terse.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"hello\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + } + ] + }, + { + "name": "03 \u00b7 Usage & credits", + "item": [ + { + "name": "My usage", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/usage?window=24h", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "usage" + ], + "query": [ + { + "key": "window", + "value": "24h" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.test(\"has cost\", () => pm.expect(json.cost_usd).to.exist);", + "pm.test(\"requests counted\", () => pm.expect(json.requests).to.be.at.least(1));" + ] + } + } + ] + }, + { + "name": "My credits", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/credits", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "credits" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.test(\"currency USD\", () => pm.expect(json.currency).to.eql(\"USD\"));" + ] + } + } + ] + } + ] + }, + { + "name": "04 \u00b7 Admin \u2014 providers", + "item": [ + { + "name": "List providers", + "request": { + "method": "GET", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/providers", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "providers" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"array\", () => pm.expect(pm.response.json()).to.be.an(\"array\"));" + ] + } + } + ] + }, + { + "name": "Create provider", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/providers", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"slug\": \"postman-{{$timestamp}}\",\n \"display_name\": \"Postman test\",\n \"type\": \"OPENAI\",\n \"base_url\": \"https://api.together.xyz/v1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "const json = pm.response.json();", + "pm.collectionVariables.set(\"createdProviderId\", json.id);", + "pm.test(\"credential state reported\", () => pm.expect(json.credential_configured).to.be.a(\"boolean\"));" + ] + } + } + ] + }, + { + "name": "Update provider", + "request": { + "method": "PATCH", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/providers/{{createdProviderId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "providers", + "{{createdProviderId}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"display_name\": \"Postman test (updated)\",\n \"enabled\": true\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + }, + { + "name": "Delete provider", + "request": { + "method": "DELETE", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/providers/{{createdProviderId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "providers", + "{{createdProviderId}}" + ] + }, + "description": "Removes the throwaway provider created above." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + } + ] + }, + { + "name": "05 \u00b7 Admin \u2014 routes", + "item": [ + { + "name": "List routes", + "request": { + "method": "GET", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/routes?model={{publicModel}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "routes" + ], + "query": [ + { + "key": "model", + "value": "{{publicModel}}" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + }, + { + "name": "Create route", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/routes", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "routes" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"public_model_id\": \"postman-route-{{$timestamp}}\",\n \"provider_id\": \"{{providerId}}\",\n \"upstream_model_id\": \"meta-llama/llama-4-scout-17b-16e-instruct\",\n \"input_price_per_million\": 0.11,\n \"output_price_per_million\": 0.34,\n \"priority\": 1\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.collectionVariables.set(\"createdRouteId\", pm.response.json().id);" + ] + } + } + ] + }, + { + "name": "Update route", + "request": { + "method": "PATCH", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/routes/{{createdRouteId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "routes", + "{{createdRouteId}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"priority\": 2,\n \"enabled\": true\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + }, + { + "name": "Delete route", + "request": { + "method": "DELETE", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/routes/{{createdRouteId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "routes", + "{{createdRouteId}}" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + } + ] + }, + { + "name": "06 \u00b7 Admin \u2014 keys", + "item": [ + { + "name": "List keys", + "request": { + "method": "GET", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/keys", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "keys" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"array\", () => pm.expect(pm.response.json()).to.be.an(\"array\"));" + ] + } + } + ] + }, + { + "name": "Top up key", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/keys/{{keyId}}/topup", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "keys", + "{{keyId}}", + "topup" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"amount_usd\": 5\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"limit grew\", () => pm.expect(pm.response.json().credit_limit_usd).to.be.above(5));" + ] + } + } + ] + }, + { + "name": "Revoke key", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/keys/{{keyId}}/revoke", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "keys", + "{{keyId}}", + "revoke" + ] + }, + "description": "Run last: the setup key stops working after this." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"disabled\", () => pm.expect(pm.response.json().enabled).to.eql(false));" + ] + } + } + ] + } + ] + }, + { + "name": "07 \u00b7 Admin \u2014 reporting", + "item": [ + { + "name": "Stats", + "request": { + "method": "GET", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/stats?window=24h", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "stats" + ], + "query": [ + { + "key": "window", + "value": "24h" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"has latency\", () => pm.expect(pm.response.json().latency_ms).to.exist);" + ] + } + } + ] + }, + { + "name": "Usage rows", + "request": { + "method": "GET", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/usage?window=24h&limit=50", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "usage" + ], + "query": [ + { + "key": "window", + "value": "24h" + }, + { + "key": "limit", + "value": "50" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));" + ] + } + } + ] + }, + { + "name": "Sync models", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Admin-Token", + "value": "{{adminToken}}" + } + ], + "url": { + "raw": "{{baseUrl}}/admin/models/sync", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "admin", + "models", + "sync" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200 OK\", () => pm.response.to.have.status(200));", + "pm.test(\"reports counts\", () => pm.expect(pm.response.json().created).to.be.a(\"number\"));" + ] + } + } + ] + } + ] + } + ] +} diff --git a/docs/path.postman_environment.json b/docs/path.postman_environment.json new file mode 100644 index 0000000..3ba7ec5 --- /dev/null +++ b/docs/path.postman_environment.json @@ -0,0 +1,18 @@ +{ + "name": "Path \u2014 local", + "values": [ + { + "key": "baseUrl", + "value": "http://localhost:8080", + "type": "default", + "enabled": true + }, + { + "key": "adminToken", + "value": "test-admin", + "type": "secret", + "enabled": true + } + ], + "_postman_variable_scope": "environment" +} diff --git a/pom.xml b/pom.xml index dd07843..b0e004a 100644 --- a/pom.xml +++ b/pom.xml @@ -11,11 +11,35 @@ - com.anindra + io.github.an1ndra path - 0.1.0 + 0.2.0 path - OpenAI-compatible multi-provider LLM gateway with routing, credits and usage metering + Provider-agnostic LLM gateway: routing, fallback, credits and cache-aware token metering for OpenAI, Anthropic and any OpenAI-compatible API + https://github.com/an1ndra/path + + + + GNU General Public License v3.0 + https://www.gnu.org/licenses/gpl-3.0.txt + repo + + + + + + an1ndra + Anindra + https://github.com/an1ndra + + + + + https://github.com/an1ndra/path + scm:git:https://github.com/an1ndra/path.git + scm:git:ssh://git@github.com/an1ndra/path.git + HEAD + 21 @@ -30,6 +54,10 @@ org.springframework.boot spring-boot-starter-data-jpa + + org.springframework.boot + spring-boot-starter-data-redis + org.springframework.boot spring-boot-starter-validation @@ -45,6 +73,41 @@ true + + com.knuddels + jtokkit + 1.1.0 + + + + org.flywaydb + flyway-core + + + + io.micrometer + micrometer-registry-prometheus + + + io.micrometer + micrometer-tracing-bridge-otel + + + io.opentelemetry + opentelemetry-exporter-otlp + + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.8.17 + + + org.flywaydb + flyway-database-postgresql + runtime + + com.h2database h2 @@ -61,6 +124,12 @@ spring-boot-starter-test test + + com.code-intelligence + jazzer-junit + 0.30.0 + test + @@ -69,6 +138,119 @@ org.springframework.boot spring-boot-maven-plugin + + org.cyclonedx + cyclonedx-maven-plugin + 2.9.3 + + + package + + makeAggregateBom + + + + + json + bom + false + + + + + + + release + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + none + false + true + + + + attach-javadocs + + jar + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + true + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.2.8 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.12.0 + true + + central + + false + uploaded + + + + + + diff --git a/src/main/java/com/anindra/path/admin/AdminController.java b/src/main/java/com/anindra/path/admin/AdminController.java index fcf1030..ad40107 100644 --- a/src/main/java/com/anindra/path/admin/AdminController.java +++ b/src/main/java/com/anindra/path/admin/AdminController.java @@ -11,11 +11,19 @@ @RestController @RequestMapping("/admin") @RequiredArgsConstructor +@io.swagger.v3.oas.annotations.tags.Tag(name = "Admin", description = "Configuration, keys, operations and reporting") public class AdminController { private final AdminService admin; private final AdminReportingService reporting; private final ModelSyncService modelSync; + private final ProviderProbeService probes; + private final AuditService audit; + private final PromptLogService promptLogs; + private final TenancyService tenancy; + private final AliasService aliasService; + private final com.anindra.path.config.GatewayProperties props; + private final com.anindra.path.gateway.SemanticCacheService semantic; // ------------------------------------------------------------- providers @@ -26,18 +34,23 @@ public List> providers() { @PostMapping("/providers") public Map createProvider(@RequestBody AdminService.ProviderInput input) { - return admin.createProvider(input); + var result = admin.createProvider(input); + audit.record("provider.create", String.valueOf(result.get("id")), input); + return result; } @PatchMapping("/providers/{id}") public Map updateProvider(@PathVariable Long id, @RequestBody AdminService.ProviderInput input) { - return admin.updateProvider(id, input); + var result = admin.updateProvider(id, input); + audit.record("provider.update", String.valueOf(result.get("id")), input); + return result; } @DeleteMapping("/providers/{id}") public Map deleteProvider(@PathVariable Long id) { admin.deleteProvider(id); + audit.record("provider.delete", "provider:" + id, null); return Map.of("deleted", true, "id", id); } @@ -50,42 +63,53 @@ public List> routes(@RequestParam(required = false) String m @PostMapping("/routes") public Map createRoute(@RequestBody AdminService.RouteInput input) { - return admin.createRoute(input); + var result = admin.createRoute(input); + audit.record("route.create", String.valueOf(result.get("id")), input); + return result; } @PatchMapping("/routes/{id}") public Map updateRoute(@PathVariable Long id, @RequestBody AdminService.RouteInput input) { - return admin.updateRoute(id, input); + var result = admin.updateRoute(id, input); + audit.record("route.update", String.valueOf(result.get("id")), input); + return result; } @DeleteMapping("/routes/{id}") public Map deleteRoute(@PathVariable Long id) { admin.deleteRoute(id); + audit.record("route.delete", "route:" + id, null); return Map.of("deleted", true, "id", id); } // ------------------------------------------------------------------ keys @GetMapping("/keys") - public List> keys() { - return admin.listKeys(); + public List> keys(@RequestParam(required = false) Long projectId) { + return admin.listKeys(projectId); } @PostMapping("/keys") public Map createKey(@RequestBody AdminService.KeyInput input) { - return admin.createKey(input); + var result = admin.createKey(input); + audit.record("key.create", String.valueOf(result.get("id")), input); + return result; } @PatchMapping("/keys/{id}") public Map updateKey(@PathVariable Long id, @RequestBody AdminService.KeyInput input) { - return admin.updateKey(id, input); + var result = admin.updateKey(id, input); + audit.record("key.update", String.valueOf(result.get("id")), input); + return result; } @PostMapping("/keys/{id}/revoke") public Map revokeKey(@PathVariable Long id) { - return admin.revokeKey(id); + var result = admin.revokeKey(id); + audit.record("key.revoke", String.valueOf(result.get("id")), null); + return result; } @PostMapping("/keys/{id}/topup") @@ -93,18 +117,193 @@ public Map topUp(@PathVariable Long id, @RequestBody Map rotateMasterKey(@RequestBody Map body) { + var result = admin.rotateMasterKey(body.get("new_key")); + audit.record("master-key.rotate", "master-key", null); + return result; + } + + @GetMapping("/settings") + public Map settings() { + Map guardrails = new java.util.LinkedHashMap<>(); + guardrails.put("prompt_log_redaction", props.getGuardrails().isPromptLogRedaction()); + guardrails.put("redact_before_send", props.getGuardrails().isRedactBeforeSend()); + guardrails.put("moderation", props.getGuardrails().isModeration()); + + Map out = new java.util.LinkedHashMap<>(); + out.put("prompt_logging", props.getPromptLogging().isEnabled()); + out.put("guardrails", guardrails); + out.put("rate_limit_backend", props.getRateLimit().getBackend()); + out.put("public_models", props.isPublicModels()); + out.put("demo_key_bootstrap", props.isBootstrapDemoKey()); + return out; + } + + // --------------------------------------------------------------- aliases + + @com.fasterxml.jackson.databind.annotation.JsonNaming( + com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy.class) + public record AliasInput(String alias, List models, String description) { + } + + @GetMapping("/aliases") + public List> aliases() { + return aliasService.list(); + } + + @PostMapping("/aliases") + public Map createAlias(@RequestBody AliasInput input) { + var created = aliasService.create(input.alias(), input.models(), input.description()); + audit.record("alias.create", "alias:" + created.get("alias"), input); + return created; + } + + @DeleteMapping("/aliases/{alias}") + public Map deleteAlias(@PathVariable String alias) { + aliasService.delete(alias); + audit.record("alias.delete", "alias:" + alias, null); + return Map.of("deleted", true, "alias", alias); + } + + // -------------------------------------------------------------- tenancy + + @com.fasterxml.jackson.databind.annotation.JsonNaming( + com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy.class) + public record ProjectInput(String slug, String name, + @com.fasterxml.jackson.annotation.JsonAlias("creditLimitUsd") java.math.BigDecimal creditLimitUsd, + @com.fasterxml.jackson.annotation.JsonAlias("dailyLimitUsd") java.math.BigDecimal dailyLimitUsd, + @com.fasterxml.jackson.annotation.JsonAlias("monthlyLimitUsd") java.math.BigDecimal monthlyLimitUsd, + @com.fasterxml.jackson.annotation.JsonAlias("rpmLimit") Integer rpmLimit, + @com.fasterxml.jackson.annotation.JsonAlias("maxConcurrentRequests") + Integer maxConcurrentRequests) { + } + + public record TokenInput(String name, String scopes) { + } + + @GetMapping("/projects") + public List> projects() { + return tenancy.listProjects(); + } + + @PostMapping("/projects") + public Map createProject(@RequestBody ProjectInput input) { + var created = tenancy.createProject(input.slug(), input.name(), + input.creditLimitUsd(), input.dailyLimitUsd(), input.monthlyLimitUsd(), + input.rpmLimit(), input.maxConcurrentRequests()); + audit.record("project.create", "project:" + created.get("id"), input); + return created; + } + + @PatchMapping("/projects/{id}") + public Map updateProject(@PathVariable Long id, @RequestBody ProjectInput input) { + var updated = tenancy.updateProjectLimits(id, input.creditLimitUsd(), + input.dailyLimitUsd(), input.monthlyLimitUsd(), + input.rpmLimit(), input.maxConcurrentRequests()); + audit.record("project.update", "project:" + id, input); + return updated; + } + + @GetMapping("/projects/{id}/usage") + public Map projectUsage(@PathVariable Long id) { + return tenancy.projectSummary(id); + } + + @DeleteMapping("/projects/{id}") + public Map deleteProject(@PathVariable Long id) { + tenancy.deleteProject(id); + audit.record("project.delete", "project:" + id, null); + return Map.of("deleted", true, "id", id); + } + + @GetMapping("/tokens") + public List> tokens() { + return tenancy.listTokens(); + } + + @PostMapping("/tokens") + public Map createToken(@RequestBody TokenInput input) { + var created = tenancy.createToken(input.name(), input.scopes()); + audit.record("token.create", "token:" + created.token().getId(), input); + Map view = new java.util.LinkedHashMap<>(); + view.put("id", created.token().getId()); + view.put("name", created.token().getName()); + view.put("scopes", created.token().getScopes()); + view.put("token", created.plaintext()); + view.put("warning", "Store this token now. It is not shown again."); + return view; + } + + @PostMapping("/tokens/{id}/revoke") + public Map revokeToken(@PathVariable Long id) { + var revoked = tenancy.revokeToken(id); + audit.record("token.revoke", "token:" + id, null); + return revoked; + } + // ------------------------------------------------------------- reporting @GetMapping("/stats") - public Map stats(@RequestParam(defaultValue = "24h") String window) { - return reporting.stats(window); + public Map stats(@RequestParam(defaultValue = "24h") String window, + @RequestParam(required = false) Long projectId) { + Map out = reporting.stats(window, projectId); + Object cache = out.get("cache"); + if (cache instanceof Map existing) { + @SuppressWarnings("unchecked") + Map typed = (Map) existing; + typed.put("semantic_hits", semantic.hitCount()); + } + return out; } @GetMapping("/usage") public Map usage(@RequestParam(required = false) Long keyId, + @RequestParam(required = false) Long projectId, + @RequestParam(required = false) String endUser, @RequestParam(defaultValue = "24h") String window, @RequestParam(defaultValue = "100") int limit) { - return reporting.usage(keyId, window, limit); + return reporting.usage(keyId, projectId, endUser, window, limit); + } + + @PostMapping("/providers/{id}/probe") + public Map probe(@PathVariable Long id) { + return probes.probe(id); + } + + @GetMapping("/providers/health") + public List> providerHealth() { + return probes.summary(); + } + + @GetMapping(value = "/usage/export", produces = "text/csv") + public org.springframework.http.ResponseEntity exportUsage( + @RequestParam(required = false) Long keyId, + @RequestParam(required = false) Long projectId, + @RequestParam(required = false) String endUser, + @RequestParam(defaultValue = "30d") String window, + @RequestParam(defaultValue = "10000") int limit) { + java.time.Duration parsed = com.anindra.path.api.Durations.cap( + com.anindra.path.api.Durations.parse(window, java.time.Duration.ofDays(30)), + java.time.Duration.ofDays(365)); + java.time.Instant to = java.time.Instant.now(); + java.time.Instant from = to.minus(parsed); + String csv = reporting.usageCsv(keyId, projectId, endUser, from, to, limit); + return org.springframework.http.ResponseEntity.ok() + .header("Content-Disposition", "attachment; filename=\"usage-" + to + ".csv\"") + .body(csv); + } + + @GetMapping("/audit") + public List> auditLog(@RequestParam(required = false) String action, + @RequestParam(defaultValue = "100") int limit) { + return audit.list(action, limit); + } + + @GetMapping("/prompt-logs") + public List> promptLogs(@RequestParam(required = false) Long keyId, + @RequestParam(defaultValue = "50") int limit) { + return promptLogs.list(keyId, limit); } @PostMapping("/models/sync") diff --git a/src/main/java/com/anindra/path/admin/AdminReportingService.java b/src/main/java/com/anindra/path/admin/AdminReportingService.java index 67c5306..6a3ff35 100644 --- a/src/main/java/com/anindra/path/admin/AdminReportingService.java +++ b/src/main/java/com/anindra/path/admin/AdminReportingService.java @@ -23,14 +23,31 @@ public class AdminReportingService { private final UsageRecordRepository usage; private final AdminService adminService; + private final com.anindra.path.repo.ClientKeyRepository keys; - public Map usage(Long keyId, String window, int limit) { + public Map usage(Long keyId, Long projectId, String endUser, String window, + int limit) { Duration parsed = Durations.cap(Durations.parse(window, Duration.ofHours(24)), Duration.ofDays(365)); int capped = Math.max(1, Math.min(limit, 500)); - List records = keyId != null - ? usage.findByClientKeyIdOrderByCreatedAtDesc(keyId, PageRequest.of(0, capped)) - : usage.findByCreatedAtAfterOrderByCreatedAtDesc( - java.time.Instant.now().minus(parsed), PageRequest.of(0, capped)); + java.time.Instant from = java.time.Instant.now().minus(parsed); + + List records; + if (endUser != null && !endUser.isBlank()) { + records = keyId != null + ? usage.findByClientKeyIdAndEndUserOrderByCreatedAtDesc( + keyId, endUser, PageRequest.of(0, capped)) + : usage.findAll(PageRequest.of(0, capped)).stream() + .filter(r -> endUser.equals(r.getEndUser())).toList(); + } else if (keyId != null) { + records = usage.findByClientKeyIdOrderByCreatedAtDesc(keyId, PageRequest.of(0, capped)); + } else if (projectId != null) { + List keyIds = projectKeyIds(projectId); + records = keyIds.isEmpty() ? List.of() + : usage.findByClientKeyIdInAndCreatedAtAfterOrderByCreatedAtDesc( + keyIds, from, PageRequest.of(0, capped)); + } else { + records = usage.findByCreatedAtAfterOrderByCreatedAtDesc(from, PageRequest.of(0, capped)); + } List> rows = new ArrayList<>(); for (UsageRecord r : records) { @@ -39,6 +56,7 @@ public Map usage(Long keyId, String window, int limit) { row.put("at", r.getCreatedAt().toString()); row.put("key_id", r.getClientKeyId()); row.put("key_name", r.getClientKeyName()); + row.put("end_user", r.getEndUser()); row.put("model", r.getPublicModelId()); row.put("provider", r.getProviderSlug()); row.put("upstream_model", r.getUpstreamModelId()); @@ -75,13 +93,166 @@ public Map usage(Long keyId, String window, int limit) { Map out = new LinkedHashMap<>(); out.put("object", "list"); out.put("window", parsed.toString()); + out.put("project_id", projectId); + out.put("end_user", endUser); out.put("count", rows.size()); out.put("data", rows); return out; } - public Map stats(String window) { + /** Comma-separated statement of calls, one row per request, oldest first. */ + public String usageCsv(Long keyId, Long projectId, String endUser, java.time.Instant from, + java.time.Instant to, int limit) { + int capped = Math.max(1, Math.min(limit, 50_000)); + List rows; + if (projectId != null) { + List keyIds = projectKeyIds(projectId); + rows = keyIds.isEmpty() ? List.of() + : usage.findByClientKeyIdInAndCreatedAtBetweenOrderByCreatedAtAsc( + keyIds, from, to, PageRequest.of(0, capped)); + } else if (endUser != null && !endUser.isBlank()) { + rows = usage.findByCreatedAtBetweenOrderByCreatedAtAsc(from, to, PageRequest.of(0, capped)) + .stream().filter(r -> endUser.equals(r.getEndUser())).toList(); + } else { + rows = usage.findByCreatedAtBetweenOrderByCreatedAtAsc(from, to, PageRequest.of(0, capped)); + } + + StringBuilder csv = new StringBuilder(); + csv.append("created_at,key_id,key_name,model,provider,upstream_model,prompt_tokens,") + .append("completion_tokens,cached_input_tokens,cache_write_tokens,reasoning_tokens,") + .append("input_cost_usd,cached_input_cost_usd,cache_write_cost_usd,output_cost_usd,") + .append("cost_usd,latency_ms,ttft_ms,streamed,status,error_code,end_user\n"); + + for (UsageRecord r : rows) { + if (keyId != null && !keyId.equals(r.getClientKeyId())) { + continue; + } + csv.append(r.getCreatedAt()).append(',') + .append(r.getClientKeyId() == null ? "" : r.getClientKeyId()).append(',') + .append(escape(r.getClientKeyName())).append(',') + .append(escape(r.getPublicModelId())).append(',') + .append(escape(r.getProviderSlug())).append(',') + .append(escape(r.getUpstreamModelId())).append(',') + .append(r.getPromptTokens()).append(',') + .append(r.getCompletionTokens()).append(',') + .append(r.getCachedInputTokens()).append(',') + .append(r.getCacheWriteTokens()).append(',') + .append(r.getReasoningTokens()).append(',') + .append(r.getInputCostUsd()).append(',') + .append(r.getCachedInputCostUsd()).append(',') + .append(r.getCacheWriteCostUsd()).append(',') + .append(r.getOutputCostUsd()).append(',') + .append(r.getCostUsd()).append(',') + .append(r.getLatencyMs()).append(',') + .append(r.getTtftMs() == null ? "" : r.getTtftMs()).append(',') + .append(r.isStreamed()).append(',') + .append(r.getStatus().name().toLowerCase()).append(',') + .append(escape(r.getErrorCode())).append(',') + .append(escape(r.getEndUser())) + .append('\n'); + } + return csv.toString(); + } + + private static String escape(String value) { + if (value == null) { + return ""; + } + if (value.indexOf(',') < 0 && value.indexOf('"') < 0 && value.indexOf('\n') < 0) { + return value; + } + return '"' + value.replace("\"", "\"\"") + '"'; + } + + public Map stats(String window, Long projectId) { Duration parsed = Durations.cap(Durations.parse(window, Duration.ofHours(24)), Duration.ofDays(365)); - return adminService.stats(parsed); + if (projectId == null) { + return adminService.stats(parsed); + } + return projectStats(projectId, parsed); + } + + /** Aggregates in Java from the project's rows; bounded so a huge project cannot OOM. */ + private Map projectStats(Long projectId, Duration window) { + java.time.Instant from = java.time.Instant.now().minus(window); + List keyIds = projectKeyIds(projectId); + List rows = keyIds.isEmpty() ? List.of() + : usage.findByClientKeyIdInAndCreatedAtAfterOrderByCreatedAtDesc( + keyIds, from, PageRequest.of(0, 50_000)); + + List latencies = rows.stream() + .filter(r -> r.getStatus() == UsageRecord.Status.SUCCESS) + .map(UsageRecord::getLatencyMs).sorted().toList(); + List ttfts = rows.stream().map(UsageRecord::getTtftMs) + .filter(java.util.Objects::nonNull).sorted().toList(); + List throughput = rows.stream().map(UsageRecord::getTokensPerSecond) + .filter(java.util.Objects::nonNull).sorted().toList(); + + long promptTokens = rows.stream().mapToLong(UsageRecord::getPromptTokens).sum(); + long cachedTokens = rows.stream().mapToLong(UsageRecord::getCachedInputTokens).sum(); + java.math.BigDecimal spend = rows.stream().map(UsageRecord::getCostUsd) + .filter(java.util.Objects::nonNull) + .reduce(java.math.BigDecimal.ZERO, java.math.BigDecimal::add); + java.math.BigDecimal savings = rows.stream().map(UsageRecord::getCachedSavingsUsd) + .filter(java.util.Objects::nonNull) + .reduce(java.math.BigDecimal.ZERO, java.math.BigDecimal::add); + + Map out = new java.util.LinkedHashMap<>(); + out.put("window", window.toString()); + out.put("project_id", projectId); + out.put("requests", rows.size()); + out.put("total_tokens", rows.stream().mapToLong(UsageRecord::getTotalTokens).sum()); + out.put("cost_usd", spend); + out.put("latency_ms", Map.of("p50", percentile(latencies, 0.50), + "p95", percentile(latencies, 0.95), "p99", percentile(latencies, 0.99))); + out.put("ttft_ms", Map.of("p50", percentile(ttfts, 0.50), "p95", percentile(ttfts, 0.95))); + + Map cache = new java.util.LinkedHashMap<>(); + cache.put("prompt_tokens", promptTokens); + cache.put("cached_input_tokens", cachedTokens); + cache.put("hit_ratio", promptTokens == 0 ? 0.0 + : java.math.BigDecimal.valueOf(cachedTokens) + .divide(java.math.BigDecimal.valueOf(promptTokens), 4, + java.math.RoundingMode.HALF_UP).doubleValue()); + cache.put("savings_usd", savings); + out.put("cache", cache); + out.put("fallback_requests", rows.stream().filter(r -> r.getAttempts() > 1).count()); + out.put("estimated_usage_requests", rows.stream().filter(UsageRecord::isUsageEstimated).count()); + + Map byProvider = new java.util.LinkedHashMap<>(); + rows.stream().filter(r -> r.getProviderSlug() != null) + .collect(java.util.stream.Collectors.groupingBy(UsageRecord::getProviderSlug)) + .forEach((slug, list) -> byProvider.put(slug, Map.of( + "requests", list.size(), + "cost_usd", list.stream().map(UsageRecord::getCostUsd) + .filter(java.util.Objects::nonNull) + .reduce(java.math.BigDecimal.ZERO, java.math.BigDecimal::add)))); + out.put("by_provider", byProvider); + + List> byModel = new java.util.ArrayList<>(); + rows.stream().filter(r -> r.getPublicModelId() != null) + .collect(java.util.stream.Collectors.groupingBy(UsageRecord::getPublicModelId)) + .forEach((model, list) -> byModel.add(Map.of( + "model", model, + "requests", list.size(), + "tokens", list.stream().mapToLong(UsageRecord::getTotalTokens).sum(), + "cost_usd", list.stream().map(UsageRecord::getCostUsd) + .filter(java.util.Objects::nonNull) + .reduce(java.math.BigDecimal.ZERO, java.math.BigDecimal::add)))); + out.put("by_model", byModel); + return out; + } + + private List projectKeyIds(Long projectId) { + return keys.findByProjectIdOrderByCreatedAtDesc(projectId).stream() + .map(com.anindra.path.domain.ClientKey::getId).toList(); + } + + private static long percentile(List sorted, double p) { + if (sorted.isEmpty()) { + return 0; + } + int index = (int) Math.ceil(p * sorted.size()) - 1; + return sorted.get(Math.max(0, Math.min(index, sorted.size() - 1))); } } diff --git a/src/main/java/com/anindra/path/admin/AdminService.java b/src/main/java/com/anindra/path/admin/AdminService.java index cb38105..8405e96 100644 --- a/src/main/java/com/anindra/path/admin/AdminService.java +++ b/src/main/java/com/anindra/path/admin/AdminService.java @@ -7,6 +7,7 @@ import com.anindra.path.domain.ProviderType; import com.anindra.path.gateway.ApiKeyService; import com.anindra.path.gateway.ProviderCredentials; +import com.anindra.path.gateway.ProviderCircuitBreaker; import com.anindra.path.gateway.UsageService; import com.anindra.path.repo.ClientKeyRepository; import com.anindra.path.repo.ModelRouteRepository; @@ -32,9 +33,14 @@ @RequiredArgsConstructor public class AdminService { + private final com.anindra.path.gateway.RoutingMetrics routingMetrics; + @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) public record ProviderInput(String slug, @JsonAlias("displayName") String displayName, String type, @JsonAlias("baseUrl") String baseUrl, @JsonAlias("apiKey") String apiKey, + @JsonAlias("apiVersion") String apiVersion, + @JsonAlias("requestTimeoutMs") Integer requestTimeoutMs, + @JsonAlias("streamIdleTimeoutMs") Integer streamIdleTimeoutMs, Boolean enabled) { } @@ -47,15 +53,30 @@ public record RouteInput(@JsonAlias("publicModelId") String publicModelId, @JsonAlias("cachedInputPricePerMillion") BigDecimal cachedInputPricePerMillion, @JsonAlias("cacheWritePricePerMillion") BigDecimal cacheWritePricePerMillion, Integer priority, + Integer weight, + @JsonAlias("cacheEnabled") Boolean cacheEnabled, + @JsonAlias("cacheTtlSeconds") Integer cacheTtlSeconds, + @JsonAlias("semanticCacheEnabled") Boolean semanticCacheEnabled, + @JsonAlias("semanticThreshold") java.math.BigDecimal semanticThreshold, @JsonAlias("contextLength") Integer contextLength, @JsonAlias("supportsStreaming") Boolean supportsStreaming, @JsonAlias("supportsTools") Boolean supportsTools, + String strategy, + @JsonAlias("latencySloMs") Integer latencySloMs, Boolean enabled, String description) { } @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) public record KeyInput(String name, @JsonAlias("creditLimitUsd") BigDecimal creditLimitUsd, - @JsonAlias("rpmLimit") Integer rpmLimit, Boolean enabled) { + @JsonAlias("dailyLimitUsd") BigDecimal dailyLimitUsd, + @JsonAlias("monthlyLimitUsd") BigDecimal monthlyLimitUsd, + @JsonAlias("endUserDailyLimitUsd") BigDecimal endUserDailyLimitUsd, + @JsonAlias("maxOutputTokens") Integer maxOutputTokens, + @JsonAlias("rpmLimit") Integer rpmLimit, + @JsonAlias("maxConcurrentRequests") Integer maxConcurrentRequests, + @JsonAlias("allowedModels") String allowedModels, + @JsonAlias("projectId") Long projectId, + Boolean enabled) { } private final ProviderRepository providers; @@ -65,6 +86,7 @@ public record KeyInput(String name, @JsonAlias("creditLimitUsd") BigDecimal cred private final ProviderCredentials credentials; private final ApiKeyService apiKeys; private final UsageService usage; + private final ProviderCircuitBreaker breaker; // -------------------------------------------------------------- providers @@ -89,7 +111,10 @@ public Map createProvider(ProviderInput in) { ? in.slug().trim() : in.displayName().trim()); p.setType(parseType(in.type())); p.setBaseUrl(requireBaseUrl(in.baseUrl())); + p.setApiVersion(in.apiVersion() == null || in.apiVersion().isBlank() ? null : in.apiVersion().trim()); p.setEnabled(in.enabled() == null || in.enabled()); + p.setRequestTimeoutMs(in.requestTimeoutMs()); + p.setStreamIdleTimeoutMs(in.streamIdleTimeoutMs()); if (in.apiKey() != null && !in.apiKey().isBlank()) { p.setApiKeyCipher(cipher.encrypt(in.apiKey().trim())); } @@ -110,9 +135,19 @@ public Map updateProvider(Long id, ProviderInput in) { if (in.baseUrl() != null && !in.baseUrl().isBlank()) { p.setBaseUrl(requireBaseUrl(in.baseUrl())); } + if (in.apiVersion() != null) { + // Blank clears it: the provider goes back to the plain OpenAI shape. + p.setApiVersion(in.apiVersion().isBlank() ? null : in.apiVersion().trim()); + } if (in.enabled() != null) { p.setEnabled(in.enabled()); } + if (in.requestTimeoutMs() != null) { + p.setRequestTimeoutMs(in.requestTimeoutMs()); + } + if (in.streamIdleTimeoutMs() != null) { + p.setStreamIdleTimeoutMs(in.streamIdleTimeoutMs()); + } if (in.apiKey() != null) { p.setApiKeyCipher(in.apiKey().isBlank() ? null : cipher.encrypt(in.apiKey().trim())); } @@ -136,10 +171,15 @@ private Map providerView(Provider p) { view.put("display_name", p.getDisplayName()); view.put("type", p.getType().name()); view.put("base_url", p.getBaseUrl()); + view.put("api_version", p.getApiVersion()); view.put("enabled", p.isEnabled()); view.put("credential_source", credentials.source(p)); view.put("credential_configured", credentials.apiKey(p) != null); view.put("credential_hint", credentials.hint(p)); + view.put("request_timeout_ms", p.getRequestTimeoutMs()); + view.put("stream_idle_timeout_ms", p.getStreamIdleTimeoutMs()); + view.put("circuit_state", breaker.state(p.getId()).name().toLowerCase()); + view.put("circuit_retry_after_seconds", breaker.retryAfterSeconds(p.getId())); view.put("created_at", p.getCreatedAt().toString()); view.put("route_count", routes.findByProviderId(p.getId()).size()); return view; @@ -167,6 +207,47 @@ private static String requireBaseUrl(String baseUrl) { return trimmed.endsWith("/") ? trimmed.substring(0, trimmed.length() - 1) : trimmed; } + // ------------------------------------------------------------ master key + + /** + * Re-encrypts every stored provider credential under a new master key. All + * ciphertexts are written before the key is swapped, so a failure leaves the old + * key valid. + */ + @Transactional + public Map rotateMasterKey(String newKeyBase64) { + byte[] newKey; + try { + newKey = SecretCipher.parseKey(newKeyBase64 == null ? "" : newKeyBase64); + } catch (Exception e) { + throw ApiException.badRequest( + "new_key must be a base64 encoded 32-byte key (openssl rand -base64 32).", + "new_key"); + } + + java.util.List all = providers.findAll(); + java.util.Map reencrypted = new java.util.LinkedHashMap<>(); + for (Provider provider : all) { + if (provider.getApiKeyCipher() == null || provider.getApiKeyCipher().isBlank()) { + continue; + } + String plaintext = cipher.decrypt(provider.getApiKeyCipher()); + reencrypted.put(provider.getId(), SecretCipher.encryptWith(newKey, plaintext)); + } + + for (var entry : reencrypted.entrySet()) { + providers.findById(entry.getKey()).ifPresent(p -> p.setApiKeyCipher(entry.getValue())); + } + providers.flush(); + cipher.rotate(newKey); + + Map out = new LinkedHashMap<>(); + out.put("rotated", reencrypted.size()); + out.put("providers", all.size()); + out.put("note", "Update GATEWAY_MASTER_KEY before the next restart."); + return out; + } + // ----------------------------------------------------------------- routes public List> listRoutes(String publicModelId) { @@ -244,10 +325,46 @@ public Map updateRoute(Long id, RouteInput in) { return routeView(route, slug); } + /** Unknown strategies are a 400, not a silent fallback to priority. */ + private static String validateStrategy(String strategy) { + String value = strategy.trim().toLowerCase(); + if (!java.util.Set.of("priority", "fastest", "cheapest-viable").contains(value)) { + throw ApiException.badRequest( + "'strategy' must be one of priority, fastest, cheapest-viable.", "strategy"); + } + return value; + } + private static void applyOptionals(ModelRoute route, RouteInput in) { if (in.priority() != null) { route.setPriority(in.priority()); } + if (in.weight() != null) { + route.setWeight(in.weight()); + } + if (in.strategy() != null) { + route.setStrategy(validateStrategy(in.strategy())); + } + if (in.latencySloMs() != null) { + route.setLatencySloMs(in.latencySloMs()); + } + if (in.cacheEnabled() != null) { + route.setCacheEnabled(in.cacheEnabled()); + } + if (in.cacheTtlSeconds() != null) { + route.setCacheTtlSeconds(in.cacheTtlSeconds()); + } + if (in.semanticCacheEnabled() != null) { + route.setSemanticCacheEnabled(in.semanticCacheEnabled()); + } + if (in.semanticThreshold() != null) { + double threshold = in.semanticThreshold().doubleValue(); + if (threshold < 0.5 || threshold > 1.0) { + throw ApiException.badRequest( + "'semantic_threshold' must be between 0.5 and 1.0.", "semantic_threshold"); + } + route.setSemanticThreshold(in.semanticThreshold()); + } if (in.contextLength() != null) { route.setContextLength(in.contextLength()); } @@ -289,21 +406,44 @@ private Map routeView(ModelRoute r, String providerSlug) { view.put("supports_streaming", r.isSupportsStreaming()); view.put("supports_tools", r.isSupportsTools()); view.put("enabled", r.isEnabled()); + view.put("weight", r.getWeight()); + view.put("semantic_cache_enabled", r.isSemanticCacheEnabled()); + view.put("semantic_threshold", r.getSemanticThreshold()); + view.put("strategy", r.getStrategy()); + double ewma = routingMetrics.latencyMs(providerSlug, r.getPublicModelId()); + view.put("ewma_latency_ms", Double.isNaN(ewma) ? null : Math.round(ewma)); + view.put("ewma_error_rate", routingMetrics.errorRate(providerSlug, r.getPublicModelId())); + view.put("latency_slo_ms", r.getLatencySloMs()); + view.put("cache_enabled", r.isCacheEnabled()); + view.put("cache_ttl_seconds", r.getCacheTtlSeconds()); view.put("description", r.getDescription()); return view; } // ------------------------------------------------------------------- keys - public List> listKeys() { + public List> listKeys(Long projectId) { List> out = new ArrayList<>(); - clientKeys.findAllByOrderByCreatedAtDesc().forEach(k -> out.add(keyView(k))); + var found = projectId == null + ? clientKeys.findAllByOrderByCreatedAtDesc() + : clientKeys.findByProjectIdOrderByCreatedAtDesc(projectId); + found.forEach(k -> out.add(keyView(k))); return out; } @Transactional public Map createKey(KeyInput in) { ApiKeyService.CreatedKey created = apiKeys.create(in.name(), in.creditLimitUsd(), in.rpmLimit()); + if (in.maxConcurrentRequests() != null) { + created.key().setMaxConcurrentRequests(in.maxConcurrentRequests()); + } + created.key().setDailyLimitUsd(in.dailyLimitUsd()); + created.key().setMonthlyLimitUsd(in.monthlyLimitUsd()); + created.key().setMaxOutputTokens(in.maxOutputTokens()); + created.key().setAllowedModels(in.allowedModels()); + created.key().setEndUserDailyLimitUsd(in.endUserDailyLimitUsd()); + created.key().setProjectId(in.projectId()); + clientKeys.save(created.key()); Map view = keyView(created.key()); view.put("key", created.plaintext()); view.put("warning", "Store this key now. It is not shown again."); @@ -326,6 +466,30 @@ public Map updateKey(Long id, KeyInput in) { } key.setRpmLimit(in.rpmLimit()); } + if (in.maxConcurrentRequests() != null) { + if (in.maxConcurrentRequests() < 0) { + throw ApiException.badRequest("maxConcurrentRequests cannot be negative", "maxConcurrentRequests"); + } + key.setMaxConcurrentRequests(in.maxConcurrentRequests()); + } + if (in.dailyLimitUsd() != null) { + key.setDailyLimitUsd(in.dailyLimitUsd()); + } + if (in.monthlyLimitUsd() != null) { + key.setMonthlyLimitUsd(in.monthlyLimitUsd()); + } + if (in.endUserDailyLimitUsd() != null) { + key.setEndUserDailyLimitUsd(in.endUserDailyLimitUsd()); + } + if (in.maxOutputTokens() != null) { + key.setMaxOutputTokens(in.maxOutputTokens()); + } + if (in.allowedModels() != null) { + key.setAllowedModels(in.allowedModels()); + } + if (in.projectId() != null) { + key.setProjectId(in.projectId()); + } if (in.enabled() != null) { key.setEnabled(in.enabled()); } @@ -365,6 +529,13 @@ private Map keyView(ClientKey k) { view.put("masked_key", k.getKeyPrefix() + "..." + k.getKeySuffix()); view.put("enabled", k.isEnabled()); view.put("rpm_limit", k.getRpmLimit()); + view.put("max_concurrent_requests", k.getMaxConcurrentRequests()); + view.put("daily_limit_usd", k.getDailyLimitUsd()); + view.put("monthly_limit_usd", k.getMonthlyLimitUsd()); + view.put("end_user_daily_limit_usd", k.getEndUserDailyLimitUsd()); + view.put("max_output_tokens", k.getMaxOutputTokens()); + view.put("allowed_models", k.getAllowedModels()); + view.put("project_id", k.getProjectId()); view.put("credit_limit_usd", k.getCreditLimitUsd()); view.put("used_usd", k.getUsedUsd()); view.put("created_at", k.getCreatedAt().toString()); diff --git a/src/main/java/com/anindra/path/admin/AlertService.java b/src/main/java/com/anindra/path/admin/AlertService.java new file mode 100644 index 0000000..07a80cd --- /dev/null +++ b/src/main/java/com/anindra/path/admin/AlertService.java @@ -0,0 +1,92 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Fire-and-forget operational alerts (Slack-compatible JSON) with a per-event cooldown. + * A blank webhook URL makes every call a no-op. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class AlertService { + + private final GatewayProperties props; + private final ObjectMapper mapper; + private final HttpClient http = HttpClient.newHttpClient(); + private final Map lastSent = new ConcurrentHashMap<>(); + private final Map sentOnce = new ConcurrentHashMap<>(); + + /** + * Fires at most once per {@code dedupeKey}, regardless of the cooldown. Used for budget + * thresholds, where the key carries the period (project, window, day or month) so a + * busy project cannot emit thousands of identical events. + */ + public void notifyOnce(String dedupeKey, String event, Map payload) { + String url = props.getAlerts().getWebhookUrl(); + if (url == null || url.isBlank()) { + return; + } + if (sentOnce.putIfAbsent(dedupeKey, System.currentTimeMillis()) != null) { + return; + } + notify(event, payload); + } + + @org.springframework.scheduling.annotation.Scheduled(fixedDelay = 3_600_000) + void forgetOldDedupeKeys() { + long cutoff = System.currentTimeMillis() - java.time.Duration.ofDays(2).toMillis(); + sentOnce.entrySet().removeIf(e -> e.getValue() < cutoff); + } + + public void notify(String event, Map payload) { + String url = props.getAlerts().getWebhookUrl(); + if (url == null || url.isBlank()) { + return; + } + long now = System.currentTimeMillis(); + long cooldownMs = props.getAlerts().getCooldown().toMillis(); + Long previous = lastSent.putIfAbsent(event, now); + if (previous != null) { + if (now - previous < cooldownMs) { + return; + } + lastSent.put(event, now); + } + + Map body = new LinkedHashMap<>(); + body.put("source", "path"); + body.put("event", event); + body.put("at", Instant.now().toString()); + body.putAll(payload); + + Thread.ofVirtual().name("alert-webhook").start(() -> send(url, body)); + } + + private void send(String url, Map body) { + try { + String json = mapper.writeValueAsString(body); + HttpRequest request = HttpRequest.newBuilder(URI.create(url)) + .timeout(java.time.Duration.ofSeconds(5)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(json)) + .build(); + http.send(request, HttpResponse.BodyHandlers.discarding()); + } catch (Exception e) { + log.warn("Alert webhook failed for {}: {}", body.get("event"), e.getMessage()); + } + } +} diff --git a/src/main/java/com/anindra/path/admin/AliasService.java b/src/main/java/com/anindra/path/admin/AliasService.java new file mode 100644 index 0000000..0ee725a --- /dev/null +++ b/src/main/java/com/anindra/path/admin/AliasService.java @@ -0,0 +1,157 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.ModelAlias; +import com.anindra.path.repo.ModelAliasRepository; +import com.anindra.path.repo.ModelRouteRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Operator-defined aliases: a name a product can depend on, pointing at an ordered chain + * of models. Kept separate from {@link com.anindra.path.gateway.ModelAliasService}, whose + * four capability aliases are recomputed from the route table and can therefore move. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class AliasService { + + /** Names that would collide with the computed capability aliases. */ + private static final Set COMPUTED = Set.of("cheap", "fast", "long-context", "vision"); + + private static final int MAX_HOPS = 10; + + private final ModelAliasRepository aliases; + private final ModelRouteRepository routes; + + public List> list() { + List> out = new ArrayList<>(); + for (ModelAlias alias : aliases.findAllByOrderByAliasAsc()) { + out.add(view(alias)); + } + return out; + } + + @Transactional + public Map create(String alias, List models, String description) { + String name = normalizeName(alias); + if (aliases.findByAlias(name).isPresent()) { + throw new ApiException(409, "invalid_request_error", "alias_exists", + "An alias named '" + name + "' already exists."); + } + List chain = validateChain(name, models); + if (COMPUTED.contains(name)) { + // Allowed, because an operator may want a stable version of a computed alias, + // but it shadows globally and that must be visible. + log.warn("Alias '{}' shadows the computed capability alias of the same name", name); + } + ModelAlias created = aliases.save(new ModelAlias(name, chain, + description == null || description.isBlank() ? null : description.trim())); + return view(created); + } + + @Transactional + public void delete(String alias) { + ModelAlias existing = aliases.findByAlias(normalizeName(alias)) + .orElseThrow(() -> ApiException.notFound( + "No alias named '" + alias + "'.", "alias_not_found")); + aliases.delete(existing); + } + + /** @return the alias chain when the name is an operator-defined alias. */ + public List chainFor(String model) { + return aliases.findByAlias(model).map(ModelAlias::modelList).orElse(List.of()); + } + + public boolean exists(String model) { + return aliases.findByAlias(model).isPresent(); + } + + private static String normalizeName(String alias) { + if (alias == null || alias.isBlank()) { + throw ApiException.badRequest("'alias' is required.", "alias"); + } + String name = alias.trim().toLowerCase(); + if (!name.matches("[a-z0-9][a-z0-9._-]{0,79}")) { + throw ApiException.badRequest( + "'alias' must be 1-80 characters of a-z, 0-9, dot, dash or underscore.", + "alias"); + } + return name; + } + + /** + * Rejects an alias that would not resolve: an unknown model, a reference to itself, or + * a chain that passes through another alias and comes back. Failing at write time is + * the whole point — a broken alias must not turn into a 500 on a customer request. + */ + private List validateChain(String name, List models) { + if (models == null || models.isEmpty()) { + throw ApiException.badRequest("'models' must list at least one model id.", "models"); + } + List cleaned = models.stream() + .filter(m -> m != null && !m.isBlank()) + .map(String::trim) + .toList(); + if (cleaned.isEmpty()) { + throw ApiException.badRequest("'models' must list at least one model id.", "models"); + } + if (cleaned.size() > MAX_HOPS) { + throw ApiException.badRequest( + "'models' accepts at most " + MAX_HOPS + " entries.", "models"); + } + + for (String model : cleaned) { + if (model.equals(name)) { + throw ApiException.badRequest("An alias cannot point at itself.", "models"); + } + if (routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc(model).isEmpty() + && aliases.findByAlias(model).isEmpty()) { + throw ApiException.badRequest( + "'" + model + "' is not a model this gateway serves.", "models"); + } + } + assertNoCycle(name, cleaned, 0); + return cleaned; + } + + private void assertNoCycle(String name, List chain, int depth) { + if (depth > MAX_HOPS) { + throw ApiException.badRequest( + "The alias chain is deeper than " + MAX_HOPS + " hops.", "models"); + } + for (String model : chain) { + List nested = aliases.findByAlias(model) + .map(ModelAlias::modelList).orElse(List.of()); + if (nested.isEmpty()) { + continue; + } + if (nested.contains(name)) { + throw ApiException.badRequest( + "That chain loops back to '" + name + "' through '" + model + "'.", "models"); + } + assertNoCycle(name, nested, depth + 1); + } + } + + private static Map view(ModelAlias alias) { + Map out = new LinkedHashMap<>(); + out.put("id", alias.getId()); + out.put("alias", alias.getAlias()); + out.put("models", alias.modelList()); + out.put("resolves_to", alias.modelList().isEmpty() ? null : alias.modelList().get(0)); + out.put("description", alias.getDescription()); + out.put("shadows_computed", COMPUTED.contains(alias.getAlias())); + out.put("created_at", alias.getCreatedAt().toString()); + return out; + } +} diff --git a/src/main/java/com/anindra/path/admin/AuditService.java b/src/main/java/com/anindra/path/admin/AuditService.java new file mode 100644 index 0000000..b92b6c1 --- /dev/null +++ b/src/main/java/com/anindra/path/admin/AuditService.java @@ -0,0 +1,62 @@ +package com.anindra.path.admin; + +import com.anindra.path.domain.AdminAudit; +import com.anindra.path.repo.AdminAuditRepository; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Records every mutating admin action; read back through /admin/audit. */ +@Service +@RequiredArgsConstructor +@Slf4j +public class AuditService { + + private static final int MAX_DETAILS = 4000; + + private final AdminAuditRepository audit; + private final ObjectMapper mapper; + + public void record(String action, String target, Object details) { + try { + AdminAudit entry = new AdminAudit(); + entry.setActor("shared-admin"); + entry.setAction(action); + entry.setTarget(target); + String json = details == null ? null : mapper.writeValueAsString(details); + entry.setDetails(json != null && json.length() > MAX_DETAILS + ? json.substring(0, MAX_DETAILS) : json); + audit.save(entry); + } catch (Exception e) { + // Auditing must never break the operation it describes. + log.warn("Could not record audit entry {}: {}", action, e.getMessage()); + } + } + + public List> list(String action, int limit) { + int capped = Math.max(1, Math.min(limit, 500)); + List rows = action == null || action.isBlank() + ? audit.findAllByOrderByCreatedAtDesc(PageRequest.of(0, capped)) + : audit.findByActionOrderByCreatedAtDesc(action, PageRequest.of(0, capped)); + + List> out = new ArrayList<>(); + for (AdminAudit row : rows) { + Map item = new LinkedHashMap<>(); + item.put("id", row.getId()); + item.put("at", row.getCreatedAt().toString()); + item.put("actor", row.getActor()); + item.put("action", row.getAction()); + item.put("target", row.getTarget()); + item.put("details", row.getDetails()); + out.add(item); + } + return out; + } +} diff --git a/src/main/java/com/anindra/path/admin/ModelSyncService.java b/src/main/java/com/anindra/path/admin/ModelSyncService.java index f971f25..be0a986 100644 --- a/src/main/java/com/anindra/path/admin/ModelSyncService.java +++ b/src/main/java/com/anindra/path/admin/ModelSyncService.java @@ -131,7 +131,10 @@ public Map sync() { route.setCachedInputPricePerMillion(price.cached()); route.setCacheWritePricePerMillion(price.write()); } else { - route.setDescription("Imported by /admin/models/sync; set a price before use."); + // Never publish an unpriced model: it would serve traffic for free. + route.setEnabled(false); + route.setDescription("Imported by /admin/models/sync with no known price; " + + "set a price and enable it."); } routes.save(route); created++; diff --git a/src/main/java/com/anindra/path/admin/PromptLogService.java b/src/main/java/com/anindra/path/admin/PromptLogService.java new file mode 100644 index 0000000..4aff85f --- /dev/null +++ b/src/main/java/com/anindra/path/admin/PromptLogService.java @@ -0,0 +1,105 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.PromptLog; +import com.anindra.path.repo.PromptLogRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.PageRequest; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Off by default. When enabled per gateway, stores redacted request/response bodies for + * a limited retention window so support can reproduce a customer call. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class PromptLogService { + + private static final Pattern SECRET = Pattern.compile( + "(?i)(sk-[a-z0-9]{10,}|gsk_[a-z0-9]{10,}|sk-ant-[a-z0-9_-]{10,}|bearer\\s+[a-z0-9._-]{10,})"); + + private final PromptLogRepository logs; + private final GatewayProperties props; + private final com.anindra.path.gateway.PromptSanitizer sanitizer; + + public boolean enabled() { + return props.getPromptLogging().isEnabled(); + } + + public void record(Long clientKeyId, String requestId, String model, String request, String response) { + if (!enabled()) { + return; + } + try { + int max = props.getPromptLogging().getMaxBodyBytes(); + PromptLog entry = new PromptLog(); + entry.setClientKeyId(clientKeyId); + entry.setRequestId(requestId); + entry.setPublicModelId(model); + entry.setRequestBody(truncate(scrub(request), max)); + entry.setResponseBody(truncate(scrub(response), max)); + entry.setExpiresAt(Instant.now().plus(props.getPromptLogging().getRetention())); + logs.save(entry); + } catch (Exception e) { + log.warn("Could not store prompt log: {}", e.getMessage()); + } + } + + public List> list(Long clientKeyId, int limit) { + int capped = Math.max(1, Math.min(limit, 200)); + List rows = clientKeyId == null + ? logs.findAllByOrderByCreatedAtDesc(PageRequest.of(0, capped)) + : logs.findByClientKeyIdOrderByCreatedAtDesc(clientKeyId, PageRequest.of(0, capped)); + + List> out = new ArrayList<>(); + for (PromptLog row : rows) { + Map item = new LinkedHashMap<>(); + item.put("id", row.getId()); + item.put("at", row.getCreatedAt().toString()); + item.put("key_id", row.getClientKeyId()); + item.put("request_id", row.getRequestId()); + item.put("model", row.getPublicModelId()); + item.put("request", row.getRequestBody()); + item.put("response", row.getResponseBody()); + out.add(item); + } + return out; + } + + @Scheduled(fixedDelay = 3_600_000) + @Transactional + void purgeExpired() { + long removed = logs.deleteByExpiresAtBefore(Instant.now()); + if (removed > 0) { + log.info("Purged {} expired prompt logs", removed); + } + } + + static String redact(String body) { + return body == null ? null : SECRET.matcher(body).replaceAll("[redacted]"); + } + + /** Credentials first, then personal identifiers: a token inside an email is still PII. */ + private String scrub(String body) { + String scrubbed = redact(body); + return sanitizer.redactPromptLogs() ? sanitizer.redactText(scrubbed) : scrubbed; + } + + private static String truncate(String body, int maxBytes) { + if (body == null || body.length() <= maxBytes) { + return body; + } + return body.substring(0, maxBytes) + "…[truncated]"; + } +} diff --git a/src/main/java/com/anindra/path/admin/ProviderProbeService.java b/src/main/java/com/anindra/path/admin/ProviderProbeService.java new file mode 100644 index 0000000..e966706 --- /dev/null +++ b/src/main/java/com/anindra/path/admin/ProviderProbeService.java @@ -0,0 +1,96 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.anindra.path.gateway.ProviderCircuitBreaker; +import com.anindra.path.gateway.ProviderCredentials; +import com.anindra.path.gateway.UpstreamClient; +import com.anindra.path.repo.ProviderRepository; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** On-demand reachability check for a provider, plus a fleet-wide summary. */ +@Service +@RequiredArgsConstructor +public class ProviderProbeService { + + private final ProviderRepository providers; + private final ProviderCredentials credentials; + private final ProviderCircuitBreaker breaker; + private final UpstreamClient upstream; + private final ObjectMapper mapper; + + public Map probe(Long providerId) { + Provider provider = providers.findById(providerId) + .orElseThrow(() -> ApiException.notFound("No provider with id " + providerId, "provider_not_found")); + String apiKey = credentials.apiKey(provider); + + Map out = new LinkedHashMap<>(); + out.put("provider", provider.getSlug()); + out.put("circuit_state", breaker.state(providerId).name().toLowerCase()); + if (apiKey == null) { + out.put("ok", false); + out.put("error", "no credential configured"); + return out; + } + + Map headers = new LinkedHashMap<>(); + if (provider.getType() == ProviderType.ANTHROPIC) { + headers.put("x-api-key", apiKey); + headers.put("anthropic-version", "2023-06-01"); + } else { + headers.putAll(com.anindra.path.gateway.ProviderUrls.headers(provider, apiKey, false)); + } + + long start = System.nanoTime(); + try { + UpstreamClient.Response response = upstream.get( + com.anindra.path.gateway.ProviderUrls.models(provider), headers); + long latency = (System.nanoTime() - start) / 1_000_000; + out.put("ok", response.ok()); + out.put("status", response.status()); + out.put("latency_ms", latency); + if (response.ok()) { + JsonNode data = mapper.readTree(response.body()).path("data"); + out.put("model_count", data.isArray() ? data.size() : 0); + } else { + out.put("error", abbreviate(response.body())); + } + } catch (Exception e) { + out.put("ok", false); + out.put("latency_ms", (System.nanoTime() - start) / 1_000_000); + out.put("error", e.getClass().getSimpleName() + ": " + e.getMessage()); + } + return out; + } + + public List> summary() { + List> out = new ArrayList<>(); + for (Provider provider : providers.findAllByOrderBySlugAsc()) { + Map row = new LinkedHashMap<>(); + row.put("id", provider.getId()); + row.put("provider", provider.getSlug()); + row.put("enabled", provider.isEnabled()); + row.put("credential_configured", credentials.apiKey(provider) != null); + row.put("circuit_state", breaker.state(provider.getId()).name().toLowerCase()); + row.put("retry_after_seconds", breaker.retryAfterSeconds(provider.getId())); + out.add(row); + } + return out; + } + + private static String abbreviate(String value) { + if (value == null) { + return ""; + } + return value.length() <= 200 ? value : value.substring(0, 200) + "…"; + } +} diff --git a/src/main/java/com/anindra/path/admin/TenancyService.java b/src/main/java/com/anindra/path/admin/TenancyService.java new file mode 100644 index 0000000..401be85 --- /dev/null +++ b/src/main/java/com/anindra/path/admin/TenancyService.java @@ -0,0 +1,238 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.AdminToken; +import com.anindra.path.domain.Project; +import com.anindra.path.gateway.ApiKeyService; +import com.anindra.path.repo.AdminTokenRepository; +import com.anindra.path.repo.ClientKeyRepository; +import com.anindra.path.repo.ProjectRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** Projects group keys and usage; admin tokens carry read or read+write scopes. */ +@Service +@RequiredArgsConstructor +public class TenancyService { + + public record Principal(String name, Set scopes) { + public boolean canWrite() { + return scopes.contains("write"); + } + } + + public record CreatedToken(AdminToken token, String plaintext) { + } + + private static final String ALPHABET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + private static final SecureRandom RANDOM = new SecureRandom(); + + private final ProjectRepository projects; + private final AdminTokenRepository tokens; + private final ClientKeyRepository keys; + private final com.anindra.path.gateway.UsageService usage; + + // ------------------------------------------------------------- projects + + public Map projectSummary(Long projectId) { + Project project = projects.findById(projectId) + .orElseThrow(() -> ApiException.notFound("No project with id " + projectId, "project_not_found")); + java.time.Instant dayStart = java.time.LocalDate.now(java.time.ZoneOffset.UTC) + .atStartOfDay(java.time.ZoneOffset.UTC).toInstant(); + java.time.Instant monthStart = java.time.LocalDate.now(java.time.ZoneOffset.UTC) + .withDayOfMonth(1).atStartOfDay(java.time.ZoneOffset.UTC).toInstant(); + + Map out = new LinkedHashMap<>(); + out.put("project", project.getSlug()); + out.put("key_count", keys.findByProjectIdOrderByCreatedAtDesc(projectId).size()); + out.put("spend_today_usd", usage.spendForProject(projectId, dayStart)); + out.put("spend_month_usd", usage.spendForProject(projectId, monthStart)); + out.put("spend_total_usd", usage.spendForProject(projectId, java.time.Instant.EPOCH)); + out.put("credit_limit_usd", project.getCreditLimitUsd()); + out.put("daily_limit_usd", project.getDailyLimitUsd()); + out.put("monthly_limit_usd", project.getMonthlyLimitUsd()); + out.put("rpm_limit", project.getRpmLimit()); + out.put("max_concurrent_requests", project.getMaxConcurrentRequests()); + return out; + } + + public List> listProjects() { + List> out = new ArrayList<>(); + for (Project project : projects.findAllByOrderBySlugAsc()) { + Map row = new LinkedHashMap<>(); + row.put("id", project.getId()); + row.put("slug", project.getSlug()); + row.put("name", project.getName()); + row.put("created_at", project.getCreatedAt().toString()); + row.put("credit_limit_usd", project.getCreditLimitUsd()); + row.put("daily_limit_usd", project.getDailyLimitUsd()); + row.put("monthly_limit_usd", project.getMonthlyLimitUsd()); + row.put("rpm_limit", project.getRpmLimit()); + row.put("max_concurrent_requests", project.getMaxConcurrentRequests()); + row.put("key_count", keys.findByProjectIdOrderByCreatedAtDesc(project.getId()).size()); + out.add(row); + } + return out; + } + + @Transactional + public Map createProject(String slug, String name, + java.math.BigDecimal creditLimit, + java.math.BigDecimal dailyLimit, + java.math.BigDecimal monthlyLimit, + Integer rpmLimit, Integer maxConcurrentRequests) { + if (slug == null || slug.isBlank()) { + throw ApiException.badRequest("slug is required", "slug"); + } + if (projects.findBySlug(slug.trim()).isPresent()) { + throw new ApiException(409, "invalid_request_error", "project_exists", + "A project with slug '" + slug + "' already exists."); + } + Project project = new Project(slug.trim().toLowerCase(), + name == null || name.isBlank() ? slug.trim() : name.trim()); + project.setCreditLimitUsd(creditLimit); + project.setDailyLimitUsd(dailyLimit); + project.setMonthlyLimitUsd(monthlyLimit); + project.setRpmLimit(positive(rpmLimit, "rpm_limit")); + project.setMaxConcurrentRequests(positive(maxConcurrentRequests, "max_concurrent_requests")); + projects.save(project); + return listProjects().stream() + .filter(p -> project.getId().equals(p.get("id"))) + .findFirst().orElseThrow(); + } + + @Transactional + public Map updateProjectLimits(Long id, java.math.BigDecimal creditLimit, + java.math.BigDecimal dailyLimit, + java.math.BigDecimal monthlyLimit, + Integer rpmLimit, Integer maxConcurrentRequests) { + Project project = projects.findById(id) + .orElseThrow(() -> ApiException.notFound("No project with id " + id, "project_not_found")); + if (creditLimit != null) { + project.setCreditLimitUsd(creditLimit); + } + if (dailyLimit != null) { + project.setDailyLimitUsd(dailyLimit); + } + if (monthlyLimit != null) { + project.setMonthlyLimitUsd(monthlyLimit); + } + if (rpmLimit != null) { + project.setRpmLimit(positive(rpmLimit, "rpm_limit")); + } + if (maxConcurrentRequests != null) { + project.setMaxConcurrentRequests(positive(maxConcurrentRequests, "max_concurrent_requests")); + } + projects.save(project); + return projectSummary(id); + } + + /** 0 means unlimited, so only negatives are rejected. */ + private static Integer positive(Integer value, String field) { + if (value == null) { + return null; + } + if (value < 0) { + throw ApiException.badRequest("'" + field + "' cannot be negative.", field); + } + return value; + } + + @Transactional + public void deleteProject(Long id) { + Project project = projects.findById(id) + .orElseThrow(() -> ApiException.notFound("No project with id " + id, "project_not_found")); + if (!keys.findByProjectIdOrderByCreatedAtDesc(id).isEmpty()) { + throw new ApiException(409, "invalid_request_error", "project_not_empty", + "Move or delete the project's keys before deleting it."); + } + projects.delete(project); + } + + // --------------------------------------------------------------- tokens + + public List> listTokens() { + List> out = new ArrayList<>(); + for (AdminToken token : tokens.findAllByOrderByCreatedAtDesc()) { + Map row = new LinkedHashMap<>(); + row.put("id", token.getId()); + row.put("name", token.getName()); + row.put("masked_token", token.getTokenPrefix() + "..." + token.getTokenSuffix()); + row.put("scopes", token.getScopes()); + row.put("enabled", token.isEnabled()); + row.put("created_at", token.getCreatedAt().toString()); + out.add(row); + } + return out; + } + + @Transactional + public CreatedToken createToken(String name, String scopes) { + if (name == null || name.isBlank()) { + throw ApiException.badRequest("name is required", "name"); + } + String normalized = normalizeScopes(scopes); + String secret = "adm_" + random(40); + AdminToken token = new AdminToken(); + token.setName(name.trim()); + token.setTokenHash(ApiKeyService.hash(secret)); + token.setTokenPrefix(secret.substring(0, 12)); + token.setTokenSuffix(secret.substring(secret.length() - 4)); + token.setScopes(normalized); + tokens.save(token); + return new CreatedToken(token, secret); + } + + @Transactional + public Map revokeToken(Long id) { + AdminToken token = tokens.findById(id) + .orElseThrow(() -> ApiException.notFound("No token with id " + id, "token_not_found")); + token.setEnabled(false); + tokens.save(token); + return listTokens().stream().filter(t -> id.equals(t.get("id"))).findFirst().orElseThrow(); + } + + public Optional authenticate(String rawToken) { + return tokens.findByTokenHash(ApiKeyService.hash(rawToken)) + .filter(AdminToken::isEnabled) + .map(token -> new Principal(token.getName(), scopesOf(token.getScopes()))); + } + + private static String normalizeScopes(String scopes) { + Set parsed = scopesOf(scopes); + if (parsed.isEmpty()) { + throw ApiException.badRequest("scopes must contain 'read' and/or 'write'", "scopes"); + } + return parsed.stream().sorted().collect(Collectors.joining(",")); + } + + private static Set scopesOf(String scopes) { + if (scopes == null || scopes.isBlank()) { + return Set.of("read"); + } + return java.util.Arrays.stream(scopes.split(",")) + .map(String::trim) + .map(String::toLowerCase) + .filter(s -> s.equals("read") || s.equals("write")) + .collect(Collectors.toSet()); + } + + private static String random(int length) { + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sb.append(ALPHABET.charAt(RANDOM.nextInt(ALPHABET.length()))); + } + return sb.toString(); + } +} diff --git a/src/main/java/com/anindra/path/api/ApiException.java b/src/main/java/com/anindra/path/api/ApiException.java index c87fd72..3492a50 100644 --- a/src/main/java/com/anindra/path/api/ApiException.java +++ b/src/main/java/com/anindra/path/api/ApiException.java @@ -54,6 +54,13 @@ public static ApiException insufficientCredits(String message) { return new ApiException(402, "billing_error", "insufficient_credits", message); } + public static ApiException tooManyConcurrent(int limit) { + ApiException e = new ApiException(429, "rate_limit_error", "concurrency_limit_exceeded", + "Too many concurrent requests for this key (limit " + limit + ")."); + e.retryAfterSeconds = 1; + return e; + } + public static ApiException upstream(int status, String message, String code) { return new ApiException(status >= 400 ? status : 502, "upstream_error", code, message); } diff --git a/src/main/java/com/anindra/path/api/CapturingResponseWrapper.java b/src/main/java/com/anindra/path/api/CapturingResponseWrapper.java new file mode 100644 index 0000000..a40f252 --- /dev/null +++ b/src/main/java/com/anindra/path/api/CapturingResponseWrapper.java @@ -0,0 +1,95 @@ +package com.anindra.path.api; + +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponseWrapper; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * Tees a streamed response into memory so it can be replayed for an Idempotency-Key. + * Stops capturing past {@code maxBytes} and reports overflow, in which case the response + * is not stored and a retry will call the provider again. + */ +public class CapturingResponseWrapper extends HttpServletResponseWrapper { + + private final ByteArrayOutputStream captured = new ByteArrayOutputStream(); + private final int maxBytes; + private boolean overflow; + private ServletOutputStream output; + + public CapturingResponseWrapper(HttpServletResponse response, int maxBytes) { + super(response); + this.maxBytes = maxBytes; + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (output == null) { + output = new Tee(super.getOutputStream()); + } + return output; + } + + public boolean overflowed() { + return overflow; + } + + public String body() { + return captured.toString(StandardCharsets.UTF_8); + } + + private final class Tee extends ServletOutputStream { + + private final ServletOutputStream delegate; + + private Tee(ServletOutputStream delegate) { + this.delegate = delegate; + } + + @Override + public void write(int b) throws IOException { + delegate.write(b); + if (captured.size() < maxBytes) { + captured.write(b); + } else { + overflow = true; + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + delegate.write(b, off, len); + int room = maxBytes - captured.size(); + if (room > 0) { + captured.write(b, off, Math.min(room, len)); + } + if (len > room) { + overflow = true; + } + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public boolean isReady() { + return delegate.isReady(); + } + + @Override + public void setWriteListener(WriteListener listener) { + delegate.setWriteListener(listener); + } + } +} diff --git a/src/main/java/com/anindra/path/api/ChatCompletionsController.java b/src/main/java/com/anindra/path/api/ChatCompletionsController.java new file mode 100644 index 0000000..19121db --- /dev/null +++ b/src/main/java/com/anindra/path/api/ChatCompletionsController.java @@ -0,0 +1,351 @@ +package com.anindra.path.api; + +import com.anindra.path.config.AuthFilter; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.gateway.ApiKeyService; +import com.anindra.path.gateway.ChatRequestSpec; +import com.anindra.path.gateway.ChatService; +import com.anindra.path.gateway.ConcurrencyLimiter; +import com.anindra.path.gateway.IdempotencyService; +import com.anindra.path.gateway.ModelCatalogService; +import com.anindra.path.gateway.RouteResolver; +import com.anindra.path.gateway.StreamService; +import com.anindra.path.gateway.UsageService; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.micrometer.tracing.Span; +import io.micrometer.tracing.Tracer; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** OpenAI-compatible public surface. */ +@RestController +@RequestMapping("/v1") +@RequiredArgsConstructor +@Tag(name = "Chat", description = "OpenAI-compatible chat, catalogue and usage") +public class ChatCompletionsController { + + public static final String IDEMPOTENCY_HEADER = "Idempotency-Key"; + + private final ChatService chatService; + private final StreamService streamService; + private final com.anindra.path.gateway.EmbeddingsService embeddings; + private final com.anindra.path.gateway.TokenCountService tokenCount; + private final com.anindra.path.gateway.CreditGuard creditGuard; + private final com.anindra.path.gateway.PromptSanitizer sanitizer; + private final com.anindra.path.gateway.ModerationService moderation; + private final ModelCatalogService catalog; + private final UsageService usageService; + private final ApiKeyService apiKeys; + private final IdempotencyService idempotency; + private final ConcurrencyLimiter concurrency; + private final com.anindra.path.gateway.DrainState drain; + private final GatewayProperties props; + private final com.anindra.path.admin.PromptLogService promptLogs; + private final Tracer tracer; + + @Operation(summary = "Create a chat completion", description = "Streaming when stream=true") + @PostMapping("/chat/completions") + public ResponseEntity chatCompletions( + @RequestBody ObjectNode body, + @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key, + HttpServletRequest request, + HttpServletResponse response) throws IOException { + + if (key == null) { + throw ApiException.unauthorized( + "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + } + + drain.begin(); + try { + int limit = key.getMaxConcurrentRequests() != null + ? key.getMaxConcurrentRequests() : props.getDefaultMaxConcurrentRequests(); + Integer projectLimit = (Integer) request.getAttribute(AuthFilter.PROJECT_CONCURRENCY_ATTR); + if (!concurrency.tryAcquire(key, limit, projectLimit)) { + throw ApiException.tooManyConcurrent( + projectLimit != null && projectLimit > 0 ? projectLimit : limit); + } + try { + Span span = tracer.nextSpan().name("chat.completions").start(); + try (Tracer.SpanInScope scope = tracer.withSpan(span)) { + span.tag("gen_ai.request.model", body.path("model").asText("unknown")); + span.tag("gen_ai.request.stream", String.valueOf(body.path("stream").asBoolean(false))); + return handle(body, key, request, response); + } finally { + span.end(); + } + } finally { + concurrency.release(key); + } + } finally { + drain.end(); + } + } + + private ResponseEntity handle(ObjectNode body, ClientKey key, HttpServletRequest request, + HttpServletResponse response) throws IOException { + if (sanitizer.redactBeforeSend()) { + // Redact first: the prompt the provider receives is then the prompt we hash, + // store and replay, so a retry cannot resurrect the original text. + body = sanitizer.redactBody(body); + } + ChatRequestSpec spec = ChatRequestSpec.parse(body); + moderation.check(spec, key); + String idempotencyKey = normalizeIdempotencyKey(request.getHeader(IDEMPOTENCY_HEADER)); + String requestHash = idempotencyKey == null ? null : ApiKeyService.hash(body.toString()); + + if (idempotencyKey != null) { + var replay = idempotency.begin(key.getId(), idempotencyKey, requestHash); + if (replay.isPresent()) { + IdempotencyService.Replay stored = replay.get(); + return ResponseEntity.status(stored.httpStatus()) + .contentType(MediaType.parseMediaType(stored.contentType())) + .header("X-Idempotent-Replay", "true") + .body(stored.body()); + } + } + + if (spec.stream()) { + return streamCompletion(spec, key, idempotencyKey, requestHash, body.toString(), response); + } + + ChatService.Result result; + try { + result = chatService.complete(spec, key); + } catch (RuntimeException e) { + abandon(idempotencyKey, key); + throw e; + } + + if (idempotencyKey != null) { + idempotency.complete(key.getId(), idempotencyKey, 200, + MediaType.APPLICATION_JSON_VALUE, result.body().toString()); + } + if (promptLogs.enabled()) { + promptLogs.record(key.getId(), org.slf4j.MDC.get("requestId"), spec.publicModel(), + body.toString(), result.body().toString()); + } + + return ResponseEntity.ok() + .header("X-Gateway-Provider", result.providerSlug()) + .header("X-Gateway-Upstream-Model", result.upstreamModel()) + .header("X-Gateway-Cost-Usd", result.costUsd().toPlainString()) + .header("X-Gateway-Latency-Ms", String.valueOf(result.latencyMs())) + .header("X-Path-Cache", result.cacheState()) + .body(result.body()); + } + + private ResponseEntity streamCompletion(ChatRequestSpec spec, ClientKey key, String idempotencyKey, + String requestHash, String requestBody, + HttpServletResponse response) + throws IOException { + + StreamService.Prepared prepared; + try { + prepared = streamService.prepare(spec, key); + } catch (RuntimeException e) { + abandon(idempotencyKey, key); + throw e; + } + + try { + List chain = prepared.chain(); + boolean capture = idempotencyKey != null || promptLogs.enabled(); + if (!capture) { + streamService.stream(response, spec, key, chain); + return null; + } + + CapturingResponseWrapper wrapper = + new CapturingResponseWrapper(response, props.getIdempotency().getMaxResponseBytes()); + boolean completed; + try { + completed = streamService.stream(wrapper, spec, key, chain); + } catch (RuntimeException e) { + abandon(idempotencyKey, key); + throw e; + } + + if (completed && !wrapper.overflowed()) { + if (idempotencyKey != null) { + idempotency.complete(key.getId(), idempotencyKey, 200, + "text/event-stream", wrapper.body()); + } + if (promptLogs.enabled()) { + promptLogs.record(key.getId(), org.slf4j.MDC.get("requestId"), spec.publicModel(), + requestBody, wrapper.body()); + } + } else { + // Not replayable: release the claim so a retry can run. + abandon(idempotencyKey, key); + } + return null; + } finally { + creditGuard.release(key, prepared.reservation()); + } + } + + private void abandon(String idempotencyKey, ClientKey key) { + if (idempotencyKey != null) { + idempotency.abandon(key.getId(), idempotencyKey); + } + } + + private static String normalizeIdempotencyKey(String raw) { + if (raw == null || raw.isBlank()) { + return null; + } + String value = raw.trim(); + if (value.length() > 255) { + throw ApiException.badRequest( + "Idempotency-Key must be at most 255 characters.", IDEMPOTENCY_HEADER); + } + return value; + } + + @Operation(summary = "Create embeddings", description = "OpenAI-compatible providers only") + @PostMapping("/embeddings") + public ResponseEntity embeddings( + @RequestBody ObjectNode body, + @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key, + HttpServletRequest request) { + if (key == null) { + throw ApiException.unauthorized( + "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + } + + String idempotencyKey = normalizeIdempotencyKey(request.getHeader(IDEMPOTENCY_HEADER)); + String requestHash = idempotencyKey == null ? null : ApiKeyService.hash(body.toString()); + if (idempotencyKey != null) { + var replay = idempotency.begin(key.getId(), idempotencyKey, requestHash); + if (replay.isPresent()) { + IdempotencyService.Replay stored = replay.get(); + return ResponseEntity.status(stored.httpStatus()) + .contentType(MediaType.parseMediaType(stored.contentType())) + .header("X-Idempotent-Replay", "true") + .body(stored.body()); + } + } + + drain.begin(); + try { + int limit = key.getMaxConcurrentRequests() != null + ? key.getMaxConcurrentRequests() : props.getDefaultMaxConcurrentRequests(); + Integer projectLimit = (Integer) request.getAttribute(AuthFilter.PROJECT_CONCURRENCY_ATTR); + if (!concurrency.tryAcquire(key, limit, projectLimit)) { + throw ApiException.tooManyConcurrent( + projectLimit != null && projectLimit > 0 ? projectLimit : limit); + } + try { + var result = embeddings.embed(body, key); + if (idempotencyKey != null) { + idempotency.complete(key.getId(), idempotencyKey, 200, + MediaType.APPLICATION_JSON_VALUE, result.body().toString()); + } + return ResponseEntity.ok() + .header("X-Gateway-Provider", result.providerSlug()) + .header("X-Gateway-Upstream-Model", result.upstreamModel()) + .header("X-Gateway-Cost-Usd", result.costUsd().toPlainString()) + .body(result.body()); + } catch (RuntimeException e) { + abandon(idempotencyKey, key); + throw e; + } finally { + concurrency.release(key); + } + } finally { + drain.end(); + } + } + + @Operation(summary = "Count prompt tokens for a model", + description = "Free, authenticated; respects the model allowlist and rate limits") + @PostMapping("/tokens/count") + public Map tokenCount( + @RequestBody ObjectNode body, + @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key, + HttpServletRequest request) { + if (key == null) { + throw ApiException.unauthorized( + "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + } + drain.begin(); + try { + int limit = key.getMaxConcurrentRequests() != null + ? key.getMaxConcurrentRequests() : props.getDefaultMaxConcurrentRequests(); + Integer projectLimit = (Integer) request.getAttribute(AuthFilter.PROJECT_CONCURRENCY_ATTR); + if (!concurrency.tryAcquire(key, limit, projectLimit)) { + throw ApiException.tooManyConcurrent( + projectLimit != null && projectLimit > 0 ? projectLimit : limit); + } + try { + return tokenCount.count(body, key); + } finally { + concurrency.release(key); + } + } finally { + drain.end(); + } + } + + @Operation(summary = "List the model catalogue") + @GetMapping("/models") + public Map models() { + return catalog.list(); + } + + @GetMapping({"/models/{*modelId}"}) + public Map model(@PathVariable String modelId) { + String id = modelId.startsWith("/") ? modelId.substring(1) : modelId; + return catalog.find(id).orElseThrow(() -> + ApiException.notFound("The model '" + id + "' does not exist.", "model_not_found")); + } + + @Operation(summary = "Token, cost and cache usage for the calling key") + @GetMapping("/usage") + public Map usage( + @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key, + @RequestParam(defaultValue = "24h") String window) { + requireKey(key); + Duration parsed = Durations.cap(Durations.parse(window, Duration.ofHours(24)), Duration.ofDays(365)); + return usageService.keySummary(key, parsed); + } + + @GetMapping("/credits") + public Map credits( + @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key) { + requireKey(key); + BigDecimal remaining = apiKeys.remaining(key); + Map out = new LinkedHashMap<>(); + out.put("object", "credits"); + out.put("key_id", key.getId()); + out.put("key_name", key.getName()); + out.put("currency", "USD"); + out.put("credit_limit_usd", key.getCreditLimitUsd()); + out.put("used_usd", key.getUsedUsd()); + out.put("remaining_usd", remaining); + out.put("unlimited", remaining == null); + return out; + } + + private static void requireKey(ClientKey key) { + if (key == null) { + throw ApiException.unauthorized( + "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + } + } +} diff --git a/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java b/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java index 99548d8..0532637 100644 --- a/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java +++ b/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.http.converter.HttpMessageNotReadableException; import org.springframework.web.bind.MissingServletRequestParameterException; @@ -17,6 +18,7 @@ @RestControllerAdvice @RequiredArgsConstructor +@Slf4j public class GlobalExceptionHandler { private final ObjectMapper mapper; @@ -32,10 +34,26 @@ public ResponseEntity> handle(ApiException e) { @ExceptionHandler(HttpMessageNotReadableException.class) public ResponseEntity> badJson(HttpMessageNotReadableException e) { + if (causedByPayloadTooLarge(e)) { + return ResponseEntity.status(413) + .body(envelope("payload_too_large", "invalid_request_error", + "Request body is larger than the configured limit.", null)); + } return ResponseEntity.badRequest() .body(envelope("invalid_json", "invalid_request_error", "Request body is not valid JSON.", null)); } + private static boolean causedByPayloadTooLarge(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof com.anindra.path.config.PayloadTooLargeException) { + return true; + } + current = current.getCause(); + } + return false; + } + @ExceptionHandler({NoResourceFoundException.class, NoHandlerFoundException.class}) public ResponseEntity> notFound(Exception e) { return ResponseEntity.status(404) @@ -50,9 +68,12 @@ public ResponseEntity> badParam(Exception e) { @ExceptionHandler(Exception.class) public ResponseEntity> unexpected(Exception e) { + String errorId = java.util.UUID.randomUUID().toString(); + log.error("Unhandled error {}: {}", errorId, e.getMessage(), e); return ResponseEntity.internalServerError() .body(envelope("internal_error", "server_error", - e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(), null)); + "Internal error. Quote reference " + errorId + " when contacting support.", + null)); } public static Map envelope(String code, String type, String message, String param) { diff --git a/src/main/java/com/anindra/path/api/OpenAiController.java b/src/main/java/com/anindra/path/api/OpenAiController.java deleted file mode 100644 index fdbb974..0000000 --- a/src/main/java/com/anindra/path/api/OpenAiController.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.anindra.path.api; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.anindra.path.config.AuthFilter; -import com.anindra.path.domain.ClientKey; -import com.anindra.path.gateway.ApiKeyService; -import com.anindra.path.gateway.ChatRequestSpec; -import com.anindra.path.gateway.ChatService; -import com.anindra.path.gateway.ModelCatalogService; -import com.anindra.path.gateway.StreamService; -import com.anindra.path.gateway.UsageService; -import jakarta.servlet.http.HttpServletResponse; -import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.io.IOException; -import java.math.BigDecimal; -import java.time.Duration; -import java.util.LinkedHashMap; -import java.util.Map; - -/** OpenAI-compatible public surface. */ -@RestController -@RequestMapping("/v1") -@RequiredArgsConstructor -public class OpenAiController { - - private final ChatService chatService; - private final StreamService streamService; - private final ModelCatalogService catalog; - private final UsageService usageService; - private final ApiKeyService apiKeys; - - @PostMapping("/chat/completions") - public ResponseEntity chatCompletions( - @RequestBody ObjectNode body, - @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key, - HttpServletResponse response) throws IOException { - - if (key == null) { - throw ApiException.unauthorized( - "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); - } - - ChatRequestSpec spec = ChatRequestSpec.parse(body); - - if (spec.stream()) { - // Resolve and authorise before any bytes are written so errors keep their status. - var chain = streamService.prepare(spec, key); - streamService.stream(response, spec, key, chain); - return null; - } - - ChatService.Result result = chatService.complete(spec, key); - return ResponseEntity.ok() - .header("X-Gateway-Provider", result.providerSlug()) - .header("X-Gateway-Upstream-Model", result.upstreamModel()) - .header("X-Gateway-Cost-Usd", result.costUsd().toPlainString()) - .header("X-Gateway-Latency-Ms", String.valueOf(result.latencyMs())) - .body(result.body()); - } - - @GetMapping("/models") - public Map models() { - return catalog.list(); - } - - @GetMapping({"/models/{*modelId}"}) - public Map model(@PathVariable String modelId) { - String id = modelId.startsWith("/") ? modelId.substring(1) : modelId; - return catalog.find(id).orElseThrow(() -> - ApiException.notFound("The model '" + id + "' does not exist.", "model_not_found")); - } - - @GetMapping("/usage") - public Map usage( - @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key, - @RequestParam(defaultValue = "24h") String window) { - requireKey(key); - Duration parsed = Durations.cap(Durations.parse(window, Duration.ofHours(24)), Duration.ofDays(365)); - return usageService.keySummary(key, parsed); - } - - @GetMapping("/credits") - public Map credits( - @RequestAttribute(name = AuthFilter.CLIENT_KEY_ATTR, required = false) ClientKey key) { - requireKey(key); - BigDecimal remaining = apiKeys.remaining(key); - Map out = new LinkedHashMap<>(); - out.put("object", "credits"); - out.put("key_id", key.getId()); - out.put("key_name", key.getName()); - out.put("currency", "USD"); - out.put("credit_limit_usd", key.getCreditLimitUsd()); - out.put("used_usd", key.getUsedUsd()); - out.put("remaining_usd", remaining); - out.put("unlimited", remaining == null); - return out; - } - - private static void requireKey(ClientKey key) { - if (key == null) { - throw ApiException.unauthorized( - "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); - } - } -} diff --git a/src/main/java/com/anindra/path/config/AdminAuthLimiter.java b/src/main/java/com/anindra/path/config/AdminAuthLimiter.java new file mode 100644 index 0000000..3603a2a --- /dev/null +++ b/src/main/java/com/anindra/path/config/AdminAuthLimiter.java @@ -0,0 +1,57 @@ +package com.anindra.path.config; + +import lombok.RequiredArgsConstructor; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Slows down guessing of the admin token: too many failed attempts from one address + * within the window return 429 instead of another cheap comparison. + */ +@Component +@RequiredArgsConstructor +public class AdminAuthLimiter { + + private record Entry(long windowStart, int failures) { + } + + private final GatewayProperties props; + private final Map entries = new ConcurrentHashMap<>(); + + public long retryAfterSeconds(String address) { + Entry entry = entries.get(address); + if (entry == null) { + return 0; + } + long windowMs = props.getAdminAuth().getWindow().toMillis(); + long elapsed = System.currentTimeMillis() - entry.windowStart(); + if (elapsed >= windowMs || entry.failures() < props.getAdminAuth().getMaxFailures()) { + return 0; + } + return Math.max(1, (windowMs - elapsed) / 1000); + } + + public void onFailure(String address) { + long now = System.currentTimeMillis(); + long windowMs = props.getAdminAuth().getWindow().toMillis(); + entries.compute(address, (key, current) -> { + if (current == null || now - current.windowStart() >= windowMs) { + return new Entry(now, 1); + } + return new Entry(current.windowStart(), current.failures() + 1); + }); + } + + public void onSuccess(String address) { + entries.remove(address); + } + + @Scheduled(fixedDelay = 300_000) + void evictStale() { + long cutoff = System.currentTimeMillis() - props.getAdminAuth().getWindow().toMillis(); + entries.entrySet().removeIf(e -> e.getValue().windowStart() < cutoff); + } +} diff --git a/src/main/java/com/anindra/path/config/AuthFilter.java b/src/main/java/com/anindra/path/config/AuthFilter.java index 8e12109..10c654c 100644 --- a/src/main/java/com/anindra/path/config/AuthFilter.java +++ b/src/main/java/com/anindra/path/config/AuthFilter.java @@ -4,7 +4,7 @@ import com.anindra.path.api.GlobalExceptionHandler; import com.anindra.path.domain.ClientKey; import com.anindra.path.gateway.ApiKeyService; -import com.anindra.path.gateway.RateLimiter; +import com.anindra.path.gateway.RateLimitStore; import jakarta.servlet.FilterChain; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -33,10 +33,16 @@ public class AuthFilter extends OncePerRequestFilter { public static final String CLIENT_KEY_ATTR = "gateway.clientKey"; + /** Project concurrency cap for the resolved key, or absent when unlimited. */ + public static final String PROJECT_CONCURRENCY_ATTR = "gateway.projectConcurrency"; + public static final String ADMIN_PRINCIPAL_ATTR = "gateway.adminPrincipal"; private final ApiKeyService apiKeys; - private final RateLimiter rateLimiter; + private final RateLimitStore rateLimiter; + private final com.anindra.path.admin.TenancyService tenancy; + private final AdminAuthLimiter adminAuthLimiter; private final GatewayProperties props; + private final com.anindra.path.repo.ProjectRepository projects; private final ObjectMapper mapper; @Override @@ -67,6 +73,7 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse "Rate limit exceeded (" + key.getRpmLimit() + " requests/minute). Retry in " + retryAfter + "s.", retryAfter); } + enforceProjectLimit(request, key); request.setAttribute(CLIENT_KEY_ATTR, key); } chain.doFilter(request, response); @@ -75,6 +82,33 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse } } + /** + * A project cap sits above the per-key one: forty keys at 120 rpm should still be + * bounded by whatever the project allows. The concurrency cap is stashed on the + * request because only the controller can hold a permit for the life of a stream. + */ + private void enforceProjectLimit(HttpServletRequest request, ClientKey key) { + if (key.getProjectId() == null) { + return; + } + com.anindra.path.domain.Project project = projects.findById(key.getProjectId()).orElse(null); + if (project == null) { + return; + } + Integer projectRpm = project.getRpmLimit(); + if (projectRpm != null && projectRpm > 0) { + int retryAfter = rateLimiter.acquire("project:" + project.getId(), projectRpm); + if (retryAfter > 0) { + throw ApiException.rateLimited( + "Project '" + project.getSlug() + "' rate limit exceeded (" + projectRpm + + " requests/minute). Retry in " + retryAfter + "s.", retryAfter); + } + } + if (project.getMaxConcurrentRequests() != null && project.getMaxConcurrentRequests() > 0) { + request.setAttribute(PROJECT_CONCURRENCY_ATTR, project.getMaxConcurrentRequests()); + } + } + private boolean isPublicModelsRequest(HttpServletRequest request) { return props.isPublicModels() && "GET".equalsIgnoreCase(request.getMethod()) @@ -95,13 +129,47 @@ private ClientKey resolveClientKey(HttpServletRequest request) { private void requireAdmin(HttpServletRequest request) { String expected = props.getAdminToken(); - if (expected == null || expected.isBlank()) { - throw ApiException.forbidden("The admin API is disabled. Set GATEWAY_ADMIN_TOKEN to enable it."); - } String provided = request.getHeader("X-Admin-Token"); - if (provided == null || !constantTimeEquals(expected, provided)) { + if (provided == null || provided.isBlank()) { throw ApiException.unauthorized("Invalid or missing X-Admin-Token.", "invalid_admin_token"); } + + String address = clientAddress(request); + long retryAfter = adminAuthLimiter.retryAfterSeconds(address); + if (retryAfter > 0) { + throw ApiException.rateLimited( + "Too many failed admin authentication attempts. Retry in " + retryAfter + "s.", + (int) retryAfter); + } + + com.anindra.path.admin.TenancyService.Principal principal; + if (expected != null && !expected.isBlank() && constantTimeEquals(expected, provided)) { + adminAuthLimiter.onSuccess(address); + principal = new com.anindra.path.admin.TenancyService.Principal( + "bootstrap", java.util.Set.of("read", "write")); + } else { + principal = tenancy.authenticate(provided).orElse(null); + if (principal == null) { + adminAuthLimiter.onFailure(address); + throw ApiException.unauthorized("Invalid or missing X-Admin-Token.", "invalid_admin_token"); + } + adminAuthLimiter.onSuccess(address); + } + + request.setAttribute(ADMIN_PRINCIPAL_ATTR, principal); + boolean mutating = !"GET".equalsIgnoreCase(request.getMethod()) + && !"HEAD".equalsIgnoreCase(request.getMethod()); + if (mutating && !principal.canWrite()) { + throw ApiException.forbidden("This admin token has read-only scope."); + } + } + + private static String clientAddress(HttpServletRequest request) { + String forwarded = request.getHeader("X-Forwarded-For"); + if (forwarded != null && !forwarded.isBlank()) { + return forwarded.split(",")[0].trim(); + } + return request.getRemoteAddr(); } private static boolean constantTimeEquals(String a, String b) { diff --git a/src/main/java/com/anindra/path/config/GatewayProperties.java b/src/main/java/com/anindra/path/config/GatewayProperties.java index 3642203..d23d0cf 100644 --- a/src/main/java/com/anindra/path/config/GatewayProperties.java +++ b/src/main/java/com/anindra/path/config/GatewayProperties.java @@ -34,13 +34,121 @@ public class GatewayProperties { private String masterKeyPath = "./data/master.key"; /** Create a demo client key on boot when the key table is empty. */ - private boolean bootstrapDemoKey = true; + private boolean bootstrapDemoKey = false; private int defaultRpmLimit = 120; + /** Concurrent in-flight requests allowed per key unless the key overrides it. */ + private int defaultMaxConcurrentRequests = 8; + + /** Request bodies larger than this are rejected with 413. */ + private long maxRequestBytes = 1_048_576; + + /** Hard cap on max_tokens applied to every request; 0 disables. */ + private int maxOutputTokens = 0; + + /** How long to wait for in-flight streams during shutdown. */ + private Duration drainTimeout = Duration.ofSeconds(30); + + /** Usage rows older than this are deleted; zero keeps them forever. */ + private Duration usageRetention = Duration.ofDays(180); + /** Null means unlimited unless set. */ private BigDecimal defaultCreditLimitUsd; + private Breaker breaker = new Breaker(); + private Idempotency idempotency = new Idempotency(); + private Cache cache = new Cache(); + private Guardrails guardrails = new Guardrails(); + private Alerts alerts = new Alerts(); + private PromptLogging promptLogging = new PromptLogging(); + private RateLimit rateLimit = new RateLimit(); + private AdminAuth adminAuth = new AdminAuth(); + + @Getter + @Setter + public static class Breaker { + /** Consecutive failures before a provider is parked. */ + private int failureThreshold = 5; + /** How long a provider stays parked after opening. */ + private Duration openDuration = Duration.ofSeconds(30); + /** Routes tried per request, including the first. */ + private int maxAttempts = 3; + } + + @Getter + @Setter + public static class AdminAuth { + /** Failed admin auth attempts allowed per window before 429s start. */ + private int maxFailures = 10; + private Duration window = Duration.ofMinutes(5); + } + + @Getter + @Setter + public static class RateLimit { + /** "memory" (single node) or "valkey" (shared across instances). */ + private String backend = "memory"; + private String prefix = "path"; + } + + @Getter + @Setter + public static class PromptLogging { + /** Off by default: storing prompts creates a PII/secret store. */ + private boolean enabled = false; + private Duration retention = Duration.ofDays(7); + private int maxBodyBytes = 65_536; + } + + @Getter + @Setter + public static class Alerts { + /** Slack-compatible webhook; blank disables alerting. */ + private String webhookUrl = ""; + private Duration cooldown = Duration.ofHours(1); + /** Warn when a key has less than this fraction of its credit left. */ + private double budgetThreshold = 0.8; + } + + @Getter + @Setter + public static class Guardrails { + /** Redact personal identifiers before a prompt is written to prompt_logs. */ + private boolean promptLogRedaction = true; + + /** Redact the prompt a provider receives. Off by default: it changes what the model sees. */ + private boolean redactBeforeSend = false; + + /** Call the provider's moderation endpoint before the completion. */ + private boolean moderation = false; + + private String moderationModel = "omni-moderation-latest"; + } + + @Getter + @Setter + public static class Cache { + /** Responses larger than this are not cached. */ + private int maxBodyBytes = 2_097_152; + + /** Public model used to embed prompts for the semantic tier; blank disables it. */ + private String semanticModel = ""; + + /** How many entries one key+model may hold before the oldest is dropped. */ + private int semanticMaxEntries = 200; + } + + @Getter + @Setter + public static class Idempotency { + private Duration ttl = Duration.ofHours(24); + /** A PENDING claim older than this is treated as abandoned and reused. */ + private Duration pendingTimeout = Duration.ofMinutes(2); + /** Responses larger than this are not stored for replay. */ + private int maxResponseBytes = 2_097_152; + } + private Providers providers = new Providers(); @Getter @@ -49,6 +157,7 @@ public static class Providers { private ProviderKey openai = new ProviderKey(); private ProviderKey anthropic = new ProviderKey(); private ProviderKey groq = new ProviderKey(); + private ProviderKey gemini = new ProviderKey(); } @Getter diff --git a/src/main/java/com/anindra/path/config/LimitedInputStream.java b/src/main/java/com/anindra/path/config/LimitedInputStream.java new file mode 100644 index 0000000..ed0e6d8 --- /dev/null +++ b/src/main/java/com/anindra/path/config/LimitedInputStream.java @@ -0,0 +1,60 @@ +package com.anindra.path.config; + +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletInputStream; + +import java.io.IOException; + +/** Counts bytes as they are read, so a chunked body cannot bypass the limit. */ +class LimitedInputStream extends ServletInputStream { + + private final ServletInputStream delegate; + private final long limit; + private long read; + + LimitedInputStream(ServletInputStream delegate, long limit) { + this.delegate = delegate; + this.limit = limit; + } + + @Override + public int read() throws IOException { + int value = delegate.read(); + if (value >= 0 && ++read > limit) { + throw new PayloadTooLargeException("Request body exceeded " + limit + " bytes"); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int count = delegate.read(buffer, offset, length); + if (count > 0) { + read += count; + if (read > limit) { + throw new PayloadTooLargeException("Request body exceeded " + limit + " bytes"); + } + } + return count; + } + + @Override + public boolean isFinished() { + return delegate.isFinished(); + } + + @Override + public boolean isReady() { + return delegate.isReady(); + } + + @Override + public void setReadListener(ReadListener listener) { + delegate.setReadListener(listener); + } + + @Override + public void close() throws IOException { + delegate.close(); + } +} diff --git a/src/main/java/com/anindra/path/config/PayloadTooLargeException.java b/src/main/java/com/anindra/path/config/PayloadTooLargeException.java new file mode 100644 index 0000000..a960ce5 --- /dev/null +++ b/src/main/java/com/anindra/path/config/PayloadTooLargeException.java @@ -0,0 +1,10 @@ +package com.anindra.path.config; + +import java.io.IOException; + +/** Signals that a request body crossed the configured byte limit. */ +public class PayloadTooLargeException extends IOException { + public PayloadTooLargeException(String message) { + super(message); + } +} diff --git a/src/main/java/com/anindra/path/config/RateLimitConfig.java b/src/main/java/com/anindra/path/config/RateLimitConfig.java new file mode 100644 index 0000000..1dc5f48 --- /dev/null +++ b/src/main/java/com/anindra/path/config/RateLimitConfig.java @@ -0,0 +1,25 @@ +package com.anindra.path.config; + +import com.anindra.path.gateway.InMemoryRateLimitStore; +import com.anindra.path.gateway.RateLimitStore; +import com.anindra.path.gateway.ValkeyRateLimitStore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.StringRedisTemplate; + +@Configuration +public class RateLimitConfig { + + @Bean + @ConditionalOnProperty(name = "gateway.rate-limit.backend", havingValue = "memory", matchIfMissing = true) + public RateLimitStore inMemoryRateLimitStore() { + return new InMemoryRateLimitStore(); + } + + @Bean + @ConditionalOnProperty(name = "gateway.rate-limit.backend", havingValue = "valkey") + public RateLimitStore valkeyRateLimitStore(StringRedisTemplate redis, GatewayProperties props) { + return new ValkeyRateLimitStore(redis, props.getRateLimit().getPrefix()); + } +} diff --git a/src/main/java/com/anindra/path/config/RequestIdFilter.java b/src/main/java/com/anindra/path/config/RequestIdFilter.java new file mode 100644 index 0000000..97ae89e --- /dev/null +++ b/src/main/java/com/anindra/path/config/RequestIdFilter.java @@ -0,0 +1,43 @@ +package com.anindra.path.config; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.MDC; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.util.UUID; + +/** + * Gives every request a correlation id: echoes an inbound X-Request-Id or generates one, + * exposes it on the response, and puts it in the MDC so structured logs carry it. + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class RequestIdFilter extends OncePerRequestFilter { + + public static final String HEADER = "X-Request-Id"; + public static final String MDC_KEY = "requestId"; + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + String inbound = request.getHeader(HEADER); + String requestId = inbound == null || inbound.isBlank() + ? UUID.randomUUID().toString() + : inbound.trim().substring(0, Math.min(128, inbound.trim().length())); + + MDC.put(MDC_KEY, requestId); + response.setHeader(HEADER, requestId); + try { + chain.doFilter(request, response); + } finally { + MDC.remove(MDC_KEY); + } + } +} diff --git a/src/main/java/com/anindra/path/config/RequestSizeFilter.java b/src/main/java/com/anindra/path/config/RequestSizeFilter.java new file mode 100644 index 0000000..863feb7 --- /dev/null +++ b/src/main/java/com/anindra/path/config/RequestSizeFilter.java @@ -0,0 +1,69 @@ +package com.anindra.path.config; + +import com.anindra.path.api.ApiException; +import com.anindra.path.api.GlobalExceptionHandler; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; + +/** + * Rejects oversized request bodies before they are read into memory. + * + * Tomcat's {@code maxPostSize} only covers form posts, so JSON bodies were previously + * unbounded. Requests without a Content-Length (chunked) are left to the container limit. + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 5) +@RequiredArgsConstructor +public class RequestSizeFilter extends OncePerRequestFilter { + + private final GatewayProperties props; + private final ObjectMapper mapper; + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + + long length = request.getContentLengthLong(); + if (length > props.getMaxRequestBytes()) { + response.setStatus(413); + response.setContentType("application/json;charset=UTF-8"); + mapper.writeValue(response.getOutputStream(), GlobalExceptionHandler.envelope( + "payload_too_large", "invalid_request_error", + "Request body is " + length + " bytes; the limit is " + + props.getMaxRequestBytes() + ".", null)); + response.flushBuffer(); + return; + } + if (length < 0) { + // No Content-Length (chunked): enforce the limit while reading. + chain.doFilter(new LimitedRequest(request, props.getMaxRequestBytes()), response); + return; + } + chain.doFilter(request, response); + } + + private static final class LimitedRequest extends jakarta.servlet.http.HttpServletRequestWrapper { + + private final long limit; + + private LimitedRequest(HttpServletRequest request, long limit) { + super(request); + this.limit = limit; + } + + @Override + public jakarta.servlet.ServletInputStream getInputStream() throws IOException { + return new LimitedInputStream(super.getInputStream(), limit); + } + } +} diff --git a/src/main/java/com/anindra/path/domain/AdminAudit.java b/src/main/java/com/anindra/path/domain/AdminAudit.java new file mode 100644 index 0000000..5adcee3 --- /dev/null +++ b/src/main/java/com/anindra/path/domain/AdminAudit.java @@ -0,0 +1,36 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +/** One mutating admin action, for "who changed this and when". */ +@Entity +@Table(name = "admin_audit", indexes = @Index(name = "ix_audit_created", columnList = "createdAt")) +@Getter +@Setter +@NoArgsConstructor +public class AdminAudit { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false, length = 64) + private String actor; + + @Column(nullable = false, length = 64) + private String action; + + @Column(length = 128) + private String target; + + @Column(columnDefinition = "text") + private String details; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); +} diff --git a/src/main/java/com/anindra/path/domain/AdminToken.java b/src/main/java/com/anindra/path/domain/AdminToken.java new file mode 100644 index 0000000..972e641 --- /dev/null +++ b/src/main/java/com/anindra/path/domain/AdminToken.java @@ -0,0 +1,43 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +/** A scoped admin credential. The plaintext is shown once and stored hashed. */ +@Entity +@Table(name = "admin_tokens") +@Getter +@Setter +@NoArgsConstructor +public class AdminToken { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false, length = 128) + private String name; + + @Column(nullable = false, unique = true, length = 64) + private String tokenHash; + + @Column(nullable = false, length = 24) + private String tokenPrefix; + + @Column(nullable = false, length = 8) + private String tokenSuffix; + + /** Comma-separated: "read", "write". Write implies read. */ + @Column(nullable = false, length = 32) + private String scopes = "read"; + + @Column(nullable = false) + private boolean enabled = true; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); +} diff --git a/src/main/java/com/anindra/path/domain/ClientKey.java b/src/main/java/com/anindra/path/domain/ClientKey.java index 8f21b5e..23b2205 100644 --- a/src/main/java/com/anindra/path/domain/ClientKey.java +++ b/src/main/java/com/anindra/path/domain/ClientKey.java @@ -40,12 +40,41 @@ public class ClientKey { @Column(precision = 18, scale = 6) private BigDecimal creditLimitUsd; + @Column(precision = 18, scale = 6) + private BigDecimal dailyLimitUsd; + + @Column(precision = 18, scale = 6) + private BigDecimal monthlyLimitUsd; + + /** Clamps max_tokens on every request; null leaves the client's value alone. */ + private Integer maxOutputTokens; + @Column(nullable = false, precision = 18, scale = 6) private BigDecimal usedUsd = BigDecimal.ZERO; + /** Pre-authorised worst-case spend for in-flight requests. */ + @Column(nullable = false, precision = 18, scale = 6) + private BigDecimal reservedUsd = BigDecimal.ZERO; + + private Instant reservedUpdatedAt; + @Column(nullable = false) private int rpmLimit = 120; + /** Null means use the gateway default. */ + private Integer maxConcurrentRequests; + + /** Owning project; null means unassigned. */ + private Long projectId; + + /** Comma-separated model ids or globs (`gpt-*`); blank means every model. */ + @Column(length = 1024) + private String allowedModels; + + /** Per-end-user daily cap applied to any request carrying an end_user. */ + @Column(precision = 18, scale = 6) + private BigDecimal endUserDailyLimitUsd; + @Column(nullable = false) private Instant createdAt = Instant.now(); diff --git a/src/main/java/com/anindra/path/domain/EndUserReservation.java b/src/main/java/com/anindra/path/domain/EndUserReservation.java new file mode 100644 index 0000000..91979e5 --- /dev/null +++ b/src/main/java/com/anindra/path/domain/EndUserReservation.java @@ -0,0 +1,33 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.math.BigDecimal; +import java.time.Instant; + +/** In-flight spend for one end user on one key, so concurrent calls cannot overshoot. */ +@Entity +@Table(name = "end_user_reservations") +@Getter +@Setter +@NoArgsConstructor +public class EndUserReservation { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private Long clientKeyId; + + @Column(nullable = false, length = 128) + private String endUser; + + @Column(nullable = false, precision = 18, scale = 6) + private BigDecimal reservedUsd = BigDecimal.ZERO; + + private Instant updatedAt; +} diff --git a/src/main/java/com/anindra/path/domain/IdempotencyRecord.java b/src/main/java/com/anindra/path/domain/IdempotencyRecord.java new file mode 100644 index 0000000..f91c69d --- /dev/null +++ b/src/main/java/com/anindra/path/domain/IdempotencyRecord.java @@ -0,0 +1,61 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +/** + * One idempotency claim. A PENDING row exists for the duration of the upstream call; + * it becomes COMPLETED with the replayable response, or is deleted on failure. + */ +@Entity +@Table(name = "idempotency_records", + uniqueConstraints = @UniqueConstraint(name = "uk_idempotency", + columnNames = {"clientKeyId", "idempotencyKey"}), + indexes = @Index(name = "ix_idempotency_expiry", columnList = "expiresAt")) +@Getter +@Setter +@NoArgsConstructor +public class IdempotencyRecord { + + public enum Status {PENDING, COMPLETED} + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private Long clientKeyId; + + /** SHA-256 of the Idempotency-Key header. */ + @Column(nullable = false, length = 64) + private String idempotencyKey; + + /** SHA-256 of the request body, to detect key reuse with a different payload. */ + @Column(nullable = false, length = 64) + private String requestHash; + + @Enumerated(EnumType.STRING) + @JdbcTypeCode(SqlTypes.VARCHAR) + @Column(nullable = false, length = 16) + private Status status = Status.PENDING; + + private int httpStatus; + + @Column(length = 128) + private String contentType; + + @Column(columnDefinition = "text") + private String body; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); + + @Column(nullable = false) + private Instant expiresAt; +} diff --git a/src/main/java/com/anindra/path/domain/ModelAlias.java b/src/main/java/com/anindra/path/domain/ModelAlias.java new file mode 100644 index 0000000..e9e2924 --- /dev/null +++ b/src/main/java/com/anindra/path/domain/ModelAlias.java @@ -0,0 +1,57 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +/** + * An operator-defined alias: a stable, ordered chain of public model ids. Unlike the + * capability aliases, which are recomputed from prices on every request, this one keeps + * meaning what it meant when it was created. + */ +@Entity +@Table(name = "model_aliases") +@Getter +@Setter +@NoArgsConstructor +public class ModelAlias { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false, unique = true, length = 80) + private String alias; + + /** Comma-separated public model ids, in fallback order. */ + @Column(nullable = false, length = 1000) + private String models; + + @Column(length = 400) + private String description; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); + + public ModelAlias(String alias, List models, String description) { + this.alias = alias; + setModelList(models); + this.description = description; + } + + public List modelList() { + return Arrays.stream(models.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + } + + public void setModelList(List values) { + this.models = String.join(",", values); + } +} diff --git a/src/main/java/com/anindra/path/domain/ModelRoute.java b/src/main/java/com/anindra/path/domain/ModelRoute.java index 34fd6f6..e14402f 100644 --- a/src/main/java/com/anindra/path/domain/ModelRoute.java +++ b/src/main/java/com/anindra/path/domain/ModelRoute.java @@ -63,6 +63,32 @@ public class ModelRoute { @Column(nullable = false) private boolean enabled = true; + /** Opt-in: replay identical deterministic requests from the response cache. */ + /** Traffic share within a public model; equal weights split evenly. */ + @Column(nullable = false) + private int weight = 100; + + /** priority | fastest | cheapest-viable */ + @Column(nullable = false, length = 32) + private String strategy = "priority"; + + /** For cheapest-viable: the p95 a route must beat to stay preferred. */ + private Integer latencySloMs; + + @Column(nullable = false) + private boolean cacheEnabled = false; + + /** Second cache tier: a near-duplicate prompt may reuse an entry. Opt-in per route. */ + @Column(nullable = false) + private boolean semanticCacheEnabled = false; + + /** Cosine similarity a candidate must reach. 0.95 is deliberately strict. */ + @Column(precision = 4, scale = 3, nullable = false) + private java.math.BigDecimal semanticThreshold = new java.math.BigDecimal("0.950"); + + @Column(nullable = false) + private int cacheTtlSeconds = 3600; + @Column(length = 512) private String description; diff --git a/src/main/java/com/anindra/path/domain/Project.java b/src/main/java/com/anindra/path/domain/Project.java new file mode 100644 index 0000000..0cb0512 --- /dev/null +++ b/src/main/java/com/anindra/path/domain/Project.java @@ -0,0 +1,51 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.math.BigDecimal; +import java.time.Instant; + +/** A tenant grouping for keys and usage. */ +@Entity +@Table(name = "projects") +@Getter +@Setter +@NoArgsConstructor +public class Project { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false, unique = true, length = 64) + private String slug; + + @Column(nullable = false, length = 128) + private String name; + + @Column(precision = 18, scale = 6) + private BigDecimal creditLimitUsd; + + @Column(precision = 18, scale = 6) + private BigDecimal dailyLimitUsd; + + @Column(precision = 18, scale = 6) + private BigDecimal monthlyLimitUsd; + + /** Requests per minute across every key in the project; null means unlimited. */ + private Integer rpmLimit; + + /** In-flight requests across the project; null means unlimited. */ + private Integer maxConcurrentRequests; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); + + public Project(String slug, String name) { + this.slug = slug; + this.name = name; + } +} diff --git a/src/main/java/com/anindra/path/domain/PromptLog.java b/src/main/java/com/anindra/path/domain/PromptLog.java new file mode 100644 index 0000000..03e270d --- /dev/null +++ b/src/main/java/com/anindra/path/domain/PromptLog.java @@ -0,0 +1,41 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +/** Opt-in request/response capture for support, redacted before storage. */ +@Entity +@Table(name = "prompt_logs", indexes = @Index(name = "ix_prompt_log_created", columnList = "createdAt")) +@Getter +@Setter +@NoArgsConstructor +public class PromptLog { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private Long clientKeyId; + + @Column(length = 64) + private String requestId; + + @Column(length = 128) + private String publicModelId; + + @Column(columnDefinition = "text") + private String requestBody; + + @Column(columnDefinition = "text") + private String responseBody; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); + + @Column(nullable = false) + private Instant expiresAt; +} diff --git a/src/main/java/com/anindra/path/domain/Provider.java b/src/main/java/com/anindra/path/domain/Provider.java index f427913..384b5ac 100644 --- a/src/main/java/com/anindra/path/domain/Provider.java +++ b/src/main/java/com/anindra/path/domain/Provider.java @@ -1,6 +1,8 @@ package com.anindra.path.domain; import jakarta.persistence.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -25,6 +27,7 @@ public class Provider { private String displayName; @Enumerated(EnumType.STRING) + @JdbcTypeCode(SqlTypes.VARCHAR) @Column(nullable = false, length = 32) private ProviderType type; @@ -32,6 +35,14 @@ public class Provider { @Column(nullable = false, length = 512) private String baseUrl; + /** + * When set, this provider is an Azure OpenAI resource: the deployment name goes in the + * path, this value goes in {@code ?api-version}, and auth uses the {@code api-key} + * header. Null means a plain OpenAI-compatible endpoint. + */ + @Column(length = 32) + private String apiVersion; + /** AES-GCM encrypted upstream credential. Null falls back to the environment. */ @Column(length = 2048) private String apiKeyCipher; @@ -39,6 +50,11 @@ public class Provider { @Column(nullable = false) private boolean enabled = true; + /** Null falls back to the gateway default. */ + private Integer requestTimeoutMs; + + private Integer streamIdleTimeoutMs; + @Column(nullable = false) private Instant createdAt = Instant.now(); diff --git a/src/main/java/com/anindra/path/domain/ResponseCacheEntry.java b/src/main/java/com/anindra/path/domain/ResponseCacheEntry.java new file mode 100644 index 0000000..068efb0 --- /dev/null +++ b/src/main/java/com/anindra/path/domain/ResponseCacheEntry.java @@ -0,0 +1,62 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.math.BigDecimal; +import java.time.Instant; + +/** A stored exact-match response, replayable without calling the provider. */ +@Entity +@Table(name = "response_cache_entries", + indexes = @Index(name = "ix_cache_expiry", columnList = "expiresAt")) +@Getter +@Setter +@NoArgsConstructor +public class ResponseCacheEntry { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false, unique = true, length = 64) + private String cacheKey; + + @Column(nullable = false) + private Long clientKeyId; + + @Column(length = 128) + private String publicModelId; + + @Column(length = 64) + private String providerSlug; + + @Column(length = 128) + private String contentType; + + @Column(columnDefinition = "text") + private String body; + + /** Comma-separated vector of the prompt that produced this entry; null for exact-only. */ + @Column(columnDefinition = "text") + private String embedding; + + @Column(length = 120) + private String embeddingModel; + + private int promptTokens; + private int completionTokens; + private int totalTokens; + + /** What this response cost when it was generated; reported as savings on a hit. */ + @Column(precision = 18, scale = 8) + private BigDecimal costUsd = BigDecimal.ZERO; + + @Column(nullable = false) + private Instant createdAt = Instant.now(); + + @Column(nullable = false) + private Instant expiresAt; +} diff --git a/src/main/java/com/anindra/path/domain/UsageRecord.java b/src/main/java/com/anindra/path/domain/UsageRecord.java index cc38140..fed4732 100644 --- a/src/main/java/com/anindra/path/domain/UsageRecord.java +++ b/src/main/java/com/anindra/path/domain/UsageRecord.java @@ -1,6 +1,8 @@ package com.anindra.path.domain; import jakarta.persistence.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -33,6 +35,10 @@ public enum Status {SUCCESS, ERROR} @Column(length = 128) private String clientKeyName; + /** Opaque identifier the caller supplied for its own end user; null when absent. */ + @Column(length = 128) + private String endUser; + @Column(length = 128) private String publicModelId; @@ -72,7 +78,7 @@ public enum Status {SUCCESS, ERROR} @Column(precision = 18, scale = 8) private BigDecimal cachedSavingsUsd = BigDecimal.ZERO; - @Column(precision = 18, scale = 6) + @Column(precision = 18, scale = 8) private BigDecimal costUsd = BigDecimal.ZERO; /** True when the provider reported no usage block and the numbers were inferred. */ @@ -102,6 +108,7 @@ public enum Status {SUCCESS, ERROR} private boolean streamed; @Enumerated(EnumType.STRING) + @JdbcTypeCode(SqlTypes.VARCHAR) @Column(nullable = false, length = 16) private Status status = Status.SUCCESS; diff --git a/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java b/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java index 6d62683..8b9c1c9 100644 --- a/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java +++ b/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.anindra.path.api.ApiException; import com.anindra.path.domain.ModelRoute; import com.anindra.path.domain.Provider; import com.anindra.path.domain.ProviderType; @@ -31,6 +32,11 @@ public class AnthropicAdapter implements ProviderAdapter { private static final String ANTHROPIC_VERSION = "2023-06-01"; + /** OpenAI parameters with no Anthropic equivalent; silently dropping them changes behaviour. */ + private static final List UNSUPPORTED = List.of( + "logit_bias", "frequency_penalty", "presence_penalty", "seed", + "logprobs", "top_logprobs", "modalities", "audio", "prediction"); + private final ObjectMapper mapper; @Override @@ -44,6 +50,7 @@ public ProviderType type() { public UpstreamRequest build(ChatRequestSpec spec, ModelRoute route, Provider provider, String apiKey, boolean stream) { ObjectNode body = spec.body(); + validateParameters(body); ObjectNode out = mapper.createObjectNode(); out.put("model", route.getUpstreamModelId()); @@ -58,6 +65,11 @@ public UpstreamRequest build(ChatRequestSpec spec, ModelRoute route, Provider pr out.put("system", system); } + JsonNode user = body.get("user"); + if (user != null && user.isTextual() && !user.asText().isBlank()) { + out.putObject("metadata").put("user_id", user.asText()); + } + out.set("messages", convertMessages(body.get("messages"))); JsonNode stop = body.get("stop"); @@ -113,6 +125,27 @@ public UpstreamRequest build(ChatRequestSpec spec, ModelRoute route, Provider pr return new UpstreamRequest(join(provider.getBaseUrl(), "/messages"), headers, out.toString()); } + private static void validateParameters(ObjectNode body) { + for (String parameter : UNSUPPORTED) { + if (body.hasNonNull(parameter)) { + throw unsupported(parameter); + } + } + if (body.hasNonNull("n") && body.get("n").asInt(1) != 1) { + throw unsupported("n"); + } + JsonNode format = body.get("response_format"); + if (format != null && !"text".equals(format.path("type").asText("text"))) { + throw unsupported("response_format"); + } + } + + private static ApiException unsupported(String parameter) { + return new ApiException(422, "invalid_request_error", "unsupported_parameter", + "Parameter '" + parameter + "' is not supported by the anthropic wire format.", + parameter); + } + private static String toolChoiceMode(JsonNode toolChoice) { if (toolChoice == null) { return "auto"; @@ -192,10 +225,17 @@ private ArrayNode convertMessages(JsonNode messages) { for (JsonNode part : content) { String type = part.path("type").asText("text"); if (type.equals("text")) { - blocks.addObject().put("type", "text").put("text", part.path("text").asText()); + ObjectNode textBlock = blocks.addObject() + .put("type", "text").put("text", part.path("text").asText()); + if (part.has("cache_control")) { + textBlock.set("cache_control", part.get("cache_control")); + } } else if (type.equals("image_url")) { ObjectNode image = imageBlock(part.path("image_url").path("url").asText("")); if (image != null) { + if (part.has("cache_control")) { + image.set("cache_control", part.get("cache_control")); + } blocks.add(image); } } diff --git a/src/main/java/com/anindra/path/gateway/ApiKeyService.java b/src/main/java/com/anindra/path/gateway/ApiKeyService.java index ac67ecd..160de96 100644 --- a/src/main/java/com/anindra/path/gateway/ApiKeyService.java +++ b/src/main/java/com/anindra/path/gateway/ApiKeyService.java @@ -5,6 +5,8 @@ import com.anindra.path.domain.ClientKey; import com.anindra.path.repo.ClientKeyRepository; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -22,6 +24,7 @@ @Service @RequiredArgsConstructor +@Slf4j public class ApiKeyService { private static final String PREFIX = "sk-gw-"; @@ -92,11 +95,38 @@ public void touch(Long keyId) { }); } + /** @return true when the reservation was granted. */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean reserve(Long keyId, BigDecimal amount) { + if (amount == null || amount.signum() <= 0) { + return true; + } + return keys.reserve(keyId, amount, Instant.now()) > 0; + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void release(Long keyId, BigDecimal amount) { + if (amount != null && amount.signum() > 0) { + keys.release(keyId, amount); + } + } + + /** Frees reservations left behind by a crashed instance. */ + @Scheduled(fixedDelay = 300_000) + @Transactional + void releaseStaleReservations() { + int released = keys.releaseStale(Instant.now().minusSeconds(900)); + if (released > 0) { + log.info("Released stale budget reservations on {} key(s)", released); + } + } + public BigDecimal remaining(ClientKey key) { if (key.getCreditLimitUsd() == null) { return null; } - return key.getCreditLimitUsd().subtract(key.getUsedUsd()); + BigDecimal reserved = key.getReservedUsd() == null ? BigDecimal.ZERO : key.getReservedUsd(); + return key.getCreditLimitUsd().subtract(key.getUsedUsd()).subtract(reserved); } public static String hash(String raw) { diff --git a/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java b/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java index 808f74e..6f2fe86 100644 --- a/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java +++ b/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java @@ -26,13 +26,14 @@ public final class ChatRequestSpec { private final List fallbackModels; private final boolean stream; private final boolean includeUsage; - private final Integer maxTokens; + private Integer maxTokens; private final List providerOrder; private final boolean allowFallbacks; + private final String endUser; private ChatRequestSpec(ObjectNode body, String publicModel, List fallbackModels, boolean stream, boolean includeUsage, Integer maxTokens, - List providerOrder, boolean allowFallbacks) { + List providerOrder, boolean allowFallbacks, String endUser) { this.body = body; this.publicModel = publicModel; this.fallbackModels = fallbackModels; @@ -41,6 +42,7 @@ private ChatRequestSpec(ObjectNode body, String publicModel, List fallba this.maxTokens = maxTokens; this.providerOrder = providerOrder; this.allowFallbacks = allowFallbacks; + this.endUser = endUser; } public static ChatRequestSpec parse(ObjectNode body) { @@ -48,6 +50,7 @@ public static ChatRequestSpec parse(ObjectNode body) { if (messages == null || !messages.isArray() || messages.isEmpty()) { throw ApiException.badRequest("'messages' must be a non-empty array.", "messages"); } + boolean hasConversationalTurn = false; for (JsonNode message : messages) { if (!message.isObject() || !message.hasNonNull("role")) { throw ApiException.badRequest("Every message needs a 'role'.", "messages"); @@ -56,12 +59,19 @@ public static ChatRequestSpec parse(ObjectNode body) { if (!ROLES.contains(role)) { throw ApiException.badRequest("Unsupported message role: " + role, "messages"); } + if (!role.equals("system") && !role.equals("developer")) { + hasConversationalTurn = true; + } JsonNode content = message.get("content"); if (content != null && !content.isNull() && !content.isTextual() && !content.isArray()) { throw ApiException.badRequest( "message.content must be a string or an array of content parts.", "messages"); } } + if (!hasConversationalTurn) { + throw ApiException.badRequest( + "'messages' must contain at least one non-system message.", "messages"); + } List models = new ArrayList<>(); JsonNode modelsNode = body.get("models"); @@ -112,7 +122,7 @@ public static ChatRequestSpec parse(ObjectNode body) { } return new ChatRequestSpec(body, model, models, stream, includeUsage, maxTokens, - providerOrder, allowFallbacks); + providerOrder, allowFallbacks, readEndUser(body)); } /** Deep copy with gateway-only fields stripped, ready to forward. */ @@ -122,12 +132,55 @@ public ObjectNode forwardableBody() { return copy; } + /** + * The caller's own end-user id. "end_user" is Path-specific and wins; "user" is + * accepted because it is what OpenAI clients already send. + */ + private static String readEndUser(ObjectNode body) { + String value = firstText(body, "end_user", "user"); + if (value == null) { + return null; + } + value = value.trim(); + if (value.isEmpty()) { + return null; + } + if (value.length() > 128) { + throw ApiException.badRequest("'end_user' must be at most 128 characters.", "end_user"); + } + return value; + } + + private static String firstText(ObjectNode body, String... names) { + for (String name : names) { + JsonNode node = body.get(name); + if (node != null && node.isTextual() && !node.asText().isBlank()) { + return node.asText(); + } + } + return null; + } + + public String endUser() { return endUser; } + public ObjectNode body() { return body; } public String publicModel() { return publicModel; } public List fallbackModels() { return fallbackModels; } public boolean stream() { return stream; } public boolean includeUsage() { return includeUsage; } public Integer maxTokens() { return maxTokens; } + + /** Applies a per-key or global output cap, rewriting the body the way a client would. */ + public void clampMaxTokens(Integer cap) { + if (cap == null || cap <= 0) { + return; + } + if (maxTokens == null || maxTokens > cap) { + maxTokens = cap; + body.put("max_tokens", cap); + body.remove("max_completion_tokens"); + } + } public List providerOrder() { return providerOrder; } public boolean allowFallbacks() { return allowFallbacks; } @@ -175,23 +228,57 @@ private static String textOf(JsonNode content) { return sb.toString(); } - /** Rough prompt size used only to pre-authorise spend. */ + /** + * Prompt size used to pre-authorise spend. Counts real tokens when the upstream model + * has a known encoding, otherwise falls back to the chars/4 heuristic. + */ + public int estimatePromptTokens(String model) { + return TokenCounter.count(body, model).tokens(); + } + public int estimatePromptTokens() { - int chars = 0; - for (JsonNode message : messages()) { - JsonNode content = message.get("content"); - if (content != null) { - chars += textOf(content).length(); + return estimatePromptTokens(null); + } + + /** + * Builds a spec for token counting, where a system-only prompt is legitimate and the + * chat-completion rules (at least one conversational turn) do not apply. + */ + public static ChatRequestSpec forCounting(ObjectNode body) { + List models = new ArrayList<>(); + JsonNode modelsNode = body.get("models"); + if (modelsNode != null && modelsNode.isArray()) { + modelsNode.forEach(n -> { + if (n.isTextual() && !n.asText().isBlank()) { + models.add(n.asText()); + } + }); + } + String model = body.path("model").isTextual() ? body.get("model").asText() : null; + if (model == null || model.isBlank()) { + model = models.isEmpty() ? null : models.get(0); + } + if (model == null || model.isBlank()) { + throw ApiException.badRequest("'model' is required.", "model"); + } + + List providerOrder = new ArrayList<>(); + boolean allowFallbacks = true; + JsonNode provider = body.get("provider"); + if (provider != null && provider.isObject()) { + JsonNode order = provider.get("order"); + if (order != null && order.isArray()) { + order.forEach(n -> { + if (n.isTextual()) { + providerOrder.add(n.asText()); + } + }); } - JsonNode toolCalls = message.get("tool_calls"); - if (toolCalls != null) { - chars += toolCalls.toString().length(); + if (provider.has("allow_fallbacks")) { + allowFallbacks = provider.get("allow_fallbacks").asBoolean(true); } } - JsonNode tools = body.get("tools"); - if (tools != null) { - chars += tools.toString().length(); - } - return Math.max(1, chars / 4); + return new ChatRequestSpec(body, model, models, false, false, null, providerOrder, + allowFallbacks, readEndUser(body)); } } diff --git a/src/main/java/com/anindra/path/gateway/ChatService.java b/src/main/java/com/anindra/path/gateway/ChatService.java index 14a3c36..1de73d3 100644 --- a/src/main/java/com/anindra/path/gateway/ChatService.java +++ b/src/main/java/com/anindra/path/gateway/ChatService.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; import com.anindra.path.domain.ClientKey; import com.anindra.path.domain.Provider; import com.anindra.path.domain.ProviderType; @@ -11,6 +12,7 @@ import java.io.IOException; import java.math.BigDecimal; +import java.time.Duration; import java.util.ArrayList; import java.util.EnumMap; import java.util.List; @@ -22,43 +24,109 @@ public class ChatService { public record Result(JsonNode body, String providerSlug, String upstreamModel, - BigDecimal costUsd, long latencyMs) { + BigDecimal costUsd, long latencyMs, String cacheState) { } private final RouteResolver resolver; private final ProviderCredentials credentials; private final UpstreamClient upstream; private final UsageService usage; + private final ApiKeyService apiKeys; private final CreditGuard creditGuard; + private final ProviderCircuitBreaker breaker; + private final RoutingMetrics metrics; + private final ResponseCacheService cache; + private final SemanticCacheService semantic; + private final GatewayProperties props; private final ObjectMapper mapper; private final Map adapters = new EnumMap<>(ProviderType.class); public ChatService(RouteResolver resolver, ProviderCredentials credentials, UpstreamClient upstream, - UsageService usage, CreditGuard creditGuard, ObjectMapper mapper, + UsageService usage, ApiKeyService apiKeys, CreditGuard creditGuard, + ProviderCircuitBreaker breaker, RoutingMetrics metrics, + ResponseCacheService cache, SemanticCacheService semantic, + GatewayProperties props, ObjectMapper mapper, List adapterList) { this.resolver = resolver; this.credentials = credentials; this.upstream = upstream; this.usage = usage; + this.apiKeys = apiKeys; this.creditGuard = creditGuard; + this.breaker = breaker; + this.metrics = metrics; + this.cache = cache; + this.semantic = semantic; + this.props = props; this.mapper = mapper; adapterList.forEach(a -> adapters.put(a.type(), a)); } public Result complete(ChatRequestSpec spec, ClientKey key) { - List chain = resolver.resolve(spec); - creditGuard.check(key, spec, chain.get(0)); - + List chain = resolver.resolve(spec, key.getId()); + CreditGuard.Reservation reserved = creditGuard.check(key, spec, chain.get(0)); + try { List failures = new ArrayList<>(); ApiException lastError = null; long started = System.nanoTime(); int attempts = 0; String fallbackFrom = null; - for (int i = 0; i < chain.size(); i++) { + boolean cacheEligible = ResponseCacheService.eligible(spec, chain.get(0).route()); + String cacheKey = cacheEligible + ? cache.key(key.getId(), spec.publicModel(), spec.body().toString()) : null; + double[] promptVector = null; + if (cacheKey != null) { + var hit = cache.lookup(cacheKey); + if (hit.isPresent()) { + ResponseCacheService.Entry entry = hit.get(); + usage.recordCacheHit(key, spec, chain.get(0), entry, false); + try { + return new Result(mapper.readTree(entry.body()), entry.providerSlug(), + chain.get(0).route().getUpstreamModelId(), BigDecimal.ZERO, + elapsedMs(started), "hit"); + } catch (IOException e) { + log.warn("Cached body for {} is unreadable; falling through", spec.publicModel()); + } + } + + // Second tier: only for routes that opted in, and only after an exact miss. + if (chain.get(0).route().isSemanticCacheEnabled() && semantic.enabled()) { + promptVector = semantic.embed(spec.body().toString(), key); + double threshold = chain.get(0).route().getSemanticThreshold() == null + ? 0.95 : chain.get(0).route().getSemanticThreshold().doubleValue(); + var near = semantic.lookup(key.getId(), spec.publicModel(), promptVector, threshold); + if (near.isPresent()) { + ResponseCacheService.Entry entry = entryOf(near.get().entry()); + usage.recordCacheHit(key, spec, chain.get(0), entry, false); + try { + return new Result(mapper.readTree(entry.body()), entry.providerSlug(), + chain.get(0).route().getUpstreamModelId(), BigDecimal.ZERO, + elapsedMs(started), "semantic"); + } catch (IOException e) { + log.warn("Cached body for {} is unreadable; falling through", spec.publicModel()); + } + } + } + } + + long available = chain.stream().filter(r -> breaker.isAvailable(r.provider().getId())).count(); + int maxAttempts = Math.min(chain.size(), Math.max(1, props.getBreaker().getMaxAttempts())); + + for (int i = 0; i < chain.size() && attempts < maxAttempts; i++) { RouteResolver.ResolvedRoute rr = chain.get(i); ProviderAdapter adapter = adapterFor(rr.provider()); String apiKey = credentials.apiKey(rr.provider()); + Long providerId = rr.provider().getId(); + + if (available > 0 && !breaker.isAvailable(providerId)) { + lastError = new ApiException(503, "upstream_error", "circuit_open", + "Provider '" + rr.providerSlug() + "' is temporarily unavailable."); + failures.add(rr.providerSlug() + ": circuit open"); + fallbackFrom = fallbackFrom == null ? rr.providerSlug() : fallbackFrom; + continue; + } + attempts++; if (apiKey == null) { @@ -72,11 +140,20 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { try { UpstreamRequest request = adapter.build(spec, rr.route(), rr.provider(), apiKey, false); - UpstreamClient.Response response = upstream.postJson(request.url(), request.headers(), request.body()); + Duration timeout = rr.provider().getRequestTimeoutMs() != null + ? Duration.ofMillis(rr.provider().getRequestTimeoutMs()) + : props.getRequestTimeout(); + UpstreamClient.Response response = upstream.postJson( + request.url(), request.headers(), request.body(), timeout); if (!response.ok()) { - UpstreamException failure = UpstreamClient.mapError(response.status(), response.body()); + UpstreamException failure = UpstreamClient.mapError( + response.status(), response.body(), response.headers()); log.warn("Upstream {} -> {}: {}", rr.providerSlug(), failure.status(), failure.getMessage()); + metrics.recordFailure(rr.providerSlug(), spec.publicModel()); + if (failure.retryable()) { + recordBreakerFailure(providerId, failure); + } if (failure.retryable() && i < chain.size() - 1) { failures.add(rr.providerSlug() + ": " + failure.getMessage()); lastError = toApiException(failure); @@ -95,7 +172,7 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { TokenUsage tokens = adapter.parseUsage(upstreamJson); if (!tokens.reported()) { - tokens = estimate(spec, normalized); + tokens = estimate(spec, normalized, rr.route().getUpstreamModelId()); } String requestId = firstNonBlank( @@ -108,12 +185,27 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { long latency = elapsedMs(started); BigDecimal cost = usage.recordSuccess(key, spec, rr, tokens, UsageService.CallMeta.nonStream(latency, requestId, finishReason, attempts, fallbackFrom)); - return new Result(normalized, rr.providerSlug(), rr.route().getUpstreamModelId(), cost, latency); + breaker.onSuccess(providerId); + metrics.recordSuccess(rr.providerSlug(), spec.publicModel(), latency); + if (cacheKey != null) { + double[] vector = promptVector; + if (vector == null && rr.route().isSemanticCacheEnabled() && semantic.enabled()) { + vector = semantic.embed(spec.body().toString(), key); + } + cache.store(cacheKey, key.getId(), spec.publicModel(), rr.providerSlug(), + rr.route(), "application/json", normalized.toString(), tokens, cost, + semantic.encode(vector), vector == null ? null : semantic.model()); + } + return new Result(normalized, rr.providerSlug(), rr.route().getUpstreamModelId(), cost, + latency, cacheKey == null ? "off" : "miss"); } catch (IOException e) { + breaker.onFailure(providerId); + metrics.recordFailure(rr.providerSlug(), spec.publicModel()); + String reason = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); lastError = ApiException.upstream(502, - "Could not reach " + rr.providerSlug() + ": " + e.getMessage(), "upstream_network_error"); - failures.add(rr.providerSlug() + ": " + e.getMessage()); + "Could not reach " + rr.providerSlug() + ": " + reason, "upstream_network_error"); + failures.add(rr.providerSlug() + ": " + reason); fallbackFrom = fallbackFrom == null ? rr.providerSlug() : fallbackFrom; if (i == chain.size() - 1) { break; @@ -133,16 +225,34 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { usage.recordError(key, spec, null, null, elapsedMs(started), false, attempts, lastError.code(), lastError.getMessage()); throw lastError; + } finally { + creditGuard.release(key, reserved); + } + } + + private static ResponseCacheService.Entry entryOf( + com.anindra.path.domain.ResponseCacheEntry record) { + return new ResponseCacheService.Entry(record.getContentType(), record.getBody(), + record.getProviderSlug(), record.getCostUsd(), record.getPromptTokens(), + record.getCompletionTokens(), record.getTotalTokens()); } /** Used only when a provider returns no usage block at all. */ - private static TokenUsage estimate(ChatRequestSpec spec, JsonNode normalized) { - int prompt = spec.estimatePromptTokens(); + private static TokenUsage estimate(ChatRequestSpec spec, JsonNode normalized, String upstreamModel) { + int prompt = spec.estimatePromptTokens(upstreamModel); JsonNode content = normalized.path("choices").path(0).path("message").path("content"); int completion = content.isTextual() ? Math.max(1, content.asText().length() / 4) : 0; return TokenUsage.estimated(prompt, completion); } + private void recordBreakerFailure(Long providerId, UpstreamException failure) { + if (failure.status() == 429 && failure.retryAfterSeconds() > 0) { + breaker.onRateLimited(providerId, java.time.Duration.ofSeconds(failure.retryAfterSeconds())); + } else { + breaker.onFailure(providerId); + } + } + private ProviderAdapter adapterFor(Provider provider) { ProviderAdapter adapter = adapters.get(provider.getType()); if (adapter == null) { diff --git a/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java b/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java new file mode 100644 index 0000000..f66cb5a --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java @@ -0,0 +1,103 @@ +package com.anindra.path.gateway; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; + +/** + * Caps concurrent in-flight requests per client key. + * + * The rate limiter bounds requests per minute; this bounds requests at an instant, which + * is what stops one key from opening thousands of streams and starving the pool. + */ +@Component +public class ConcurrencyLimiter { + + private record Entry(int limit, Semaphore semaphore) { + } + + private final Map entries = new ConcurrentHashMap<>(); + + public static String keyBucket(Long keyId) { + return "key:" + keyId; + } + + public static String projectBucket(Long projectId) { + return "project:" + projectId; + } + + /** + * Takes the key's permit and, when the key belongs to a project with a limit, the + * project's permit too. Either both are held or neither is. + * + * @return true when permits were taken; callers must pass the same key to {@code release} + */ + public boolean tryAcquire(com.anindra.path.domain.ClientKey key, int keyLimit, + Integer projectLimit) { + if (!tryAcquire(keyBucket(key.getId()), keyLimit)) { + return false; + } + if (projectLimit == null || projectLimit <= 0 || key.getProjectId() == null) { + return true; + } + if (tryAcquire(projectBucket(key.getProjectId()), projectLimit)) { + return true; + } + release(keyBucket(key.getId())); + return false; + } + + /** Releases both permits taken by {@link #tryAcquire(ClientKey, int, Integer)}. */ + public void release(com.anindra.path.domain.ClientKey key) { + release(keyBucket(key.getId())); + if (key.getProjectId() != null) { + release(projectBucket(key.getProjectId())); + } + } + + /** @return true when a permit was taken; callers must release it. */ + public boolean tryAcquire(Long keyId, int limit) { + return tryAcquire(keyBucket(keyId), limit); + } + + /** @return true when a permit was taken; callers must release the same bucket. */ + public boolean tryAcquire(String bucket, int limit) { + if (limit <= 0) { + return true; + } + Entry entry = entries.compute(bucket, (id, current) -> + current == null || current.limit() != limit + ? new Entry(limit, new Semaphore(limit)) + : current); + return entry.semaphore().tryAcquire(); + } + + public void release(Long keyId) { + release(keyBucket(keyId)); + } + + public void release(String bucket) { + Entry entry = entries.get(bucket); + if (entry != null && entry.semaphore().availablePermits() < entry.limit()) { + entry.semaphore().release(); + } + } + + public int inFlight(Long keyId) { + return inFlight(keyBucket(keyId)); + } + + public int inFlight(String bucket) { + Entry entry = entries.get(bucket); + return entry == null ? 0 : entry.limit() - entry.semaphore().availablePermits(); + } + + @Scheduled(fixedDelay = 600_000) + void evictIdle() { + entries.entrySet().removeIf(e -> + e.getValue().semaphore().availablePermits() == e.getValue().limit()); + } +} diff --git a/src/main/java/com/anindra/path/gateway/CreditGuard.java b/src/main/java/com/anindra/path/gateway/CreditGuard.java index 734e3f0..465d3a6 100644 --- a/src/main/java/com/anindra/path/gateway/CreditGuard.java +++ b/src/main/java/com/anindra/path/gateway/CreditGuard.java @@ -1,15 +1,21 @@ package com.anindra.path.gateway; import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; import com.anindra.path.domain.ClientKey; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.Map; /** - * Refuses a request before any upstream call when the key is out of credit or the - * worst-case cost of the call could exceed what is left. + * Refuses a request before any upstream call when the key is out of credit, has + * exhausted its daily or monthly window, or when the worst-case cost of the call + * could exceed what is left. Also applies the output-token cap. */ @Component @RequiredArgsConstructor @@ -17,30 +23,236 @@ public class CreditGuard { private static final int DEFAULT_MAX_OUTPUT = 1024; private static final int ESTIMATE_CAP = 8192; + private static final int EXPECTED_OUTPUT = 512; private final ApiKeyService apiKeys; + private final EndUserBudget endUserBudget; private final UsageService usage; + private final GatewayProperties props; + private final com.anindra.path.admin.AlertService alerts; + private final com.anindra.path.repo.ProjectRepository projects; + + /** + * What was reserved for a call: the key's credit and, when the request carries an + * end user with a cap, that end user's daily budget. + */ + public record Reservation(BigDecimal keyUsd, BigDecimal endUserUsd, String endUser) { + public static Reservation none() { + return new Reservation(BigDecimal.ZERO, BigDecimal.ZERO, null); + } + } + + public Reservation check(ClientKey key, ChatRequestSpec spec, RouteResolver.ResolvedRoute route) { + if (!allows(key, spec.publicModel())) { + throw new ApiException(403, "permission_error", "model_not_allowed", + "This API key is not allowed to use model '" + spec.publicModel() + "'."); + } + Integer cap = key.getMaxOutputTokens() != null && key.getMaxOutputTokens() > 0 + ? key.getMaxOutputTokens() + : (props.getMaxOutputTokens() > 0 ? props.getMaxOutputTokens() : null); + spec.clampMaxTokens(cap); + + Instant now = Instant.now(); + Instant dayStart = LocalDate.now(ZoneOffset.UTC).atStartOfDay(ZoneOffset.UTC).toInstant(); + Instant monthStart = LocalDate.now(ZoneOffset.UTC).withDayOfMonth(1) + .atStartOfDay(ZoneOffset.UTC).toInstant(); + + enforceWindow("Daily", key.getDailyLimitUsd(), usage.spendFor(key.getId(), dayStart), dayStart, now); + enforceWindow("Monthly", key.getMonthlyLimitUsd(), usage.spendFor(key.getId(), monthStart), monthStart, now); + + if (key.getProjectId() != null) { + projects.findById(key.getProjectId()).ifPresent(project -> { + enforceProjectWindow(project, "daily", project.getDailyLimitUsd(), + usage.spendForProject(project.getId(), dayStart), dayStart, now); + enforceProjectWindow(project, "monthly", project.getMonthlyLimitUsd(), + usage.spendForProject(project.getId(), monthStart), monthStart, now); + if (project.getCreditLimitUsd() != null && project.getCreditLimitUsd().signum() > 0) { + BigDecimal projectSpend = usage.spendForProject(project.getId(), + Instant.EPOCH); + warnProject(project, "lifetime", project.getCreditLimitUsd(), projectSpend, + "lifetime"); + if (projectSpend.compareTo(project.getCreditLimitUsd()) >= 0) { + exhaustedProject(project, "lifetime", project.getCreditLimitUsd(), + projectSpend, "lifetime"); + throw ApiException.insufficientCredits( + "Project '" + project.getSlug() + "' has exhausted its credit limit of " + + project.getCreditLimitUsd().toPlainString() + " USD."); + } + } + }); + } + + // Estimate before the balance checks: the end-user cap applies even to a key with + // no credit limit of its own, so it cannot sit behind the remaining == null return. + int promptEstimate = spec.estimatePromptTokens(route.route().getUpstreamModelId()); + int outputEstimate = spec.maxTokens() != null ? spec.maxTokens() : DEFAULT_MAX_OUTPUT; + outputEstimate = Math.min(outputEstimate, ESTIMATE_CAP); + BigDecimal worstCase = usage.cost(route.route(), + TokenUsage.of(promptEstimate, outputEstimate)).totalUsd(); + // Reserve what a typical completion would cost, not the absolute cap: a small + // balance should not be blocked by a generous max_tokens it will never use. + int expectedOutput = Math.min(outputEstimate, EXPECTED_OUTPUT); + BigDecimal expected = usage.cost(route.route(), + TokenUsage.of(promptEstimate, expectedOutput)).totalUsd(); - public void check(ClientKey key, ChatRequestSpec spec, RouteResolver.ResolvedRoute route) { BigDecimal remaining = apiKeys.remaining(key); if (remaining == null) { - return; + // Unlimited key: it still has to fit its end-user caps. + BigDecimal endUserOnly = reserveEndUser(key, spec, worstCase, expected, dayStart); + return new Reservation(BigDecimal.ZERO, endUserOnly, spec.endUser()); + } + if (key.getCreditLimitUsd() != null && key.getCreditLimitUsd().signum() > 0) { + double left = remaining.divide(key.getCreditLimitUsd(), 4, java.math.RoundingMode.HALF_UP).doubleValue(); + if (left < props.getAlerts().getBudgetThreshold()) { + alerts.notify("key.budget_low", Map.of( + "key_id", key.getId(), + "key_name", key.getName(), + "remaining_usd", remaining, + "limit_usd", key.getCreditLimitUsd(), + "fraction", left)); + } } if (remaining.signum() <= 0) { throw ApiException.insufficientCredits( "This API key has no credit left (" + key.getUsedUsd() + " of " + key.getCreditLimitUsd() + " USD used). Top up the key to continue."); } - int promptEstimate = spec.estimatePromptTokens(); - int outputEstimate = spec.maxTokens() != null ? spec.maxTokens() : DEFAULT_MAX_OUTPUT; - outputEstimate = Math.min(outputEstimate, ESTIMATE_CAP); - BigDecimal worstCase = usage.cost(route.route(), - TokenUsage.of(promptEstimate, outputEstimate)).totalUsd(); - if (worstCase.compareTo(remaining) > 0) { + if (expected.compareTo(remaining) > 0) { throw ApiException.insufficientCredits( - "This request could cost up to " + worstCase.toPlainString() + "This request is expected to cost about " + expected.toPlainString() + " USD but only " + remaining.toPlainString() + " USD of credit remains. Lower max_tokens or top up the key."); } + BigDecimal endUserUsd = reserveEndUser(key, spec, worstCase, expected, dayStart); + BigDecimal reserve = worstCase.min(remaining); + if (!apiKeys.reserve(key.getId(), reserve)) { + endUserBudget.release(key.getId(), spec.endUser(), endUserUsd); + throw ApiException.insufficientCredits( + "Credit is exhausted by concurrent requests. Retry shortly or top up the key."); + } + return new Reservation(reserve, endUserUsd, spec.endUser()); + } + + /** + * Reserves against the key's per-end-user daily cap. The spend read and the atomic + * increment are separate, which bounds an overshoot to one in-flight request — the + * same guarantee the key-level reservation gives. + */ + private BigDecimal reserveEndUser(ClientKey key, ChatRequestSpec spec, BigDecimal worstCase, + BigDecimal expected, Instant dayStart) { + BigDecimal limit = key.getEndUserDailyLimitUsd(); + if (limit == null || limit.signum() <= 0 || spec.endUser() == null) { + return BigDecimal.ZERO; + } + BigDecimal spent = usage.spendForEndUser(key.getId(), spec.endUser(), dayStart); + BigDecimal headroom = limit.subtract(spent); + if (expected.compareTo(headroom) > 0) { + throw ApiException.insufficientCredits( + "End user '" + spec.endUser() + "' is expected to cost about " + + expected.toPlainString() + " USD but only " + headroom.max(BigDecimal.ZERO) + .toPlainString() + " USD of its daily budget of " + limit.toPlainString() + + " USD remains."); + } + BigDecimal reserve = worstCase.min(headroom); + if (!endUserBudget.reserve(key.getId(), spec.endUser(), reserve, headroom)) { + throw ApiException.insufficientCredits( + "End user '" + spec.endUser() + "' has budget in flight by concurrent requests. " + + "Retry shortly."); + } + return reserve; + } + + /** Releases both reservations taken by {@link #check}. */ + public void release(ClientKey key, Reservation reservation) { + if (reservation == null) { + return; + } + apiKeys.release(key.getId(), reservation.keyUsd()); + endUserBudget.release(key.getId(), reservation.endUser(), reservation.endUserUsd()); + } + + private void enforceProjectWindow(com.anindra.path.domain.Project project, String window, + BigDecimal limit, BigDecimal spent, Instant periodStart, + Instant now) { + if (limit == null || limit.signum() <= 0) { + return; + } + warnProject(project, window, limit, spent, periodKey(window, periodStart)); + if (spent.compareTo(limit) >= 0) { + exhaustedProject(project, window, limit, spent, periodKey(window, periodStart)); + throw ApiException.insufficientCredits( + "Project '" + project.getSlug() + "' " + window + " budget of " + + limit.toPlainString() + " USD is exhausted (" + spent.toPlainString() + + " USD spent). Raise the limit or wait for the window to reset."); + } + } + + private void warnProject(com.anindra.path.domain.Project project, String window, + BigDecimal limit, BigDecimal spent, String period) { + double remainingFraction = limit.subtract(spent) + .divide(limit, 4, java.math.RoundingMode.HALF_UP).doubleValue(); + if (remainingFraction >= props.getAlerts().getBudgetThreshold()) { + return; + } + alerts.notifyOnce("project.budget_low:" + project.getId() + ":" + period, + "project.budget_low", projectPayload(project, window, limit, spent)); + } + + private void exhaustedProject(com.anindra.path.domain.Project project, String window, + BigDecimal limit, BigDecimal spent, String period) { + alerts.notifyOnce("project.budget_exhausted:" + project.getId() + ":" + period, + "project.budget_exhausted", projectPayload(project, window, limit, spent)); + } + + private static Map projectPayload(com.anindra.path.domain.Project project, + String window, BigDecimal limit, + BigDecimal spent) { + Map payload = new java.util.LinkedHashMap<>(); + payload.put("project_id", project.getId()); + payload.put("project", project.getSlug()); + payload.put("window", window); + payload.put("limit_usd", limit); + payload.put("spent_usd", spent); + payload.put("remaining_usd", limit.subtract(spent).max(BigDecimal.ZERO)); + return payload; + } + + private static String periodKey(String window, Instant periodStart) { + return window + "@" + periodStart; + } + + static boolean allows(ClientKey key, String model) { + String allowed = key.getAllowedModels(); + if (allowed == null || allowed.isBlank()) { + return true; + } + for (String raw : allowed.split(",")) { + String pattern = raw.trim(); + if (pattern.isEmpty()) { + continue; + } + if (pattern.equals("*") || pattern.equalsIgnoreCase(model)) { + return true; + } + if (pattern.endsWith("*")) { + String prefix = pattern.substring(0, pattern.length() - 1); + if (model.regionMatches(true, 0, prefix, 0, prefix.length())) { + return true; + } + } + } + return false; + } + + private static void enforceWindow(String label, BigDecimal limit, BigDecimal spent, + Instant windowStart, Instant now) { + if (limit == null || limit.signum() <= 0) { + return; + } + if (spent.compareTo(limit) >= 0) { + throw ApiException.insufficientCredits( + label + " budget of " + limit.toPlainString() + " USD is exhausted (" + + spent.toPlainString() + " USD spent since " + windowStart + ")."); + } } } diff --git a/src/main/java/com/anindra/path/gateway/DrainState.java b/src/main/java/com/anindra/path/gateway/DrainState.java new file mode 100644 index 0000000..b07309b --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/DrainState.java @@ -0,0 +1,65 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Stops accepting work on shutdown and waits for in-flight streams to finish. + * + * Without this, Spring's 20s graceful-shutdown window cuts any generation still + * running; with it, the readiness probe goes DOWN immediately and new requests are + * refused while existing streams drain for up to {@code gateway.drain-timeout}. + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class DrainState { + + private final GatewayProperties props; + private final AtomicBoolean draining = new AtomicBoolean(false); + private final AtomicInteger inFlight = new AtomicInteger(0); + + public boolean isDraining() { + return draining.get(); + } + + public int inFlight() { + return inFlight.get(); + } + + /** Registers a request; refuses new work once draining has begun. */ + public void begin() { + if (draining.get()) { + throw new ApiException(503, "server_error", "shutting_down", + "The gateway is shutting down. Retry against another instance."); + } + inFlight.incrementAndGet(); + } + + public void end() { + inFlight.decrementAndGet(); + } + + @EventListener(ContextClosedEvent.class) + public void drain() { + draining.set(true); + long deadline = System.currentTimeMillis() + props.getDrainTimeout().toMillis(); + while (inFlight.get() > 0 && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + log.info("Shutdown drain finished with {} request(s) still in flight", inFlight.get()); + } +} diff --git a/src/main/java/com/anindra/path/gateway/EmbeddingsService.java b/src/main/java/com/anindra/path/gateway/EmbeddingsService.java new file mode 100644 index 0000000..c5ea245 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/EmbeddingsService.java @@ -0,0 +1,112 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * POST /v1/embeddings for OpenAI-compatible providers. Routes through the same chain + * resolution as chat, so aliases, weights and fallback order all apply. + */ +@Service +@RequiredArgsConstructor +public class EmbeddingsService { + + public record Result(JsonNode body, String providerSlug, String upstreamModel, BigDecimal costUsd) { + } + + private final RouteResolver resolver; + private final ProviderCredentials credentials; + private final UpstreamClient upstream; + private final UsageService usage; + private final ApiKeyService apiKeys; + private final CreditGuard creditGuard; + private final ObjectMapper mapper; + + public Result embed(ObjectNode request, ClientKey key) { + JsonNode input = request.get("input"); + if (input == null || input.isNull()) { + throw ApiException.badRequest("'input' is required.", "input"); + } + String model = request.path("model").asText(""); + if (model.isBlank()) { + throw ApiException.badRequest("'model' is required.", "model"); + } + + ChatRequestSpec spec = routingSpec(model); + List chain = resolver.resolve(spec, key.getId()); + CreditGuard.Reservation reserved = creditGuard.check(key, spec, chain.get(0)); + + RouteResolver.ResolvedRoute resolved = chain.stream() + .filter(r -> r.provider().getType() == ProviderType.OPENAI) + .findFirst() + .orElseThrow(() -> new ApiException(501, "invalid_request_error", "unsupported_endpoint", + "Embeddings are only supported for OpenAI-compatible providers.")); + + Provider provider = resolved.provider(); + String apiKey = credentials.apiKey(provider); + if (apiKey == null) { + throw ApiException.upstream(502, + "Provider '" + provider.getSlug() + "' has no API key configured.", + "provider_not_configured"); + } + + long started = System.nanoTime(); + try { + return doEmbed(request, key, spec, resolved, provider, apiKey, model, started); + } finally { + creditGuard.release(key, reserved); + } + } + + private Result doEmbed(ObjectNode request, ClientKey key, ChatRequestSpec spec, + RouteResolver.ResolvedRoute resolved, Provider provider, String apiKey, + String model, long started) { + ObjectNode body = request.deepCopy(); + body.put("model", resolved.route().getUpstreamModelId()); + Map headers = ProviderUrls.headers(provider, apiKey, false); + try { + UpstreamClient.Response response = upstream.postJson( + ProviderUrls.embeddings(provider, resolved.route().getUpstreamModelId()), + headers, body.toString()); + if (!response.ok()) { + throw ChatService.toApiException(UpstreamClient.mapError( + response.status(), response.body(), response.headers())); + } + JsonNode parsed = mapper.readTree(response.body()); + if (parsed instanceof ObjectNode object) { + object.put("model", model); + } + int prompt = parsed.path("usage").path("prompt_tokens").asInt(0); + BigDecimal cost = usage.recordSuccess(key, spec, resolved, TokenUsage.of(prompt, 0), + UsageService.CallMeta.nonStream(ChatService.elapsedMs(started), null, "stop", 1, null)); + return new Result(parsed, provider.getSlug(), resolved.route().getUpstreamModelId(), cost); + } catch (IOException e) { + throw ApiException.upstream(502, "Could not reach " + provider.getSlug() + ": " + e.getMessage(), + "upstream_network_error"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ApiException.upstream(502, "Request interrupted", "upstream_interrupted"); + } + } + + /** RouteResolver speaks ChatRequestSpec; embeddings only need the model and fallbacks. */ + private ChatRequestSpec routingSpec(String model) { + ObjectNode synthetic = mapper.createObjectNode(); + synthetic.put("model", model); + synthetic.putArray("messages").addObject().put("role", "user").put("content", ""); + return ChatRequestSpec.parse(synthetic); + } +} diff --git a/src/main/java/com/anindra/path/gateway/EndUserBudget.java b/src/main/java/com/anindra/path/gateway/EndUserBudget.java new file mode 100644 index 0000000..2dcac0e --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/EndUserBudget.java @@ -0,0 +1,63 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.EndUserReservation; +import com.anindra.path.repo.EndUserReservationRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigDecimal; +import java.time.Instant; + +/** + * Per-end-user budget reservations, so two concurrent calls for the same customer cannot + * both pass a check that only one of them fits. + * + *

Spend itself lives in {@code usage_records}; this table only holds what is in flight. + */ +@Service +@RequiredArgsConstructor +public class EndUserBudget { + + private final EndUserReservationRepository reservations; + + /** @return true when the amount was reserved; false when it would exceed the headroom. */ + @Transactional + public boolean reserve(Long keyId, String endUser, BigDecimal amount, BigDecimal headroom) { + if (amount == null || amount.signum() <= 0) { + return true; + } + ensureRow(keyId, endUser); + return reservations.reserve(keyId, endUser, amount, headroom, Instant.now()) > 0; + } + + @Transactional + public void release(Long keyId, String endUser, BigDecimal amount) { + if (endUser == null || amount == null || amount.signum() <= 0) { + return; + } + reservations.release(keyId, endUser, amount, Instant.now()); + } + + /** Test and admin helper: drop in-flight amounts for one end user. */ + @Transactional + public void clear(Long keyId, String endUser) { + reservations.clear(keyId, endUser, Instant.now()); + } + + private void ensureRow(Long keyId, String endUser) { + if (reservations.findByClientKeyIdAndEndUser(keyId, endUser).isPresent()) { + return; + } + EndUserReservation row = new EndUserReservation(); + row.setClientKeyId(keyId); + row.setEndUser(endUser); + row.setReservedUsd(BigDecimal.ZERO); + row.setUpdatedAt(Instant.now()); + try { + reservations.save(row); + } catch (org.springframework.dao.DataIntegrityViolationException raced) { + // Another request created it first; the row exists either way. + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/IdempotencyService.java b/src/main/java/com/anindra/path/gateway/IdempotencyService.java new file mode 100644 index 0000000..f845f38 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/IdempotencyService.java @@ -0,0 +1,105 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.IdempotencyRecord; +import com.anindra.path.repo.IdempotencyRecordRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.util.Optional; + +/** + * Idempotent replay for chat completions. + * + * The first request with a given {@code Idempotency-Key} writes a PENDING claim and runs; + * a repeat with the same body replays the stored response, a repeat with a different body + * is rejected, and a concurrent duplicate gets 409 rather than a second upstream call. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class IdempotencyService { + + public record Replay(int httpStatus, String contentType, String body) { + } + + private final IdempotencyRecordRepository records; + private final GatewayProperties props; + + /** + * @return the stored response when this key was already completed, empty when the + * caller should proceed (a PENDING claim has been written). + */ + @Transactional + public Optional begin(Long clientKeyId, String rawKey, String requestHash) { + String key = ApiKeyService.hash(rawKey); + Optional existing = records.findByClientKeyIdAndIdempotencyKey(clientKeyId, key); + if (existing.isPresent()) { + IdempotencyRecord record = existing.get(); + if (record.getStatus() == IdempotencyRecord.Status.PENDING) { + java.time.Duration staleAfter = props.getIdempotency().getPendingTimeout(); + boolean abandoned = record.getCreatedAt().isBefore(Instant.now().minus(staleAfter)); + if (!abandoned) { + throw new ApiException(409, "invalid_request_error", "idempotency_in_progress", + "A request with this Idempotency-Key is still in progress."); + } + // The instance that claimed this key died mid-request; let the retry run. + log.warn("Reclaiming stale PENDING idempotency claim {}", record.getId()); + records.delete(record); + records.flush(); + } else { + if (!record.getRequestHash().equals(requestHash)) { + throw new ApiException(409, "invalid_request_error", "idempotency_conflict", + "This Idempotency-Key was already used with a different request body."); + } + return Optional.of(new Replay(record.getHttpStatus(), record.getContentType(), record.getBody())); + } + } + + IdempotencyRecord claim = new IdempotencyRecord(); + claim.setClientKeyId(clientKeyId); + claim.setIdempotencyKey(key); + claim.setRequestHash(requestHash); + claim.setStatus(IdempotencyRecord.Status.PENDING); + claim.setExpiresAt(Instant.now().plus(props.getIdempotency().getTtl())); + try { + records.saveAndFlush(claim); + } catch (DataIntegrityViolationException e) { + throw new ApiException(409, "invalid_request_error", "idempotency_in_progress", + "A request with this Idempotency-Key is still in progress."); + } + return Optional.empty(); + } + + @Transactional + public void complete(Long clientKeyId, String rawKey, int httpStatus, String contentType, String body) { + records.findByClientKeyIdAndIdempotencyKey(clientKeyId, ApiKeyService.hash(rawKey)) + .ifPresent(record -> { + record.setStatus(IdempotencyRecord.Status.COMPLETED); + record.setHttpStatus(httpStatus); + record.setContentType(contentType); + record.setBody(body); + }); + } + + @Transactional + public void abandon(Long clientKeyId, String rawKey) { + records.findByClientKeyIdAndIdempotencyKey(clientKeyId, ApiKeyService.hash(rawKey)) + .ifPresent(records::delete); + } + + @Scheduled(fixedDelay = 3_600_000) + @Transactional + void purgeExpired() { + long removed = records.deleteByExpiresAtBefore(Instant.now()); + if (removed > 0) { + log.info("Purged {} expired idempotency records", removed); + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/RateLimiter.java b/src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java similarity index 57% rename from src/main/java/com/anindra/path/gateway/RateLimiter.java rename to src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java index 9ee1c29..21b70d1 100644 --- a/src/main/java/com/anindra/path/gateway/RateLimiter.java +++ b/src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java @@ -1,31 +1,28 @@ package com.anindra.path.gateway; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - import java.time.Instant; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** - * Fixed one-minute window limiter, one bucket per client key. Deliberately in-memory: - * a single gateway node enforces it exactly; for a cluster, replace with Redis. + * Fixed one-minute window per key. Exact on a single instance; a cluster needs + * {@link ValkeyRateLimitStore} instead. */ -@Component -public class RateLimiter { +public class InMemoryRateLimitStore implements RateLimitStore { - private record Window(long minute, int used) {} + private record Window(long minute, int used) { + } - private final Map windows = new ConcurrentHashMap<>(); + private final Map windows = new ConcurrentHashMap<>(); - /** @return 0 when the call may proceed, otherwise the Retry-After seconds. */ - public int acquire(Long keyId, int rpmLimit) { + @Override + public int acquire(String bucket, int rpmLimit) { if (rpmLimit <= 0) { return 0; } long minute = Instant.now().getEpochSecond() / 60; int[] retryAfter = {0}; - windows.compute(keyId, (id, current) -> { + windows.compute(bucket, (id, current) -> { if (current == null || current.minute() != minute) { return new Window(minute, 1); } @@ -38,7 +35,7 @@ public int acquire(Long keyId, int rpmLimit) { return retryAfter[0]; } - @Scheduled(fixedDelay = 300_000) + @org.springframework.scheduling.annotation.Scheduled(fixedDelay = 300_000) void evictStale() { long cutoff = Instant.now().getEpochSecond() / 60 - 10; windows.entrySet().removeIf(e -> e.getValue().minute() < cutoff); diff --git a/src/main/java/com/anindra/path/gateway/ModelAliasService.java b/src/main/java/com/anindra/path/gateway/ModelAliasService.java new file mode 100644 index 0000000..8f1a328 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ModelAliasService.java @@ -0,0 +1,80 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.repo.ModelRouteRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Capability aliases resolved from the live route table, so clients can ask for + * "cheap" or "long-context" instead of hardcoding a vendor model id. + */ +@Service +@RequiredArgsConstructor +public class ModelAliasService { + + public static final String CHEAP = "cheap"; + public static final String FAST = "fast"; + public static final String LONG_CONTEXT = "long-context"; + public static final String VISION = "vision"; + + private final ModelRouteRepository routes; + + public boolean isAlias(String model) { + return List.of(CHEAP, FAST, LONG_CONTEXT, VISION).contains(model); + } + + /** Maps an alias to a concrete public model id, or empty when nothing matches. */ + public Optional resolve(String alias) { + List enabled = routes.findByEnabledTrueOrderByPublicModelIdAscPriorityAsc(); + if (enabled.isEmpty()) { + return Optional.empty(); + } + return switch (alias) { + case CHEAP -> enabled.stream() + .filter(r -> r.getInputPricePerMillion() != null && r.getInputPricePerMillion().signum() >= 0) + .min(Comparator.comparing(ModelRoute::getInputPricePerMillion)) + .map(ModelRoute::getPublicModelId); + case FAST -> enabled.stream() + .filter(r -> r.getOutputPricePerMillion() != null) + .min(Comparator.comparing(ModelRoute::getOutputPricePerMillion)) + .map(ModelRoute::getPublicModelId); + case LONG_CONTEXT -> enabled.stream() + .filter(r -> r.getContextLength() != null) + .max(Comparator.comparing(ModelRoute::getContextLength)) + .map(ModelRoute::getPublicModelId); + case VISION -> enabled.stream() + .filter(r -> r.getDescription() != null + && r.getDescription().toLowerCase().contains("vision")) + .min(Comparator.comparing(ModelRoute::getPriority)) + .map(ModelRoute::getPublicModelId); + default -> Optional.empty(); + }; + } + + /** Public catalogue entries for the aliases, so clients can discover them. */ + public List> describeAll() { + return List.of(CHEAP, FAST, LONG_CONTEXT, VISION).stream() + .map(alias -> { + Map entry = new LinkedHashMap<>(); + entry.put("id", alias); + entry.put("object", "model"); + entry.put("owned_by", "path"); + entry.put("resolves_to", resolve(alias).orElse(null)); + entry.put("description", "Capability alias resolved from the live route table"); + return entry; + }) + .toList(); + } + + static BigDecimal orZero(BigDecimal value) { + return value == null ? BigDecimal.ZERO : value; + } +} diff --git a/src/main/java/com/anindra/path/gateway/ModelCatalogService.java b/src/main/java/com/anindra/path/gateway/ModelCatalogService.java index 625e0c5..ed62ef6 100644 --- a/src/main/java/com/anindra/path/gateway/ModelCatalogService.java +++ b/src/main/java/com/anindra/path/gateway/ModelCatalogService.java @@ -26,6 +26,8 @@ public class ModelCatalogService { private final ModelRouteRepository routes; private final ProviderRepository providers; + private final ModelAliasService aliases; + private final com.anindra.path.admin.AliasService operatorAliases; public Map list() { Map byId = providers.findAll().stream() @@ -43,6 +45,19 @@ public Map list() { List> data = new ArrayList<>(); grouped.forEach((modelId, modelRoutes) -> data.add(describe(modelId, modelRoutes, byId))); + data.addAll(aliases.describeAll()); + operatorAliases.list().forEach(alias -> { + Map entry = new LinkedHashMap<>(); + entry.put("id", alias.get("alias")); + entry.put("object", "model"); + entry.put("owned_by", "path"); + entry.put("resolves_to", alias.get("resolves_to")); + entry.put("models", alias.get("models")); + entry.put("description", alias.get("description") == null + ? "Operator-defined alias" : alias.get("description")); + data.add(entry); + }); + Map out = new LinkedHashMap<>(); out.put("object", "list"); out.put("data", data); @@ -50,6 +65,14 @@ public Map list() { } public Optional> find(String modelId) { + if (operatorAliases.exists(modelId)) { + List chain = operatorAliases.chainFor(modelId); + return chain.isEmpty() ? Optional.empty() : find(chain.get(0)); + } + if (aliases.isAlias(modelId)) { + return aliases.resolve(modelId).flatMap(this::find).or(() -> + Optional.of(resolveAliasEntry(modelId))); + } List modelRoutes = routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc(modelId); if (modelRoutes.isEmpty()) { return Optional.empty(); @@ -59,6 +82,15 @@ public Optional> find(String modelId) { return Optional.of(describe(modelId, modelRoutes, byId)); } + private Map resolveAliasEntry(String alias) { + Map entry = new LinkedHashMap<>(); + entry.put("id", alias); + entry.put("object", "model"); + entry.put("owned_by", "path"); + entry.put("resolves_to", null); + return entry; + } + private Map describe(String modelId, List modelRoutes, Map byId) { List> routing = new ArrayList<>(); diff --git a/src/main/java/com/anindra/path/gateway/ModerationService.java b/src/main/java/com/anindra/path/gateway/ModerationService.java new file mode 100644 index 0000000..a2be23c --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ModerationService.java @@ -0,0 +1,112 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Pre-flight moderation for OpenAI-shaped providers. Off by default, because it doubles + * latency and cost for the majority of benign traffic. + * + *

A flagged request is refused with 400 {@code content_flagged} and the categories; + * it is never silently dropped. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class ModerationService { + + private final GatewayProperties props; + private final RouteResolver resolver; + private final ProviderCredentials credentials; + private final UpstreamClient upstream; + private final ObjectMapper mapper; + + public boolean enabled() { + return props.getGuardrails().isModeration(); + } + + public void check(ChatRequestSpec spec, ClientKey key) { + if (!enabled()) { + return; + } + String text = TokenCounter.promptText(spec.body()); + if (text.isBlank()) { + return; + } + + List chain = resolver.resolve(spec, key.getId()); + RouteResolver.ResolvedRoute route = chain.stream() + .filter(r -> r.provider().getType() == ProviderType.OPENAI) + .findFirst() + .orElse(null); + if (route == null) { + // Moderation is an OpenAI-shaped surface; providers that lack one are not + // silently moderated by a different provider's key. + return; + } + + Provider provider = route.provider(); + String apiKey = credentials.apiKey(provider); + if (apiKey == null) { + log.warn("Moderation skipped: provider '{}' has no credential", provider.getSlug()); + return; + } + + ObjectNode payload = mapper.createObjectNode(); + payload.put("model", props.getGuardrails().getModerationModel()); + payload.put("input", text); + + UpstreamClient.Response response; + try { + Map headers = ProviderUrls.headers(provider, apiKey, false); + headers.put("Content-Type", "application/json"); + response = upstream.postJson(ProviderUrls.moderation(provider), + headers, + payload.toString(), Duration.ofMillis(provider.getRequestTimeoutMs() == null + ? 5_000 : Math.min(provider.getRequestTimeoutMs(), 10_000))); + } catch (Exception e) { + // A moderation outage must not take the gateway down; log and continue. + log.warn("Moderation call failed for '{}': {}", provider.getSlug(), e.getMessage()); + return; + } + if (!response.ok()) { + log.warn("Moderation for '{}' returned {}", provider.getSlug(), response.status()); + return; + } + + try { + JsonNode body = mapper.readTree(response.body()); + JsonNode result = body.path("results").path(0); + if (!result.path("flagged").asBoolean(false)) { + return; + } + List categories = new ArrayList<>(); + result.path("categories").fields().forEachRemaining(entry -> { + if (entry.getValue().asBoolean(false)) { + categories.add(entry.getKey()); + } + }); + throw new ApiException(400, "invalid_request_error", "content_flagged", + "The request was blocked by moderation (" + String.join(", ", categories) + + ").", "messages"); + } catch (ApiException e) { + throw e; + } catch (Exception e) { + log.warn("Could not parse moderation response: {}", e.getMessage()); + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/OpenAiAdapter.java b/src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java similarity index 94% rename from src/main/java/com/anindra/path/gateway/OpenAiAdapter.java rename to src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java index b34451c..a36dbbe 100644 --- a/src/main/java/com/anindra/path/gateway/OpenAiAdapter.java +++ b/src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java @@ -25,7 +25,7 @@ @Component @RequiredArgsConstructor @Slf4j -public class OpenAiAdapter implements ProviderAdapter { +public class OpenAiCompatibleAdapter implements ProviderAdapter { private final ObjectMapper mapper; @@ -46,10 +46,9 @@ public UpstreamRequest build(ChatRequestSpec spec, ModelRoute route, Provider pr streamOptions.put("include_usage", true); } - Map headers = new LinkedHashMap<>(); - headers.put("Authorization", "Bearer " + apiKey); - headers.put("Accept", stream ? "text/event-stream" : "application/json"); - return new UpstreamRequest(join(provider.getBaseUrl(), "/chat/completions"), headers, body.toString()); + Map headers = ProviderUrls.headers(provider, apiKey, stream); + return new UpstreamRequest(ProviderUrls.chat(provider, route.getUpstreamModelId()), + headers, body.toString()); } @Override diff --git a/src/main/java/com/anindra/path/gateway/PiiRedactor.java b/src/main/java/com/anindra/path/gateway/PiiRedactor.java new file mode 100644 index 0000000..ee82eeb --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/PiiRedactor.java @@ -0,0 +1,148 @@ +package com.anindra.path.gateway; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Finds and replaces common personal identifiers with stable tokens. + * + *

Pure and side-effect free: the same input always produces the same output, which is + * what makes it safe to run on the request path. Tokens are numbered per call + * ({@code [EMAIL_1]}), so a response that echoes a token stays readable and two distinct + * values never collapse into one. + * + *

Order matters. Card-like digit runs are matched before phone numbers, because a + * 16-digit card also looks like a very long phone number. + */ +public final class PiiRedactor { + + private PiiRedactor() { + } + + /** @param counts how many of each kind was replaced, keyed by the token name. */ + public record Result(String text, Map counts) { + public boolean redacted() { + return !counts.isEmpty(); + } + + public int total() { + return counts.values().stream().mapToInt(Integer::intValue).sum(); + } + } + + private static final Pattern EMAIL = Pattern.compile( + "[A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z]{2,}"); + + /** 13-19 digits, optionally grouped by spaces or dashes. Validated with Luhn below. */ + private static final Pattern CARD = Pattern.compile( + "(? counts = new LinkedHashMap<>(); + String out = text; + out = replace(out, "EMAIL", EMAIL, m -> true, counts); + out = replace(out, "CARD", CARD, PiiRedactor::luhnValid, counts); + out = replace(out, "AADHAAR", AADHAAR, m -> true, counts); + out = replace(out, "PAN", PAN, m -> true, counts); + out = replace(out, "PHONE", PHONE, PiiRedactor::looksLikePhone, counts); + return new Result(out, counts); + } + + private static String replace(String text, String token, Pattern pattern, + Function accept, Map counts) { + Matcher matcher = pattern.matcher(text); + StringBuilder out = new StringBuilder(); + int n = 0; + while (matcher.find()) { + if (!accept.apply(matcher)) { + matcher.appendReplacement(out, Matcher.quoteReplacement(matcher.group())); + continue; + } + n++; + counts.merge(token, 1, Integer::sum); + matcher.appendReplacement(out, Matcher.quoteReplacement("[" + token + "_" + n + "]")); + } + matcher.appendTail(out); + return out.toString(); + } + + /** A 13-19 digit run is only a card if it passes the checksum. */ + static boolean luhnValid(Matcher matcher) { + String digits = matcher.group().replaceAll("[^0-9]", ""); + if (digits.length() < 13 || digits.length() > 19) { + return false; + } + int sum = 0; + boolean doubleIt = false; + for (int i = digits.length() - 1; i >= 0; i--) { + int d = digits.charAt(i) - '0'; + if (doubleIt) { + d *= 2; + if (d > 9) { + d -= 9; + } + } + sum += d; + doubleIt = !doubleIt; + } + return sum % 10 == 0; + } + + /** + * Rejects things that look numeric but are not phone numbers: IDs, timestamps, order + * numbers and long decimal fractions. A phone number has 8-15 digits and is not part + * of a longer run. + */ + static boolean looksLikePhone(Matcher matcher) { + String whole = matcher.group().trim(); + String digits = whole.replaceAll("[^0-9]", ""); + if (digits.length() < 8 || digits.length() > 15) { + return false; + } + // A date is the common false positive: 2026-11-17 has eight digits and separators. + if (whole.matches("\\d{4}-\\d{2}-\\d{2}.*")) { + return false; + } + // So is a version or a decimal fraction: 1.2.3.4567 must stay readable. + if (whole.contains(".") && !whole.contains("@")) { + return false; + } + // A leading zero with too few digits is an id, not a phone number. + if (whole.startsWith("0") && digits.length() < 10) { + return false; + } + if (!whole.startsWith("+")) { + boolean hasSeparator = whole.matches(".*[ ().-].*"); + if (!hasSeparator) { + // Contiguous digits are only a phone when they look like a mobile number. + // This is what keeps a 10-digit epoch timestamp ("1700000000") readable. + if (digits.length() < 10 || digits.length() > 13) { + return false; + } + return digits.charAt(0) >= '6' && digits.charAt(0) <= '9'; + } + } + return true; + } +} diff --git a/src/main/java/com/anindra/path/gateway/PromptSanitizer.java b/src/main/java/com/anindra/path/gateway/PromptSanitizer.java new file mode 100644 index 0000000..05dd779 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/PromptSanitizer.java @@ -0,0 +1,74 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +/** + * Applies {@link PiiRedactor} to a request body: message contents, tool results and tool + * definitions. Used both for "redact before send" and for prompt logs, so what is stored + * and what a provider sees are redacted by the same rules. + */ +@Component +@RequiredArgsConstructor +public class PromptSanitizer { + + private final GatewayProperties props; + + public boolean redactBeforeSend() { + return props.getGuardrails().isRedactBeforeSend(); + } + + public boolean redactPromptLogs() { + return props.getGuardrails().isPromptLogRedaction(); + } + + /** @return a redacted deep copy; the input is never modified. */ + public ObjectNode redactBody(ObjectNode body) { + ObjectNode copy = body.deepCopy(); + JsonNode messages = copy.get("messages"); + if (messages != null && messages.isArray()) { + for (JsonNode message : messages) { + if (message instanceof ObjectNode node) { + redactField(node, "content"); + redactField(node, "tool_calls"); + redactField(node, "name"); + } + } + } + redactField(copy, "tools"); + redactField(copy, "prompt"); + redactField(copy, "input"); + redactField(copy, "system"); + return copy; + } + + public String redactText(String text) { + return PiiRedactor.redact(text).text(); + } + + private void redactField(ObjectNode node, String field) { + JsonNode value = node.get(field); + if (value == null) { + return; + } + if (value.isTextual()) { + node.put(field, redactText(value.asText())); + } else if (value.isArray()) { + ArrayNode array = (ArrayNode) value; + for (int i = 0; i < array.size(); i++) { + JsonNode part = array.get(i); + if (part.isTextual()) { + array.set(i, TextNode.valueOf(redactText(part.asText()))); + } else if (part instanceof ObjectNode obj) { + redactField(obj, "text"); + redactField(obj, "content"); + } + } + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/ProviderCircuitBreaker.java b/src/main/java/com/anindra/path/gateway/ProviderCircuitBreaker.java new file mode 100644 index 0000000..e504b47 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ProviderCircuitBreaker.java @@ -0,0 +1,113 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Per-provider circuit breaker. + * + * A provider that keeps failing is parked for {@code open-duration}, during which routes + * on it are skipped in microseconds instead of costing every request a connect timeout. + * After the window a probe request is allowed; success closes the circuit, failure reopens it. + * An upstream 429 with {@code Retry-After} parks the provider for exactly that long. + */ +@Component +@RequiredArgsConstructor +public class ProviderCircuitBreaker { + + public enum State {CLOSED, OPEN, HALF_OPEN} + + private static final class Entry { + State state = State.CLOSED; + int failures; + long openUntilMs; + } + + private final Map entries = new ConcurrentHashMap<>(); + private final GatewayProperties props; + private final com.anindra.path.admin.AlertService alerts; + + /** Read-only projection used by the admin API. */ + public Map snapshot() { + Map out = new LinkedHashMap<>(); + entries.forEach((providerId, entry) -> out.put(String.valueOf(providerId), Map.of( + "state", entry.state.name().toLowerCase(), + "failures", entry.failures, + "retry_after_seconds", retryAfterSeconds(providerId)))); + return out; + } + + public State state(Long providerId) { + Entry entry = entries.get(providerId); + return entry == null ? State.CLOSED : entry.state; + } + + public long retryAfterSeconds(Long providerId) { + Entry entry = entries.get(providerId); + if (entry == null || entry.state != State.OPEN) { + return 0; + } + return Math.max(0, (entry.openUntilMs - System.currentTimeMillis()) / 1000); + } + + /** True when a request may be sent to this provider right now. */ + public boolean isAvailable(Long providerId) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + if (entry.state != State.OPEN) { + return true; + } + if (System.currentTimeMillis() >= entry.openUntilMs) { + entry.state = State.HALF_OPEN; + return true; + } + return false; + } + } + + public void onSuccess(Long providerId) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + entry.state = State.CLOSED; + entry.failures = 0; + entry.openUntilMs = 0; + } + } + + public void onFailure(Long providerId) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + entry.failures++; + if (entry.state == State.HALF_OPEN + || entry.failures >= Math.max(1, props.getBreaker().getFailureThreshold())) { + openUntil(entry, props.getBreaker().getOpenDuration()); + alerts.notify("provider.circuit_open", Map.of( + "provider_id", providerId, + "failures", entry.failures, + "retry_after_seconds", Math.max(0, (entry.openUntilMs - System.currentTimeMillis()) / 1000))); + } + } + } + + /** Upstream asked us to back off; park the provider for exactly that long. */ + public void onRateLimited(Long providerId, Duration retryAfter) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + openUntil(entry, retryAfter == null ? props.getBreaker().getOpenDuration() : retryAfter); + alerts.notify("provider.rate_limited", Map.of( + "provider_id", providerId, + "retry_after_seconds", Math.max(0, (entry.openUntilMs - System.currentTimeMillis()) / 1000))); + } + } + + private static void openUntil(Entry entry, Duration duration) { + entry.state = State.OPEN; + entry.openUntilMs = System.currentTimeMillis() + Math.max(1, duration.toMillis()); + } +} diff --git a/src/main/java/com/anindra/path/gateway/ProviderCredentials.java b/src/main/java/com/anindra/path/gateway/ProviderCredentials.java index cba8221..879754b 100644 --- a/src/main/java/com/anindra/path/gateway/ProviderCredentials.java +++ b/src/main/java/com/anindra/path/gateway/ProviderCredentials.java @@ -62,6 +62,7 @@ private String configKey(String slug) { case "openai" -> props.getProviders().getOpenai().getApiKey(); case "anthropic" -> props.getProviders().getAnthropic().getApiKey(); case "groq" -> props.getProviders().getGroq().getApiKey(); + case "gemini" -> props.getProviders().getGemini().getApiKey(); default -> null; }; return value == null || value.isBlank() ? null : value.trim(); diff --git a/src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java b/src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java new file mode 100644 index 0000000..3948076 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java @@ -0,0 +1,58 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.Provider; +import com.anindra.path.repo.ProviderRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; +import org.springframework.boot.actuate.health.Status; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Surfaces provider circuit state in /actuator/health. + * + * Reports DEGRADED (not DOWN) when a provider is parked: a single failing upstream should + * not take the gateway out of a load-balancer rotation, because fallback still works. + */ +@Component +@RequiredArgsConstructor +public class ProviderHealthIndicator implements HealthIndicator { + + /** Boot ranks unknown status codes below UP, so DEGRADED would never win the aggregate. */ + private static final Status ALL_PARKED = + new Status("OUT_OF_SERVICE", "every enabled provider is circuit-open"); + + private final ProviderRepository providers; + private final ProviderCircuitBreaker breaker; + + @Override + public Health health() { + Map details = new LinkedHashMap<>(); + int enabled = 0; + int open = 0; + + for (Provider provider : providers.findByEnabledTrueOrderBySlugAsc()) { + ProviderCircuitBreaker.State state = breaker.state(provider.getId()); + Map entry = new LinkedHashMap<>(); + entry.put("circuit", state.name().toLowerCase()); + entry.put("retry_after_seconds", breaker.retryAfterSeconds(provider.getId())); + details.put(provider.getSlug(), entry); + enabled++; + if (state == ProviderCircuitBreaker.State.OPEN) { + open++; + } + } + + // One parked provider still has fallback, so the gateway stays in rotation. + // Only a total outage is OUT_OF_SERVICE (503). + boolean totalOutage = enabled > 0 && open == enabled; + return (totalOutage ? Health.status(ALL_PARKED) : Health.up()) + .withDetail("open_circuits", open) + .withDetail("enabled_providers", enabled) + .withDetail("providers", details) + .build(); + } +} diff --git a/src/main/java/com/anindra/path/gateway/ProviderUrls.java b/src/main/java/com/anindra/path/gateway/ProviderUrls.java new file mode 100644 index 0000000..147711d --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ProviderUrls.java @@ -0,0 +1,74 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.Provider; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * URL and auth shapes for a provider. + * + *

A provider with {@code apiVersion} set is an Azure OpenAI resource: the model id is a + * deployment name in the path, the version is a query parameter, and the credential goes + * in {@code api-key} rather than a bearer header. Everything else is the plain OpenAI + * shape. + */ +public final class ProviderUrls { + + private ProviderUrls() { + } + + public static boolean isAzure(Provider provider) { + return provider.getApiVersion() != null && !provider.getApiVersion().isBlank(); + } + + public static String chat(Provider provider, String model) { + return isAzure(provider) + ? azurePath(provider, model, "/chat/completions") + : join(provider.getBaseUrl(), "/chat/completions"); + } + + public static String embeddings(Provider provider, String model) { + return isAzure(provider) + ? azurePath(provider, model, "/embeddings") + : join(provider.getBaseUrl(), "/embeddings"); + } + + /** Azure has no /models; a deployment listing is the closest equivalent. */ + public static String models(Provider provider) { + return isAzure(provider) + ? join(provider.getBaseUrl(), "/openai/models?api-version=" + provider.getApiVersion()) + : join(provider.getBaseUrl(), "/models"); + } + + public static String moderation(Provider provider) { + return isAzure(provider) + ? join(provider.getBaseUrl(), + "/openai/deployments/" + provider.getApiVersion() + "/moderations") + : join(provider.getBaseUrl(), "/moderations"); + } + + public static Map headers(Provider provider, String apiKey, boolean stream) { + Map headers = new LinkedHashMap<>(); + if (isAzure(provider)) { + headers.put("api-key", apiKey); + } else { + headers.put("Authorization", "Bearer " + apiKey); + } + headers.put("Accept", stream ? "text/event-stream" : "application/json"); + return headers; + } + + private static String azurePath(Provider provider, String deployment, String suffix) { + String base = provider.getBaseUrl().endsWith("/") + ? provider.getBaseUrl().substring(0, provider.getBaseUrl().length() - 1) + : provider.getBaseUrl(); + return base + "/openai/deployments/" + deployment + suffix + + "?api-version=" + provider.getApiVersion(); + } + + private static String join(String base, String path) { + String trimmed = base.endsWith("/") ? base.substring(0, base.length() - 1) : base; + return trimmed + path; + } +} diff --git a/src/main/java/com/anindra/path/gateway/RateLimitStore.java b/src/main/java/com/anindra/path/gateway/RateLimitStore.java new file mode 100644 index 0000000..9d08841 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/RateLimitStore.java @@ -0,0 +1,15 @@ +package com.anindra.path.gateway; + +/** Pluggable RPM limiter: in-memory for one node, Valkey for a cluster. */ +public interface RateLimitStore { + + /** + * @param bucket "key:<id>" or "project:<id>" + * @return 0 when the call may proceed, otherwise the Retry-After seconds. + */ + int acquire(String bucket, int rpmLimit); + + default int acquire(Long keyId, int rpmLimit) { + return acquire("key:" + keyId, rpmLimit); + } +} diff --git a/src/main/java/com/anindra/path/gateway/ResponseCacheService.java b/src/main/java/com/anindra/path/gateway/ResponseCacheService.java new file mode 100644 index 0000000..daef6e0 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ResponseCacheService.java @@ -0,0 +1,107 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.ResponseCacheEntry; +import com.anindra.path.repo.ResponseCacheEntryRepository; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Optional; + +/** + * Exact-match response cache. + * + * Only deterministic requests qualify: the route must opt in, the request must set + * {@code temperature: 0} explicitly, and it must not use tools (tool calls are not + * side-effect free to replay). Keys are scoped to the client key so one tenant can + * never read another's cached completion. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class ResponseCacheService { + + public record Entry(String contentType, String body, String providerSlug, BigDecimal costUsd, + int promptTokens, int completionTokens, int totalTokens) { + } + + private final ResponseCacheEntryRepository entries; + private final GatewayProperties props; + + public static boolean eligible(ChatRequestSpec spec, ModelRoute route) { + if (!route.isCacheEnabled() || spec.hasTools()) { + return false; + } + JsonNode temperature = spec.body().get("temperature"); + return temperature != null && temperature.isNumber() && temperature.asDouble() == 0.0; + } + + public String key(Long clientKeyId, String model, String requestBody) { + return ApiKeyService.hash(clientKeyId + "|" + model + "|" + requestBody); + } + + public Optional lookup(String cacheKey) { + return entries.findByCacheKey(cacheKey).flatMap(record -> { + if (record.getExpiresAt().isBefore(Instant.now())) { + entries.delete(record); + return Optional.empty(); + } + return Optional.of(new Entry(record.getContentType(), record.getBody(), + record.getProviderSlug(), record.getCostUsd(), + record.getPromptTokens(), record.getCompletionTokens(), record.getTotalTokens())); + }); + } + + @Transactional + public void store(String cacheKey, Long clientKeyId, String publicModel, String providerSlug, + ModelRoute route, String contentType, String body, TokenUsage usage, + BigDecimal costUsd) { + store(cacheKey, clientKeyId, publicModel, providerSlug, route, contentType, body, usage, + costUsd, null, null); + } + + /** @param embedding prompt vector for the semantic tier; null stores an exact-only entry */ + @Transactional + public void store(String cacheKey, Long clientKeyId, String publicModel, String providerSlug, + ModelRoute route, String contentType, String body, TokenUsage usage, + BigDecimal costUsd, String embedding, String embeddingModel) { + if (body == null || body.isBlank() + || body.getBytes(StandardCharsets.UTF_8).length > props.getCache().getMaxBodyBytes()) { + return; + } + entries.findByCacheKey(cacheKey).ifPresent(entries::delete); + + ResponseCacheEntry record = new ResponseCacheEntry(); + record.setCacheKey(cacheKey); + record.setClientKeyId(clientKeyId); + record.setPublicModelId(publicModel); + record.setProviderSlug(providerSlug); + record.setContentType(contentType); + record.setBody(body); + record.setEmbedding(embedding); + record.setEmbeddingModel(embedding == null ? null : embeddingModel); + record.setPromptTokens(usage == null ? 0 : usage.prompt()); + record.setCompletionTokens(usage == null ? 0 : usage.completion()); + record.setTotalTokens(usage == null ? 0 : usage.total()); + record.setCostUsd(costUsd == null ? BigDecimal.ZERO : costUsd); + record.setExpiresAt(Instant.now().plusSeconds(Math.max(1, route.getCacheTtlSeconds()))); + entries.save(record); + } + + @Scheduled(fixedDelay = 1_800_000) + @Transactional + void purgeExpired() { + long removed = entries.deleteByExpiresAtBefore(Instant.now()); + if (removed > 0) { + log.info("Purged {} expired cache entries", removed); + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/RouteResolver.java b/src/main/java/com/anindra/path/gateway/RouteResolver.java index e4115f7..c84097f 100644 --- a/src/main/java/com/anindra/path/gateway/RouteResolver.java +++ b/src/main/java/com/anindra/path/gateway/RouteResolver.java @@ -26,8 +26,94 @@ public String providerSlug() { private final ModelRouteRepository routes; private final ProviderRepository providers; + private final ModelAliasService aliases; + private final com.anindra.path.admin.AliasService operatorAliases; + private final RoutingMetrics metrics; public List resolve(ChatRequestSpec spec) { + return resolve(spec, null); + } + + /** + * Reorders a resolved group by the route's strategy. The declared priority is still + * the default and the fallback spine: this only changes which route is tried first. + */ + private List adaptive(List group, String publicModel) { + if (group.size() < 2) { + return group; + } + String strategy = group.get(0).route().getStrategy(); + if (strategy == null || "priority".equalsIgnoreCase(strategy)) { + return group; + } + + java.util.Comparator byLatency = java.util.Comparator.comparingDouble( + r -> { + double ewma = metrics.latencyMs(r.providerSlug(), publicModel); + return Double.isNaN(ewma) ? Double.MAX_VALUE : ewma; + }); + java.util.Comparator byPrice = java.util.Comparator.comparing( + r -> r.route().getInputPricePerMillion() == null + ? new java.math.BigDecimal("999999") + : r.route().getInputPricePerMillion()); + java.util.Comparator byPriority = + java.util.Comparator.comparingInt(r -> r.route().getPriority()); + + List ordered = new ArrayList<>(group); + if ("fastest".equalsIgnoreCase(strategy)) { + ordered.sort(byLatency.thenComparing(byPrice).thenComparing(byPriority)); + return ordered; + } + if ("cheapest-viable".equalsIgnoreCase(strategy)) { + Integer slo = group.get(0).route().getLatencySloMs(); + if (slo == null || slo <= 0) { + ordered.sort(byPrice.thenComparing(byPriority)); + return ordered; + } + List viable = new ArrayList<>(); + List tooSlow = new ArrayList<>(); + for (ResolvedRoute route : ordered) { + double ewma = metrics.latencyMs(route.providerSlug(), publicModel); + // Unknown latency is treated as viable: routing away from a provider that + // has never been called would be a cold-start deadlock. + if (Double.isNaN(ewma) || ewma <= slo) { + viable.add(route); + } else { + tooSlow.add(route); + } + } + viable.sort(byPrice.thenComparing(byPriority)); + tooSlow.sort(byLatency.thenComparing(byPrice)); + viable.addAll(tooSlow); + return viable; + } + return group; + } + + /** Expands operator-defined aliases, following nested ones, bounded to avoid a loop. */ + private List expandOperatorAliases(List models) { + List expanded = new ArrayList<>(); + for (String model : models) { + expandInto(model, expanded, 0); + } + return expanded; + } + + private void expandInto(String model, List out, int depth) { + List chain = operatorAliases.chainFor(model); + if (chain.isEmpty() || depth > 5) { + if (!out.contains(model)) { + out.add(model); + } + return; + } + for (String hop : chain) { + expandInto(hop, out, depth + 1); + } + } + + /** @param keyId used for stable canary assignment; null means no weighting. */ + public List resolve(ChatRequestSpec spec, Long keyId) { List wantedModels = new ArrayList<>(); if (!spec.fallbackModels().isEmpty()) { wantedModels.addAll(spec.fallbackModels()); @@ -35,12 +121,17 @@ public List resolve(ChatRequestSpec spec) { if (!wantedModels.contains(spec.publicModel())) { wantedModels.add(0, spec.publicModel()); } + // An operator-defined alias is a chain, not a model: expand it in place so the + // declared order becomes the fallback order. + wantedModels = expandOperatorAliases(wantedModels); List chain = new ArrayList<>(); Set seen = new LinkedHashSet<>(); for (String model : wantedModels) { - for (ModelRoute route : routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc(model)) { + String target = aliases.isAlias(model) ? aliases.resolve(model).orElse(model) : model; + List group = new ArrayList<>(); + for (ModelRoute route : routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc(target)) { Provider provider = providers.findById(route.getProviderId()).orElse(null); if (provider == null || !provider.isEnabled()) { continue; @@ -49,8 +140,9 @@ public List resolve(ChatRequestSpec spec) { if (!seen.add(key)) { continue; } - chain.add(new ResolvedRoute(route, provider)); + group.add(new ResolvedRoute(route, provider)); } + chain.addAll(adaptive(weighted(group, keyId, target), target)); } if (!spec.providerOrder().isEmpty()) { @@ -72,4 +164,39 @@ public List resolve(ChatRequestSpec spec) { } return chain; } + + /** Stable per-key weighted shuffle: the chosen route leads, the rest stay as fallback. */ + private static List weighted(List group, Long keyId, String model) { + if (group.size() < 2 || keyId == null) { + return group; + } + // Weighting is opt-in: a chain of default-weight routes keeps priority order, + // otherwise fallback would become a lottery. + boolean configured = group.stream().anyMatch(r -> r.route().getWeight() != 100); + if (!configured) { + return group; + } + int total = group.stream().mapToInt(r -> Math.max(0, r.route().getWeight())).sum(); + if (total <= 0) { + return group; + } + int point = (int) Math.floorMod(keyId * 131L + model.hashCode(), (long) total); + int cursor = 0; + int chosen = 0; + for (int i = 0; i < group.size(); i++) { + cursor += Math.max(0, group.get(i).route().getWeight()); + if (point < cursor) { + chosen = i; + break; + } + } + List ordered = new ArrayList<>(group.size()); + ordered.add(group.get(chosen)); + for (int i = 0; i < group.size(); i++) { + if (i != chosen) { + ordered.add(group.get(i)); + } + } + return ordered; + } } diff --git a/src/main/java/com/anindra/path/gateway/RoutingMetrics.java b/src/main/java/com/anindra/path/gateway/RoutingMetrics.java new file mode 100644 index 0000000..90dfe50 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/RoutingMetrics.java @@ -0,0 +1,96 @@ +package com.anindra.path.gateway; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Exponentially weighted latency and error rate per (provider, public model). + * + *

Kept in memory on purpose, like the circuit breaker: it is an ordering hint, not a + * source of truth, and a restart should only cost a cold start. That limitation is + * documented in docs/LIMITS.md. + */ +@Component +public class RoutingMetrics { + + /** Weight of a new sample. 0.2 gives a ~5-sample half-life: fast to react, not jittery. */ + private static final double ALPHA = 0.2; + + private static final class Sample { + double latencyMs; + double errorRate; + long count; + } + + private final Map samples = new ConcurrentHashMap<>(); + + private static String key(String providerSlug, String publicModel) { + return providerSlug + "|" + publicModel; + } + + public void recordSuccess(String providerSlug, String publicModel, long latencyMs) { + if (providerSlug == null || publicModel == null) { + return; + } + update(providerSlug, publicModel, Math.max(1, latencyMs), 0.0); + } + + public void recordFailure(String providerSlug, String publicModel) { + if (providerSlug == null || publicModel == null) { + return; + } + update(providerSlug, publicModel, 0, 1.0); + } + + private void update(String providerSlug, String publicModel, long latencyMs, double error) { + samples.compute(key(providerSlug, publicModel), (k, current) -> { + Sample next = current == null ? new Sample() : current; + if (next.count == 0) { + next.latencyMs = latencyMs; + next.errorRate = error; + } else { + // A failure carries no latency sample; keep the last latency rather than + // dragging it toward zero. + if (latencyMs > 0) { + next.latencyMs = ALPHA * latencyMs + (1 - ALPHA) * next.latencyMs; + } + next.errorRate = ALPHA * error + (1 - ALPHA) * next.errorRate; + } + next.count++; + return next; + }); + } + + /** @return EWMA latency in ms, or {@link Double#NaN} when there is no sample yet. */ + public double latencyMs(String providerSlug, String publicModel) { + Sample sample = samples.get(key(providerSlug, publicModel)); + return sample == null || sample.count == 0 ? Double.NaN : sample.latencyMs; + } + + public double errorRate(String providerSlug, String publicModel) { + Sample sample = samples.get(key(providerSlug, publicModel)); + return sample == null ? 0.0 : sample.errorRate; + } + + public long sampleCount(String providerSlug, String publicModel) { + Sample sample = samples.get(key(providerSlug, publicModel)); + return sample == null ? 0 : sample.count; + } + + public void reset() { + samples.clear(); + } + + /** Drops state for providers that have not been used in a while. */ + @Scheduled(fixedDelay = 600_000) + void evictIdle() { + // Bounded by distinct (provider, model) pairs, which is small; nothing to do unless + // the catalogue churns. Kept as a hook so growth is not unbounded by accident. + if (samples.size() > 10_000) { + samples.clear(); + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/SemanticCacheService.java b/src/main/java/com/anindra/path/gateway/SemanticCacheService.java new file mode 100644 index 0000000..54e19c9 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/SemanticCacheService.java @@ -0,0 +1,170 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.ResponseCacheEntry; +import com.anindra.path.repo.ResponseCacheEntryRepository; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.LongAdder; + +/** + * Second cache tier: a near-duplicate prompt may reuse a stored completion. + * + *

The safety of the whole thing rests on staying inside the exact cache's rules — the + * route opted into caching, the request is {@code temperature: 0} and tool-free, and the + * entry is scoped to one client key. On top of that this tier is explicitly opt-in per + * route, because a hit here answers a different prompt. + * + *

Cost: the prompt has to be embedded, which is an upstream call. That is why the tier + * is off by default and why it is only consulted after the exact lookup misses. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class SemanticCacheService { + + /** @param similarity the cosine distance the match was accepted at, for the response header */ + public record Hit(ResponseCacheEntry entry, double similarity) { + } + + private final ResponseCacheEntryRepository entries; + private final EmbeddingsService embeddings; + private final GatewayProperties props; + private final ObjectMapper mapper; + private final LongAdder hits = new LongAdder(); + + public boolean enabled() { + String model = props.getCache().getSemanticModel(); + return model != null && !model.isBlank(); + } + + public long hitCount() { + return hits.sum(); + } + + /** + * @return the prompt vector, or null when the tier is off or embedding failed. A + * failure here must never fail the request: the gateway simply behaves as if + * the tier were absent. + */ + public double[] embed(String text, ClientKey key) { + if (!enabled()) { + return null; + } + ObjectNode body = mapper.createObjectNode(); + body.put("model", props.getCache().getSemanticModel()); + body.put("input", text); + try { + EmbeddingsService.Result result = embeddings.embed(body, key); + JsonNode data = result.body().path("data").path(0).path("embedding"); + if (!data.isArray() || data.isEmpty()) { + return null; + } + double[] vector = new double[data.size()]; + for (int i = 0; i < data.size(); i++) { + vector[i] = data.get(i).asDouble(); + } + return vector; + } catch (Exception e) { + log.debug("Semantic cache skipped: could not embed the prompt ({})", e.getMessage()); + return null; + } + } + + /** + * Scans the candidate entries for this key and model. The candidate set is bounded by + * {@code semantic-max-entries}, so this stays a small comparison rather than a table + * scan; a larger deployment wants an ANN index, which is called out in docs/LIMITS.md. + */ + public Optional lookup(Long clientKeyId, String publicModel, double[] vector, double threshold) { + if (vector == null || vector.length == 0) { + return Optional.empty(); + } + List candidates = entries.findByClientKeyIdAndPublicModelIdAndEmbeddingIsNotNull( + clientKeyId, publicModel, + PageRequest.of(0, Math.max(1, props.getCache().getSemanticMaxEntries()))); + + Instant now = Instant.now(); + double best = -1; + ResponseCacheEntry bestEntry = null; + for (ResponseCacheEntry candidate : candidates) { + if (candidate.getExpiresAt().isBefore(now)) { + continue; + } + double[] other = parse(candidate.getEmbedding()); + if (other == null) { + continue; + } + double similarity = cosine(vector, other); + if (similarity >= threshold && similarity > best) { + best = similarity; + bestEntry = candidate; + } + } + if (bestEntry == null) { + return Optional.empty(); + } + hits.increment(); + return Optional.of(new Hit(bestEntry, best)); + } + + public String encode(double[] vector) { + if (vector == null || vector.length == 0) { + return null; + } + List parts = new ArrayList<>(vector.length); + for (double value : vector) { + parts.add(String.valueOf(value)); + } + return String.join(",", parts); + } + + static double cosine(double[] a, double[] b) { + if (a.length != b.length || a.length == 0) { + return -1; + } + double dot = 0; + double normA = 0; + double normB = 0; + for (int i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + if (normA == 0 || normB == 0) { + return -1; + } + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); + } + + private static double[] parse(String encoded) { + if (encoded == null || encoded.isBlank()) { + return null; + } + String[] parts = encoded.split(","); + double[] vector = new double[parts.length]; + try { + for (int i = 0; i < parts.length; i++) { + vector[i] = Double.parseDouble(parts[i]); + } + } catch (NumberFormatException e) { + return null; + } + return vector; + } + + public String model() { + return props.getCache().getSemanticModel(); + } +} diff --git a/src/main/java/com/anindra/path/gateway/StreamService.java b/src/main/java/com/anindra/path/gateway/StreamService.java index 733bfea..beaea4b 100644 --- a/src/main/java/com/anindra/path/gateway/StreamService.java +++ b/src/main/java/com/anindra/path/gateway/StreamService.java @@ -15,7 +15,9 @@ import java.io.IOException; import java.io.OutputStream; +import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.time.Instant; import java.util.EnumMap; import java.util.List; @@ -40,39 +42,70 @@ public class StreamService { private final UpstreamClient upstream; private final UsageService usage; private final CreditGuard creditGuard; + private final ProviderCircuitBreaker breaker; + private final RoutingMetrics metrics; + private final ResponseCacheService cache; private final ObjectMapper mapper; private final GatewayProperties props; private final Map adapters = new EnumMap<>(ProviderType.class); public StreamService(RouteResolver resolver, ProviderCredentials credentials, UpstreamClient upstream, - UsageService usage, CreditGuard creditGuard, ObjectMapper mapper, + UsageService usage, CreditGuard creditGuard, ProviderCircuitBreaker breaker, + RoutingMetrics metrics, ResponseCacheService cache, ObjectMapper mapper, GatewayProperties props, List adapterList) { this.resolver = resolver; this.credentials = credentials; this.upstream = upstream; this.usage = usage; this.creditGuard = creditGuard; + this.breaker = breaker; + this.metrics = metrics; + this.cache = cache; this.mapper = mapper; this.props = props; adapterList.forEach(a -> adapters.put(a.type(), a)); } + /** Route chain plus the budget reserved for it; release the reservation when done. */ + public record Prepared(List chain, + CreditGuard.Reservation reservation) { + } + /** Resolve the fallback chain and authorise spend. Throws before any bytes are sent. */ - public List prepare(ChatRequestSpec spec, ClientKey key) { - List chain = resolver.resolve(spec); - creditGuard.check(key, spec, chain.get(0)); - return chain; + public Prepared prepare(ChatRequestSpec spec, ClientKey key) { + List chain = resolver.resolve(spec, key.getId()); + CreditGuard.Reservation reserved = creditGuard.check(key, spec, chain.get(0)); + return new Prepared(chain, reserved); } /** Blocks for the whole stream. Safe on a virtual thread. */ - public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey key, - List chain) throws IOException { + public boolean stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey key, + List chain) throws IOException { response.setStatus(200); response.setContentType("text/event-stream;charset=UTF-8"); response.setHeader("Cache-Control", "no-cache, no-transform"); response.setHeader("Connection", "keep-alive"); response.setHeader("X-Accel-Buffering", "no"); - OutputStream out = response.getOutputStream(); + + boolean cacheEligible = ResponseCacheService.eligible(spec, chain.get(0).route()); + String cacheKey = cacheEligible + ? cache.key(key.getId(), spec.publicModel(), spec.body().toString()) : null; + if (cacheKey != null) { + var hit = cache.lookup(cacheKey); + if (hit.isPresent()) { + response.setHeader("X-Path-Cache", "hit"); + OutputStream cached = response.getOutputStream(); + cached.write(hit.get().body().getBytes(StandardCharsets.UTF_8)); + cached.flush(); + usage.recordCacheHit(key, spec, chain.get(0), hit.get(), true); + return true; + } + } + response.setHeader("X-Path-Cache", cacheEligible ? "miss" : "off"); + + OutputStream raw = response.getOutputStream(); + Tee tee = cacheEligible ? new Tee(raw, props.getCache().getMaxBodyBytes()) : null; + OutputStream out = tee == null ? raw : tee; out.write(": open\n\n".getBytes(StandardCharsets.UTF_8)); out.flush(); @@ -85,11 +118,23 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey String requestId = null; String finishReason = null; int generatedChars = 0; + long available = chain.stream().filter(r -> breaker.isAvailable(r.provider().getId())).count(); + int maxAttempts = Math.min(chain.size(), Math.max(1, props.getBreaker().getMaxAttempts())); - for (int i = 0; i < chain.size(); i++) { + for (int i = 0; i < chain.size() && attempts < maxAttempts; i++) { RouteResolver.ResolvedRoute rr = chain.get(i); Provider provider = rr.provider(); ProviderAdapter adapter = adapters.get(provider.getType()); + Long providerId = provider.getId(); + + if (available > 0 && !breaker.isAvailable(providerId)) { + lastError = new ApiException(503, "upstream_error", "circuit_open", + "Provider '" + provider.getSlug() + "' is temporarily unavailable."); + failures.add(provider.getSlug() + ": circuit open"); + fallbackFrom = fallbackFrom == null ? provider.getSlug() : fallbackFrom; + continue; + } + attempts++; if (adapter == null) { @@ -121,8 +166,11 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey stream.headers().get("openai-request-id")); } + Duration idleTimeout = provider.getStreamIdleTimeoutMs() != null + ? Duration.ofMillis(provider.getStreamIdleTimeoutMs()) + : props.getStreamIdleTimeout(); while (true) { - String line = stream.poll(props.getStreamIdleTimeout()); + String line = stream.poll(idleTimeout); if (line == null) { break; } @@ -153,7 +201,8 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey TokenUsage tokens = translator.usage(); if (!tokens.reported()) { - tokens = TokenUsage.estimated(spec.estimatePromptTokens(), + tokens = TokenUsage.estimated( + spec.estimatePromptTokens(rr.route().getUpstreamModelId()), Math.max(1, generatedChars / 4)); } if (spec.includeUsage() && tokens.reported()) { @@ -168,9 +217,23 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey log.info("stream ok model={} provider={} tokens={} cached={} cost={} ttft={}ms latency={}ms", spec.publicModel(), provider.getSlug(), tokens.total(), tokens.cached(), cost.toPlainString(), ttftMs, latency); - return; + breaker.onSuccess(providerId); + metrics.recordSuccess(provider.getSlug(), spec.publicModel(), latency); + if (tee != null && cacheKey != null && !tee.overflowed()) { + cache.store(cacheKey, key.getId(), spec.publicModel(), provider.getSlug(), + rr.route(), "text/event-stream", tee.body(), tokens, cost); + } + return true; } catch (UpstreamException e) { + metrics.recordFailure(provider.getSlug(), spec.publicModel()); + if (e.retryable()) { + if (e.status() == 429 && e.retryAfterSeconds() > 0) { + breaker.onRateLimited(providerId, java.time.Duration.ofSeconds(e.retryAfterSeconds())); + } else { + breaker.onFailure(providerId); + } + } lastError = ChatService.toApiException(e); if (!committed && e.retryable() && i < chain.size() - 1) { log.warn("Stream route {} failed before first chunk ({}), trying next", @@ -182,11 +245,14 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey usage.recordError(key, spec, provider.getSlug(), rr.route().getUpstreamModelId(), ChatService.elapsedMs(started), true, attempts, lastError.code(), lastError.getMessage()); emitError(out, lastError); - return; + return false; } catch (TimeoutException e) { + breaker.onFailure(providerId); lastError = new ApiException(504, "upstream_error", "upstream_timeout", - "The upstream sent nothing for " + props.getStreamIdleTimeout().toSeconds() + "s."); + "The upstream sent nothing for " + (provider.getStreamIdleTimeoutMs() != null + ? provider.getStreamIdleTimeoutMs() / 1000 + "s" + : props.getStreamIdleTimeout().toSeconds() + "s") + "."); if (!committed && i < chain.size() - 1) { failures.add(provider.getSlug() + ": idle timeout"); fallbackFrom = fallbackFrom == null ? provider.getSlug() : fallbackFrom; @@ -195,9 +261,10 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey usage.recordError(key, spec, provider.getSlug(), rr.route().getUpstreamModelId(), ChatService.elapsedMs(started), true, attempts, lastError.code(), lastError.getMessage()); emitError(out, lastError); - return; + return false; } catch (IOException e) { + breaker.onFailure(providerId); if (!committed) { lastError = ApiException.upstream(502, "Could not reach " + provider.getSlug() + ": " + e.getMessage(), @@ -218,11 +285,11 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey } catch (IOException clientGone) { log.debug("Client disconnected after upstream failure"); } - return; + return false; } catch (InterruptedException e) { Thread.currentThread().interrupt(); - return; + return false; } finally { if (stream != null) { @@ -240,6 +307,62 @@ public void stream(HttpServletResponse response, ChatRequestSpec spec, ClientKey usage.recordError(key, spec, null, null, ChatService.elapsedMs(started), true, attempts, lastError.code(), lastError.getMessage()); emitError(out, lastError); + return false; + } + + /** Writes through while capturing the SSE payload for the response cache. */ + private static final class Tee extends OutputStream { + + private final OutputStream delegate; + private final StringBuilder captured = new StringBuilder(); + private final int maxBytes; + private boolean overflow; + + private Tee(OutputStream delegate, int maxBytes) { + this.delegate = delegate; + this.maxBytes = maxBytes; + } + + @Override + public void write(int b) throws IOException { + delegate.write(b); + append(new byte[]{(byte) b}, 0, 1); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + delegate.write(b, off, len); + append(b, off, len); + } + + private void append(byte[] b, int off, int len) { + if (overflow) { + return; + } + if (captured.length() + len > maxBytes) { + overflow = true; + return; + } + captured.append(new String(b, off, len, StandardCharsets.UTF_8)); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private boolean overflowed() { + return overflow; + } + + private String body() { + return captured.toString(); + } } private void send(OutputStream out, String data) throws IOException { diff --git a/src/main/java/com/anindra/path/gateway/TokenCountService.java b/src/main/java/com/anindra/path/gateway/TokenCountService.java new file mode 100644 index 0000000..71e0dc5 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/TokenCountService.java @@ -0,0 +1,77 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * {@code POST /v1/tokens/count}: answers "will this prompt fit?" without spending a call. + * + *

Free, but authenticated and bounded exactly like a chat request: the model allowlist, + * the drain state, the concurrency limiter and the RPM limiter all apply, so it cannot be + * used to enumerate the catalogue or to bypass a key's restrictions. + */ +@Service +@RequiredArgsConstructor +public class TokenCountService { + + private final RouteResolver resolver; + private final GatewayProperties props; + + public Map count(ObjectNode body, ClientKey key) { + JsonNode messages = body.get("messages"); + if (messages == null || !messages.isArray() || messages.isEmpty()) { + throw ApiException.badRequest("'messages' must be a non-empty array.", "messages"); + } + ChatRequestSpec spec = ChatRequestSpec.forCounting(body); + if (!CreditGuard.allows(key, spec.publicModel())) { + throw new ApiException(403, "permission_error", "model_not_allowed", + "This API key is not allowed to use model '" + spec.publicModel() + "'."); + } + + List chain = resolver.resolve(spec, key.getId()); + if (chain.isEmpty()) { + throw ApiException.notFound( + "The model '" + spec.publicModel() + "' does not exist.", "model_not_found"); + } + + RouteResolver.ResolvedRoute first = chain.get(0); + String upstreamModel = first.route().getUpstreamModelId(); + TokenCounter.Count count = TokenCounter.count(body, upstreamModel); + Integer contextWindow = first.route().getContextLength(); + + int maxOutput = key.getMaxOutputTokens() != null && key.getMaxOutputTokens() > 0 + ? key.getMaxOutputTokens() + : props.getMaxOutputTokens(); + if (maxOutput <= 0) { + // No ceiling configured: report what the route itself allows. + maxOutput = Math.max(0, (contextWindow == null ? 0 : contextWindow) - count.tokens()); + } + + Map fits = new LinkedHashMap<>(); + fits.put("context_window", contextWindow); + fits.put("max_output_tokens", maxOutput); + if (contextWindow != null) { + fits.put("fits", count.tokens() + maxOutput <= contextWindow); + fits.put("remaining_input_tokens", Math.max(0, contextWindow - maxOutput - count.tokens())); + } + + Map out = new LinkedHashMap<>(); + out.put("object", "token_count"); + out.put("model", spec.publicModel()); + out.put("upstream_model", upstreamModel); + out.put("provider", first.providerSlug()); + out.put("prompt_tokens", count.tokens()); + out.put("estimated", count.estimated()); + out.put("fits", fits); + return out; + } +} diff --git a/src/main/java/com/anindra/path/gateway/TokenCounter.java b/src/main/java/com/anindra/path/gateway/TokenCounter.java new file mode 100644 index 0000000..fb5a8d2 --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/TokenCounter.java @@ -0,0 +1,86 @@ +package com.anindra.path.gateway; + +import com.fasterxml.jackson.databind.JsonNode; +import com.knuddels.jtokkit.Encodings; +import com.knuddels.jtokkit.api.Encoding; +import com.knuddels.jtokkit.api.EncodingRegistry; + +/** + * Token counting shared by pre-authorisation and {@code POST /v1/tokens/count}. + * + *

Counts real tokens when the upstream model has a known encoding and falls back to + * {@code chars / 4} otherwise. The estimate only ever gates spend; billing uses the + * provider-reported usage. + */ +public final class TokenCounter { + + private static final EncodingRegistry TOKENIZERS = Encodings.newLazyEncodingRegistry(); + + private TokenCounter() { + } + + /** @param estimated true when the chars/4 heuristic was used instead of a real encoding. */ + public record Count(int tokens, boolean estimated) { + } + + public static Count count(JsonNode body, String model) { + String text = promptText(body); + if (model != null && !model.isBlank()) { + try { + Encoding encoding = TOKENIZERS.getEncodingForModel(model).orElse(null); + if (encoding != null) { + return new Count(Math.max(1, encoding.countTokens(text)), false); + } + } catch (Exception ignored) { + // unknown model or unsupported encoding: fall through to the heuristic + } + } + return new Count(Math.max(1, text.length() / 4), true); + } + + /** + * The text a prompt costs for: roles, contents, tool calls and the tool definitions, + * which is where naive estimates are most wrong. + */ + public static String promptText(JsonNode body) { + StringBuilder sb = new StringBuilder(); + JsonNode messages = body.path("messages"); + if (messages.isArray()) { + for (JsonNode message : messages) { + JsonNode role = message.get("role"); + if (role != null) { + sb.append(role.asText()).append(' '); + } + JsonNode content = message.get("content"); + if (content != null) { + sb.append(textOf(content)); + } + JsonNode toolCalls = message.get("tool_calls"); + if (toolCalls != null) { + sb.append(toolCalls.toString()); + } + sb.append('\n'); + } + } + JsonNode tools = body.get("tools"); + if (tools != null) { + sb.append(tools.toString()); + } + return sb.toString(); + } + + private static String textOf(JsonNode content) { + if (content.isTextual()) { + return content.asText(); + } + StringBuilder sb = new StringBuilder(); + if (content.isArray()) { + for (JsonNode part : content) { + if (part.path("type").asText().equals("text") && part.hasNonNull("text")) { + sb.append(part.get("text").asText()); + } + } + } + return sb.toString(); + } +} diff --git a/src/main/java/com/anindra/path/gateway/UpstreamClient.java b/src/main/java/com/anindra/path/gateway/UpstreamClient.java index 7438d0f..376444e 100644 --- a/src/main/java/com/anindra/path/gateway/UpstreamClient.java +++ b/src/main/java/com/anindra/path/gateway/UpstreamClient.java @@ -30,9 +30,11 @@ public String header(String name) { private final HttpClient http; private final GatewayProperties props; + private final io.micrometer.tracing.Tracer tracer; - public UpstreamClient(GatewayProperties props) { + public UpstreamClient(GatewayProperties props, io.micrometer.tracing.Tracer tracer) { this.props = props; + this.tracer = tracer; this.http = HttpClient.newBuilder() .connectTimeout(props.getConnectTimeout()) .followRedirects(HttpClient.Redirect.NORMAL) @@ -43,16 +45,23 @@ public UpstreamClient(GatewayProperties props) { public Response get(String url, Map headers) throws IOException, InterruptedException { HttpRequest.Builder b = base(url, Duration.ofSeconds(30)).GET(); headers.forEach(b::header); + addTraceparent(b); HttpResponse resp = http.send(b.build(), HttpResponse.BodyHandlers.ofString()); return new Response(resp.statusCode(), resp.body(), flatten(resp)); } public Response postJson(String url, Map headers, String body) throws IOException, InterruptedException { - HttpRequest.Builder b = base(url, props.getRequestTimeout()) + return postJson(url, headers, body, props.getRequestTimeout()); + } + + public Response postJson(String url, Map headers, String body, Duration timeout) + throws IOException, InterruptedException { + HttpRequest.Builder b = base(url, timeout) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)); headers.forEach(b::header); + addTraceparent(b); HttpResponse resp = http.send(b.build(), HttpResponse.BodyHandlers.ofString()); return new Response(resp.statusCode(), resp.body(), flatten(resp)); } @@ -65,6 +74,7 @@ public LineStream openStream(String url, Map headers, String bod .header("Accept", "text/event-stream") .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)); headers.forEach(b::header); + addTraceparent(b); HttpResponse resp = http.send(b.build(), HttpResponse.BodyHandlers.ofInputStream()); if (resp.statusCode() >= 400) { @@ -72,7 +82,7 @@ public LineStream openStream(String url, Map headers, String bod try (InputStream in = resp.body()) { errorBody = new String(in.readAllBytes(), StandardCharsets.UTF_8); } - throw mapError(resp.statusCode(), errorBody); + throw mapError(resp.statusCode(), errorBody, flatten(resp)); } return new LineStream(resp.body(), flatten(resp)); } @@ -87,16 +97,32 @@ private static Map flatten(HttpResponse response) { return headers; } + /** W3C trace context, so provider-side logs join the same trace. */ + private void addTraceparent(HttpRequest.Builder builder) { + io.micrometer.tracing.Span span = tracer.currentSpan(); + if (span != null) { + var context = span.context(); + builder.header("traceparent", + "00-" + context.traceId() + "-" + context.spanId() + "-01"); + } + } + private HttpRequest.Builder base(String url, Duration timeout) { return HttpRequest.newBuilder(URI.create(url)).timeout(timeout); } public static UpstreamException mapError(int status, String body) { + return mapError(status, body, Map.of()); + } + + private static final com.fasterxml.jackson.databind.ObjectMapper ERROR_MAPPER = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public static UpstreamException mapError(int status, String body, Map headers) { String message = body; String code = "upstream_error"; try { - var mapper = new com.fasterxml.jackson.databind.ObjectMapper(); - var node = mapper.readTree(body); + var node = ERROR_MAPPER.readTree(body); if (node.has("error")) { var err = node.get("error"); if (err.isTextual()) { @@ -121,7 +147,23 @@ public static UpstreamException mapError(int status, String body) { if (status == 429) { code = "upstream_rate_limited"; } - return new UpstreamException(status, code, truncate(message, 800)); + return new UpstreamException(status, code, truncate(message, 800), retryAfterSeconds(headers)); + } + + /** Parses Retry-After in seconds. HTTP-date form is treated as 5s. */ + private static int retryAfterSeconds(Map headers) { + if (headers == null) { + return 0; + } + String raw = headers.get("retry-after"); + if (raw == null || raw.isBlank()) { + return 0; + } + try { + return Math.max(0, Math.min(3600, Integer.parseInt(raw.trim()))); + } catch (NumberFormatException e) { + return 5; + } } private static String truncate(String s, int max) { diff --git a/src/main/java/com/anindra/path/gateway/UpstreamException.java b/src/main/java/com/anindra/path/gateway/UpstreamException.java index b850b31..c0a7561 100644 --- a/src/main/java/com/anindra/path/gateway/UpstreamException.java +++ b/src/main/java/com/anindra/path/gateway/UpstreamException.java @@ -5,16 +5,25 @@ public class UpstreamException extends RuntimeException { private final int status; private final String providerCode; + private final int retryAfterSeconds; public UpstreamException(int status, String providerCode, String message) { + this(status, providerCode, message, 0); + } + + public UpstreamException(int status, String providerCode, String message, int retryAfterSeconds) { super(message); this.status = status; this.providerCode = providerCode; + this.retryAfterSeconds = retryAfterSeconds; } public int status() { return status; } public String providerCode() { return providerCode; } + /** Upstream's Retry-After hint in seconds, or 0 when absent. */ + public int retryAfterSeconds() { return retryAfterSeconds; } + /** 429 and 5xx are worth trying on another route; 400/401/404 are not. */ public boolean retryable() { return status == 429 || status == 408 || status >= 500; diff --git a/src/main/java/com/anindra/path/gateway/UsageService.java b/src/main/java/com/anindra/path/gateway/UsageService.java index 5bd00e2..3cf1d6e 100644 --- a/src/main/java/com/anindra/path/gateway/UsageService.java +++ b/src/main/java/com/anindra/path/gateway/UsageService.java @@ -4,7 +4,9 @@ import com.anindra.path.domain.ModelRoute; import com.anindra.path.domain.UsageRecord; import com.anindra.path.repo.UsageRecordRepository; +import io.micrometer.core.instrument.MeterRegistry; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -21,6 +23,7 @@ @Service @RequiredArgsConstructor +@Slf4j public class UsageService { private static final BigDecimal MILLION = BigDecimal.valueOf(1_000_000L); @@ -28,6 +31,8 @@ public class UsageService { private final UsageRecordRepository usage; private final ApiKeyService apiKeys; + private final MeterRegistry metrics; + private final com.anindra.path.config.GatewayProperties props; /** Itemised cost of one call. */ public record CostBreakdown(BigDecimal inputUsd, BigDecimal cachedInputUsd, BigDecimal cacheWriteUsd, @@ -93,6 +98,7 @@ public BigDecimal recordSuccess(ClientKey key, ChatRequestSpec spec, RouteResolv UsageRecord record = new UsageRecord(); record.setClientKeyId(key.getId()); record.setClientKeyName(key.getName()); + record.setEndUser(spec.endUser()); record.setPublicModelId(spec.publicModel()); record.setProviderSlug(resolved.providerSlug()); record.setUpstreamModelId(resolved.route().getUpstreamModelId()); @@ -121,11 +127,39 @@ public BigDecimal recordSuccess(ClientKey key, ChatRequestSpec spec, RouteResolv record.setFallbackFrom(meta.fallbackFrom()); record.setStatus(UsageRecord.Status.SUCCESS); usage.save(record); + recordMetrics(record); apiKeys.charge(key.getId(), cost.totalUsd()); return cost.totalUsd(); } + /** Records a cache hit: no upstream call, no charge, savings carried for reporting. */ + @Transactional + public void recordCacheHit(ClientKey key, ChatRequestSpec spec, RouteResolver.ResolvedRoute resolved, + ResponseCacheService.Entry entry, boolean streamed) { + UsageRecord record = new UsageRecord(); + record.setClientKeyId(key.getId()); + record.setClientKeyName(key.getName()); + record.setEndUser(spec.endUser()); + record.setPublicModelId(spec.publicModel()); + record.setProviderSlug(entry.providerSlug()); + record.setUpstreamModelId(resolved.route().getUpstreamModelId()); + record.setPromptTokens(entry.promptTokens()); + record.setCompletionTokens(entry.completionTokens()); + record.setTotalTokens(entry.totalTokens()); + record.setCostUsd(BigDecimal.ZERO); + record.setCachedSavingsUsd(entry.costUsd() == null ? BigDecimal.ZERO : entry.costUsd()); + record.setLatencyMs(0); + record.setStreamed(streamed); + record.setAttempts(0); + record.setFinishReason("cache"); + record.setStatus(UsageRecord.Status.SUCCESS); + usage.save(record); + metrics.counter("path.cache.hits", + "model", String.valueOf(record.getPublicModelId())).increment(); + recordMetrics(record); + } + @Transactional public void recordError(ClientKey key, ChatRequestSpec spec, String providerSlug, String upstreamModel, long latencyMs, boolean streamed, int attempts, String code, String message) { @@ -144,6 +178,27 @@ public void recordError(ClientKey key, ChatRequestSpec spec, String providerSlug record.setErrorCode(code); record.setErrorMessage(message != null && message.length() > 500 ? message.substring(0, 500) : message); usage.save(record); + recordMetrics(record); + } + + /** Feeds the Prometheus endpoint. Tag values stay low-cardinality (model, provider, status). */ + private void recordMetrics(UsageRecord record) { + String model = String.valueOf(record.getPublicModelId()); + String provider = String.valueOf(record.getProviderSlug()); + metrics.counter("path.requests", + "model", model, + "provider", provider, + "status", record.getStatus().name().toLowerCase(), + "streamed", String.valueOf(record.isStreamed())).increment(); + metrics.timer("path.latency", "provider", provider) + .record(java.time.Duration.ofMillis(record.getLatencyMs())); + metrics.counter("path.tokens", "type", "prompt").increment(record.getPromptTokens()); + metrics.counter("path.tokens", "type", "completion").increment(record.getCompletionTokens()); + metrics.counter("path.tokens", "type", "cached").increment(record.getCachedInputTokens()); + if (record.getCostUsd() != null) { + metrics.counter("path.cost.usd", + "model", model, "provider", provider).increment(record.getCostUsd().doubleValue()); + } } private static BigDecimal throughput(TokenUsage usage, CallMeta meta) { @@ -162,6 +217,54 @@ private static BigDecimal throughput(TokenUsage usage, CallMeta meta) { .divide(BigDecimal.valueOf(window), 2, RoundingMode.HALF_UP); } + /** Deletes usage older than the retention window; zero keeps history forever. */ + @org.springframework.scheduling.annotation.Scheduled(cron = "0 30 3 * * *") + @Transactional + void purgeOldUsage() { + java.time.Duration retention = props.getUsageRetention(); + if (retention == null || retention.isZero() || retention.isNegative()) { + return; + } + long removed = usage.deleteByCreatedAtBefore(Instant.now().minus(retention)); + if (removed > 0) { + log.info("Purged {} usage rows older than {}", removed, retention); + } + } + + /** Spend since a timestamp, used by the daily/monthly windows. */ + public BigDecimal spendFor(Long keyId, Instant from) { + return nz(usage.sumCostForKeySince(keyId, from)); + } + + /** Spend across every key in a project since a timestamp. */ + /** Spend for one end user on one key since {@code from}; 0 when the id is absent. */ + public BigDecimal spendForEndUser(Long keyId, String endUser, Instant from) { + if (endUser == null || endUser.isBlank()) { + return BigDecimal.ZERO; + } + BigDecimal spend = usage.sumCostForKeyAndEndUserSince(keyId, endUser, from); + return spend == null ? BigDecimal.ZERO : spend; + } + + /** Per-end-user rollup for /v1/usage, so a product can bill its own customers. */ + public List> endUserBreakdown(Long keyId, Duration window) { + Instant from = Instant.now().minus(window); + List> out = new ArrayList<>(); + for (Object[] row : usage.groupByEndUserSince(keyId, from)) { + Map entry = new LinkedHashMap<>(); + entry.put("end_user", row[0]); + entry.put("requests", ((Number) row[1]).longValue()); + entry.put("total_tokens", ((Number) row[2]).longValue()); + entry.put("cost_usd", row[3]); + out.add(entry); + } + return out; + } + + public BigDecimal spendForProject(Long projectId, Instant from) { + return nz(usage.sumCostForProjectSince(projectId, from)); + } + public Map keySummary(ClientKey key, Duration window) { Instant from = Instant.now().minus(window); long requests = usage.countByClientKeyIdAndCreatedAtAfter(key.getId(), from); @@ -191,6 +294,7 @@ public Map keySummary(ClientKey key, Duration window) { "cached_input", nz(usage.sumCachedCostForKeySince(key.getId(), from)), "cache_write", nz(usage.sumCacheWriteCostForKeySince(key.getId(), from)), "output", nz(usage.sumOutputCostForKeySince(key.getId(), from)))); + out.put("end_users", endUserBreakdown(key.getId(), window)); out.put("credit_limit_usd", key.getCreditLimitUsd()); out.put("used_usd", key.getUsedUsd()); out.put("remaining_usd", apiKeys.remaining(key)); diff --git a/src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java b/src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java new file mode 100644 index 0000000..c3a916f --- /dev/null +++ b/src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java @@ -0,0 +1,44 @@ +package com.anindra.path.gateway; + +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.core.script.RedisScript; + +import java.util.List; + +/** + * Fixed one-minute window in Valkey, shared by every instance. + * + * Valkey speaks the same protocol as Redis, so Spring Data Redis and Lettuce work + * unchanged; only the host/port differ. + */ +public class ValkeyRateLimitStore implements RateLimitStore { + + private static final RedisScript INCR_WITH_TTL = new DefaultRedisScript<>( + "local current = redis.call('INCR', KEYS[1]) " + + "if current == 1 then redis.call('EXPIRE', KEYS[1], 60) end " + + "return current", + Long.class); + + private final StringRedisTemplate redis; + private final String prefix; + + public ValkeyRateLimitStore(StringRedisTemplate redis, String prefix) { + this.redis = redis; + this.prefix = prefix; + } + + @Override + public int acquire(String bucket, int rpmLimit) { + if (rpmLimit <= 0) { + return 0; + } + String redisKey = prefix + ":rpm:" + bucket; + Long count = redis.execute(INCR_WITH_TTL, List.of(redisKey)); + if (count == null || count <= rpmLimit) { + return 0; + } + Long ttl = redis.getExpire(redisKey); + return (int) Math.max(1, ttl == null || ttl <= 0 ? 60 : ttl); + } +} diff --git a/src/main/java/com/anindra/path/repo/AdminAuditRepository.java b/src/main/java/com/anindra/path/repo/AdminAuditRepository.java new file mode 100644 index 0000000..ba1a961 --- /dev/null +++ b/src/main/java/com/anindra/path/repo/AdminAuditRepository.java @@ -0,0 +1,14 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.AdminAudit; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface AdminAuditRepository extends JpaRepository { + + List findAllByOrderByCreatedAtDesc(Pageable pageable); + + List findByActionOrderByCreatedAtDesc(String action, Pageable pageable); +} diff --git a/src/main/java/com/anindra/path/repo/AdminTokenRepository.java b/src/main/java/com/anindra/path/repo/AdminTokenRepository.java new file mode 100644 index 0000000..fd34d43 --- /dev/null +++ b/src/main/java/com/anindra/path/repo/AdminTokenRepository.java @@ -0,0 +1,14 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.AdminToken; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface AdminTokenRepository extends JpaRepository { + + Optional findByTokenHash(String tokenHash); + + List findAllByOrderByCreatedAtDesc(); +} diff --git a/src/main/java/com/anindra/path/repo/ClientKeyRepository.java b/src/main/java/com/anindra/path/repo/ClientKeyRepository.java index 9953e98..7605b2a 100644 --- a/src/main/java/com/anindra/path/repo/ClientKeyRepository.java +++ b/src/main/java/com/anindra/path/repo/ClientKeyRepository.java @@ -2,10 +2,13 @@ import com.anindra.path.domain.ClientKey; import jakarta.persistence.LockModeType; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Query; +import java.math.BigDecimal; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -15,8 +18,30 @@ public interface ClientKeyRepository extends JpaRepository { List findAllByOrderByCreatedAtDesc(); + List findByProjectIdOrderByCreatedAtDesc(Long projectId); + long countByEnabledTrue(); + /** + * Atomically reserves budget for an in-flight request. The WHERE clause makes the + * check-and-increment one statement, so concurrent requests cannot overshoot the limit. + */ + @Modifying(clearAutomatically = true) + @Query("update ClientKey k set k.reservedUsd = coalesce(k.reservedUsd, 0) + :amount, " + + "k.reservedUpdatedAt = :now " + + "where k.id = :id and (k.creditLimitUsd is null " + + "or k.usedUsd + coalesce(k.reservedUsd, 0) + :amount <= k.creditLimitUsd)") + int reserve(Long id, BigDecimal amount, Instant now); + + @Modifying(clearAutomatically = true) + @Query("update ClientKey k set k.reservedUsd = case when coalesce(k.reservedUsd, 0) < :amount " + + "then 0 else k.reservedUsd - :amount end where k.id = :id") + int release(Long id, BigDecimal amount); + + @Modifying(clearAutomatically = true) + @Query("update ClientKey k set k.reservedUsd = 0 where k.reservedUpdatedAt < :cutoff") + int releaseStale(Instant cutoff); + /** Row lock so concurrent charges on one key cannot lose an update. */ @Lock(LockModeType.PESSIMISTIC_WRITE) @Query("select k from ClientKey k where k.id = :id") diff --git a/src/main/java/com/anindra/path/repo/EndUserReservationRepository.java b/src/main/java/com/anindra/path/repo/EndUserReservationRepository.java new file mode 100644 index 0000000..d5b8ec0 --- /dev/null +++ b/src/main/java/com/anindra/path/repo/EndUserReservationRepository.java @@ -0,0 +1,35 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.EndUserReservation; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.Optional; + +public interface EndUserReservationRepository extends JpaRepository { + + Optional findByClientKeyIdAndEndUser(Long clientKeyId, String endUser); + + /** + * Atomically takes {@code amount} when the result still fits the headroom + * (limit minus spend already reported). Returns 0 when it would not. + */ + @Modifying + @Query("update EndUserReservation r set r.reservedUsd = r.reservedUsd + :amount, r.updatedAt = :now " + + "where r.clientKeyId = :keyId and r.endUser = :endUser " + + "and r.reservedUsd + :amount <= :headroom") + int reserve(Long keyId, String endUser, BigDecimal amount, BigDecimal headroom, Instant now); + + @Modifying + @Query("update EndUserReservation r set r.reservedUsd = r.reservedUsd - :amount, r.updatedAt = :now " + + "where r.clientKeyId = :keyId and r.endUser = :endUser and r.reservedUsd >= :amount") + int release(Long keyId, String endUser, BigDecimal amount, Instant now); + + @Modifying + @Query("update EndUserReservation r set r.reservedUsd = 0, r.updatedAt = :now " + + "where r.clientKeyId = :keyId and r.endUser = :endUser") + int clear(Long keyId, String endUser, Instant now); +} diff --git a/src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java b/src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java new file mode 100644 index 0000000..1030cdd --- /dev/null +++ b/src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java @@ -0,0 +1,14 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.IdempotencyRecord; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.time.Instant; +import java.util.Optional; + +public interface IdempotencyRecordRepository extends JpaRepository { + + Optional findByClientKeyIdAndIdempotencyKey(Long clientKeyId, String idempotencyKey); + + long deleteByExpiresAtBefore(Instant cutoff); +} diff --git a/src/main/java/com/anindra/path/repo/ModelAliasRepository.java b/src/main/java/com/anindra/path/repo/ModelAliasRepository.java new file mode 100644 index 0000000..904f254 --- /dev/null +++ b/src/main/java/com/anindra/path/repo/ModelAliasRepository.java @@ -0,0 +1,16 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.ModelAlias; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface ModelAliasRepository extends JpaRepository { + + Optional findByAlias(String alias); + + List findAllByOrderByAliasAsc(); + + void deleteByAlias(String alias); +} diff --git a/src/main/java/com/anindra/path/repo/ProjectRepository.java b/src/main/java/com/anindra/path/repo/ProjectRepository.java new file mode 100644 index 0000000..23e8d88 --- /dev/null +++ b/src/main/java/com/anindra/path/repo/ProjectRepository.java @@ -0,0 +1,14 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.Project; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface ProjectRepository extends JpaRepository { + + Optional findBySlug(String slug); + + List findAllByOrderBySlugAsc(); +} diff --git a/src/main/java/com/anindra/path/repo/PromptLogRepository.java b/src/main/java/com/anindra/path/repo/PromptLogRepository.java new file mode 100644 index 0000000..a89c8d5 --- /dev/null +++ b/src/main/java/com/anindra/path/repo/PromptLogRepository.java @@ -0,0 +1,17 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.PromptLog; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.time.Instant; +import java.util.List; + +public interface PromptLogRepository extends JpaRepository { + + List findAllByOrderByCreatedAtDesc(Pageable pageable); + + List findByClientKeyIdOrderByCreatedAtDesc(Long clientKeyId, Pageable pageable); + + long deleteByExpiresAtBefore(Instant cutoff); +} diff --git a/src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java b/src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java new file mode 100644 index 0000000..7c7c3ce --- /dev/null +++ b/src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java @@ -0,0 +1,20 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.ResponseCacheEntry; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +public interface ResponseCacheEntryRepository extends JpaRepository { + + Optional findByCacheKey(String cacheKey); + + List findByClientKeyIdAndPublicModelIdAndEmbeddingIsNotNull( + Long clientKeyId, String publicModelId, org.springframework.data.domain.Pageable pageable); + + long deleteByExpiresAtBefore(Instant cutoff); + + long deleteByClientKeyId(Long clientKeyId); +} diff --git a/src/main/java/com/anindra/path/repo/UsageRecordRepository.java b/src/main/java/com/anindra/path/repo/UsageRecordRepository.java index 6d90135..55183ac 100644 --- a/src/main/java/com/anindra/path/repo/UsageRecordRepository.java +++ b/src/main/java/com/anindra/path/repo/UsageRecordRepository.java @@ -15,16 +15,43 @@ public interface UsageRecordRepository extends JpaRepository List findByCreatedAtAfterOrderByCreatedAtDesc(Instant from, Pageable pageable); + List findByCreatedAtBetweenOrderByCreatedAtAsc(Instant from, Instant to, Pageable pageable); + + List findByClientKeyIdInAndCreatedAtAfterOrderByCreatedAtDesc( + java.util.Collection keyIds, Instant from, Pageable pageable); + + List findByClientKeyIdInAndCreatedAtBetweenOrderByCreatedAtAsc( + java.util.Collection keyIds, Instant from, Instant to, Pageable pageable); + long countByCreatedAtAfter(Instant from); + long deleteByCreatedAtBefore(Instant cutoff); + long countByClientKeyIdAndCreatedAtAfter(Long clientKeyId, Instant from); + @Query("select coalesce(sum(u.costUsd), 0) from UsageRecord u " + + "where u.clientKeyId in (select k.id from ClientKey k where k.projectId = :projectId) " + + "and u.createdAt >= :from") + BigDecimal sumCostForProjectSince(Long projectId, Instant from); + @Query("select coalesce(sum(u.costUsd), 0) from UsageRecord u where u.createdAt >= :from") BigDecimal sumCostSince(Instant from); @Query("select coalesce(sum(u.costUsd), 0) from UsageRecord u where u.clientKeyId = :keyId and u.createdAt >= :from") BigDecimal sumCostForKeySince(Long keyId, Instant from); + @Query("select coalesce(sum(u.costUsd), 0) from UsageRecord u " + + "where u.clientKeyId = :keyId and u.endUser = :endUser and u.createdAt >= :from") + BigDecimal sumCostForKeyAndEndUserSince(Long keyId, String endUser, Instant from); + + @Query("select u.endUser, count(u), coalesce(sum(u.totalTokens), 0), coalesce(sum(u.costUsd), 0) " + + "from UsageRecord u where u.clientKeyId = :keyId and u.createdAt >= :from " + + "and u.endUser is not null group by u.endUser order by sum(u.costUsd) desc") + List groupByEndUserSince(Long keyId, Instant from); + + List findByClientKeyIdAndEndUserOrderByCreatedAtDesc( + Long clientKeyId, String endUser, Pageable pageable); + @Query("select coalesce(sum(u.promptTokens), 0) from UsageRecord u where u.clientKeyId = :keyId and u.createdAt >= :from") Long sumPromptTokensForKeySince(Long keyId, Instant from); diff --git a/src/main/java/com/anindra/path/secret/SecretCipher.java b/src/main/java/com/anindra/path/secret/SecretCipher.java index e4391dc..e8ffe50 100644 --- a/src/main/java/com/anindra/path/secret/SecretCipher.java +++ b/src/main/java/com/anindra/path/secret/SecretCipher.java @@ -24,7 +24,7 @@ public class SecretCipher { private static final int IV_BYTES = 12; private static final int TAG_BITS = 128; - private final SecretKeySpec key; + private volatile SecretKeySpec key; private final SecureRandom random = new SecureRandom(); public SecretCipher(GatewayProperties props) { @@ -62,6 +62,15 @@ private static byte[] loadOrCreateKey(GatewayProperties props) { } public String encrypt(String plaintext) { + return encryptWith(key, plaintext, random); + } + + /** Encrypts under an explicit key; used when rotating the master key. */ + public static String encryptWith(byte[] rawKey, String plaintext) { + return encryptWith(new SecretKeySpec(rawKey, "AES"), plaintext, new SecureRandom()); + } + + private static String encryptWith(SecretKeySpec key, String plaintext, SecureRandom random) { if (plaintext == null) { return null; } @@ -80,6 +89,20 @@ public String encrypt(String plaintext) { } } + /** Validates and decodes a base64 32-byte master key. */ + public static byte[] parseKey(String base64) { + byte[] raw = Base64.getDecoder().decode(base64.trim()); + if (raw.length != 32) { + throw new IllegalArgumentException("master key must decode to exactly 32 bytes"); + } + return raw; + } + + /** Switches the active key; callers must have re-encrypted stored ciphertext first. */ + public void rotate(byte[] rawKey) { + this.key = new SecretKeySpec(rawKey, "AES"); + } + public String decrypt(String encoded) { if (encoded == null || encoded.isBlank()) { return null; diff --git a/src/main/java/com/anindra/path/seed/DataSeeder.java b/src/main/java/com/anindra/path/seed/DataSeeder.java index aa5f5f9..39dcd0c 100644 --- a/src/main/java/com/anindra/path/seed/DataSeeder.java +++ b/src/main/java/com/anindra/path/seed/DataSeeder.java @@ -33,6 +33,9 @@ public class DataSeeder implements ApplicationRunner { private final ClientKeyRepository clientKeys; private final ApiKeyService apiKeys; private final GatewayProperties props; + private final SeedState seedState; + private final com.anindra.path.repo.ProjectRepository projectsRepository; + private final org.springframework.core.env.Environment environment; @Override @Transactional @@ -42,10 +45,17 @@ public void run(ApplicationArguments args) { if (fresh && routes.count() == 0) { seedRoutes(bySlug); } - if (props.isBootstrapDemoKey() && clientKeys.count() == 0) { + if (projectsRepository.count() == 0) { + projectsRepository.save(new com.anindra.path.domain.Project("default", "Default project")); + } + boolean prod = environment.acceptsProfiles(org.springframework.core.env.Profiles.of("prod")); + if (props.isBootstrapDemoKey() && !prod && clientKeys.count() == 0) { var created = apiKeys.create("demo-local", null, props.getDefaultRpmLimit()); log.info("Created demo API key (no credit limit): {}", created.plaintext()); + } else if (prod && props.isBootstrapDemoKey()) { + log.warn("Ignoring gateway.bootstrap-demo-key in the prod profile; no demo key created."); } + seedState.markReady(); log.info("Gateway ready. Admin API: {} | Public API: /v1/chat/completions, /v1/models", props.getAdminToken() == null || props.getAdminToken().isBlank() ? "disabled (set GATEWAY_ADMIN_TOKEN)" @@ -60,6 +70,9 @@ private Map ensureProviders() { new Provider("anthropic", "Anthropic", ProviderType.ANTHROPIC, "https://api.anthropic.com/v1")))); bySlug.put("groq", providers.findBySlug("groq").orElseGet(() -> providers.save( new Provider("groq", "Groq", ProviderType.OPENAI, "https://api.groq.com/openai/v1")))); + bySlug.put("gemini", providers.findBySlug("gemini").orElseGet(() -> providers.save( + new Provider("gemini", "Google Gemini", ProviderType.OPENAI, + "https://generativelanguage.googleapis.com/v1beta/openai")))); return bySlug; } diff --git a/src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java b/src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java new file mode 100644 index 0000000..e702e0f --- /dev/null +++ b/src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java @@ -0,0 +1,31 @@ +package com.anindra.path.seed; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; +import org.springframework.stereotype.Component; + +/** + * Reports OUT_OF_SERVICE until the first-boot seed has committed, so load balancers + * and CI wait loops do not send traffic to a gateway with no providers or routes. + */ +@Component +@RequiredArgsConstructor +public class SeedReadinessIndicator implements HealthIndicator { + + private final SeedState state; + private final com.anindra.path.gateway.DrainState drain; + + @Override + public Health health() { + if (drain.isDraining()) { + return Health.outOfService() + .withDetail("seeded", state.isReady()) + .withDetail("draining", true) + .build(); + } + return state.isReady() + ? Health.up().withDetail("seeded", true).build() + : Health.outOfService().withDetail("seeded", false).build(); + } +} diff --git a/src/main/java/com/anindra/path/seed/SeedState.java b/src/main/java/com/anindra/path/seed/SeedState.java new file mode 100644 index 0000000..286b71f --- /dev/null +++ b/src/main/java/com/anindra/path/seed/SeedState.java @@ -0,0 +1,26 @@ +package com.anindra.path.seed; + +import org.springframework.stereotype.Component; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Tracks whether first-boot seeding has finished. + * + * Tomcat starts listening during context refresh, before {@code ApplicationRunner}s + * execute, so without this flag a readiness probe can succeed against a half-seeded + * database. + */ +@Component +public class SeedState { + + private final AtomicBoolean ready = new AtomicBoolean(false); + + public void markReady() { + ready.set(true); + } + + public boolean isReady() { + return ready.get(); + } +} diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml new file mode 100644 index 0000000..56bd4c1 --- /dev/null +++ b/src/main/resources/application-prod.yml @@ -0,0 +1,19 @@ +# Production profile: structured JSON logs and conservative tracing. +logging: + structured: + format: + console: ecs + ecs: + service: + name: path + version: 0.1.0 + +# The spec describes every endpoint; keep it off on a public deployment unless asked. +springdoc: + api-docs: + enabled: ${GATEWAY_API_DOCS:false} + +management: + tracing: + sampling: + probability: ${GATEWAY_TRACE_SAMPLE:0} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 90a295f..518bf7b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -16,13 +16,21 @@ spring: password: ${DB_PASSWORD:} hikari: maximum-pool-size: 20 + flyway: + enabled: true + locations: classpath:db/migration + # An existing pre-Flyway schema is marked as V1 instead of being recreated. + baseline-on-migrate: true + baseline-version: 1 jpa: open-in-view: false hibernate: - ddl-auto: update + ddl-auto: validate properties: hibernate: jdbc.time_zone: UTC + # Store enums as varchar so one migration works on H2 and Postgres. + type.preferred_enum_jdbc_type: STRING jackson: default-property-inclusion: non_null mvc: @@ -43,9 +51,45 @@ gateway: # 32-byte base64 key. When empty, a key is generated and stored at master-key-path. master-key: ${GATEWAY_MASTER_KEY:} master-key-path: ${GATEWAY_MASTER_KEY_PATH:./data/master.key} - bootstrap-demo-key: ${GATEWAY_BOOTSTRAP_DEMO_KEY:true} + bootstrap-demo-key: ${GATEWAY_BOOTSTRAP_DEMO_KEY:false} default-rpm-limit: ${GATEWAY_DEFAULT_RPM:120} default-credit-limit-usd: ${GATEWAY_DEFAULT_CREDIT_USD:} + default-max-concurrent: ${GATEWAY_DEFAULT_MAX_CONCURRENT:8} + max-request-bytes: ${GATEWAY_MAX_REQUEST_BYTES:1048576} + max-output-tokens: ${GATEWAY_MAX_OUTPUT_TOKENS:0} + drain-timeout: ${GATEWAY_DRAIN_TIMEOUT:30s} + breaker: + failure-threshold: ${GATEWAY_BREAKER_FAILURES:5} + open-duration: ${GATEWAY_BREAKER_OPEN:30s} + max-attempts: ${GATEWAY_MAX_ATTEMPTS:3} + cache: + semantic-model: ${GATEWAY_SEMANTIC_MODEL:} + semantic-max-entries: ${GATEWAY_SEMANTIC_MAX_ENTRIES:200} + max-body-bytes: ${GATEWAY_CACHE_MAX_BYTES:2097152} + admin-auth: + max-failures: ${GATEWAY_ADMIN_AUTH_FAILURES:10} + window: ${GATEWAY_ADMIN_AUTH_WINDOW:5m} + guardrails: + prompt-log-redaction: ${GATEWAY_GUARDRAILS_LOG_REDACTION:true} + redact-before-send: ${GATEWAY_GUARDRAILS_REDACT:false} + moderation: ${GATEWAY_GUARDRAILS_MODERATION:false} + moderation-model: ${GATEWAY_GUARDRAILS_MODERATION_MODEL:omni-moderation-latest} + rate-limit: + backend: ${GATEWAY_RATE_LIMIT_BACKEND:memory} + prefix: ${GATEWAY_RATE_LIMIT_PREFIX:path} + prompt-logging: + enabled: ${GATEWAY_PROMPT_LOGGING:false} + retention: ${GATEWAY_PROMPT_LOG_RETENTION:7d} + max-body-bytes: ${GATEWAY_PROMPT_LOG_MAX_BYTES:65536} + alerts: + webhook-url: ${GATEWAY_ALERT_WEBHOOK:} + cooldown: ${GATEWAY_ALERT_COOLDOWN:1h} + budget-threshold: ${GATEWAY_ALERT_BUDGET_THRESHOLD:0.8} + usage-retention: ${GATEWAY_USAGE_RETENTION:180d} + idempotency: + ttl: ${GATEWAY_IDEMPOTENCY_TTL:24h} + pending-timeout: ${GATEWAY_IDEMPOTENCY_PENDING:2m} + max-response-bytes: ${GATEWAY_IDEMPOTENCY_MAX_BYTES:2097152} # Upstream credentials. A key stored through the admin API takes precedence. providers: openai: @@ -54,15 +98,39 @@ gateway: api-key: ${ANTHROPIC_API_KEY:} groq: api-key: ${GROQ_API_KEY:} + gemini: + api-key: ${GEMINI_API_KEY:} management: endpoints: web: exposure: - include: health,info,metrics + include: health,info,metrics,prometheus endpoint: health: show-details: when-authorized + group: + providers: + include: provider + show-details: always + prometheus: + metrics: + export: + enabled: true + health: + redis: + enabled: false + tracing: + sampling: + probability: ${GATEWAY_TRACE_SAMPLE:0} + +springdoc: + api-docs: + enabled: ${GATEWAY_API_DOCS:true} + path: /v3/api-docs + swagger-ui: + enabled: ${GATEWAY_SWAGGER_UI:false} + path: /swagger-ui.html logging: level: diff --git a/src/main/resources/db/migration/V10__project_budgets.sql b/src/main/resources/db/migration/V10__project_budgets.sql new file mode 100644 index 0000000..1925e1b --- /dev/null +++ b/src/main/resources/db/migration/V10__project_budgets.sql @@ -0,0 +1,5 @@ +-- Project-level budgets and rollups (docs/LIMITS.md: projects grouped keys but not money). + +alter table projects add column credit_limit_usd numeric(18,6); +alter table projects add column daily_limit_usd numeric(18,6); +alter table projects add column monthly_limit_usd numeric(18,6); diff --git a/src/main/resources/db/migration/V11__project_limits.sql b/src/main/resources/db/migration/V11__project_limits.sql new file mode 100644 index 0000000..b982723 --- /dev/null +++ b/src/main/resources/db/migration/V11__project_limits.sql @@ -0,0 +1,5 @@ +-- Project-level rate limits (issue #65). A project that owns forty keys could push +-- 4,800 rpm with nothing to cap it; these are the caps. + +alter table projects add column rpm_limit integer; +alter table projects add column max_concurrent_requests integer; diff --git a/src/main/resources/db/migration/V12__model_aliases.sql b/src/main/resources/db/migration/V12__model_aliases.sql new file mode 100644 index 0000000..72921d3 --- /dev/null +++ b/src/main/resources/db/migration/V12__model_aliases.sql @@ -0,0 +1,11 @@ +-- Operator-defined model aliases (issue #68). The capability aliases (cheap, fast, +-- long-context, vision) stay computed; these are stable chains a product can depend on. + +create table model_aliases ( + id bigint generated by default as identity, + alias varchar(80) not null unique, + models varchar(1000) not null, + description varchar(400), + created_at timestamp(6) with time zone not null, + primary key (id) +); diff --git a/src/main/resources/db/migration/V13__route_strategy.sql b/src/main/resources/db/migration/V13__route_strategy.sql new file mode 100644 index 0000000..2878cae --- /dev/null +++ b/src/main/resources/db/migration/V13__route_strategy.sql @@ -0,0 +1,6 @@ +-- Adaptive routing (issue #63). Routing was static: a degraded-but-alive provider kept +-- the traffic until it 5xxed. strategy defaults to 'priority', so nothing changes until +-- an operator opts in. + +alter table model_routes add column strategy varchar(32) not null default 'priority'; +alter table model_routes add column latency_slo_ms integer; diff --git a/src/main/resources/db/migration/V14__end_user_attribution.sql b/src/main/resources/db/migration/V14__end_user_attribution.sql new file mode 100644 index 0000000..6e1c6c4 --- /dev/null +++ b/src/main/resources/db/migration/V14__end_user_attribution.sql @@ -0,0 +1,8 @@ +-- End-user attribution (issue #64). A key and a project are operator concepts; without +-- an end-user axis the gateway cannot bill a product's own customers. + +alter table usage_records add column end_user varchar(128); +alter table client_keys add column end_user_daily_limit_usd numeric(18,6); + +create index idx_usage_key_end_user_created + on usage_records (client_key_id, end_user, created_at); diff --git a/src/main/resources/db/migration/V15__end_user_reservations.sql b/src/main/resources/db/migration/V15__end_user_reservations.sql new file mode 100644 index 0000000..32a3af3 --- /dev/null +++ b/src/main/resources/db/migration/V15__end_user_reservations.sql @@ -0,0 +1,12 @@ +-- Reserve end-user budget the way key credit is reserved (issue #64), so concurrent +-- requests for one end user cannot overshoot the cap between the check and the charge. + +create table end_user_reservations ( + id bigint generated by default as identity, + client_key_id bigint not null, + end_user varchar(128) not null, + reserved_usd numeric(18,6) not null default 0, + updated_at timestamp(6) with time zone, + primary key (id), + constraint uk_end_user_reservation unique (client_key_id, end_user) +); diff --git a/src/main/resources/db/migration/V16__provider_api_version.sql b/src/main/resources/db/migration/V16__provider_api_version.sql new file mode 100644 index 0000000..750443f --- /dev/null +++ b/src/main/resources/db/migration/V16__provider_api_version.sql @@ -0,0 +1,5 @@ +-- Azure OpenAI (issue #73). Azure serves the same wire format at a different path, with +-- the deployment in the URL and the version in a query parameter. One column is enough; +-- a separate provider type would duplicate every route and every adapter lookup. + +alter table providers add column api_version varchar(32); diff --git a/src/main/resources/db/migration/V17__semantic_cache.sql b/src/main/resources/db/migration/V17__semantic_cache.sql new file mode 100644 index 0000000..bb659d4 --- /dev/null +++ b/src/main/resources/db/migration/V17__semantic_cache.sql @@ -0,0 +1,10 @@ +-- Semantic cache tier (issue #71). Opt-in per route, on top of the exact-match cache. + +alter table model_routes add column semantic_cache_enabled boolean not null default false; +alter table model_routes add column semantic_threshold numeric(4,3) not null default 0.950; + +alter table response_cache_entries add column embedding text; +alter table response_cache_entries add column embedding_model varchar(120); + +create index ix_cache_semantic on response_cache_entries + (client_key_id, public_model_id, expires_at); diff --git a/src/main/resources/db/migration/V1__baseline.sql b/src/main/resources/db/migration/V1__baseline.sql new file mode 100644 index 0000000..99c0e2b --- /dev/null +++ b/src/main/resources/db/migration/V1__baseline.sql @@ -0,0 +1,116 @@ +-- Baseline schema for Path. +-- +-- Generated from the JPA entities and written in portable SQL so the same migration +-- runs on H2 (PostgreSQL mode) and on PostgreSQL: +-- * bigint generated by default as identity (PG 10+, H2 2.x) +-- * timestamp(6) with time zone (both) +-- * enums stored as varchar + check (deliberately not native enum types) +-- +-- Hibernate runs with ddl-auto=validate, so any change here must be matched by an +-- entity change, and any entity change needs a new migration. + +create table providers ( + id bigint generated by default as identity, + slug varchar(64) not null unique, + display_name varchar(128) not null, + type varchar(32) not null check (type in ('OPENAI', 'ANTHROPIC')), + base_url varchar(512) not null, + api_key_cipher varchar(2048), + enabled boolean not null, + created_at timestamp(6) with time zone not null, + primary key (id) +); + +create table model_routes ( + id bigint generated by default as identity, + public_model_id varchar(128) not null, + provider_id bigint not null, + upstream_model_id varchar(256) not null, + input_price_per_million numeric(18,6), + output_price_per_million numeric(18,6), + cached_input_price_per_million numeric(18,6), + cache_write_price_per_million numeric(18,6), + priority integer not null, + context_length integer, + supports_streaming boolean not null, + supports_tools boolean not null, + enabled boolean not null, + description varchar(512), + primary key (id), + constraint uk_route unique (public_model_id, provider_id, upstream_model_id) +); + +create table client_keys ( + id bigint generated by default as identity, + name varchar(128) not null, + key_hash varchar(64) not null unique, + key_prefix varchar(24) not null, + key_suffix varchar(8) not null, + enabled boolean not null, + credit_limit_usd numeric(18,6), + used_usd numeric(18,6) not null, + rpm_limit integer not null, + max_concurrent_requests integer, + created_at timestamp(6) with time zone not null, + last_used_at timestamp(6) with time zone, + revoked_at timestamp(6) with time zone, + primary key (id) +); + +create table usage_records ( + id bigint generated by default as identity, + created_at timestamp(6) with time zone not null, + client_key_id bigint, + client_key_name varchar(128), + public_model_id varchar(128), + provider_slug varchar(64), + upstream_model_id varchar(256), + prompt_tokens integer not null, + completion_tokens integer not null, + total_tokens integer not null, + cached_input_tokens integer not null, + cache_write_tokens integer not null, + reasoning_tokens integer not null, + audio_input_tokens integer not null, + audio_output_tokens integer not null, + input_cost_usd numeric(18,8), + cached_input_cost_usd numeric(18,8), + cache_write_cost_usd numeric(18,8), + output_cost_usd numeric(18,8), + cached_savings_usd numeric(18,8), + cost_usd numeric(18,8), + usage_estimated boolean not null, + ttft_ms bigint, + tokens_per_second numeric(10,2), + upstream_request_id varchar(128), + finish_reason varchar(32), + attempts integer not null, + fallback_from varchar(64), + latency_ms bigint not null, + streamed boolean not null, + status varchar(16) not null check (status in ('SUCCESS', 'ERROR')), + error_code varchar(64), + error_message varchar(512), + primary key (id) +); + +create table idempotency_records ( + id bigint generated by default as identity, + client_key_id bigint not null, + idempotency_key varchar(64) not null, + request_hash varchar(64) not null, + status varchar(16) not null check (status in ('PENDING', 'COMPLETED')), + http_status integer not null, + content_type varchar(128), + body text, + created_at timestamp(6) with time zone not null, + expires_at timestamp(6) with time zone not null, + primary key (id), + constraint uk_idempotency unique (client_key_id, idempotency_key) +); + +create index ix_route_public on model_routes (public_model_id); +create index ix_usage_created on usage_records (created_at); +create index ix_usage_client on usage_records (client_key_id); +create index ix_usage_model on usage_records (public_model_id); +create index ix_idempotency_expiry on idempotency_records (expires_at); diff --git a/src/main/resources/db/migration/V2__response_cache.sql b/src/main/resources/db/migration/V2__response_cache.sql new file mode 100644 index 0000000..730cfbc --- /dev/null +++ b/src/main/resources/db/migration/V2__response_cache.sql @@ -0,0 +1,24 @@ +-- Exact-match response cache (issue #10). +-- Opt-in per route; only deterministic requests (temperature 0, no tools) are stored. + +alter table model_routes add column cache_enabled boolean not null default false; +alter table model_routes add column cache_ttl_seconds integer not null default 3600; + +create table response_cache_entries ( + id bigint generated by default as identity, + cache_key varchar(64) not null unique, + client_key_id bigint not null, + public_model_id varchar(128), + provider_slug varchar(64), + content_type varchar(128), + body text, + prompt_tokens integer not null, + completion_tokens integer not null, + total_tokens integer not null, + cost_usd numeric(18,8), + created_at timestamp(6) with time zone not null, + expires_at timestamp(6) with time zone not null, + primary key (id) +); + +create index ix_cache_expiry on response_cache_entries (expires_at); diff --git a/src/main/resources/db/migration/V3__budgets_and_timeouts.sql b/src/main/resources/db/migration/V3__budgets_and_timeouts.sql new file mode 100644 index 0000000..820a6e6 --- /dev/null +++ b/src/main/resources/db/migration/V3__budgets_and_timeouts.sql @@ -0,0 +1,8 @@ +-- Spend windows, output caps and per-provider timeouts (issues #16, #21). + +alter table client_keys add column daily_limit_usd numeric(18,6); +alter table client_keys add column monthly_limit_usd numeric(18,6); +alter table client_keys add column max_output_tokens integer; + +alter table providers add column request_timeout_ms integer; +alter table providers add column stream_idle_timeout_ms integer; diff --git a/src/main/resources/db/migration/V4__routing_weights.sql b/src/main/resources/db/migration/V4__routing_weights.sql new file mode 100644 index 0000000..067e421 --- /dev/null +++ b/src/main/resources/db/migration/V4__routing_weights.sql @@ -0,0 +1,3 @@ +-- Canary / A-B routing (issue #30): traffic weight per route. + +alter table model_routes add column weight integer not null default 100; diff --git a/src/main/resources/db/migration/V5__audit_and_alerts.sql b/src/main/resources/db/migration/V5__audit_and_alerts.sql new file mode 100644 index 0000000..748d002 --- /dev/null +++ b/src/main/resources/db/migration/V5__audit_and_alerts.sql @@ -0,0 +1,13 @@ +-- Admin audit trail (issue #27). + +create table admin_audit ( + id bigint generated by default as identity, + actor varchar(64) not null, + action varchar(64) not null, + target varchar(128), + details text, + created_at timestamp(6) with time zone not null, + primary key (id) +); + +create index ix_audit_created on admin_audit (created_at); diff --git a/src/main/resources/db/migration/V6__prompt_logs.sql b/src/main/resources/db/migration/V6__prompt_logs.sql new file mode 100644 index 0000000..88d749f --- /dev/null +++ b/src/main/resources/db/migration/V6__prompt_logs.sql @@ -0,0 +1,15 @@ +-- Opt-in prompt/response logging (issue #34). + +create table prompt_logs ( + id bigint generated by default as identity, + client_key_id bigint, + request_id varchar(64), + public_model_id varchar(128), + request_body text, + response_body text, + created_at timestamp(6) with time zone not null, + expires_at timestamp(6) with time zone not null, + primary key (id) +); + +create index ix_prompt_log_created on prompt_logs (created_at); diff --git a/src/main/resources/db/migration/V7__key_model_allowlist.sql b/src/main/resources/db/migration/V7__key_model_allowlist.sql new file mode 100644 index 0000000..1ed0a90 --- /dev/null +++ b/src/main/resources/db/migration/V7__key_model_allowlist.sql @@ -0,0 +1,3 @@ +-- Per-key model allowlist (issue #15). + +alter table client_keys add column allowed_models varchar(1024); diff --git a/src/main/resources/db/migration/V8__budget_reservations.sql b/src/main/resources/db/migration/V8__budget_reservations.sql new file mode 100644 index 0000000..947a73e --- /dev/null +++ b/src/main/resources/db/migration/V8__budget_reservations.sql @@ -0,0 +1,4 @@ +-- Budget reservation so concurrent requests cannot overshoot a credit limit (issue #20). + +alter table client_keys add column reserved_usd numeric(18,6) not null default 0; +alter table client_keys add column reserved_updated_at timestamp(6) with time zone; diff --git a/src/main/resources/db/migration/V9__projects_and_admin_tokens.sql b/src/main/resources/db/migration/V9__projects_and_admin_tokens.sql new file mode 100644 index 0000000..fa560ef --- /dev/null +++ b/src/main/resources/db/migration/V9__projects_and_admin_tokens.sql @@ -0,0 +1,23 @@ +-- Tenancy and scoped admin credentials (issue #33). + +create table projects ( + id bigint generated by default as identity, + slug varchar(64) not null unique, + name varchar(128) not null, + created_at timestamp(6) with time zone not null, + primary key (id) +); + +create table admin_tokens ( + id bigint generated by default as identity, + name varchar(128) not null, + token_hash varchar(64) not null unique, + token_prefix varchar(24) not null, + token_suffix varchar(8) not null, + scopes varchar(32) not null, + enabled boolean not null, + created_at timestamp(6) with time zone not null, + primary key (id) +); + +alter table client_keys add column project_id bigint; diff --git a/src/test/java/com/anindra/path/AnthropicAdapterTest.java b/src/test/java/com/anindra/path/AnthropicAdapterTest.java index 31fcdbc..a1241ad 100644 --- a/src/test/java/com/anindra/path/AnthropicAdapterTest.java +++ b/src/test/java/com/anindra/path/AnthropicAdapterTest.java @@ -12,6 +12,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class AnthropicAdapterTest { @@ -93,6 +94,71 @@ void translatesSystemMessagesImagesToolsAndToolResults() throws Exception { assertThat(tools.get(0).get("input_schema").get("type").asText()).isEqualTo("object"); } + @Test + void rejectsParametersAnthropicCannotExpress() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"claude-3.5-haiku","seed":42, + "messages":[{"role":"user","content":"hi"}]} + """); + + assertThatThrownBy(() -> adapter.build(spec, route, provider, "k", false)) + .isInstanceOf(com.anindra.path.api.ApiException.class) + .hasMessageContaining("seed") + .hasFieldOrPropertyWithValue("code", "unsupported_parameter"); + } + + @Test + void rejectsMultipleChoices() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"claude-3.5-haiku","n":2, + "messages":[{"role":"user","content":"hi"}]} + """); + + assertThatThrownBy(() -> adapter.build(spec, route, provider, "k", false)) + .hasMessageContaining("'n'"); + } + + @Test + void allowsTextResponseFormat() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"claude-3.5-haiku","response_format":{"type":"text"}, + "messages":[{"role":"user","content":"hi"}]} + """); + + UpstreamRequest request = adapter.build(spec, route, provider, "k", false); + assertThat(request.body()).isNotEmpty(); + } + + @Test + void mapsUserToMetadata() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"claude-3.5-haiku","user":"tenant-42", + "messages":[{"role":"user","content":"hi"}]} + """); + + UpstreamRequest request = adapter.build(spec, route, provider, "k", false); + JsonNode body = mapper.readTree(request.body()); + assertThat(body.get("metadata").get("user_id").asText()).isEqualTo("tenant-42"); + } + + @Test + void preservesCacheControlOnContentBlocks() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"claude-3.5-haiku","max_tokens":50, + "messages":[{"role":"user","content":[ + {"type":"text","text":"long stable prefix","cache_control":{"type":"ephemeral"}}, + {"type":"text","text":"question"}]}]} + """); + + JsonNode body = mapper.readTree( + adapter.build(spec, route, provider, "k", false).body()); + JsonNode firstBlock = body.path("messages").get(0).path("content").get(0); + + assertThat(firstBlock.path("type").asText()).isEqualTo("text"); + assertThat(firstBlock.path("cache_control").path("type").asText()).isEqualTo("ephemeral"); + assertThat(body.path("messages").get(0).path("content").get(1).has("cache_control")).isFalse(); + } + @Test void translatesAResponseWithToolUse() throws Exception { ChatRequestSpec spec = spec(""" diff --git a/src/test/java/com/anindra/path/GatewayApiTest.java b/src/test/java/com/anindra/path/GatewayApiTest.java index 8af362c..c56702b 100644 --- a/src/test/java/com/anindra/path/GatewayApiTest.java +++ b/src/test/java/com/anindra/path/GatewayApiTest.java @@ -17,7 +17,7 @@ @SpringBootTest(properties = { "spring.datasource.url=jdbc:h2:mem:gwtest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", - "spring.jpa.hibernate.ddl-auto=create-drop", + "spring.jpa.hibernate.ddl-auto=validate", "gateway.admin-token=test-admin", "gateway.bootstrap-demo-key=false", "gateway.master-key-path=./target/test-master.key" @@ -34,6 +34,9 @@ class GatewayApiTest { @Autowired private ClientKeyRepository clientKeys; + @Autowired + private com.anindra.path.admin.TenancyService tenancy; + private String key; @BeforeEach @@ -98,6 +101,55 @@ void invalidMessageRoleIs400() throws Exception { .andExpect(jsonPath("$.error.code").value("invalid_request")); } + @Test + void oversizedBodyIsRejectedBeforeParsing() throws Exception { + String big = "x".repeat(1_100_000); + mvc.perform(post("/v1/chat/completions") + .header("Authorization", "Bearer " + key) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"" + + big + "\"}]}")) + .andExpect(status().isPayloadTooLarge()) + .andExpect(jsonPath("$.error.code").value("payload_too_large")); + } + + @Test + void openApiSpecIsPublished() throws Exception { + mvc.perform(get("/v3/api-docs")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.paths").exists()) + .andExpect(jsonPath("$.paths['/v1/chat/completions']").exists()); + } + + @Test + void everyResponseCarriesARequestId() throws Exception { + mvc.perform(get("/v1/models")) + .andExpect(status().isOk()) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .header().exists("X-Request-Id")); + } + + @Test + void readOnlyAdminTokensCannotMutate() throws Exception { + String readOnly = tenancy.createToken("read-only-test", "read").plaintext(); + + mvc.perform(get("/admin/providers").header("X-Admin-Token", readOnly)) + .andExpect(status().isOk()); + + mvc.perform(post("/admin/keys") + .header("X-Admin-Token", readOnly) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"name\":\"nope\"}")) + .andExpect(status().isForbidden()); + + String full = tenancy.createToken("full-test", "write").plaintext(); + mvc.perform(post("/admin/keys") + .header("X-Admin-Token", full) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"name\":\"allowed\"}")) + .andExpect(status().isOk()); + } + @Test void adminRequiresToken() throws Exception { mvc.perform(get("/admin/providers")) diff --git a/src/test/java/com/anindra/path/UsageCostTest.java b/src/test/java/com/anindra/path/UsageCostTest.java index b9e9d69..e3925c9 100644 --- a/src/test/java/com/anindra/path/UsageCostTest.java +++ b/src/test/java/com/anindra/path/UsageCostTest.java @@ -11,7 +11,7 @@ class UsageCostTest { - private final UsageService service = new UsageService(null, null); + private final UsageService service = new UsageService(null, null, new io.micrometer.core.instrument.simple.SimpleMeterRegistry(), new com.anindra.path.config.GatewayProperties()); private final ObjectMapper mapper = new ObjectMapper(); private static ModelRoute route(String input, String cached, String output, String write) { @@ -71,7 +71,7 @@ void reasoningTokensAreBilledAtTheOutputRate() { @Test void openAiUsageParsesCacheAndReasoningDetails() throws Exception { - OpenAiAdapter adapter = new OpenAiAdapter(mapper); + OpenAiCompatibleAdapter adapter = new OpenAiCompatibleAdapter(mapper); JsonNode upstream = mapper.readTree(""" {"usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "prompt_tokens_details": {"cached_tokens": 40}, @@ -90,7 +90,7 @@ void openAiUsageParsesCacheAndReasoningDetails() throws Exception { @Test void missingUsageIsReportedAsEmpty() throws Exception { - OpenAiAdapter adapter = new OpenAiAdapter(mapper); + OpenAiCompatibleAdapter adapter = new OpenAiCompatibleAdapter(mapper); TokenUsage usage = adapter.parseUsage(mapper.readTree("{\"choices\":[]}")); assertThat(usage.reported()).isFalse(); } diff --git a/src/test/java/com/anindra/path/admin/AlertServiceTest.java b/src/test/java/com/anindra/path/admin/AlertServiceTest.java new file mode 100644 index 0000000..d6a9ca3 --- /dev/null +++ b/src/test/java/com/anindra/path/admin/AlertServiceTest.java @@ -0,0 +1,74 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +class AlertServiceTest { + + private HttpServer server; + private final AtomicInteger deliveries = new AtomicInteger(); + private final GatewayProperties props = new GatewayProperties(); + private AlertService alerts; + + @BeforeEach + void startWebhook() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/hook", exchange -> { + deliveries.incrementAndGet(); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + props.getAlerts().setWebhookUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/hook"); + props.getAlerts().setCooldown(java.time.Duration.ZERO); + alerts = new AlertService(props, new ObjectMapper()); + } + + @AfterEach + void stop() { + server.stop(0); + } + + @Test + void notifyOnceDeliversOneEventPerKey() throws Exception { + for (int i = 0; i < 5; i++) { + alerts.notifyOnce("project.budget_low:1:daily@2026-11-17", "project.budget_low", + Map.of("project", "acme")); + } + // A different period is a different alert. + alerts.notifyOnce("project.budget_low:1:daily@2026-11-18", "project.budget_low", + Map.of("project", "acme")); + + assertThat(awaitDeliveries(2)).isEqualTo(2); + } + + @Test + void noWebhookConfiguredIsANoOp() { + props.getAlerts().setWebhookUrl(""); + alerts.notifyOnce("k", "event", Map.of()); + + assertThat(deliveries.get()).isZero(); + } + + private int awaitDeliveries(int expected) throws InterruptedException { + for (int i = 0; i < 100; i++) { + if (deliveries.get() >= expected) { + // give an over-delivery a chance to show up + Thread.sleep(150); + return deliveries.get(); + } + Thread.sleep(50); + } + return deliveries.get(); + } +} diff --git a/src/test/java/com/anindra/path/admin/AliasServiceTest.java b/src/test/java/com/anindra/path/admin/AliasServiceTest.java new file mode 100644 index 0000000..153af49 --- /dev/null +++ b/src/test/java/com/anindra/path/admin/AliasServiceTest.java @@ -0,0 +1,144 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.repo.ModelAliasRepository; +import com.anindra.path.repo.ModelRouteRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:aliastest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class AliasServiceTest { + + @Autowired + private AliasService aliases; + + @Autowired + private ModelAliasRepository repository; + + @Autowired + private ModelRouteRepository routes; + + @BeforeEach + void clean() { + repository.deleteAll(); + } + + private boolean routeExists(String model) { + return !routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc(model).isEmpty(); + } + + private String anySeededModel() { + for (ModelRoute route : routes.findAll()) { + if (route.isEnabled()) { + return route.getPublicModelId(); + } + } + throw new IllegalStateException("no seeded routes"); + } + + @Test + void createsAndResolvesAChainInOrder() { + String model = anySeededModel(); + Map created = aliases.create("prod-chat", List.of(model), "stable chain"); + + assertThat(created.get("alias")).isEqualTo("prod-chat"); + assertThat(created.get("resolves_to")).isEqualTo(model); + assertThat(aliases.chainFor("prod-chat")).containsExactly(model); + + var listed = aliases.list(); + assertThat(listed).anySatisfy(row -> assertThat(row.get("alias")).isEqualTo("prod-chat")); + } + + @Test + void rejectsAnUnknownModelAtWriteTime() { + assertThatThrownBy(() -> aliases.create("broken", List.of("no-such-model"), null)) + .isInstanceOf(ApiException.class) + .hasMessageContaining("not a model this gateway serves"); + } + + @Test + void rejectsSelfReference() { + assertThatThrownBy(() -> aliases.create("loop", List.of("loop"), null)) + .isInstanceOf(ApiException.class) + .hasMessageContaining("cannot point at itself"); + } + + @Test + void allowsNestingOneAliasInsideAnother() { + String model = anySeededModel(); + aliases.create("base-alias", List.of(model), null); + + Map nested = aliases.create("outer-alias", List.of("base-alias"), null); + + assertThat(nested.get("resolves_to")).isEqualTo("base-alias"); + assertThat(aliases.chainFor("outer-alias")).containsExactly("base-alias"); + } + + @Test + void rejectsDuplicateNames() { + String model = anySeededModel(); + aliases.create("dup", List.of(model), null); + + assertThatThrownBy(() -> aliases.create("dup", List.of(model), null)) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "alias_exists"); + } + + @Test + void flagsAnAliasThatShadowsAComputedOne() { + String model = anySeededModel(); + Map created = aliases.create("cheap", List.of(model), "pinned cheapest"); + + assertThat(created.get("shadows_computed")).isEqualTo(true); + } + + @Test + void deletesAndThenReportsMissing() { + String model = anySeededModel(); + aliases.create("temp", List.of(model), null); + aliases.delete("temp"); + + assertThat(aliases.exists("temp")).isFalse(); + assertThatThrownBy(() -> aliases.delete("temp")) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "alias_not_found"); + } + + @Test + void validatesTheName() { + String model = anySeededModel(); + assertThatThrownBy(() -> aliases.create("Not Valid Name", List.of(model), null)) + .isInstanceOf(ApiException.class) + .hasMessageContaining("a-z"); + assertThatThrownBy(() -> aliases.create("empty", List.of(), null)) + .isInstanceOf(ApiException.class) + .hasMessageContaining("at least one model"); + } + + @Test + void pricesAreIrrelevantToWhetherTheChainIsValid() { + // Guards against a regression where the check used to require a price. + String model = anySeededModel(); + ModelRoute route = routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc(model).get(0); + route.setInputPricePerMillion(BigDecimal.ZERO); + routes.save(route); + + assertThat(routeExists(model)).isTrue(); + assertThat(aliases.create("free-model", List.of(model), null).get("resolves_to")).isEqualTo(model); + } +} diff --git a/src/test/java/com/anindra/path/admin/AuditServiceTest.java b/src/test/java/com/anindra/path/admin/AuditServiceTest.java new file mode 100644 index 0000000..75976f2 --- /dev/null +++ b/src/test/java/com/anindra/path/admin/AuditServiceTest.java @@ -0,0 +1,51 @@ +package com.anindra.path.admin; + +import com.anindra.path.repo.AdminAuditRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:audittest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class AuditServiceTest { + + @Autowired + private AuditService audit; + + @Autowired + private AdminAuditRepository repository; + + @BeforeEach + void clean() { + repository.deleteAll(); + } + + @Test + void recordsAndListsEntriesNewestFirst() { + audit.record("key.create", "key:1", Map.of("name", "team-a")); + audit.record("route.update", "route:4", Map.of("weight", 20)); + + var entries = audit.list(null, 10); + + assertThat(entries).hasSize(2); + assertThat(entries.get(0).get("action")).isEqualTo("route.update"); + assertThat(entries.get(1).get("target")).isEqualTo("key:1"); + } + + @Test + void filtersByAction() { + audit.record("key.create", "key:1", null); + audit.record("route.create", "route:2", null); + + assertThat(audit.list("key.create", 10)).hasSize(1); + } +} diff --git a/src/test/java/com/anindra/path/admin/PromptLogServiceTest.java b/src/test/java/com/anindra/path/admin/PromptLogServiceTest.java new file mode 100644 index 0000000..b8487b9 --- /dev/null +++ b/src/test/java/com/anindra/path/admin/PromptLogServiceTest.java @@ -0,0 +1,50 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class PromptLogServiceTest { + + private PromptLogService service() { + GatewayProperties props = new GatewayProperties(); + return new PromptLogService(null, props, + new com.anindra.path.gateway.PromptSanitizer(props)); + } + + @Test + void isDisabledByDefault() { + assertThat(service().enabled()).isFalse(); + } + + @Test + void redactsProviderKeysAndBearerTokens() { + assertThat(PromptLogService.redact("my key is sk-abcdefghijklmnopqrstuvwx")) + .isEqualTo("my key is [redacted]"); + assertThat(PromptLogService.redact("Authorization: Bearer abcdefghijklmnop1234")) + .contains("[redacted]"); + assertThat(PromptLogService.redact("gsk_abcdefghijklmnopqrst")) + .isEqualTo("[redacted]"); + } + + @Test + void piiRedactionIsOnForLogsByDefault() { + GatewayProperties props = new GatewayProperties(); + assertThat(props.getGuardrails().isPromptLogRedaction()).isTrue(); + assertThat(props.getGuardrails().isRedactBeforeSend()).isFalse(); + assertThat(props.getGuardrails().isModeration()).isFalse(); + } + + @Test + void leavesOrdinaryTextAlone() { + String text = "Explain the difference between a mutex and a semaphore."; + assertThat(PromptLogService.redact(text)).isEqualTo(text); + } + + @Test + void recordingIsANoOpWhenDisabled() { + // repository is null: a call here would NPE if the disabled guard were missing + service().record(1L, "req", "gpt-4o-mini", "{}", "{}"); + } +} diff --git a/src/test/java/com/anindra/path/admin/TenancyServiceTest.java b/src/test/java/com/anindra/path/admin/TenancyServiceTest.java new file mode 100644 index 0000000..54bc370 --- /dev/null +++ b/src/test/java/com/anindra/path/admin/TenancyServiceTest.java @@ -0,0 +1,82 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.repo.ProjectRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:tenancytest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class TenancyServiceTest { + + @Autowired + private TenancyService tenancy; + + @Autowired + private ProjectRepository projects; + + @BeforeEach + void clean() { + projects.deleteAll(); + } + + @Test + void createsProjectsAndRejectsDuplicates() { + var created = tenancy.createProject("team-a", "Team A", null, null, null, null, null); + assertThat(created.get("slug")).isEqualTo("team-a"); + + assertThatThrownBy(() -> tenancy.createProject("team-a", "Again", null, null, null, null, null)) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "project_exists"); + } + + @Test + void tokensCarryScopes() { + var readOnly = tenancy.createToken("read-only", "read"); + var readWrite = tenancy.createToken("full", "write"); + + var readPrincipal = tenancy.authenticate(readOnly.plaintext()).orElseThrow(); + assertThat(readPrincipal.canWrite()).isFalse(); + + var writePrincipal = tenancy.authenticate(readWrite.plaintext()).orElseThrow(); + assertThat(writePrincipal.canWrite()).isTrue(); + } + + @Test + void tokensAreStoredHashed() { + var created = tenancy.createToken("hashed", "read"); + + assertThat(created.token().getTokenHash()).hasSize(64) + .isNotEqualTo(created.plaintext()); + assertThat(created.token().getTokenPrefix()).isEqualTo(created.plaintext().substring(0, 12)); + } + + @Test + void revokedTokensStopAuthenticating() { + var created = tenancy.createToken("temp", "read"); + tenancy.revokeToken(created.token().getId()); + + assertThat(tenancy.authenticate(created.plaintext())).isEmpty(); + } + + @Test + void unknownTokensDoNotAuthenticate() { + assertThat(tenancy.authenticate("adm_not-a-real-token")).isEmpty(); + } + + @Test + void invalidScopesAreRejected() { + assertThatThrownBy(() -> tenancy.createToken("bad", "admin")) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "invalid_request"); + } +} diff --git a/src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java b/src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java new file mode 100644 index 0000000..00a786a --- /dev/null +++ b/src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java @@ -0,0 +1,28 @@ +package com.anindra.path.api; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class GlobalExceptionHandlerTest { + + private final GlobalExceptionHandler handler = new GlobalExceptionHandler(new ObjectMapper()); + + @Test + @SuppressWarnings("unchecked") + void unexpectedErrorsDoNotLeakInternals() { + var response = handler.unexpected(new IllegalStateException( + "could not execute statement [select * from client_keys where id=1]")); + + Map body = response.getBody(); + assertThat(body).isNotNull(); + String message = (String) ((Map) body.get("error")).get("message"); + + assertThat(message).doesNotContain("select").doesNotContain("client_keys"); + assertThat(message).contains("reference"); + assertThat(response.getStatusCode().value()).isEqualTo(500); + } +} diff --git a/src/test/java/com/anindra/path/config/AdminAuthLimiterTest.java b/src/test/java/com/anindra/path/config/AdminAuthLimiterTest.java new file mode 100644 index 0000000..4b012b0 --- /dev/null +++ b/src/test/java/com/anindra/path/config/AdminAuthLimiterTest.java @@ -0,0 +1,40 @@ +package com.anindra.path.config; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class AdminAuthLimiterTest { + + private AdminAuthLimiter limiter() { + GatewayProperties props = new GatewayProperties(); + props.getAdminAuth().setMaxFailures(2); + props.getAdminAuth().setWindow(Duration.ofMinutes(5)); + return new AdminAuthLimiter(props); + } + + @Test + void locksOutAfterTheConfiguredFailures() { + AdminAuthLimiter limiter = limiter(); + + limiter.onFailure("10.0.0.1"); + assertThat(limiter.retryAfterSeconds("10.0.0.1")).isZero(); + + limiter.onFailure("10.0.0.1"); + assertThat(limiter.retryAfterSeconds("10.0.0.1")).isGreaterThan(0); + assertThat(limiter.retryAfterSeconds("10.0.0.2")).isZero(); + } + + @Test + void successClearsTheCounter() { + AdminAuthLimiter limiter = limiter(); + limiter.onFailure("10.0.0.3"); + limiter.onFailure("10.0.0.3"); + assertThat(limiter.retryAfterSeconds("10.0.0.3")).isGreaterThan(0); + + limiter.onSuccess("10.0.0.3"); + assertThat(limiter.retryAfterSeconds("10.0.0.3")).isZero(); + } +} diff --git a/src/test/java/com/anindra/path/config/LimitedInputStreamTest.java b/src/test/java/com/anindra/path/config/LimitedInputStreamTest.java new file mode 100644 index 0000000..fc65372 --- /dev/null +++ b/src/test/java/com/anindra/path/config/LimitedInputStreamTest.java @@ -0,0 +1,51 @@ +package com.anindra.path.config; + +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletInputStream; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class LimitedInputStreamTest { + + private ServletInputStream stream(byte[] data, long limit) { + ByteArrayInputStream source = new ByteArrayInputStream(data); + return new LimitedInputStream(new ServletInputStream() { + @Override + public int read() { + return source.read(); + } + + @Override + public boolean isFinished() { + return source.available() == 0; + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setReadListener(ReadListener listener) { + // no async reads in this test + } + }, limit); + } + + @Test + void passesBodiesUnderTheLimit() throws Exception { + byte[] body = "small body".getBytes(); + assertThat(stream(body, 1024).readAllBytes()).isEqualTo(body); + } + + @Test + void abortsOnceTheLimitIsCrossed() { + // A chunked request has no Content-Length; this is the only enforcement point. + assertThatThrownBy(() -> stream(new byte[2048], 1024).readAllBytes()) + .isInstanceOf(PayloadTooLargeException.class); + } +} diff --git a/src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java b/src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java new file mode 100644 index 0000000..bf48cdb --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java @@ -0,0 +1,162 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Randomised invariants for the OpenAI → Anthropic translation, over generated + * conversations. Example-based tests only cover cases we thought of; this covers 200. + */ +class AnthropicTranslationPropertyTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final AnthropicAdapter adapter = new AnthropicAdapter(mapper); + private final Provider provider = + new Provider("anthropic", "Anthropic", ProviderType.ANTHROPIC, "https://api.anthropic.com/v1"); + private final ModelRoute route = + new ModelRoute("claude-3.5-haiku", 1L, "claude-3-5-haiku-latest", + new BigDecimal("0.80"), new BigDecimal("4.00"), 10); + + @Test + void generatedConversationsTranslateWithStableInvariants() throws Exception { + Random random = new Random(20261117L); + + for (int iteration = 0; iteration < 200; iteration++) { + ObjectNode body = generate(random); + boolean systemOnly = true; + for (JsonNode message : body.path("messages")) { + String role = message.path("role").asText(); + if (!role.equals("system") && !role.equals("developer")) { + systemOnly = false; + break; + } + } + if (systemOnly) { + // A system-only conversation is rejected rather than forwarded with an + // empty Anthropic messages array. + assertThat(org.assertj.core.api.Assertions.catchThrowable( + () -> ChatRequestSpec.parse(body))) + .isInstanceOf(com.anindra.path.api.ApiException.class); + continue; + } + ChatRequestSpec spec = ChatRequestSpec.parse(body); + + JsonNode translated = mapper.readTree( + adapter.build(spec, route, provider, "test-key", false).body()); + + assertSystemIsPreserved(body, translated); + assertMessageShape(translated); + assertToolCallsSurvive(body, translated); + } + } + + private ObjectNode generate(Random random) { + ObjectNode body = mapper.createObjectNode(); + body.put("model", "claude-3.5-haiku"); + body.put("max_tokens", 50 + random.nextInt(200)); + + StringBuilder system = new StringBuilder(); + ArrayNode messages = body.putArray("messages"); + int count = 1 + random.nextInt(6); + boolean pendingToolResult = false; + + for (int i = 0; i < count; i++) { + int pick = random.nextInt(10); + if (pick == 0) { + String text = "system rule " + i; + system.append(text); + messages.addObject().put("role", "system").put("content", text); + } else if (pick == 1) { + ObjectNode assistant = messages.addObject(); + assistant.put("role", "assistant"); + assistant.put("content", "thinking " + i); + ArrayNode calls = assistant.putArray("tool_calls"); + ObjectNode call = calls.addObject(); + call.put("id", "call_" + i); + call.put("type", "function"); + ObjectNode fn = call.putObject("function"); + fn.put("name", "lookup"); + fn.put("arguments", "{\"q\":\"" + i + "\"}"); + pendingToolResult = true; + } else if (pick == 2 && pendingToolResult) { + messages.addObject().put("role", "tool") + .put("tool_call_id", "call_" + (i - 1)).put("content", "result " + i); + pendingToolResult = false; + } else { + messages.addObject().put("role", random.nextBoolean() ? "user" : "assistant") + .put("content", "message " + i); + } + } + return body; + } + + private void assertSystemIsPreserved(ObjectNode body, JsonNode translated) { + List parts = new ArrayList<>(); + for (JsonNode message : body.path("messages")) { + if ("system".equals(message.path("role").asText())) { + parts.add(message.path("content").asText()); + } + } + String expected = String.join("\n\n", parts); + if (expected.isEmpty()) { + return; + } + assertThat(translated.path("system").asText()).isEqualTo(expected); + for (JsonNode message : translated.path("messages")) { + assertThat(message.path("role").asText()) + .as("system messages must be hoisted out of the message list") + .isNotEqualTo("system"); + } + } + + private void assertMessageShape(JsonNode translated) { + ArrayNode messages = (ArrayNode) translated.path("messages"); + assertThat(messages).isNotEmpty(); + + String previousRole = null; + for (JsonNode message : messages) { + String role = message.path("role").asText(); + assertThat(role).isIn("user", "assistant"); + assertThat(role).as("consecutive turns must be merged").isNotEqualTo(previousRole); + assertThat(message.path("content")).as("content must be block array").isNotNull(); + assertThat(message.path("content").isArray()).isTrue(); + assertThat(message.path("content")).isNotEmpty(); + previousRole = role; + } + } + + private void assertToolCallsSurvive(ObjectNode body, JsonNode translated) { + List expectedIds = new ArrayList<>(); + for (JsonNode message : body.path("messages")) { + for (JsonNode call : message.path("tool_calls")) { + expectedIds.add(call.path("id").asText()); + } + } + if (expectedIds.isEmpty()) { + return; + } + + List actualIds = new ArrayList<>(); + for (JsonNode message : translated.path("messages")) { + for (JsonNode block : message.path("content")) { + if ("tool_use".equals(block.path("type").asText())) { + actualIds.add(block.path("id").asText()); + } + } + } + assertThat(actualIds).containsExactlyElementsOf(expectedIds); + } +} diff --git a/src/test/java/com/anindra/path/gateway/BudgetReservationTest.java b/src/test/java/com/anindra/path/gateway/BudgetReservationTest.java new file mode 100644 index 0000000..5d3d0fe --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/BudgetReservationTest.java @@ -0,0 +1,79 @@ +package com.anindra.path.gateway; + +import com.anindra.path.repo.ClientKeyRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Reservations are what stop two concurrent requests from each passing the balance check + * and then both spending it. + */ +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:reservationtest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class BudgetReservationTest { + + @Autowired + private ApiKeyService apiKeys; + + @Autowired + private ClientKeyRepository keys; + + private Long keyId; + + @BeforeEach + void setUp() { + keys.deleteAll(); + keyId = apiKeys.create("reservation-test", new BigDecimal("1.00"), 100).key().getId(); + } + + @Test + void aReservationReducesTheVisibleBalance() { + var key = keys.findById(keyId).orElseThrow(); + assertThat(apiKeys.remaining(key)).isEqualByComparingTo("1.00"); + + assertThat(apiKeys.reserve(keyId, new BigDecimal("0.40"))).isTrue(); + + var after = keys.findById(keyId).orElseThrow(); + assertThat(after.getReservedUsd()).isEqualByComparingTo("0.40"); + assertThat(apiKeys.remaining(after)).isEqualByComparingTo("0.60"); + } + + @Test + void concurrentReservationsCannotOvershootTheLimit() { + // Two requests that would each fit alone, but not together. + assertThat(apiKeys.reserve(keyId, new BigDecimal("0.60"))).isTrue(); + assertThat(apiKeys.reserve(keyId, new BigDecimal("0.60"))).isFalse(); + + assertThat(keys.findById(keyId).orElseThrow().getReservedUsd()) + .isEqualByComparingTo("0.60"); + } + + @Test + void releasingRestoresTheBalance() { + assertThat(apiKeys.reserve(keyId, new BigDecimal("0.90"))).isTrue(); + apiKeys.release(keyId, new BigDecimal("0.90")); + + var key = keys.findById(keyId).orElseThrow(); + assertThat(key.getReservedUsd()).isEqualByComparingTo("0.00"); + assertThat(apiKeys.reserve(keyId, new BigDecimal("0.90"))).isTrue(); + } + + @Test + void releasingMoreThanReservedNeverGoesNegative() { + assertThat(apiKeys.reserve(keyId, new BigDecimal("0.10"))).isTrue(); + apiKeys.release(keyId, new BigDecimal("5.00")); + + assertThat(keys.findById(keyId).orElseThrow().getReservedUsd()) + .isEqualByComparingTo("0.00"); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java b/src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java new file mode 100644 index 0000000..e90f880 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java @@ -0,0 +1,67 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.junit.FuzzTest; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Fuzzes the untrusted request path: arbitrary JSON into {@link ChatRequestSpec} and then + * into the Anthropic translator. The only acceptable failure is {@link ApiException}; + * anything else (NPE, ClassCastException, OOM) is a bug. + */ +class ChatRequestFuzzTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final AnthropicAdapter adapter = new AnthropicAdapter(mapper); + private final Provider provider = + new Provider("anthropic", "Anthropic", ProviderType.ANTHROPIC, "https://api.anthropic.com/v1"); + private final ModelRoute route = + new ModelRoute("claude-3.5-haiku", 1L, "claude-3-5-haiku-latest", + new BigDecimal("0.80"), new BigDecimal("4.00"), 10); + + @FuzzTest(maxDuration = "10s") + void parserAndTranslatorOnlyRejectWithApiExceptions(FuzzedDataProvider data) { + String json = data.consumeString(4096); + JsonNode node; + try { + node = mapper.readTree(json); + } catch (Exception notJson) { + return; + } + if (!(node instanceof ObjectNode object)) { + return; + } + try { + ChatRequestSpec spec = ChatRequestSpec.parse(object); + adapter.build(spec, route, provider, "test-key", data.consumeBoolean()); + } catch (ApiException expected) { + // rejected input is fine; anything else would escape and fail the test + } + } + + @Test + void aDeeplyNestedBodyDoesNotOverflowTheStack() { + StringBuilder json = new StringBuilder("{\"model\":\"x\",\"messages\":[{\"role\":\"user\",\"content\":"); + json.append("[".repeat(200)).append("\"x\"").append("]".repeat(200)); + json.append("}]}"); + + try { + JsonNode node = mapper.readTree(json.toString()); + ChatRequestSpec.parse((ObjectNode) node); + } catch (ApiException | com.fasterxml.jackson.core.JacksonException expected) { + // either a clean rejection or a parse failure is acceptable + } + assertThat(true).isTrue(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java b/src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java new file mode 100644 index 0000000..db17ad0 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java @@ -0,0 +1,89 @@ +package com.anindra.path.gateway; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ConcurrencyLimiterTest { + + private final ConcurrencyLimiter limiter = new ConcurrencyLimiter(); + + @Test + void capsConcurrentPermitsPerKey() { + assertThat(limiter.tryAcquire(1L, 2)).isTrue(); + assertThat(limiter.tryAcquire(1L, 2)).isTrue(); + assertThat(limiter.tryAcquire(1L, 2)).isFalse(); + assertThat(limiter.inFlight(1L)).isEqualTo(2); + + limiter.release(1L); + assertThat(limiter.inFlight(1L)).isEqualTo(1); + assertThat(limiter.tryAcquire(1L, 2)).isTrue(); + } + + @Test + void aProjectCapBoundsEveryKeyInIt() { + com.anindra.path.domain.ClientKey a = projectKey(10L, 100L); + com.anindra.path.domain.ClientKey b = projectKey(11L, 100L); + com.anindra.path.domain.ClientKey other = projectKey(12L, 200L); + + assertThat(limiter.tryAcquire(a, 5, 1)).isTrue(); + // b is a different key in the same project: the project permit is already held. + assertThat(limiter.tryAcquire(b, 5, 1)).isFalse(); + // a different project is unaffected. + assertThat(limiter.tryAcquire(other, 5, 1)).isTrue(); + + limiter.release(a); + assertThat(limiter.tryAcquire(b, 5, 1)).isTrue(); + } + + @Test + void aFailedProjectPermitReleasesTheKeyPermit() { + com.anindra.path.domain.ClientKey a = projectKey(20L, 300L); + com.anindra.path.domain.ClientKey b = projectKey(21L, 300L); + assertThat(limiter.tryAcquire(a, 5, 1)).isTrue(); + + assertThat(limiter.tryAcquire(b, 5, 1)).isFalse(); + // b's key permit was handed back, so it is not counted as in flight. + assertThat(limiter.inFlight(21L)).isZero(); + } + + @Test + void aProjectWithoutALimitDoesNotTakePermits() { + com.anindra.path.domain.ClientKey key = projectKey(30L, 400L); + assertThat(limiter.tryAcquire(key, 5, null)).isTrue(); + assertThat(limiter.tryAcquire(key, 5, 0)).isTrue(); + assertThat(limiter.inFlight(ConcurrencyLimiter.projectBucket(400L))).isZero(); + } + + private static com.anindra.path.domain.ClientKey projectKey(Long id, Long projectId) { + com.anindra.path.domain.ClientKey key = new com.anindra.path.domain.ClientKey(); + key.setId(id); + key.setName("k" + id); + key.setProjectId(projectId); + return key; + } + + @Test + void keysAreIndependent() { + assertThat(limiter.tryAcquire(1L, 1)).isTrue(); + assertThat(limiter.tryAcquire(1L, 1)).isFalse(); + assertThat(limiter.tryAcquire(2L, 1)).isTrue(); + } + + @Test + void zeroMeansUnlimited() { + assertThat(limiter.tryAcquire(3L, 0)).isTrue(); + assertThat(limiter.tryAcquire(3L, 0)).isTrue(); + assertThat(limiter.inFlight(3L)).isZero(); + } + + @Test + void releaseDoesNotOverRelease() { + assertThat(limiter.tryAcquire(4L, 1)).isTrue(); + limiter.release(4L); + limiter.release(4L); + assertThat(limiter.inFlight(4L)).isZero(); + assertThat(limiter.tryAcquire(4L, 1)).isTrue(); + assertThat(limiter.tryAcquire(4L, 1)).isFalse(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/CreditGuardTest.java b/src/test/java/com/anindra/path/gateway/CreditGuardTest.java new file mode 100644 index 0000000..ed4ae93 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/CreditGuardTest.java @@ -0,0 +1,155 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class CreditGuardTest { + + private final ApiKeyService apiKeys = mock(ApiKeyService.class); + private final UsageService usage = mock(UsageService.class); + private final GatewayProperties props = new GatewayProperties(); + private final com.anindra.path.repo.ProjectRepository projects = + mock(com.anindra.path.repo.ProjectRepository.class); + private final EndUserBudget endUserBudget = mock(EndUserBudget.class); + private final CreditGuard guard = new CreditGuard(apiKeys, endUserBudget, usage, props, + new com.anindra.path.admin.AlertService(props, new ObjectMapper()), projects); + private final ObjectMapper mapper = new ObjectMapper(); + + @org.junit.jupiter.api.BeforeEach + void stubCost() { + // Estimates are computed before the balance is read, so every test that gets past + // the budget windows needs a price. + org.mockito.Mockito.lenient().when(usage.cost(any(), any())) + .thenReturn(new UsageService.CostBreakdown(BigDecimal.ZERO, BigDecimal.ZERO, + BigDecimal.ZERO, BigDecimal.ZERO, BigDecimal.ZERO, + new BigDecimal("0.0001"))); + } + + private ClientKey key(BigDecimal daily, BigDecimal monthly, Integer maxOutput) { + ClientKey key = new ClientKey(); + key.setId(1L); + key.setName("test"); + key.setDailyLimitUsd(daily); + key.setMonthlyLimitUsd(monthly); + key.setMaxOutputTokens(maxOutput); + return key; + } + + private RouteResolver.ResolvedRoute route() { + ModelRoute route = new ModelRoute("gpt-4o-mini", 1L, "gpt-4o-mini", + new BigDecimal("0.15"), new BigDecimal("0.60"), 10); + Provider provider = new Provider("openai", "OpenAI", ProviderType.OPENAI, "https://api.openai.com/v1"); + return new RouteResolver.ResolvedRoute(route, provider); + } + + private ChatRequestSpec spec(String maxTokens) throws Exception { + String json = "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]" + + (maxTokens == null ? "" : ",\"max_tokens\":" + maxTokens) + "}"; + return ChatRequestSpec.parse((ObjectNode) mapper.readTree(json)); + } + + @Test + void rejectsWhenTheDailyBudgetIsSpent() throws Exception { + when(usage.spendFor(eq(1L), any(Instant.class))).thenReturn(new BigDecimal("5.00")); + + assertThatThrownBy(() -> guard.check(key(new BigDecimal("5.00"), null, null), spec(null), route())) + .isInstanceOf(ApiException.class) + .hasMessageContaining("Daily budget"); + } + + @Test + void allowsWhenUnderTheWindow() throws Exception { + when(usage.spendFor(eq(1L), any(Instant.class))).thenReturn(new BigDecimal("1.00")); + + guard.check(key(new BigDecimal("5.00"), new BigDecimal("50.00"), null), spec(null), route()); + } + + @Test + void clampsMaxTokensToTheKeyCap() throws Exception { + ChatRequestSpec spec = spec("5000"); + when(usage.spendFor(eq(1L), any(Instant.class))).thenReturn(BigDecimal.ZERO); + + guard.check(key(null, null, 100), spec, route()); + + assertThat(spec.maxTokens()).isEqualTo(100); + assertThat(spec.body().get("max_tokens").asInt()).isEqualTo(100); + } + + @Test + void allowlistMatchesIdsAndGlobs() { + ClientKey key = key(null, null, null); + + key.setAllowedModels("gpt-4o-mini,claude-*"); + assertThat(CreditGuard.allows(key, "gpt-4o-mini")).isTrue(); + assertThat(CreditGuard.allows(key, "claude-3.5-haiku")).isTrue(); + assertThat(CreditGuard.allows(key, "llama-3.1-8b")).isFalse(); + + key.setAllowedModels("gpt-*"); + assertThat(CreditGuard.allows(key, "llama-3.1-8b")).isFalse(); + + key.setAllowedModels(null); + assertThat(CreditGuard.allows(key, "anything")).isTrue(); + } + + @Test + void reservesTheExpectedCostRatherThanTheMaxTokensCap() throws Exception { + ClientKey key = key(null, null, null); + key.setCreditLimitUsd(new BigDecimal("0.001")); + key.setUsedUsd(BigDecimal.ZERO); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("0.001")); + when(apiKeys.reserve(eq(1L), any())).thenReturn(true); + when(usage.cost(any(), any())).thenReturn( + new UsageService.CostBreakdown(BigDecimal.ZERO, BigDecimal.ZERO, BigDecimal.ZERO, + BigDecimal.ZERO, BigDecimal.ZERO, new BigDecimal("0.0100")), + new UsageService.CostBreakdown(BigDecimal.ZERO, BigDecimal.ZERO, BigDecimal.ZERO, + BigDecimal.ZERO, BigDecimal.ZERO, new BigDecimal("0.0005"))); + + // max_tokens is 8000, but a typical completion fits in the remaining balance. + BigDecimal reserved = guard.check(key, spec("8000"), route()).keyUsd(); + + assertThat(reserved).isEqualByComparingTo("0.001"); + } + + @Test + void blocksAModelOutsideTheAllowlist() throws Exception { + ClientKey key = key(null, null, null); + key.setAllowedModels("gpt-4o-mini"); + when(usage.spendFor(eq(1L), any(Instant.class))).thenReturn(BigDecimal.ZERO); + + assertThatThrownBy(() -> guard.check(key, specOf("claude-3.5-haiku"), route())) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "model_not_allowed"); + } + + private ChatRequestSpec specOf(String model) throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree( + "{\"model\":\"" + model + "\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")); + } + + @Test + void leavesSmallerMaxTokensAlone() throws Exception { + ChatRequestSpec spec = spec("40"); + when(usage.spendFor(eq(1L), any(Instant.class))).thenReturn(BigDecimal.ZERO); + + guard.check(key(null, null, 100), spec, route()); + + assertThat(spec.maxTokens()).isEqualTo(40); + } +} diff --git a/src/test/java/com/anindra/path/gateway/DrainStateTest.java b/src/test/java/com/anindra/path/gateway/DrainStateTest.java new file mode 100644 index 0000000..eb7d92b --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/DrainStateTest.java @@ -0,0 +1,54 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DrainStateTest { + + private DrainState state() { + GatewayProperties props = new GatewayProperties(); + props.setDrainTimeout(Duration.ofMillis(200)); + return new DrainState(props); + } + + @Test + void tracksInFlightRequests() { + DrainState drain = state(); + drain.begin(); + drain.begin(); + assertThat(drain.inFlight()).isEqualTo(2); + + drain.end(); + assertThat(drain.inFlight()).isEqualTo(1); + } + + @Test + void refusesNewWorkOnceDraining() { + DrainState drain = state(); + drain.drain(); + + assertThat(drain.isDraining()).isTrue(); + assertThatThrownBy(drain::begin) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "shutting_down"); + } + + @Test + void drainWaitsForInFlightWorkUpToTheTimeout() { + DrainState drain = state(); + drain.begin(); + + long started = System.currentTimeMillis(); + drain.drain(); + long elapsed = System.currentTimeMillis() - started; + + assertThat(elapsed).isGreaterThanOrEqualTo(150); + assertThat(drain.isDraining()).isTrue(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java b/src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java new file mode 100644 index 0000000..b22d393 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java @@ -0,0 +1,80 @@ +package com.anindra.path.gateway; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:enduserbudget;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class EndUserBudgetTest { + + @Autowired + private EndUserBudget budget; + + @BeforeEach + void clean() { + budget.clear(1L, "alice"); + budget.clear(1L, "bob"); + } + + @Test + void reservesAndReleasesTheSameAmount() { + assertThat(budget.reserve(1L, "alice", new BigDecimal("0.40"), new BigDecimal("1.00"))).isTrue(); + budget.release(1L, "alice", new BigDecimal("0.40")); + + // The full headroom is available again. + assertThat(budget.reserve(1L, "alice", new BigDecimal("1.00"), new BigDecimal("1.00"))).isTrue(); + budget.clear(1L, "alice"); + } + + @Test + void refusesMoreThanTheHeadroomAllows() { + assertThat(budget.reserve(1L, "alice", new BigDecimal("0.60"), new BigDecimal("1.00"))).isTrue(); + // 0.60 in flight, so 0.50 more would cross the 1.00 headroom. + assertThat(budget.reserve(1L, "alice", new BigDecimal("0.50"), new BigDecimal("1.00"))).isFalse(); + // What still fits is allowed. + assertThat(budget.reserve(1L, "alice", new BigDecimal("0.40"), new BigDecimal("1.00"))).isTrue(); + budget.clear(1L, "alice"); + } + + @Test + void concurrentReservationsCannotBothTakeTheLastOfTheBudget() { + // Two requests that each fit alone but not together. + boolean first = budget.reserve(1L, "alice", new BigDecimal("0.60"), new BigDecimal("1.00")); + boolean second = budget.reserve(1L, "alice", new BigDecimal("0.60"), new BigDecimal("1.00")); + + assertThat(first).isTrue(); + assertThat(second).isFalse(); + budget.clear(1L, "alice"); + } + + @Test + void endUsersAreIndependent() { + assertThat(budget.reserve(1L, "alice", new BigDecimal("1.00"), new BigDecimal("1.00"))).isTrue(); + assertThat(budget.reserve(1L, "bob", new BigDecimal("1.00"), new BigDecimal("1.00"))).isTrue(); + + budget.clear(1L, "alice"); + budget.clear(1L, "bob"); + } + + @Test + void anOverReleaseIsIgnoredRatherThanCreditingBudget() { + budget.reserve(1L, "alice", new BigDecimal("0.10"), new BigDecimal("1.00")); + // A caller releasing more than it reserved must not free up someone else's budget. + budget.release(1L, "alice", new BigDecimal("5.00")); + + // The 0.10 is still held, so the remaining 0.90 fits and 1.00 does not. + assertThat(budget.reserve(1L, "alice", new BigDecimal("0.90"), new BigDecimal("1.00"))).isTrue(); + assertThat(budget.reserve(1L, "alice", new BigDecimal("0.01"), new BigDecimal("1.00"))).isFalse(); + budget.clear(1L, "alice"); + } +} diff --git a/src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java b/src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java new file mode 100644 index 0000000..282ab8a --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java @@ -0,0 +1,99 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.repo.IdempotencyRecordRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:idempotencytest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class IdempotencyServiceTest { + + @Autowired + private IdempotencyService service; + + @Autowired + private IdempotencyRecordRepository records; + + @BeforeEach + void clean() { + records.deleteAll(); + } + + @Test + void replaysACompletedResponse() { + assertThat(service.begin(42L, "idem-1", "hash-a")).isEmpty(); + + service.complete(42L, "idem-1", 200, "application/json", "{\"ok\":true}"); + + IdempotencyService.Replay replay = service.begin(42L, "idem-1", "hash-a").orElseThrow(); + assertThat(replay.httpStatus()).isEqualTo(200); + assertThat(replay.contentType()).isEqualTo("application/json"); + assertThat(replay.body()).isEqualTo("{\"ok\":true}"); + } + + @Test + void rejectsTheSameKeyWithADifferentBody() { + service.begin(43L, "idem-2", "hash-a"); + service.complete(43L, "idem-2", 200, "application/json", "{}"); + + assertThatThrownBy(() -> service.begin(43L, "idem-2", "hash-b")) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "idempotency_conflict"); + } + + @Test + void rejectsAConcurrentDuplicate() { + assertThat(service.begin(44L, "idem-3", "hash-a")).isEmpty(); + + assertThatThrownBy(() -> service.begin(44L, "idem-3", "hash-a")) + .isInstanceOf(ApiException.class) + .hasFieldOrPropertyWithValue("code", "idempotency_in_progress"); + } + + @Test + void abandonedClaimCanBeReused() { + assertThat(service.begin(45L, "idem-4", "hash-a")).isEmpty(); + service.abandon(45L, "idem-4"); + + assertThat(service.begin(45L, "idem-4", "hash-a")).isEmpty(); + } + + @Test + void stalePendingClaimsAreReclaimed() { + assertThat(service.begin(50L, "idem-stale", "hash-a")).isEmpty(); + + var record = records.findAll().get(0); + record.setCreatedAt(java.time.Instant.now().minus(java.time.Duration.ofMinutes(10))); + records.save(record); + + // The instance that claimed this key died; a retry must not be blocked for 24h. + assertThat(service.begin(50L, "idem-stale", "hash-a")).isEmpty(); + } + + @Test + void keysAreScopedToTheClientKey() { + service.begin(46L, "shared-key", "hash-a"); + service.complete(46L, "shared-key", 200, "application/json", "{\"tenant\":1}"); + + // A different client key with the same header value gets its own claim. + assertThat(service.begin(47L, "shared-key", "hash-a")).isEmpty(); + } + + @Test + void storedKeyIsHashedNotPlaintext() { + service.begin(48L, "plain-text-key", "hash-a"); + var stored = records.findAll().get(0); + assertThat(stored.getIdempotencyKey()).isNotEqualTo("plain-text-key"); + assertThat(stored.getIdempotencyKey()).hasSize(64); + } +} diff --git a/src/test/java/com/anindra/path/gateway/PiiRedactorTest.java b/src/test/java/com/anindra/path/gateway/PiiRedactorTest.java new file mode 100644 index 0000000..ab2caff --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/PiiRedactorTest.java @@ -0,0 +1,88 @@ +package com.anindra.path.gateway; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class PiiRedactorTest { + + private String redact(String text) { + return PiiRedactor.redact(text).text(); + } + + @Test + void replacesEmailsWithStableTokens() { + var result = PiiRedactor.redact("write to alice@example.com and bob@corp.co.uk"); + + assertThat(result.text()).isEqualTo("write to [EMAIL_1] and [EMAIL_2]"); + assertThat(result.counts()).containsEntry("EMAIL", 2); + } + + @Test + void replacesCardNumbersThatPassTheChecksum() { + // 4242 4242 4242 4242 is a valid Luhn number (Stripe's test card). + var result = PiiRedactor.redact("charge 4242 4242 4242 4242 please"); + + assertThat(result.text()).contains("[CARD_1]"); + assertThat(result.text()).doesNotContain("4242"); + } + + @Test + void leavesDigitRunsThatAreNotCardsAlone() { + // Passes the 16-digit shape but fails Luhn, so it is an id, not a card. + assertThat(redact("order 1234 5678 9012 3456 shipped")).contains("1234 5678 9012 3456"); + } + + @Test + void replacesAadhaarAndPan() { + assertThat(redact("aadhaar 1234 5678 9012")).contains("[AADHAAR_1]"); + assertThat(redact("pan ABCDE1234F")).contains("[PAN_1]"); + } + + @Test + void replacesPhoneNumbers() { + assertThat(redact("call +91 98765 43210 now")).contains("[PHONE_1]"); + assertThat(redact("call 9876543210 now")).contains("[PHONE_1]"); + } + + @Test + void leavesTimestampsAndShortNumbersAlone() { + String text = "at 2026-11-17T10:45:24Z request 42 took 250ms"; + assertThat(redact(text)).isEqualTo(text); + } + + @Test + void tokensAreNumberedPerCallAndRepeatable() { + String once = redact("a@b.com a@b.com"); + assertThat(once).isEqualTo("[EMAIL_1] [EMAIL_2]"); + // Deterministic: the same input redacts identically every time. + assertThat(redact("a@b.com a@b.com")).isEqualTo(once); + } + + @Test + void handlesEmptyAndNullInput() { + assertThat(PiiRedactor.redact("").text()).isEmpty(); + assertThat(PiiRedactor.redact(null).text()).isNull(); + assertThat(PiiRedactor.redact(null).redacted()).isFalse(); + } + + @Test + void reportsWhatItReplaced() { + var result = PiiRedactor.redact("mail me at a@b.com or ring 9876543210"); + assertThat(result.redacted()).isTrue(); + assertThat(result.total()).isEqualTo(2); + } + + @Test + void leavesEpochTimestampsAlone() { + // A 10-digit epoch starts with 1, so it is not mistaken for a mobile number. + assertThat(redact("{\"created\":1700000000}")).isEqualTo("{\"created\":1700000000}"); + } + + @Test + void aCleanPromptIsUntouched() { + String text = "Summarise the second paragraph in one sentence."; + assertThat(PiiRedactor.redact(text).counts()).isEmpty(); + assertThat(redact(text)).isEqualTo(text); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java b/src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java new file mode 100644 index 0000000..9429841 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java @@ -0,0 +1,227 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Project; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.anindra.path.repo.ProjectRepository; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ProjectBudgetAlertTest { + + private final ApiKeyService apiKeys = mock(ApiKeyService.class); + private final UsageService usage = mock(UsageService.class); + private final ProjectRepository projects = mock(ProjectRepository.class); + private final com.anindra.path.admin.AlertService alerts = + mock(com.anindra.path.admin.AlertService.class); + private final GatewayProperties props = new GatewayProperties(); + private final EndUserBudget endUserBudget = mock(EndUserBudget.class); + private final CreditGuard guard = + new CreditGuard(apiKeys, endUserBudget, usage, props, alerts, projects); + private final ObjectMapper mapper = new ObjectMapper(); + + private Project project(String dailyLimit) { + Project project = new Project(); + project.setId(7L); + project.setSlug("acme"); + project.setName("Acme"); + project.setDailyLimitUsd(dailyLimit == null ? null : new BigDecimal(dailyLimit)); + return project; + } + + private ClientKey key(long projectId) { + ClientKey key = new ClientKey(); + key.setId(1L); + key.setName("test"); + key.setProjectId(projectId); + return key; + } + + private RouteResolver.ResolvedRoute route() { + ModelRoute route = new ModelRoute("gpt-4o-mini", 1L, "gpt-4o-mini", + new BigDecimal("0.15"), new BigDecimal("0.60"), 10); + Provider provider = new Provider("openai", "OpenAI", ProviderType.OPENAI, + "https://api.openai.com/v1"); + return new RouteResolver.ResolvedRoute(route, provider); + } + + private void stubCost() { + when(usage.cost(any(), any())).thenReturn(new UsageService.CostBreakdown( + BigDecimal.ZERO, BigDecimal.ZERO, BigDecimal.ZERO, BigDecimal.ZERO, + BigDecimal.ZERO, new BigDecimal("0.0001"))); + when(apiKeys.reserve(any(), any())).thenReturn(true); + } + + private ChatRequestSpec specFor(String endUser) throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree( + "{\"model\":\"gpt-4o-mini\",\"end_user\":\"" + endUser + "\"," + + "\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")); + } + + private ChatRequestSpec spec() throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree( + "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")); + } + + @Test + void theEndUserCapBlocksWhenItsDailyBudgetIsSpent() throws Exception { + ClientKey key = key(7L); + key.setEndUserDailyLimitUsd(new BigDecimal("1.00")); + when(projects.findById(7L)).thenReturn(Optional.of(project(null))); + when(usage.spendForEndUser(eq(1L), eq("alice"), any(Instant.class))) + .thenReturn(new BigDecimal("1.00")); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + stubCost(); + + assertThatThrownBy(() -> guard.check(key, specFor("alice"), route())) + .isInstanceOf(ApiException.class) + .hasMessageContaining("alice"); + } + + @Test + void aSecondEndUserIsUnaffectedByTheFirstOnesCap() throws Exception { + ClientKey key = key(7L); + key.setEndUserDailyLimitUsd(new BigDecimal("1.00")); + when(projects.findById(7L)).thenReturn(Optional.of(project(null))); + when(usage.spendForEndUser(eq(1L), eq("alice"), any(Instant.class))) + .thenReturn(new BigDecimal("1.00")); + when(usage.spendForEndUser(eq(1L), eq("bob"), any(Instant.class))) + .thenReturn(BigDecimal.ZERO); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + when(endUserBudget.reserve(eq(1L), eq("bob"), any(), any())).thenReturn(true); + stubCost(); + + var reservation = guard.check(key, specFor("bob"), route()); + + assertThat(reservation.endUser()).isEqualTo("bob"); + assertThat(reservation.endUserUsd()).isPositive(); + } + + @Test + void withoutACapNothingIsReservedForTheEndUser() throws Exception { + when(projects.findById(7L)).thenReturn(Optional.of(project(null))); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + stubCost(); + + var reservation = guard.check(key(7L), specFor("alice"), route()); + + assertThat(reservation.endUserUsd()).isEqualByComparingTo("0"); + verify(endUserBudget, never()).reserve(any(), any(), any(), any()); + } + + @Test + void anUnlimitedKeyStillEnforcesItsEndUserCap() throws Exception { + // Regression: the cap used to sit behind the remaining == null early return, so a + // key without a credit limit skipped it entirely. + ClientKey key = key(7L); + key.setEndUserDailyLimitUsd(new BigDecimal("1.00")); + when(projects.findById(7L)).thenReturn(Optional.of(project(null))); + when(usage.spendForEndUser(eq(1L), eq("alice"), any(Instant.class))) + .thenReturn(new BigDecimal("1.00")); + when(apiKeys.remaining(any())).thenReturn(null); + stubCost(); + + assertThatThrownBy(() -> guard.check(key, specFor("alice"), route())) + .isInstanceOf(ApiException.class) + .hasMessageContaining("alice"); + } + + @Test + void theKeyReservationIsReleasedWhenTheEndUserReservationFails() throws Exception { + ClientKey key = key(7L); + key.setEndUserDailyLimitUsd(new BigDecimal("1.00")); + when(projects.findById(7L)).thenReturn(Optional.of(project(null))); + when(usage.spendForEndUser(eq(1L), eq("alice"), any(Instant.class))) + .thenReturn(BigDecimal.ZERO); + when(endUserBudget.reserve(eq(1L), eq("alice"), any(), any())).thenReturn(true); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + stubCost(); + // after stubCost, so the key reservation is the one that fails + when(apiKeys.reserve(any(), any())).thenReturn(false); + + assertThatThrownBy(() -> guard.check(key, specFor("alice"), route())) + .isInstanceOf(ApiException.class) + .hasMessageContaining("concurrent"); + + verify(endUserBudget).release(eq(1L), eq("alice"), any()); + } + + @Test + void warnsOnceWhenTheProjectBudgetIsNearlySpent() throws Exception { + when(projects.findById(7L)).thenReturn(Optional.of(project("10.00"))); + when(usage.spendForProject(eq(7L), any(Instant.class))).thenReturn(new BigDecimal("9.00")); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + stubCost(); + + guard.check(key(7L), spec(), route()); + guard.check(key(7L), spec(), route()); + + // The dedupe key is stable per project, window and period: AlertService.notifyOnce + // swallows the repeats, so the guard may ask twice but one event goes out. + verify(alerts, times(2)).notifyOnce(contains("project.budget_low:7:daily"), eq("project.budget_low"), anyMap()); + verify(alerts, never()).notifyOnce(contains("project.budget_exhausted"), anyString(), anyMap()); + } + + @Test + void stopsTheCallAndAlertsWhenTheProjectBudgetIsGone() throws Exception { + when(projects.findById(7L)).thenReturn(Optional.of(project("10.00"))); + when(usage.spendForProject(eq(7L), any(Instant.class))).thenReturn(new BigDecimal("10.00")); + + assertThatThrownBy(() -> guard.check(key(7L), spec(), route())) + .isInstanceOf(ApiException.class) + .hasMessageContaining("acme"); + + verify(alerts, times(1)).notifyOnce(contains("project.budget_exhausted:7:daily"), + eq("project.budget_exhausted"), anyMap()); + } + + @Test + void doesNotAlertWhenTheProjectHasNoLimit() throws Exception { + when(projects.findById(7L)).thenReturn(Optional.of(project(null))); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + stubCost(); + + guard.check(key(7L), spec(), route()); + + verify(alerts, never()).notifyOnce(anyString(), anyString(), anyMap()); + } + + @Test + @SuppressWarnings("unchecked") + void thePayloadNamesTheWindowAndTheMoney() throws Exception { + when(projects.findById(7L)).thenReturn(Optional.of(project("10.00"))); + when(usage.spendForProject(eq(7L), any(Instant.class))).thenReturn(new BigDecimal("9.50")); + when(apiKeys.remaining(any())).thenReturn(new BigDecimal("10.00")); + stubCost(); + + guard.check(key(7L), spec(), route()); + verify(alerts).notifyOnce(contains("project.budget_low"), eq("project.budget_low"), + org.mockito.ArgumentMatchers.argThat((Map payload) -> + "acme".equals(payload.get("project")) + && "daily".equals(payload.get("window")) + && payload.get("remaining_usd") != null)); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java b/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java new file mode 100644 index 0000000..0dd9e3f --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java @@ -0,0 +1,78 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProviderCircuitBreakerTest { + + private ProviderCircuitBreaker breaker; + + @BeforeEach + void setUp() { + GatewayProperties props = new GatewayProperties(); + props.getBreaker().setFailureThreshold(2); + props.getBreaker().setOpenDuration(Duration.ofMillis(120)); + breaker = new ProviderCircuitBreaker(props, + new com.anindra.path.admin.AlertService(props, new com.fasterxml.jackson.databind.ObjectMapper())); + } + + @Test + void opensAfterThresholdAndClosesOnSuccessfulProbe() throws Exception { + assertThat(breaker.isAvailable(1L)).isTrue(); + + breaker.onFailure(1L); + assertThat(breaker.isAvailable(1L)).isTrue(); + + breaker.onFailure(1L); + assertThat(breaker.isAvailable(1L)).isFalse(); + assertThat(breaker.state(1L)).isEqualTo(ProviderCircuitBreaker.State.OPEN); + + Thread.sleep(180); + assertThat(breaker.isAvailable(1L)).isTrue(); + assertThat(breaker.state(1L)).isEqualTo(ProviderCircuitBreaker.State.HALF_OPEN); + + breaker.onSuccess(1L); + assertThat(breaker.state(1L)).isEqualTo(ProviderCircuitBreaker.State.CLOSED); + } + + @Test + void failureDuringHalfOpenReopensImmediately() throws Exception { + breaker.onFailure(2L); + breaker.onFailure(2L); + Thread.sleep(180); + + assertThat(breaker.isAvailable(2L)).isTrue(); + breaker.onFailure(2L); + assertThat(breaker.isAvailable(2L)).isFalse(); + assertThat(breaker.state(2L)).isEqualTo(ProviderCircuitBreaker.State.OPEN); + } + + @Test + void successResetsTheFailureCount() { + breaker.onFailure(3L); + breaker.onSuccess(3L); + breaker.onFailure(3L); + assertThat(breaker.isAvailable(3L)).isTrue(); + } + + @Test + void rateLimitParksTheProviderForRetryAfter() { + breaker.onRateLimited(4L, Duration.ofSeconds(5)); + + assertThat(breaker.isAvailable(4L)).isFalse(); + assertThat(breaker.retryAfterSeconds(4L)).isBetween(3L, 5L); + assertThat(breaker.state(4L)).isEqualTo(ProviderCircuitBreaker.State.OPEN); + } + + @Test + void unknownProviderIsClosedAndAvailable() { + assertThat(breaker.isAvailable(99L)).isTrue(); + assertThat(breaker.state(99L)).isEqualTo(ProviderCircuitBreaker.State.CLOSED); + assertThat(breaker.retryAfterSeconds(99L)).isZero(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java b/src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java new file mode 100644 index 0000000..921f4bd --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java @@ -0,0 +1,48 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.anindra.path.repo.ProviderRepository; +import org.junit.jupiter.api.Test; +import org.springframework.boot.actuate.health.Status; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ProviderHealthIndicatorTest { + + private final ProviderRepository providers = mock(ProviderRepository.class); + private final ProviderCircuitBreaker breaker = mock(ProviderCircuitBreaker.class); + + private Provider provider(long id, String slug) { + Provider provider = new Provider(slug, slug, ProviderType.OPENAI, "http://localhost"); + provider.setId(id); + return provider; + } + + @Test + void staysUpWhileFallbackExists() { + when(providers.findByEnabledTrueOrderBySlugAsc()) + .thenReturn(List.of(provider(1, "openai"), provider(2, "groq"))); + when(breaker.state(1L)).thenReturn(ProviderCircuitBreaker.State.OPEN); + when(breaker.state(2L)).thenReturn(ProviderCircuitBreaker.State.CLOSED); + + var health = new ProviderHealthIndicator(providers, breaker).health(); + + assertThat(health.getStatus()).isEqualTo(Status.UP); + assertThat(health.getDetails().get("open_circuits")).isEqualTo(1); + } + + @Test + void reportsOutOfServiceWhenEveryProviderIsParked() { + when(providers.findByEnabledTrueOrderBySlugAsc()).thenReturn(List.of(provider(1, "openai"))); + when(breaker.state(1L)).thenReturn(ProviderCircuitBreaker.State.OPEN); + + var health = new ProviderHealthIndicator(providers, breaker).health(); + + assertThat(health.getStatus().getCode()).isEqualTo("OUT_OF_SERVICE"); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java b/src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java new file mode 100644 index 0000000..3ea9a16 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java @@ -0,0 +1,73 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProviderUrlsTest { + + private Provider provider(String baseUrl, String apiVersion) { + Provider provider = new Provider("p", "p", ProviderType.OPENAI, baseUrl); + provider.setApiVersion(apiVersion); + return provider; + } + + @Test + void plainProvidersUseTheOpenAiShape() { + Provider openai = provider("https://api.openai.com/v1", null); + + assertThat(ProviderUrls.chat(openai, "gpt-4o-mini")) + .isEqualTo("https://api.openai.com/v1/chat/completions"); + assertThat(ProviderUrls.embeddings(openai, "text-embedding-3-small")) + .isEqualTo("https://api.openai.com/v1/embeddings"); + assertThat(ProviderUrls.headers(openai, "sk-test", false)) + .containsEntry("Authorization", "Bearer sk-test") + .doesNotContainKey("api-key"); + } + + @Test + void azurePutsTheDeploymentInThePathAndTheVersionInTheQuery() { + Provider azure = provider("https://acme.openai.azure.com", "2024-10-21"); + + assertThat(ProviderUrls.chat(azure, "gpt-4o-prod")) + .isEqualTo("https://acme.openai.azure.com/openai/deployments/gpt-4o-prod" + + "/chat/completions?api-version=2024-10-21"); + assertThat(ProviderUrls.embeddings(azure, "embed-prod")) + .isEqualTo("https://acme.openai.azure.com/openai/deployments/embed-prod" + + "/embeddings?api-version=2024-10-21"); + } + + @Test + void azureAuthenticatesWithTheApiKeyHeader() { + Map headers = ProviderUrls.headers( + provider("https://acme.openai.azure.com", "2024-10-21"), "secret", true); + + assertThat(headers).containsEntry("api-key", "secret") + .containsEntry("Accept", "text/event-stream") + .doesNotContainKey("Authorization"); + } + + @Test + void aTrailingSlashDoesNotDoubleUp() { + Provider azure = provider("https://acme.openai.azure.com/", "2024-10-21"); + assertThat(ProviderUrls.chat(azure, "deployment")) + .startsWith("https://acme.openai.azure.com/openai/deployments/deployment/chat") + .doesNotContain("//openai"); + } + + @Test + void probeUsesTheDeploymentListingForAzure() { + Provider azure = provider("https://acme.openai.azure.com", "2024-10-21"); + assertThat(ProviderUrls.models(azure)) + .isEqualTo("https://acme.openai.azure.com/openai/models?api-version=2024-10-21"); + } + + @Test + void blankApiVersionIsNotTreatedAsAzure() { + assertThat(ProviderUrls.isAzure(provider("https://x/v1", " "))).isFalse(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java b/src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java new file mode 100644 index 0000000..0e032a4 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java @@ -0,0 +1,69 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.ModelRoute; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ResponseCacheEligibilityTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final ResponseCacheService cache = new ResponseCacheService(null, null); + + private ModelRoute route(boolean enabled) { + ModelRoute route = new ModelRoute("gpt-4o-mini", 1L, "gpt-4o-mini", null, null, 10); + route.setCacheEnabled(enabled); + return route; + } + + private ChatRequestSpec spec(String json) throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree(json)); + } + + @Test + void requiresTheRouteToOptIn() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"gpt-4o-mini","temperature":0, + "messages":[{"role":"user","content":"hi"}]} + """); + assertThat(ResponseCacheService.eligible(spec, route(true))).isTrue(); + assertThat(ResponseCacheService.eligible(spec, route(false))).isFalse(); + } + + @Test + void requiresAnExplicitZeroTemperature() throws Exception { + ChatRequestSpec noTemp = spec(""" + {"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}]} + """); + ChatRequestSpec warm = spec(""" + {"model":"gpt-4o-mini","temperature":0.7, + "messages":[{"role":"user","content":"hi"}]} + """); + assertThat(ResponseCacheService.eligible(noTemp, route(true))).isFalse(); + assertThat(ResponseCacheService.eligible(warm, route(true))).isFalse(); + } + + @Test + void refusesRequestsWithTools() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"gpt-4o-mini","temperature":0, + "messages":[{"role":"user","content":"hi"}], + "tools":[{"type":"function","function":{"name":"f","parameters":{"type":"object"}}}]} + """); + assertThat(ResponseCacheService.eligible(spec, route(true))).isFalse(); + } + + @Test + void keyIsScopedToTheClientAndPayload() { + String keyA = cache.key(1L, "gpt-4o-mini", "{\"a\":1}"); + String keyB = cache.key(1L, "gpt-4o-mini", "{\"a\":1}"); + String otherClient = cache.key(2L, "gpt-4o-mini", "{\"a\":1}"); + String otherBody = cache.key(1L, "gpt-4o-mini", "{\"a\":2}"); + + assertThat(keyA).isEqualTo(keyB).hasSize(64); + assertThat(otherClient).isNotEqualTo(keyA); + assertThat(otherBody).isNotEqualTo(keyA); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java b/src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java new file mode 100644 index 0000000..dbabb7d --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java @@ -0,0 +1,89 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.repo.ResponseCacheEntryRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.math.BigDecimal; +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:cachetest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class ResponseCacheServiceTest { + + @Autowired + private ResponseCacheService cache; + + @Autowired + private ResponseCacheEntryRepository entries; + + @BeforeEach + void clean() { + entries.deleteAll(); + } + + private ModelRoute route(int ttlSeconds) { + ModelRoute route = new ModelRoute("gpt-4o-mini", 1L, "gpt-4o-mini", null, null, 10); + route.setCacheEnabled(true); + route.setCacheTtlSeconds(ttlSeconds); + return route; + } + + @Test + void storesAndReplaysACompletion() { + String key = cache.key(1L, "gpt-4o-mini", "{\"m\":1}"); + assertThat(cache.lookup(key)).isEmpty(); + + cache.store(key, 1L, "gpt-4o-mini", "openai", route(60), "application/json", + "{\"choices\":[]}", new TokenUsage(10, 5, 0, 0, 0, 0, 0, false), new BigDecimal("0.0001")); + + var entry = cache.lookup(key).orElseThrow(); + assertThat(entry.body()).isEqualTo("{\"choices\":[]}"); + assertThat(entry.providerSlug()).isEqualTo("openai"); + assertThat(entry.costUsd()).isEqualByComparingTo("0.0001"); + assertThat(entry.totalTokens()).isEqualTo(15); + } + + @Test + void keysAreScopedToTheClient() { + String tenantOne = cache.key(1L, "gpt-4o-mini", "{}"); + cache.store(tenantOne, 1L, "gpt-4o-mini", "openai", route(60), "application/json", "{}", + TokenUsage.of(1, 1), BigDecimal.ZERO); + + assertThat(cache.lookup(tenantOne)).isPresent(); + assertThat(cache.lookup(cache.key(2L, "gpt-4o-mini", "{}"))).isEmpty(); + } + + @Test + void expiredEntriesAreNotServed() { + String key = cache.key(1L, "gpt-4o-mini", "{\"m\":2}"); + cache.store(key, 1L, "gpt-4o-mini", "openai", route(60), "application/json", "{}", + TokenUsage.of(1, 1), BigDecimal.ZERO); + + var record = entries.findByCacheKey(key).orElseThrow(); + record.setExpiresAt(Instant.now().minusSeconds(5)); + entries.save(record); + + assertThat(cache.lookup(key)).isEmpty(); + assertThat(entries.findByCacheKey(key)).isEmpty(); + } + + @Test + void oversizedBodiesAreNotStored() { + String key = cache.key(1L, "gpt-4o-mini", "{\"m\":3}"); + String big = "x".repeat(3_000_000); + cache.store(key, 1L, "gpt-4o-mini", "openai", route(60), "application/json", big, + TokenUsage.of(1, 1), BigDecimal.ZERO); + + assertThat(cache.lookup(key)).isEmpty(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java b/src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java new file mode 100644 index 0000000..79a9e46 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java @@ -0,0 +1,121 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.anindra.path.repo.ModelRouteRepository; +import com.anindra.path.repo.ProviderRepository; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class RouteResolverAdaptiveTest { + + private final ModelRouteRepository routes = mock(ModelRouteRepository.class); + private final ProviderRepository providers = mock(ProviderRepository.class); + private final ModelAliasService aliases = mock(ModelAliasService.class); + private final com.anindra.path.admin.AliasService operatorAliases = + mock(com.anindra.path.admin.AliasService.class); + private final RoutingMetrics metrics = mock(RoutingMetrics.class); + private final RouteResolver resolver = + new RouteResolver(routes, providers, aliases, operatorAliases, metrics); + private final ObjectMapper mapper = new ObjectMapper(); + + private ModelRoute route(Long providerId, String price, int priority, String strategy, Integer slo) { + ModelRoute route = new ModelRoute("chat", providerId, "upstream-" + providerId, + new BigDecimal(price), new BigDecimal("1.00"), priority); + route.setStrategy(strategy); + route.setLatencySloMs(slo); + return route; + } + + private Provider provider(Long id, String slug) { + Provider provider = new Provider(slug, slug, ProviderType.OPENAI, "http://localhost"); + provider.setId(id); + return provider; + } + + private ChatRequestSpec spec() throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree( + "{\"model\":\"chat\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")); + } + + private void stub(ModelRoute cheap, ModelRoute dear, Provider cheapProvider, + Provider dearProvider, long cheapLatency, long dearLatency) { + when(routes.findByPublicModelIdAndEnabledTrueOrderByPriorityAscIdAsc("chat")) + .thenReturn(List.of(cheap, dear)); + when(providers.findById(1L)).thenReturn(Optional.of(cheapProvider)); + when(providers.findById(2L)).thenReturn(Optional.of(dearProvider)); + when(aliases.isAlias(any())).thenReturn(false); + when(operatorAliases.chainFor(any())).thenReturn(List.of()); + when(metrics.latencyMs("cheap", "chat")).thenReturn((double) cheapLatency); + when(metrics.latencyMs("dear", "chat")).thenReturn((double) dearLatency); + } + + @Test + void priorityLeavesTheDeclaredOrderAlone() throws Exception { + Provider cheapProvider = provider(1L, "cheap"); + Provider dearProvider = provider(2L, "dear"); + stub(route(1L, "0.10", 10, "priority", null), route(2L, "0.50", 20, "priority", null), + cheapProvider, dearProvider, 900, 100); + + assertThat(resolver.resolve(spec(), 1L)) + .extracting(RouteResolver.ResolvedRoute::providerSlug) + .containsExactly("cheap", "dear"); + } + + @Test + void fastestMovesTheLowerEwmaFirst() throws Exception { + stub(route(1L, "0.10", 10, "fastest", null), route(2L, "0.50", 20, "fastest", null), + provider(1L, "cheap"), provider(2L, "dear"), 900, 100); + + assertThat(resolver.resolve(spec(), 1L)) + .extracting(RouteResolver.ResolvedRoute::providerSlug) + .containsExactly("dear", "cheap"); + } + + @Test + void cheapestViableDemotesASlowRouteButKeepsItAsFallback() throws Exception { + stub(route(1L, "0.10", 10, "cheapest-viable", 200), route(2L, "0.50", 20, "cheapest-viable", 200), + provider(1L, "cheap"), provider(2L, "dear"), 900, 100); + + assertThat(resolver.resolve(spec(), 1L)) + .extracting(RouteResolver.ResolvedRoute::providerSlug) + .containsExactly("dear", "cheap"); + } + + @Test + void cheapestViableKeepsTheCheapRouteWhenItIsFastEnough() throws Exception { + stub(route(1L, "0.10", 10, "cheapest-viable", 200), route(2L, "0.50", 20, "cheapest-viable", 200), + provider(1L, "cheap"), provider(2L, "dear"), 90, 100); + + assertThat(resolver.resolve(spec(), 1L)) + .extracting(RouteResolver.ResolvedRoute::providerSlug) + .containsExactly("cheap", "dear"); + } + + @Test + void anUnknownLatencyCountsAsViable() throws Exception { + Provider cheapProvider = provider(1L, "cheap"); + Provider dearProvider = provider(2L, "dear"); + ModelRoute cheap = route(1L, "0.10", 10, "cheapest-viable", 200); + ModelRoute dear = route(2L, "0.50", 20, "cheapest-viable", 200); + stub(cheap, dear, cheapProvider, dearProvider, 0, 0); + when(metrics.latencyMs("cheap", "chat")).thenReturn(Double.NaN); + when(metrics.latencyMs(eq("dear"), eq("chat"))).thenReturn(100.0); + + assertThat(resolver.resolve(spec(), 1L)) + .extracting(RouteResolver.ResolvedRoute::providerSlug) + .containsExactly("cheap", "dear"); + } +} diff --git a/src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java b/src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java new file mode 100644 index 0000000..ed51bb2 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java @@ -0,0 +1,49 @@ +package com.anindra.path.gateway; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class RoutingMetricsTest { + + private final RoutingMetrics metrics = new RoutingMetrics(); + + @Test + void startsUnknown() { + assertThat(metrics.latencyMs("openai", "gpt-4o")).isNaN(); + assertThat(metrics.sampleCount("openai", "gpt-4o")).isZero(); + assertThat(metrics.errorRate("openai", "gpt-4o")).isZero(); + } + + @Test + void theFirstSampleIsTakenAsIs() { + metrics.recordSuccess("openai", "gpt-4o", 500); + assertThat(metrics.latencyMs("openai", "gpt-4o")).isEqualTo(500.0); + } + + @Test + void convergesTowardRepeatedSamples() { + metrics.recordSuccess("openai", "gpt-4o", 1000); + for (int i = 0; i < 30; i++) { + metrics.recordSuccess("openai", "gpt-4o", 100); + } + assertThat(metrics.latencyMs("openai", "gpt-4o")).isBetween(100.0, 120.0); + } + + @Test + void failuresRaiseTheErrorRateWithoutZeroingLatency() { + metrics.recordSuccess("openai", "gpt-4o", 400); + metrics.recordFailure("openai", "gpt-4o"); + + assertThat(metrics.errorRate("openai", "gpt-4o")).isGreaterThan(0.0); + assertThat(metrics.latencyMs("openai", "gpt-4o")).isEqualTo(400.0); + } + + @Test + void providersAreTrackedSeparately() { + metrics.recordSuccess("openai", "chat", 100); + metrics.recordSuccess("groq", "chat", 900); + + assertThat(metrics.latencyMs("openai", "chat")).isLessThan(metrics.latencyMs("groq", "chat")); + } +} diff --git a/src/test/java/com/anindra/path/gateway/RoutingTest.java b/src/test/java/com/anindra/path/gateway/RoutingTest.java new file mode 100644 index 0000000..ec26243 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/RoutingTest.java @@ -0,0 +1,82 @@ +package com.anindra.path.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:routingtest;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class RoutingTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + @Autowired + private RouteResolver resolver; + + @Autowired + private ModelAliasService aliases; + + @Autowired + private com.anindra.path.repo.ModelRouteRepository routes; + + private ChatRequestSpec spec(String model) throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree( + "{\"model\":\"" + model + "\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")); + } + + @Test + void aliasesFollowTheLivePriceTable() { + assertThat(aliases.resolve(ModelAliasService.CHEAP)).contains("llama-3.1-8b"); + assertThat(aliases.resolve(ModelAliasService.FAST)).contains("llama-3.1-8b"); + assertThat(aliases.resolve(ModelAliasService.LONG_CONTEXT)).contains("gpt-4.1-mini"); + assertThat(aliases.resolve(ModelAliasService.VISION)).isEmpty(); + } + + @Test + void anAliasResolvesToARealRouteChain() throws Exception { + List chain = resolver.resolve(spec("cheap"), 1L); + + assertThat(chain).isNotEmpty(); + assertThat(chain.get(0).route().getPublicModelId()).isEqualTo("llama-3.1-8b"); + assertThat(chain.get(0).providerSlug()).isEqualTo("groq"); + } + + @Test + void weightedRoutingIsStablePerKeyAndSplitsAcrossKeys() throws Exception { + // Weighting is opt-in: configure a 50/50 canary first. + var group = routes.findByPublicModelIdOrderByPriorityAsc("auto-fast"); + group.forEach(route -> route.setWeight(50)); + routes.saveAll(group); + ChatRequestSpec spec = spec("auto-fast"); + + String first = resolver.resolve(spec, 42L).get(0).providerSlug(); + String second = resolver.resolve(spec, 42L).get(0).providerSlug(); + assertThat(first).isEqualTo(second); + + Set providers = new HashSet<>(); + for (long key = 1; key <= 40; key++) { + providers.add(resolver.resolve(spec, key).get(0).providerSlug()); + } + assertThat(providers).contains("openai", "groq"); + } + + @Test + void theRestOfTheChainStaysAsFallback() throws Exception { + List chain = resolver.resolve(spec("auto-fast"), 7L); + + assertThat(chain).hasSize(2); + assertThat(chain.get(0).providerSlug()).isNotEqualTo(chain.get(1).providerSlug()); + } +} diff --git a/src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java b/src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java new file mode 100644 index 0000000..643c29b --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java @@ -0,0 +1,44 @@ +package com.anindra.path.gateway; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class SemanticCacheGeometryTest { + + @Test + void identicalVectorsAreMaximallySimilar() { + double[] v = {0.6, 0.8}; + assertThat(SemanticCacheService.cosine(v, v)).isCloseTo(1.0, org.assertj.core.data.Offset.offset(1e-9)); + } + + @Test + void orthogonalVectorsAreNotSimilar() { + assertThat(SemanticCacheService.cosine(new double[] {1, 0}, new double[] {0, 1})) + .isCloseTo(0.0, org.assertj.core.data.Offset.offset(1e-9)); + } + + @Test + void magnitudeDoesNotMatterOnlyDirection() { + double a = SemanticCacheService.cosine(new double[] {1, 1}, new double[] {1, 1}); + double b = SemanticCacheService.cosine(new double[] {10, 10}, new double[] {0.1, 0.1}); + assertThat(a).isCloseTo(b, org.assertj.core.data.Offset.offset(1e-9)); + } + + @Test + void mismatchedDimensionsAndZeroVectorsAreRejected() { + assertThat(SemanticCacheService.cosine(new double[] {1, 2}, new double[] {1})).isEqualTo(-1); + assertThat(SemanticCacheService.cosine(new double[] {0, 0}, new double[] {1, 1})).isEqualTo(-1); + assertThat(SemanticCacheService.cosine(new double[] {}, new double[] {})).isEqualTo(-1); + } + + @Test + void vectorsRoundTripThroughStorage() { + SemanticCacheService service = new SemanticCacheService(null, null, null, null); + double[] vector = {0.1, -0.25, 3.5}; + String encoded = service.encode(vector); + assertThat(encoded).isEqualTo("0.1,-0.25,3.5"); + assertThat(service.encode(null)).isNull(); + assertThat(service.encode(new double[] {})).isNull(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java b/src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java new file mode 100644 index 0000000..4f55e03 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java @@ -0,0 +1,83 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TokenCountServiceTest { + + private final RouteResolver resolver = mock(RouteResolver.class); + private final GatewayProperties props = new GatewayProperties(); + private final TokenCountService service = new TokenCountService(resolver, props); + private final ObjectMapper mapper = new ObjectMapper(); + + private ObjectNode body(String model) throws Exception { + return (ObjectNode) mapper.readTree("{\"model\":\"" + model + "\",\"messages\":[" + + "{\"role\":\"system\",\"content\":\"you are terse\"}," + + "{\"role\":\"user\",\"content\":\"hello\"}]}"); + } + + private ClientKey key(String allowed) { + ClientKey key = new ClientKey(); + key.setId(1L); + key.setName("test"); + key.setAllowedModels(allowed); + return key; + } + + private RouteResolver.ResolvedRoute route() { + ModelRoute route = new ModelRoute("gpt-4o-mini", 1L, "gpt-4o-mini", + new BigDecimal("0.15"), new BigDecimal("0.60"), 10); + route.setContextLength(128000); + Provider provider = new Provider("openai", "OpenAI", ProviderType.OPENAI, + "https://api.openai.com/v1"); + return new RouteResolver.ResolvedRoute(route, provider); + } + + @Test + @SuppressWarnings("unchecked") + void countsASystemOnlyPromptThatChatWouldReject() throws Exception { + when(resolver.resolve(any(), eq(1L))).thenReturn(List.of(route())); + + Map out = service.count(body("gpt-4o-mini"), key(null)); + + assertThat(out.get("object")).isEqualTo("token_count"); + assertThat((Integer) out.get("prompt_tokens")).isPositive(); + assertThat(out.get("estimated")).isEqualTo(false); + Map fits = (Map) out.get("fits"); + assertThat(fits.get("context_window")).isEqualTo(128000); + } + + @Test + void rejectsAModelOutsideTheKeyAllowlist() throws Exception { + assertThatThrownBy(() -> service.count(body("gpt-4o"), key("claude-*"))) + .isInstanceOf(ApiException.class) + .hasMessageContaining("not allowed"); + } + + @Test + void unknownModelIs404() throws Exception { + when(resolver.resolve(any(), eq(1L))).thenReturn(List.of()); + + assertThatThrownBy(() -> service.count(body("nope"), key(null))) + .isInstanceOf(ApiException.class) + .hasMessageContaining("does not exist"); + } +} diff --git a/src/test/java/com/anindra/path/gateway/TokenCounterTest.java b/src/test/java/com/anindra/path/gateway/TokenCounterTest.java new file mode 100644 index 0000000..e696466 --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/TokenCounterTest.java @@ -0,0 +1,64 @@ +package com.anindra.path.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TokenCounterTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + private ObjectNode body(String json) throws Exception { + return (ObjectNode) mapper.readTree(json); + } + + @Test + void countsAGrowingConversation() throws Exception { + int one = TokenCounter.count(body( + "{\"messages\":[{\"role\":\"user\",\"content\":\"hello\"}]}"), "gpt-4o-mini").tokens(); + int many = TokenCounter.count(body( + "{\"messages\":[{\"role\":\"user\",\"content\":\"hello\"}," + + "{\"role\":\"assistant\",\"content\":\"hi there, how can I help you today?\"}," + + "{\"role\":\"user\",\"content\":\"explain vector databases\"}]}"), + "gpt-4o-mini").tokens(); + + assertThat(one).isPositive(); + assertThat(many).isGreaterThan(one); + } + + @Test + void countsToolDefinitions() throws Exception { + String tool = "{\"type\":\"function\",\"function\":{\"name\":\"lookup\"," + + "\"description\":\"Look up a customer by id\"," + + "\"parameters\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"string\"}}}}}"; + int without = TokenCounter.count(body( + "{\"messages\":[{\"role\":\"user\",\"content\":\"find customer 42\"}]}"), + "gpt-4o-mini").tokens(); + int with = TokenCounter.count(body( + "{\"messages\":[{\"role\":\"user\",\"content\":\"find customer 42\"}]," + + "\"tools\":[" + tool + "]}"), "gpt-4o-mini").tokens(); + + assertThat(with).isGreaterThan(without); + } + + @Test + void fallsBackToTheHeuristicForAnUnknownModel() throws Exception { + var count = TokenCounter.count(body( + "{\"messages\":[{\"role\":\"user\",\"content\":\"some prompt text here\"}]}"), + "totally-unknown-model"); + + assertThat(count.estimated()).isTrue(); + assertThat(count.tokens()).isPositive(); + } + + @Test + void knownModelsAreNotMarkedEstimated() throws Exception { + var count = TokenCounter.count(body( + "{\"messages\":[{\"role\":\"user\",\"content\":\"some prompt text here\"}]}"), + "gpt-4o-mini"); + + assertThat(count.estimated()).isFalse(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/TokenEstimateTest.java b/src/test/java/com/anindra/path/gateway/TokenEstimateTest.java new file mode 100644 index 0000000..086a1ff --- /dev/null +++ b/src/test/java/com/anindra/path/gateway/TokenEstimateTest.java @@ -0,0 +1,43 @@ +package com.anindra.path.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TokenEstimateTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + private ChatRequestSpec spec(String content) throws Exception { + String json = "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":" + + mapper.writeValueAsString(content) + "}]}"; + return ChatRequestSpec.parse((ObjectNode) mapper.readTree(json)); + } + + @Test + void tokenisesKnownModels() throws Exception { + ChatRequestSpec spec = spec("The quick brown fox jumps over the lazy dog."); + int tokens = spec.estimatePromptTokens("gpt-4o-mini"); + + assertThat(tokens).isBetween(1, 40); + // Real token counts land close to chars/4 for plain English. + assertThat(tokens).isCloseTo(spec.estimatePromptTokens(), org.assertj.core.data.Offset.offset(15)); + } + + @Test + void fallsBackForUnknownModels() throws Exception { + ChatRequestSpec spec = spec("hello there"); + assertThat(spec.estimatePromptTokens("not-a-real-model-xyz")) + .isEqualTo(spec.estimatePromptTokens()); + } + + @Test + void scalesWithInputLength() throws Exception { + ChatRequestSpec small = spec("hi"); + ChatRequestSpec large = spec("hi ".repeat(500)); + assertThat(large.estimatePromptTokens("gpt-4o-mini")) + .isGreaterThan(small.estimatePromptTokens("gpt-4o-mini") * 10); + } +} diff --git a/src/test/java/com/anindra/path/secret/SecretCipherTest.java b/src/test/java/com/anindra/path/secret/SecretCipherTest.java new file mode 100644 index 0000000..1e0b104 --- /dev/null +++ b/src/test/java/com/anindra/path/secret/SecretCipherTest.java @@ -0,0 +1,59 @@ +package com.anindra.path.secret; + +import com.anindra.path.config.GatewayProperties; +import org.junit.jupiter.api.Test; + +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class SecretCipherTest { + + private static String key() { + byte[] raw = new byte[32]; + for (int i = 0; i < raw.length; i++) { + raw[i] = (byte) i; + } + return Base64.getEncoder().encodeToString(raw); + } + + private SecretCipher cipher() { + GatewayProperties props = new GatewayProperties(); + props.setMasterKey(key()); + return new SecretCipher(props); + } + + @Test + void encryptsAndDecrypts() { + SecretCipher cipher = cipher(); + String ciphertext = cipher.encrypt("sk-test-value"); + + assertThat(ciphertext).startsWith("v1:"); + assertThat(cipher.decrypt(ciphertext)).isEqualTo("sk-test-value"); + } + + @Test + void rotationReencryptsAndInvalidatesTheOldKey() { + SecretCipher cipher = cipher(); + String oldCiphertext = cipher.encrypt("sk-rotate-me"); + + String newKeyBase64 = Base64.getEncoder().encodeToString( + "0123456789abcdef0123456789abcdef".getBytes()); + byte[] newKey = SecretCipher.parseKey(newKeyBase64); + + String reencrypted = SecretCipher.encryptWith(newKey, cipher.decrypt(oldCiphertext)); + cipher.rotate(newKey); + + assertThat(cipher.decrypt(reencrypted)).isEqualTo("sk-rotate-me"); + assertThatThrownBy(() -> cipher.decrypt(oldCiphertext)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void rejectsWrongLengthKeys() { + assertThatThrownBy(() -> SecretCipher.parseKey( + Base64.getEncoder().encodeToString(new byte[16]))) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/tools/changelog.py b/tools/changelog.py new file mode 100644 index 0000000..bf7b624 --- /dev/null +++ b/tools/changelog.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Print the release notes for a tag, for use as a GitHub release body. + + python3 tools/changelog.py v0.1.0 > release-notes.md + python3 tools/changelog.py 0.1.0 # the v prefix is optional + +Reads `changelog/.md`. Exits non-zero when the file is missing or empty, so a tag +without notes fails the release instead of publishing a blank body. +""" +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CHANGELOG_DIR = ROOT / "changelog" + + +def find(version: str) -> Path | None: + candidates = [f"{version}.md"] + if version.startswith("v"): + candidates.append(f"{version[1:]}.md") + else: + candidates.append(f"v{version}.md") + for name in candidates: + path = CHANGELOG_DIR / name + if path.exists(): + return path + return None + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: changelog.py ", file=sys.stderr) + return 2 + + version = sys.argv[1].strip() + path = find(version) + if path is None: + print( + f"No changelog file for '{version}'. Add changelog/{version}.md before tagging " + f"(see changelog/README.md).", + file=sys.stderr, + ) + return 1 + + body = path.read_text(encoding="utf-8").strip() + real_entries = [ + line for line in body.splitlines() + if line.strip().startswith("- ") and "Nothing yet." not in line + ] + if not body or not real_entries: + print(f"changelog/{path.name} has no real entries yet.", file=sys.stderr) + return 1 + + print(body) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/demo.sh b/tools/demo.sh new file mode 100755 index 0000000..fd823b7 --- /dev/null +++ b/tools/demo.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# 60-second demo of Path against a mock upstream. No API keys required. +# +# bash tools/demo.sh +# +# Starts the mock provider and the gateway, then walks through catalogue, chat, +# streaming, the response cache and usage metering, and cleans up afterwards. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +JAR="$(ls "$ROOT"/target/path-*.jar 2>/dev/null | head -1 || true)" +MOCK_PORT="${MOCK_PORT:-9099}" +PORT="${PORT:-8080}" +ADMIN="demo-admin" +LOG="$(mktemp -d)" + +bold() { printf '\n\033[1m%s\033[0m\n' "$1"; } + +cleanup() { + [[ -n "${GATEWAY_PID:-}" ]] && kill "$GATEWAY_PID" 2>/dev/null || true + [[ -n "${MOCK_PID:-}" ]] && kill "$MOCK_PID" 2>/dev/null || true + rm -rf "$LOG" +} +trap cleanup EXIT + +if [[ -z "$JAR" ]]; then + echo "Build first: ./mvnw -DskipTests package" >&2 + exit 1 +fi + +bold "1. starting the mock upstream and the gateway" +python3 "$ROOT/tools/mock-upstream.py" "$MOCK_PORT" > "$LOG/mock.log" 2>&1 & +MOCK_PID=$! +(cd "$ROOT" && rm -rf data && exec env GATEWAY_ADMIN_TOKEN="$ADMIN" GATEWAY_BOOTSTRAP_DEMO_KEY=true \ + java -jar "$JAR" --server.port="$PORT" > "$LOG/gateway.log" 2>&1) & +GATEWAY_PID=$! + +for _ in $(seq 1 60); do + curl -sf "http://127.0.0.1:$PORT/actuator/health" > /dev/null 2>&1 && break + sleep 1 +done + +KEY="$(grep -o 'sk-gw-[A-Za-z0-9]*' "$LOG/gateway.log" | head -1)" +[[ -n "$KEY" ]] || { echo "gateway did not start; see $LOG/gateway.log" >&2; exit 1; } +AUTH=(-H "Authorization: Bearer $KEY" -H 'Content-Type: application/json') +API="http://127.0.0.1:$PORT" + +# Point the seeded providers at the mock so the demo spends nothing. +for id in 1 2 3 4; do + curl -s -X PATCH "$API/admin/providers/$id" -H "X-Admin-Token: $ADMIN" \ + -H 'Content-Type: application/json' \ + -d "{\"base_url\":\"http://127.0.0.1:$MOCK_PORT/v1\",\"api_key\":\"demo\"}" > /dev/null +done + +bold "2. the catalogue (aliases included)" +curl -s "$API/v1/models" | python3 -c " +import json,sys +for m in json.load(sys.stdin)['data']: + label = m['id'] + (' -> ' + m['resolves_to'] if m.get('resolves_to') else '') + print(' ', label) +" + +bold "3. chat through the auto-fast alias" +curl -s -D "$LOG/headers" "$API/v1/chat/completions" "${AUTH[@]}" \ + -d '{"model":"auto-fast","messages":[{"role":"user","content":"Say hi in five words."}]}' \ + | python3 -c "import json,sys; print(' ', json.load(sys.stdin)['choices'][0]['message']['content'])" +grep -iE "x-gateway-(provider|cost-usd)" "$LOG/headers" | sed 's/^/ /' + +bold "4. streaming" +curl -sN "$API/v1/chat/completions" "${AUTH[@]}" \ + -d '{"model":"gpt-4o-mini","stream":true,"messages":[{"role":"user","content":"Count to three."}]}' \ + | grep '^data:' | head -6 | sed 's/^/ /' + +bold "5. the same deterministic request twice (response cache)" +CACHED='{"model":"gpt-4o-mini","temperature":0,"messages":[{"role":"user","content":"cache me"}]}' +for attempt in 1 2; do + curl -s -o /dev/null -D "$LOG/cache" "$API/v1/chat/completions" "${AUTH[@]}" -d "$CACHED" + printf ' attempt %s: %s\n' "$attempt" "$(grep -i '^x-path-cache' "$LOG/cache" | tr -d '\r')" +done +# The route must opt in first; enable it and repeat. +ROUTE_ID="$(curl -s "$API/admin/routes?model=gpt-4o-mini" -H "X-Admin-Token: $ADMIN" \ + | python3 -c "import json,sys; print(json.load(sys.stdin)[0]['id'])")" +curl -s -X PATCH "$API/admin/routes/$ROUTE_ID" -H "X-Admin-Token: $ADMIN" \ + -H 'Content-Type: application/json' -d '{"cache_enabled":true}' > /dev/null +for attempt in 1 2; do + curl -s -o /dev/null -D "$LOG/cache" "$API/v1/chat/completions" "${AUTH[@]}" -d "$CACHED" + printf ' cached run %s: %s %s\n' "$attempt" \ + "$(grep -i '^x-path-cache' "$LOG/cache" | tr -d '\r')" \ + "$(grep -i '^x-gateway-cost-usd' "$LOG/cache" | tr -d '\r')" +done + +bold "6. usage for the key" +curl -s "$API/v1/usage?window=1h" "${AUTH[@]}" | python3 -c " +import json,sys +d = json.load(sys.stdin) +print(' requests:', d['requests'], '| tokens:', d['total_tokens'], '| cost:', d['cost_usd']) +print(' cached tokens:', d['cached_input_tokens'], '| savings:', d['cached_savings_usd']) +" + +bold "done - shutting down" diff --git a/tools/load-test.js b/tools/load-test.js new file mode 100644 index 0000000..fb20609 --- /dev/null +++ b/tools/load-test.js @@ -0,0 +1,68 @@ +// k6 load test for Path. +// +// k6 run -e BASE=http://127.0.0.1:8080 -e KEY=sk-gw-... tools/load-test.js +// +// Two scenarios: steady non-streaming traffic and a streaming burst. Thresholds fail +// the run when latency or error rate regress. +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +const BASE = __ENV.BASE || 'http://127.0.0.1:8080'; +const KEY = __ENV.KEY || ''; +const MODEL = __ENV.MODEL || 'gpt-4o-mini'; + +export const options = { + scenarios: { + steady: { + executor: 'constant-vus', + vus: Number(__ENV.VUS || 10), + duration: __ENV.DURATION || '30s', + exec: 'chat', + }, + streaming: { + executor: 'constant-arrival-rate', + rate: Number(__ENV.STREAM_RATE || 5), + timeUnit: '1s', + duration: __ENV.DURATION || '30s', + preAllocatedVUs: 10, + exec: 'streamChat', + startTime: '5s', + }, + }, + thresholds: { + http_req_failed: ['rate<0.01'], + http_req_duration: ['p(95)<2000'], + }, +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${KEY}`, +}; + +export function chat() { + const res = http.post(`${BASE}/v1/chat/completions`, JSON.stringify({ + model: MODEL, + temperature: 0, + messages: [{ role: 'user', content: 'Say hello in five words.' }], + }), { headers }); + + check(res, { + 'status is 200': (r) => r.status === 200, + 'has choices': (r) => !!r.json('choices'), + }); + sleep(0.5); +} + +export function streamChat() { + const res = http.post(`${BASE}/v1/chat/completions`, JSON.stringify({ + model: MODEL, + stream: true, + messages: [{ role: 'user', content: 'Count to five.' }], + }), { headers }); + + check(res, { + 'status is 200': (r) => r.status === 200, + 'terminates with DONE': (r) => String(r.body).includes('[DONE]'), + }); +} diff --git a/tools/mock-upstream.py b/tools/mock-upstream.py index 9dc4139..450fcba 100644 --- a/tools/mock-upstream.py +++ b/tools/mock-upstream.py @@ -13,6 +13,10 @@ * "empty" -> stream sends nothing then closes """ import json +import math +import re +import zlib +import urllib.parse import sys import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -20,6 +24,22 @@ PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 9099 +def _embedding(text, dims=32): + """Deterministic bag-of-words vector, so near-duplicate prompts are near in cosine. + + Not a real embedding model — the point is stable, testable geometry: sharing words + raises the cosine, sharing none leaves it near zero. + """ + vector = [0.0] * dims + words = re.findall(r"[a-z0-9]+", text.lower()) + for word in words: + vector[zlib.crc32(word.encode()) % dims] += 1.0 + norm = math.sqrt(sum(v * v for v in vector)) + if norm == 0: + return vector + return [round(v / norm, 6) for v in vector] + + def openai_body(model, prompt_tokens=11, completion_tokens=7): return { "id": "chatcmpl-mock123", @@ -81,6 +101,7 @@ def do_GET(self): {"id": "text-embedding-3-small", "object": "model"}, {"id": "claude-3-5-haiku-latest", "object": "model"}, {"id": "llama-3.3-70b-versatile", "object": "model"}, + {"id": "acme-unknown-model", "object": "model"}, ] self._json(200, {"object": "list", "data": data}) else: @@ -89,6 +110,21 @@ def do_GET(self): def do_POST(self): body = self._read() model = str(body.get("model", "")) + # Azure puts the version in a query string, so suffix checks need the bare path. + route = urllib.parse.urlparse(self.path).path + + # Azure shape: /openai/deployments//chat/completions?api-version=... + if "/openai/deployments/" in route: + query = urllib.parse.urlparse(self.path).query + version = urllib.parse.parse_qs(query).get("api-version", [""])[0] + if version != "2024-10-21" and "/moderations" not in route: + self._json(400, {"error": { + "message": "API version not supported: " + (version or "(none)"), + "type": "invalid_request_error", "code": "invalid_api_version"}}) + return + if not self.headers.get("api-key"): + self._json(401, {"error": {"message": "missing api-key", "code": "401"}}) + return stream = bool(body.get("stream")) if "boom" in model: @@ -104,7 +140,27 @@ def do_POST(self): self._json(400, {"error": {"message": "bad request from client", "type": "invalid_request"}}) return - if self.path.endswith("/messages"): + if route.endswith("/moderations"): + text = str(body.get("input", "")) + flagged = "flag-me" in text + self._json(200, {"id": "modr-mock", "model": body.get("model"), + "results": [{"flagged": flagged, "categories": { + "violence": flagged, "harassment": False, + "hate": False, "self-harm": False}}]}) + return + + if route.endswith("/embeddings"): + text = body.get("input") + if isinstance(text, list): + text = " ".join(str(t) for t in text) + self._json(200, { + "object": "list", + "data": [{"object": "embedding", "index": 0, + "embedding": _embedding(str(text or ""))}], + "model": model, + "usage": {"prompt_tokens": 7, "total_tokens": 7}, + }) + elif route.endswith("/messages"): self._anthropic(model, stream) else: self._openai(model, stream) diff --git a/tools/postman_to_bruno.py b/tools/postman_to_bruno.py new file mode 100644 index 0000000..05d29a7 --- /dev/null +++ b/tools/postman_to_bruno.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Regenerate the Bruno collection from the Postman collection. + + python3 tools/postman_to_bruno.py + +Reads `docs/path.postman_collection.json` and rewrites `bruno/` so the two stay in +sync. Postman-specific bits are normalised: `{{$timestamp}}` becomes a fixed name and +"Postman" wording becomes "Bruno". Scripts that captured variables in Postman are +translated to Bruno's `bru.setVar`. +""" +import json +import os +import shutil + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SOURCE = os.path.join(ROOT, "docs", "path.postman_collection.json") +TARGET = os.path.join(ROOT, "bruno") + +# Requests whose testing scripts set collection variables in Postman. +POST_RESPONSE = { + "Create API key": 'bru.setVar("apiKey", res.body.key);\nbru.setVar("keyId", res.body.id);', + "Create provider": 'bru.setVar("createdProviderId", res.body.id);', + "Create route": 'bru.setVar("createdRouteId", res.body.id);', +} + +ENVIRONMENT = """vars { + baseUrl: http://localhost:8080 + adminToken: test-admin + apiKey: + model: gpt-4o-mini + publicModel: cheap + providerId: 1 + keyId: 1 +} +""" + + +def normalise(text): + if text is None: + return None + return (text.replace("postman-{{$timestamp}}", "bruno-demo") + .replace("{{$timestamp}}", "bruno-demo") + .replace("Postman console", "Bruno's console") + .replace("Postman", "Bruno")) + + +def safe_name(name): + return name.replace("/", "-").replace(":", "-") + + +def render(item, seq): + request = item["request"] + method = request["method"].lower() + url = normalise(request["url"]["raw"]) + headers = request.get("header", []) + body = normalise(request.get("body", {}).get("raw")) + description = normalise((request.get("description") or "").strip()) + + blocks = [f"meta {{\n name: {item['name']}\n type: http\n seq: {seq}\n}}"] + + verb = f"{method} {{\n url: {url}\n" + if body: + verb += " body: json\n" + verb += " auth: none\n}" + blocks.append(verb) + + if headers: + lines = "\n".join(f" {header['key']}: {normalise(header['value'])}" for header in headers) + blocks.append(f"headers {{\n{lines}\n}}") + + if body: + blocks.append(f"body:json {{\n {body.replace(chr(10), chr(10) + ' ')}\n}}") + + if description: + blocks.append(f"docs {{\n {description.replace(chr(10), chr(10) + ' ')}\n}}") + + script = POST_RESPONSE.get(item["name"]) + if script: + spaced = script.replace("\n", "\n ") + blocks.append(f"script:post-response {{\n {spaced}\n}}") + + return "\n\n".join(blocks) + "\n" + + +def walk(items, directory): + os.makedirs(directory, exist_ok=True) + for seq, item in enumerate(items, start=1): + if "item" in item: + walk(item["item"], os.path.join(directory, safe_name(item["name"]))) + else: + path = os.path.join(directory, safe_name(item["name"]) + ".bru") + with open(path, "w", encoding="utf-8") as fh: + fh.write(render(item, seq)) + + +def main(): + # Regenerate only the generated parts; keep the hand-written README. + os.makedirs(TARGET, exist_ok=True) + for entry in os.listdir(TARGET): + if entry == "README.md": + continue + path = os.path.join(TARGET, entry) + shutil.rmtree(path) if os.path.isdir(path) else os.remove(path) + os.makedirs(os.path.join(TARGET, "environments"), exist_ok=True) + + with open(os.path.join(TARGET, "bruno.json"), "w", encoding="utf-8") as fh: + json.dump({"version": "1", "name": "Path", "type": "collection", + "ignore": ["node_modules", ".git", ".DS_Store"]}, fh, indent=2) + fh.write("\n") + + with open(os.path.join(TARGET, "environments", "Local.bru"), "w", encoding="utf-8") as fh: + fh.write(ENVIRONMENT) + + collection = json.load(open(SOURCE, encoding="utf-8")) + walk(collection["item"], TARGET) + print(f"wrote {TARGET}") + + +if __name__ == "__main__": + main() diff --git a/tools/smoke_test.py b/tools/smoke_test.py index ae906d8..da31962 100644 --- a/tools/smoke_test.py +++ b/tools/smoke_test.py @@ -12,6 +12,7 @@ import os import re import sys +import time import urllib.error import urllib.request @@ -28,7 +29,7 @@ def check(name, condition, detail=""): print(("PASS " if condition else "FAIL ") + name + ((" [" + detail + "]") if detail and not condition else "")) -def req(method, path, body=None, key=None, admin=False, timeout=30): +def req(method, path, body=None, key=None, admin=False, timeout=30, headers=None): data = json.dumps(body).encode() if body is not None else None request = urllib.request.Request(BASE + path, data=data, method=method) request.add_header("Content-Type", "application/json") @@ -36,6 +37,8 @@ def req(method, path, body=None, key=None, admin=False, timeout=30): request.add_header("Authorization", "Bearer " + key) if admin: request.add_header("X-Admin-Token", ADMIN) + for name, value in (headers or {}).items(): + request.add_header(name, value) try: with urllib.request.urlopen(request, timeout=timeout) as resp: return resp.status, dict(resp.headers), resp.read() @@ -52,11 +55,13 @@ def jreq(*args, **kwargs): return status, headers, parsed -def sse_post(path, body, key, timeout=60): +def sse_post(path, body, key, timeout=60, headers=None): """Returns (status, headers, list_of_data_payloads).""" request = urllib.request.Request(BASE + path, data=json.dumps(body).encode(), method="POST") request.add_header("Content-Type", "application/json") request.add_header("Authorization", "Bearer " + key) + for name, value in (headers or {}).items(): + request.add_header(name, value) try: resp = urllib.request.urlopen(request, timeout=timeout) except urllib.error.HTTPError as e: @@ -84,8 +89,17 @@ def demo_key(): return None +def key_works(key): + """A demo key scraped from a stale log may belong to a deleted database.""" + status, _, _ = req("GET", "/v1/models", key=key, timeout=10) + return status == 200 + + def main(): key = demo_key() + if key and not key_works(key): + print("Demo key in", LOG, "does not authenticate; creating a fresh one") + key = None if not key: status, _, body = jreq("POST", "/admin/keys", {"name": "smoke-fallback"}, admin=True) if status != 200: @@ -186,6 +200,70 @@ def main(): check("anthropic stream finish_reason stop", finish == "stop", str(finish)) check("anthropic stream [DONE]", payloads and payloads[-1] == "[DONE]") + # -------------------------------------------------------------- idempotency + idem_body = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "idem"}]} + s1, h1, b1 = jreq("POST", "/v1/chat/completions", idem_body, key=key, + headers={"Idempotency-Key": "smoke-idem-a"}) + s2, h2, b2 = jreq("POST", "/v1/chat/completions", idem_body, key=key, + headers={"Idempotency-Key": "smoke-idem-a"}) + check("idempotent replay returns the same body", s1 == 200 and s2 == 200 and b1 == b2, f"{s1}/{s2}") + check("replay is flagged with X-Idempotent-Replay", h2.get("X-Idempotent-Replay") == "true", + str(h2.get("X-Idempotent-Replay"))) + + s3, _, b3 = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "different"}]}, + key=key, headers={"Idempotency-Key": "smoke-idem-a"}) + check("reused key with a different body is rejected", + s3 == 409 and b3.get("error", {}).get("code") == "idempotency_conflict", str(s3) + str(b3)[:120]) + + stream_idem = {"model": "gpt-4o-mini", "stream": True, + "messages": [{"role": "user", "content": "idem stream"}]} + _, _, p4 = sse_post("/v1/chat/completions", stream_idem, key, headers={"Idempotency-Key": "smoke-idem-s"}) + _, _, p5 = sse_post("/v1/chat/completions", stream_idem, key, headers={"Idempotency-Key": "smoke-idem-s"}) + check("streamed replay is identical", p4 == p5 and len(p4) > 1, f"{len(p4)}/{len(p5)}") + + # ------------------------------------------------------------ response cache + _, _, route_list = jreq("GET", "/admin/routes?model=gpt-4o-mini", admin=True) + targets = [r for r in route_list if r["provider"] == "openai"] + check("found the route to cache", bool(targets)) + for route in targets: + jreq("PATCH", f"/admin/routes/{route['id']}", + {"cache_enabled": True, "cache_ttl_seconds": 60}, admin=True) + + # Unique prompt so a warm database does not already hold this answer. + marker_text = f"cached prompt {time.time_ns()}" + cached_body = {"model": "gpt-4o-mini", "temperature": 0, + "messages": [{"role": "user", "content": marker_text}]} + _, ch1, cb1 = jreq("POST", "/v1/chat/completions", cached_body, key=key) + _, ch2, cb2 = jreq("POST", "/v1/chat/completions", cached_body, key=key) + check("cache miss then hit", + ch1.get("X-Path-Cache") == "miss" and ch2.get("X-Path-Cache") == "hit", + f"{ch1.get('X-Path-Cache')}/{ch2.get('X-Path-Cache')}") + check("cached body is identical", cb1 == cb2) + check("cache hit costs nothing", float(ch2.get("X-Gateway-Cost-Usd", "1")) == 0.0, + str(ch2.get("X-Gateway-Cost-Usd"))) + + _, warm_h, _ = jreq("POST", "/v1/chat/completions", + {**cached_body, "temperature": 1}, key=key) + check("non-zero temperature bypasses the cache", warm_h.get("X-Path-Cache") == "off", + str(warm_h.get("X-Path-Cache"))) + + stream_cached = {"model": "gpt-4o-mini", "temperature": 0, "stream": True, + "messages": [{"role": "user", "content": f"cached stream {time.time_ns()}"}]} + _, sh1, sp1 = sse_post("/v1/chat/completions", stream_cached, key) + _, sh2, sp2 = sse_post("/v1/chat/completions", stream_cached, key) + check("stream cache miss then hit", + sh1.get("X-Path-Cache") == "miss" and sh2.get("X-Path-Cache") == "hit", + f"{sh1.get('X-Path-Cache')}/{sh2.get('X-Path-Cache')}") + check("stream cache replays identical chunks", sp1 == sp2 and len(sp1) > 1, + f"{len(sp1)}/{len(sp2)}") + + _, _, recent = jreq("GET", "/admin/usage?window=1h&limit=25", admin=True) + cache_rows = [r for r in recent.get("data", []) if r.get("finish_reason") == "cache"] + check("cache hits are metered at zero cost", + bool(cache_rows) and all(float(r["cost_usd"]) == 0 for r in cache_rows), + str(len(cache_rows))) + # -------------------------------------------------------- fallback by chain for payload, label in [ ({"publicModelId": "fallback-test", "providerId": providers["openai"]["id"], @@ -214,6 +292,515 @@ def main(): check("allow_fallbacks=false surfaces the 500", status == 502 and body.get("error", {}).get("code") == "upstream_error", str(status) + str(body)[:200]) + # --------------------------------------------------------------- limits, params + status, _, body = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "x" * 1_100_000}]}, key=key) + check("oversized body -> 413", status == 413 and body.get("error", {}).get("code") == "payload_too_large", + str(status)) + + status, _, body = jreq("POST", "/v1/chat/completions", { + "model": "claude-3.5-haiku", "seed": 7, + "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("anthropic rejects unsupported seed -> 422", + status == 422 and body.get("error", {}).get("code") == "unsupported_parameter", + str(status) + str(body)[:120]) + + # ----------------------------------------------------------------- budgets + _, _, daily = jreq("POST", "/admin/keys", + {"name": "daily-key", "daily_limit_usd": 0.000001, "rpm_limit": 100}, admin=True) + dk = daily.get("key") + s1, _, _ = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, key=dk) + s2, _, b2 = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, key=dk) + check("daily budget blocks the next call", + s1 == 200 and s2 == 402 and "Daily budget" in b2.get("error", {}).get("message", ""), + f"{s1}/{s2}") + + # ---------------------------------------------------------- provider probe + status, _, probe = jreq("POST", f"/admin/providers/{providers['openai']['id']}/probe", admin=True) + check("provider probe reaches the upstream", + status == 200 and probe.get("ok") is True and probe.get("model_count", 0) > 0, + str(probe)[:140]) + status, _, health = jreq("GET", "/admin/providers/health", admin=True) + check("provider health summary lists circuits", + status == 200 and any(p.get("circuit_state") for p in health), str(status)) + + # ------------------------------------------------------------- usage export + status, _, raw_csv = req("GET", "/admin/usage/export?window=30d&limit=100", admin=True) + csv_text = raw_csv.decode() + check("usage export returns a CSV header", + status == 200 and csv_text.splitlines()[0].startswith("created_at,key_id"), str(status)) + check("usage export contains rows", len(csv_text.strip().splitlines()) > 1, + str(len(csv_text.splitlines()))) + + # ------------------------------------------------- metrics, docs, tracing + status, docs_headers, docs = jreq("GET", "/v3/api-docs") + check("OpenAPI spec is served", status == 200 and "/v1/chat/completions" in json.dumps(docs), str(status)) + + status, _, prom = jreq("GET", "/actuator/prometheus") + check("Prometheus metrics are exposed", + status == 200 and "path_requests_total" in prom.get("_raw", ""), str(status)) + + status, chat_headers, _ = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "id please"}]}, key=key) + check("responses carry X-Request-Id", bool(chat_headers.get("X-Request-Id")), + str(chat_headers.get("X-Request-Id"))) + + # ----------------------------------------------------------------- aliases + status, _, created_alias = jreq("POST", "/admin/aliases", { + "alias": "smoke-alias", + "models": ["gpt-4o-mini", "claude-3.5-haiku"], + "description": "smoke chain"}, admin=True) + check("an operator alias can be created", + status == 200 and created_alias.get("resolves_to") == "gpt-4o-mini", str(created_alias)[:120]) + + status, _, body = jreq("POST", "/v1/chat/completions", { + "model": "smoke-alias", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("the alias serves traffic", status == 200 and body.get("choices"), str(status)) + + _, _, catalogue = jreq("GET", "/v1/models") + ids = [m["id"] for m in catalogue["data"]] + check("the alias appears in the catalogue", "smoke-alias" in ids, str(ids)[:120]) + + status, _, dup = jreq("POST", "/admin/aliases", { + "alias": "smoke-alias", "models": ["gpt-4o-mini"]}, admin=True) + check("a duplicate alias is rejected", status == 409, str(status)) + + status, _, bad = jreq("POST", "/admin/aliases", { + "alias": "broken-alias", "models": ["not-a-real-model"]}, admin=True) + check("an alias to an unknown model is rejected at write time", + status == 400 and "not-a-real-model" in bad["error"]["message"], str(bad)[:120]) + + status, _, _ = jreq("DELETE", "/admin/aliases/smoke-alias", admin=True) + check("an alias can be deleted", status == 200, str(status)) + status, _, gone = jreq("POST", "/v1/chat/completions", { + "model": "smoke-alias", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("a deleted alias returns 404, not 500", status == 404, str(status)) + + # --------------------------------------------------------- azure openai + status, _, azure = jreq("POST", "/admin/providers", { + "slug": "azure-test", "display_name": "Azure (mock)", + "type": "openai", "base_url": MOCK.replace("/v1", ""), + "api_key": "azure-secret", "api_version": "2024-10-21"}, admin=True) + check("an Azure OpenAI provider can be created", + status in (200, 409) and (status == 409 or azure.get("api_version") == "2024-10-21"), + str(status) + str(azure)[:120]) + if status == 409: + _, _, found = jreq("GET", "/admin/providers", admin=True) + azure = next(p for p in found if p["slug"] == "azure-test") + jreq("PATCH", f"/admin/providers/{azure['id']}", + {"api_version": "2024-10-21", "base_url": MOCK.replace("/v1", "")}, admin=True) + + payload = {"public_model_id": "azure-test", "provider_id": azure["id"], + "upstream_model_id": "deployment-prod", + "input_price_per_million": 0.15, "output_price_per_million": 0.6} + status, _, _ = jreq("POST", "/admin/routes", payload, admin=True) + if status == 409: + _, _, existing = jreq("GET", "/admin/routes?model=azure-test", admin=True) + jreq("PATCH", f"/admin/routes/{existing[0]['id']}", payload, admin=True) + + status, headers, body = jreq("POST", "/v1/chat/completions", { + "model": "azure-test", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("an Azure route serves traffic with a deployment name", + status == 200 and headers.get("X-Gateway-Provider") == "azure-test", str(status)) + + status, _, _ = jreq("POST", "/v1/embeddings", { + "model": "azure-test", "input": "hello"}, key=key) + check("embeddings work on an Azure route", status == 200, str(status)) + + # A wrong api-version must surface Azure's message, not a generic 502. + jreq("PATCH", f"/admin/providers/{azure['id']}", {"api_version": "1999-01-01"}, admin=True) + status, _, bad = jreq("POST", "/v1/chat/completions", { + "model": "azure-test", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("a bad api-version surfaces the provider message", + status >= 400 and "API version" in str(bad.get("error", {}).get("message", "")), + str(status) + str(bad)[:120]) + jreq("PATCH", f"/admin/providers/{azure['id']}", {"api_version": "2024-10-21"}, admin=True) + + # -------------------------------------------------------- adaptive routing + adaptive_payload = { + "public_model_id": "adaptive-test", "provider_id": providers["openai"]["id"], + "upstream_model_id": "gpt-4o-mini", + "input_price_per_million": 0.15, "output_price_per_million": 0.60, + "strategy": "fastest", "priority": 10} + status, _, route_row = jreq("POST", "/admin/routes", adaptive_payload, admin=True) + if status == 409: + _, _, existing = jreq("GET", "/admin/routes?model=adaptive-test", admin=True) + status, _, route_row = jreq("PATCH", f"/admin/routes/{existing[0]['id']}", + adaptive_payload, admin=True) + check("a route can declare a routing strategy", + status == 200 and route_row.get("strategy") == "fastest", str(route_row)[:120]) + status, _, bad = jreq("POST", "/admin/routes", { + "public_model_id": "adaptive-bad", "provider_id": providers["openai"]["id"], + "upstream_model_id": "gpt-4o-mini", "strategy": "telepathy"}, admin=True) + check("an unknown strategy is rejected", status == 400, str(status)) + + status, _, body = jreq("POST", "/v1/chat/completions", { + "model": "adaptive-test", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("an adaptive route still serves traffic", status == 200, str(status)) + + _, _, adaptive_rows = jreq("GET", "/admin/routes?model=adaptive-test", admin=True) + adaptive_row = adaptive_rows[0] if adaptive_rows else {} + check("route views expose the measured latency", + (adaptive_row.get("ewma_latency_ms") or 0) > 0 + and adaptive_row.get("strategy") == "fastest", str(adaptive_row)[:150]) + + # ---------------------------------------------------------------- routing + status, _, models = jreq("GET", "/v1/models", key=key) + ids = [m["id"] for m in models.get("data", [])] + check("catalogue advertises capability aliases", + all(a in ids for a in ("cheap", "fast", "long-context")), str(ids[-6:])) + + status, headers, body = jreq("POST", "/v1/chat/completions", { + "model": "cheap", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("cheap alias routes to a model", status == 200 and headers.get("X-Gateway-Provider") is not None, + str(status) + str(body)[:120]) + + status, _, routes_body = jreq("GET", "/admin/routes?model=auto-fast", admin=True) + if routes_body: + jreq("PATCH", f"/admin/routes/{routes_body[0]['id']}", {"weight": 100}, admin=True) + check("auto-fast keeps both routes", len(routes_body) == 2, str(len(routes_body))) + + # ------------------------------------------------------------------ audit + status, _, audit_rows = jreq("GET", "/admin/audit?limit=50", admin=True) + check("admin audit records mutations", + status == 200 and isinstance(audit_rows, list) + and any(e.get("action") == "key.create" for e in audit_rows), + str(status) + str(audit_rows)[:120]) + + # ------------------------------------------------------------- embeddings + payload = {"public_model_id": "embed-test", "provider_id": providers["openai"]["id"], + "upstream_model_id": "text-embedding-3-small", + "input_price_per_million": 0.02, "output_price_per_million": 0.0} + status, _, created = jreq("POST", "/admin/routes", payload, admin=True) + if status == 409: + _, _, existing = jreq("GET", "/admin/routes?model=embed-test", admin=True) + for route in existing: + jreq("PATCH", f"/admin/routes/{route['id']}", payload, admin=True) + + status, eheaders, ebody = jreq("POST", "/v1/embeddings", { + "model": "embed-test", "input": "hello"}, key=key) + check("embeddings endpoint returns vectors", + status == 200 and ebody.get("data") and ebody["data"][0].get("embedding"), + str(status) + str(ebody)[:120]) + check("embeddings are metered", float(eheaders.get("X-Gateway-Cost-Usd", "0")) > 0, + str(eheaders.get("X-Gateway-Cost-Usd"))) + + status, _, ebody = jreq("POST", "/v1/embeddings", { + "model": "claude-3.5-haiku", "input": "hello"}, key=key) + check("embeddings reject Anthropic routes", status == 501 + and ebody["error"]["code"] == "unsupported_endpoint", str(status)) + + # ------------------------------------------------- embeddings idempotency + idem = {"Idempotency-Key": "embed-" + str(int(time.time() * 1000))} + status, first_headers, first_body = jreq("POST", "/v1/embeddings", { + "model": "embed-test", "input": "idempotent"}, key=key, headers=idem) + status2, second_headers, second_body = jreq("POST", "/v1/embeddings", { + "model": "embed-test", "input": "idempotent"}, key=key, headers=idem) + check("embeddings replay a repeated Idempotency-Key", + status == 200 and status2 == 200 + and second_headers.get("X-Idempotent-Replay") == "true" + and second_body == first_body, + str(status) + str(status2) + str(second_headers.get("X-Idempotent-Replay"))) + + status, _, conflict = jreq("POST", "/v1/embeddings", { + "model": "embed-test", "input": "different body"}, key=key, headers=idem) + check("embeddings reject a different body on the same key", + status == 409 and conflict["error"]["code"] == "idempotency_conflict", str(status)) + + # ------------------------------------------------------- semantic cache + _, _, settings = jreq("GET", "/admin/settings", admin=True) + # enable both tiers on a deterministic route + _, _, cache_routes = jreq("GET", "/admin/routes?model=gpt-4o-mini", admin=True) + if cache_routes: + jreq("PATCH", f"/admin/routes/{cache_routes[0]['id']}", + {"cache_enabled": True, "semantic_cache_enabled": True, + "semantic_threshold": 0.9}, admin=True) + + base_prompt = "summarise the quarterly report for the board meeting" + _, headers_a, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "temperature": 0, + "messages": [{"role": "user", "content": base_prompt}]}, key=key) + # A warm database may already hold this prompt, so only the header's presence matters. + check("a deterministic request reports its cache state", + headers_a.get("X-Path-Cache") is not None, str(headers_a.get("X-Path-Cache"))) + + _, headers_b, body_b = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "temperature": 0, + "messages": [{"role": "user", "content": base_prompt}]}, key=key) + check("an identical request is an exact cache hit", + headers_b.get("X-Path-Cache") == "hit", str(headers_b.get("X-Path-Cache"))) + + _, headers_c, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "temperature": 0, + "messages": [{"role": "user", "content": base_prompt + " please"}]}, key=key) + check("a near duplicate is served from the semantic tier", + headers_c.get("X-Path-Cache") == "semantic", str(headers_c.get("X-Path-Cache"))) + + _, headers_d, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "temperature": 0, + "messages": [{"role": "user", "content": "what is the weather in Paris"}]}, key=key) + check("an unrelated prompt is not a semantic hit", + headers_d.get("X-Path-Cache") != "semantic", str(headers_d.get("X-Path-Cache"))) + + # Leave the route as a warm run expects to find it: exact cache on, semantic off. + if cache_routes: + jreq("PATCH", f"/admin/routes/{cache_routes[0]['id']}", + {"semantic_cache_enabled": False}, admin=True) + + _, _, stats_now = jreq("GET", "/admin/stats?window=1h", admin=True) + check("stats report semantic cache hits", + isinstance(stats_now.get("cache"), dict) and "semantic_hits" in stats_now["cache"], + str(stats_now.get("cache"))[:120]) + + # ------------------------------------------------------- end-user billing + status, _, eu_key = jreq("POST", "/admin/keys", { + "name": "end-user-key", "end_user_daily_limit_usd": 1000}, admin=True) + check("a key can carry an end-user daily cap", + status == 200 and float(eu_key.get("end_user_daily_limit_usd") or 0) == 1000.0, + str(status)) + eu_key_value = eu_key.get("key") + + status, _, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "end_user": "alice", + "messages": [{"role": "user", "content": "bill me"}]}, key=eu_key_value) + check("a chat request carrying end_user succeeds", status == 200, str(status)) + + _, _, eu_usage = jreq("GET", "/v1/usage?window=1h", key=eu_key_value) + end_users = {row["end_user"]: row for row in eu_usage.get("end_users", [])} + check("usage is attributed to the end user", + "alice" in end_users and float(end_users["alice"]["cost_usd"]) > 0, + str(eu_usage.get("end_users"))[:150]) + + _, _, filtered = jreq("GET", "/admin/usage?endUser=alice&window=1h", admin=True) + rows = filtered.get("data", []) + check("admin usage filters by end user", + filtered.get("end_user") == "alice" and rows + and all(r.get("end_user") == "alice" for r in rows), str(len(rows))) + + status, _, csv_bytes = req("GET", "/admin/usage/export?endUser=alice&window=1h", admin=True) + check("the CSV export filters by end user", status == 200 and b"end_user" in csv_bytes, str(status)) + + # A cap just above one call: alice is spent, carol is not. + jreq("PATCH", f"/admin/keys/{eu_key['id']}", {"end_user_daily_limit_usd": 0.000006}, admin=True) + # max_tokens 1 keeps the pre-flight estimate small, so the cap is exercised without + # having to spend 0.0003 USD to reach it. + status, _, alice_body = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "end_user": "alice", "max_tokens": 1, + "messages": [{"role": "user", "content": "again"}]}, key=eu_key_value) + check("the end-user cap stops a spent end user", + status == 402 and "alice" in alice_body.get("error", {}).get("message", ""), + str(status) + str(alice_body)[:120]) + + status, _, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "end_user": "carol", "max_tokens": 1, + "messages": [{"role": "user", "content": "first time"}]}, key=eu_key_value) + check("another end user on the same key is unaffected", status == 200, str(status)) + + # lift the cap again so the alias check is not blocked by it + jreq("PATCH", f"/admin/keys/{eu_key['id']}", {"end_user_daily_limit_usd": 1000}, admin=True) + + status, _, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "user": "dave", + "messages": [{"role": "user", "content": "openai style"}]}, key=eu_key_value) + _, _, dave_usage = jreq("GET", "/v1/usage?window=1h", key=eu_key_value) + dave_rows = {row["end_user"] for row in dave_usage.get("end_users", [])} + check("the OpenAI 'user' field is accepted as an alias", + status == 200 and "dave" in dave_rows, str(dave_rows)) + + # ------------------------------------------------------------ token count + status, _, counted = jreq("POST", "/v1/tokens/count", { + "model": "gpt-4o-mini", "messages": [{"role": "user", "content": "count me"}]}, key=key) + check("tokens/count reports prompt tokens", + status == 200 and counted.get("prompt_tokens", 0) > 0 + and counted.get("object") == "token_count", str(status) + str(counted)[:120]) + + tools_payload = {"model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "count me"}], + "tools": [{"type": "function", "function": { + "name": "lookup", "description": "look something up", + "parameters": {"type": "object", "properties": {}}}}]} + _, _, with_tools = jreq("POST", "/v1/tokens/count", tools_payload, key=key) + check("tokens/count counts tool definitions", + with_tools.get("prompt_tokens", 0) > counted.get("prompt_tokens", 0), + str(with_tools.get("prompt_tokens"))) + + status, _, unknown = jreq("POST", "/v1/tokens/count", { + "model": "nope-not-real", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("tokens/count rejects an unknown model", + status == 404 and unknown["error"]["code"] == "model_not_found", str(status)) + + # -------------------------------------------------------------- guardrails + _, _, settings = jreq("GET", "/admin/settings", admin=True) + guard = settings.get("guardrails", {}) + check("settings report the active guardrails", + "moderation" in guard and "redact_before_send" in guard, str(settings)[:150]) + + status, _, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "mail me at alice@example.com"}]}, key=key) + check("a prompt containing PII still succeeds", status == 200, str(status)) + + if guard.get("moderation"): + status, _, flagged = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "please flag-me"}]}, key=key) + check("moderation blocks a flagged prompt", + status == 400 and flagged.get("error", {}).get("code") == "content_flagged", + str(status) + str(flagged)[:120]) + + status, _, _ = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "a perfectly ordinary request"}]}, key=key) + check("moderation lets a clean prompt through", status == 200, str(status)) + elif settings.get("prompt_logging"): + pass + + # ------------------------------------------------------------ prompt logs + status, _, plogs = jreq("GET", "/admin/prompt-logs?limit=5", admin=True) + check("prompt log endpoint responds", status == 200 and isinstance(plogs, list), str(status)) + + if settings.get("prompt_logging") and guard.get("prompt_log_redaction"): + # The PII request is not necessarily in the newest five, so read a wider window. + _, _, wide = jreq("GET", "/admin/prompt-logs?limit=200", admin=True) + bodies = " ".join(str(row.get("request", "")) for row in wide) + check("prompt logs carry no email address", "@example.com" not in bodies, bodies[:160]) + check("prompt logs show the redaction token", "[EMAIL_" in bodies, bodies[:160]) + + # ------------------------------------------------------------- allowlist + _, _, allow_key = jreq("POST", "/admin/keys", + {"name": "allowlist-key", "allowed_models": "gpt-4o-mini"}, admin=True) + ak = allow_key.get("key") + a1, _, _ = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, key=ak) + a2, _, ab = jreq("POST", "/v1/chat/completions", + {"model": "claude-3.5-haiku", "messages": [{"role": "user", "content": "hi"}]}, key=ak) + check("model allowlist blocks other models", + a1 == 200 and a2 == 403 and ab["error"]["code"] == "model_not_allowed", f"{a1}/{a2}") + + # --------------------------------------------------------------- tenancy + status, _, _ = jreq("POST", "/admin/projects", {"slug": "smoke-team", "name": "Smoke Team"}, admin=True) + _, _, project_list = jreq("GET", "/admin/projects", admin=True) + project = next((p for p in project_list if p["slug"] == "smoke-team"), None) + check("project is created", project is not None, str(status)) + + _, _, token = jreq("POST", "/admin/tokens", {"name": "smoke-read", "scopes": "read"}, admin=True) + read_only = token.get("token") + check("scoped token is issued once", bool(read_only) and read_only.startswith("adm_"), + str(token)[:80]) + + r1, _, _ = jreq("GET", "/admin/providers", headers={"X-Admin-Token": read_only}) + r2, _, r2body = jreq("POST", "/admin/keys", {"name": "nope"}, + headers={"X-Admin-Token": read_only}) + check("read-only token can GET but not POST", + r1 == 200 and r2 == 403, f"{r1}/{r2} {r2body}"[:120]) + + if project: + status, _, scoped_key = jreq("POST", "/admin/keys", + {"name": "project-key", "project_id": project["id"]}, admin=True) + check("key can belong to a project", + status == 200 and scoped_key.get("project_id") == project["id"], + str(status) + str(scoped_key)[:100]) + _, _, scoped_keys = jreq("GET", f"/admin/keys?projectId={project['id']}", admin=True) + check("keys filter by project", + any(k.get("name") == "project-key" for k in scoped_keys), + str(len(scoped_keys))) + + _, _, summary = jreq("GET", f"/admin/projects/{project['id']}/usage", admin=True) + check("project usage rollup is available", + "spend_today_usd" in summary and "key_count" in summary, str(summary)[:120]) + + # Reset any limit from a previous run, spend once, then set the tiny budget. + jreq("PATCH", f"/admin/projects/{project['id']}", + {"daily_limit_usd": 1000, "monthly_limit_usd": 1000}, admin=True) + p1, _, _ = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + key=scoped_key.get("key")) + jreq("PATCH", f"/admin/projects/{project['id']}", + {"daily_limit_usd": 0.000001}, admin=True) + p2, _, p2body = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + key=scoped_key.get("key")) + check("project daily budget blocks its keys", + p1 == 200 and p2 == 402 and "daily budget" in p2body.get("error", {}).get("message", ""), + f"{p1}/{p2} {str(p2body)[:100]}") + + # ----------------------------------------------------- project reporting + if project: + status, _, scoped_usage = jreq("GET", f"/admin/usage?projectId={project['id']}&window=1h", admin=True) + rows = scoped_usage.get("data", []) + _, _, project_keys = jreq("GET", f"/admin/keys?projectId={project['id']}", admin=True) + project_key_ids = {k["id"] for k in project_keys} + scoped_ok = status == 200 and scoped_usage.get("project_id") == project["id"] + if rows: + scoped_ok = scoped_ok and all(r["key_id"] in project_key_ids for r in rows) + check("usage can be filtered by project", scoped_ok, str(status) + " rows=" + str(len(rows))) + + status, _, scoped_stats = jreq("GET", f"/admin/stats?projectId={project['id']}&window=1h", admin=True) + check("stats can be scoped to a project", + status == 200 and scoped_stats.get("project_id") == project["id"], + str(scoped_stats)[:120]) + + status, _, csv_bytes = req("GET", f"/admin/usage/export?projectId={project['id']}&window=1h", admin=True) + check("export can be scoped to a project", status == 200 and b"created_at" in csv_bytes, str(status)) + + # ------------------------------------------------- project rate limits + # A dedicated project: the rpm window is per minute, so reusing the budget + # project would carry a previous run's counts into this one. + slug = "smoke-rpm-" + str(int(time.time())) + _, _, rpm_project = jreq("POST", "/admin/projects", {"slug": slug, "name": "Smoke RPM"}, + admin=True) + status, _, second_key = jreq("POST", "/admin/keys", + {"name": "project-key-2", "project_id": rpm_project["id"]}, + admin=True) + check("a project can own several keys", status == 200 and second_key.get("key"), str(status)) + + jreq("PATCH", f"/admin/projects/{rpm_project['id']}", + {"rpm_limit": 2, "max_concurrent_requests": 8}, admin=True) + codes = [] + for _ in range(3): + code, _, _ = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}]}, + key=second_key.get("key")) + codes.append(code) + check("project rpm caps every key in the project", + codes[:2] == [200, 200] and codes[2] == 429, str(codes)) + + status, _, limited = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}]}, + key=second_key.get("key")) + check("the 429 names the project limit", + status == 429 and "Project" in limited.get("error", {}).get("message", ""), + str(limited)[:120]) + + jreq("PATCH", f"/admin/projects/{rpm_project['id']}", {"rpm_limit": 0}, admin=True) + status, _, _ = jreq("POST", "/v1/chat/completions", + {"model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}]}, + key=second_key.get("key")) + check("zero project rpm means unlimited", status == 200, str(status)) + + _, _, refreshed = jreq("GET", "/admin/projects", admin=True) + row = next((p for p in refreshed if p["id"] == rpm_project["id"]), {}) + check("project limits are reported by the admin API", + "rpm_limit" in row and "max_concurrent_requests" in row, str(row)[:200]) + + # -------------------------------------------------------- master key rotation + import base64 as _b64 + new_master = _b64.b64encode(b"0123456789abcdef0123456789abcdef").decode() + status, _, rotated = jreq("POST", "/admin/rotate-master-key", {"new_key": new_master}, admin=True) + check("master key rotation re-encrypts credentials", + status == 200 and rotated.get("rotated", 0) >= 1, str(rotated)[:120]) + + status, _, body = jreq("POST", "/v1/chat/completions", { + "model": "gpt-4o-mini", "messages": [{"role": "user", "content": "after rotation"}]}, key=key) + check("provider credentials still decrypt after rotation", status == 200, str(status) + str(body)[:100]) + # ------------------------------------------------------------ auth and errors status, _, body = jreq("POST", "/v1/chat/completions", { "model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, key="sk-gw-nope") @@ -320,6 +907,15 @@ def main(): check("sync recreated the removed model", sync.get("created", 0) >= 1, str(sync)[:200]) check("sync reported no provider errors", sync.get("errors") == [], str(sync.get("errors"))) + _, _, unpriced = jreq("GET", "/admin/routes?model=acme-unknown-model", admin=True) + check("unpriced synced routes stay disabled", + bool(unpriced) and all(not r["enabled"] for r in unpriced), str(unpriced)[:150]) + + status, _, body = jreq("POST", "/v1/chat/completions", { + "model": "acme-unknown-model", "messages": [{"role": "user", "content": "hi"}]}, key=key) + check("unpriced synced model is not callable", + status == 404 and body["error"]["code"] == "model_not_found", str(status)) + status, _, models = jreq("GET", "/v1/models", key=key) ids = [m["id"] for m in models.get("data", [])] check("synced catalogue gained claude alias", any(i.startswith("claude") for i in ids), str(ids))