diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index bc6477c..7fa0792 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -40,7 +40,7 @@ jobs: if: steps.gate.outputs.enabled == 'true' run: | ./mvnw -B -DskipTests package - GATEWAY_ADMIN_TOKEN=canary java -jar target/path-*.jar > /tmp/gateway.log 2>&1 & + GATEWAY_ADMIN_TOKEN=canary java -jar path-gateway/target/path-*.jar > /tmp/gateway.log 2>&1 & for i in $(seq 1 60); do curl -sf http://127.0.0.1:8080/actuator/health > /dev/null && break sleep 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 415aa23..c4eaaba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: surefire-reports - path: target/surefire-reports/ + path: path-gateway/target/surefire-reports/ retention-days: 7 helm: @@ -46,10 +46,16 @@ jobs: runs-on: ubuntu-26.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 + - uses: azure/setup-helm@2f5e7bc67f1f9972ab16657bb795f36d8f04de0e # v5.1.0 with: version: v3.16.3 + - name: Prometheus rules are valid + run: | + docker run --rm -v "$PWD/deploy/prometheus:/rules:ro" \ + --entrypoint /bin/promtool prom/prometheus:v3.1.0 \ + check rules /rules/path-rules.yml + - name: Lint run: helm lint deploy/helm/path --set existingSecret=path-credentials @@ -58,7 +64,8 @@ jobs: helm template path deploy/helm/path --set existingSecret=x > /dev/null helm template path deploy/helm/path --set existingSecret=x \ --set autoscaling.enabled=true --set postgres.enabled=true \ - --set valkey.enabled=true --set serviceMonitor.enabled=true > /dev/null + --set valkey.enabled=true --set serviceMonitor.enabled=true \ + --set prometheusRule.enabled=true > /dev/null - name: No credentials are rendered inline run: | @@ -99,7 +106,7 @@ jobs: GATEWAY_PROMPT_LOGGING=true \ GATEWAY_GUARDRAILS_MODERATION=true \ GATEWAY_SEMANTIC_MODEL=embed-test \ - java -jar target/path-*.jar \ + java -jar path-gateway/target/path-*.jar \ > /tmp/opencode/gateway.log 2>&1 & for i in $(seq 1 60); do if curl -sf http://127.0.0.1:8080/actuator/health > /dev/null; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1fec82..1985391 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,17 +40,17 @@ jobs: # --- supply chain: checksums ------------------------------------------- - name: Collect SBOM - run: cp target/classes/META-INF/sbom/application.cdx.json target/bom.json + run: cp path-gateway/target/classes/META-INF/sbom/application.cdx.json path-gateway/target/bom.json - name: Write SHA-256 checksums run: | - cd target + cd path-gateway/target sha256sum path-*.jar bom.json | sort -k2 > checksums.txt ls -l path-*.jar bom.json checksums.txt cat checksums.txt - name: Verify the checksums we just wrote - run: cd target && sha256sum -c checksums.txt + run: cd path-gateway/target && sha256sum -c checksums.txt # --- container: build, scan, push -------------------------------------- - name: Set up QEMU @@ -64,7 +64,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push container image + # Only an unversioned sha- tag is pushed before the scan. A failed scan then leaves + # nothing a deployment could pull by version, and the version number stays unused for + # the retry. The version tag and latest are created after the gate, by promoting this + # exact digest instead of rebuilding. + - name: Build and push the candidate image id: image uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: @@ -72,8 +76,7 @@ jobs: push: true platforms: linux/amd64,linux/arm64 tags: | - ghcr.io/${{ github.repository }}:${{ github.ref_name }} - ghcr.io/${{ github.repository }}:latest + ghcr.io/${{ github.repository }}:sha-${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max @@ -83,7 +86,7 @@ jobs: TRIVY_USERNAME: ${{ github.actor }} TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} with: - image-ref: ghcr.io/${{ github.repository }}:${{ github.ref_name }} + image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }} # CRITICAL blocks the release; HIGH is visible in the scan output but the # Debian base image rotates packages too often to gate on it. # @@ -96,12 +99,26 @@ jobs: exit-code: "1" ignore-unfixed: true + # The scan passed, so the same digest can carry the version. imagetools copies the + # multi-arch manifest; nothing is rebuilt and the digest recorded below is the one + # Trivy scanned. + - name: Promote the scanned digest to the version and latest + run: | + docker buildx imagetools create \ + -t "ghcr.io/${{ github.repository }}:${{ github.ref_name }}" \ + -t "ghcr.io/${{ github.repository }}:latest" \ + "ghcr.io/${{ github.repository }}:sha-${{ github.sha }}" + - name: Record the image digest run: | + digest="$(docker buildx imagetools inspect \ + --format '{{.Manifest.Digest}}' \ + "ghcr.io/${{ github.repository }}:${{ github.ref_name }}")" echo "### Container digest" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "ghcr.io/${{ github.repository }}@${{ steps.image.outputs.digest }}" >> "$GITHUB_STEP_SUMMARY" + echo "ghcr.io/${{ github.repository }}@${digest}" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "Also kept for traceability: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}" # --- Maven Central (optional) ------------------------------------------- - name: Publish to Maven Central @@ -140,7 +157,7 @@ jobs: name: ${{ github.ref_name }} body_path: /tmp/release-notes.md files: | - target/path-*.jar - target/bom.json - target/checksums.txt + path-gateway/target/path-*.jar + path-gateway/target/bom.json + path-gateway/target/checksums.txt fail_on_unmatched_files: true diff --git a/AGENTS.md b/AGENTS.md index 70b3f35..91d75da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,18 +19,32 @@ dependency: the gateway translates to each provider's native protocol. # unit + integration tests (85) ./mvnw verify -# full end-to-end suite against a mock upstream (104 checks, no real keys) +# full end-to-end suite against a mock upstream (no real keys) python3 tools/mock-upstream.py 9099 & -GATEWAY_ADMIN_TOKEN=test-admin java -jar target/path-*.jar & +GATEWAY_ADMIN_TOKEN=test-admin java -jar path-gateway/target/path-*.jar & python3 tools/smoke_test.py # run locally export OPENAI_API_KEY=... ANTHROPIC_API_KEY=... GATEWAY_ADMIN_TOKEN=... -./mvnw spring-boot:run +./mvnw -pl path-gateway spring-boot:run ``` The seeder prints a demo `sk-gw-...` key on the first boot only. +## Modules + +`./mvnw` is a Maven reactor, not a single project: + +| Module | Artifact | Contents | +|---|---|---| +| `path-gateway/` | `io.github.an1ndra:path` | the gateway jar, the runnable Spring Boot application | +| `path-spring-ai/` | `io.github.an1ndra:path-spring-ai` | a Spring AI `ChatModel`/`EmbeddingModel` over the gateway | + +The gateway is the published root artifact and never depends on Spring AI; the adapter is +the only module that knows about it. `./mvnw verify` builds both. The Dockerfile builds only +`-pl path-gateway`, so the image does not carry the adapter's dependencies. `tools/` and +`deploy/` stay at the repository root. + ## Layout ``` @@ -176,6 +190,12 @@ a smoke check. Never claim a fix without running both. `v*.*.*` tag → jar + SBOM + checksums + multi-arch `ghcr.io/an1ndra/path`. The workflow publishes `changelog/.md` verbatim as the release body and fails if the file is missing or empty (`tools/changelog.py`). +- **The image is scanned before it is published.** The workflow pushes only + `ghcr.io/:sha-`; Trivy gates that tag; only then does + `docker buildx imagetools create` copy the same digest to `` and `latest`. A failed + scan therefore leaves a `sha-` tag and an unused version number, not a released image. + The digest in the workflow summary is the one that was scanned. Do not move the version + tag back into the build step. ## Issue labels @@ -202,6 +222,15 @@ is a guess, not a commitment — re-tag when reality disagrees. ## Gotchas +- **IntelliJ: point Maven home at a real install, not "Use Maven wrapper".** The wrapper here + is `distributionType=only-script` (no `maven-wrapper.jar`), and IntelliJ's Maven server + resolves its own home instead of running `mvnw`. It then bootstraps Maven without the + plexus artifact bindings and fails with + `Guice/MissingImplementation: No implementation for ArtifactHandlerManager was bound`. + Set Settings -> Build Tools -> Maven -> Maven home to the wrapper's extracted 3.10.1 + (`~/.m2/wrapper/dists/apache-maven-3.10.1/`) or to the IDE's bundled Maven. + The CLI `./mvnw` is unaffected. + - **Flyway owns the schema; `ddl-auto` is `validate`.** Add `V__name.sql` for every entity change and never edit an applied migration. `V1__baseline.sql` is written in portable SQL: `bigint generated by default as identity`, `timestamp(6) with time zone`, diff --git a/CHANGELOG.md b/CHANGELOG.md index 974dd68..4b109d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ verbatim as the body of the matching GitHub release. | Version | Date | Notes | |---|---|---| +| v0.2.3 | 2026-11-20 | [changelog/v0.2.3.md](changelog/v0.2.3.md) | | v0.2.2 | 2026-11-17 | [changelog/v0.2.2.md](changelog/v0.2.2.md) | | v0.2.1 | 2026-11-17 | [changelog/v0.2.1.md](changelog/v0.2.1.md) | | v0.2.0 | 2026-11-17 | [changelog/v0.2.0.md](changelog/v0.2.0.md) | diff --git a/Dockerfile b/Dockerfile index 8aaa443..fa0f4a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,14 +23,24 @@ RUN set -eux; \ mvn -v COPY pom.xml ./ +COPY path-gateway/pom.xml path-gateway/pom.xml +COPY path-spring-ai/pom.xml path-spring-ai/pom.xml RUN mvn -B -q -DskipTests dependency:go-offline -COPY src/ src/ -RUN mvn -B -q -DskipTests package +COPY path-gateway/src/ path-gateway/src/ +# The gateway does not depend on the adapter, so the image build skips the Spring AI module. +RUN mvn -B -q -DskipTests -pl path-gateway package FROM eclipse-temurin:21-jre WORKDIR /app -COPY --from=build /app/target/path-*.jar app.jar + +# curl is here for the container health check below: Dokploy (and Docker) can act on it, +# and it is one small, patched package rather than the whole JDK toolchain. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /app/path-gateway/target/path-*.jar app.jar # Run as an unprivileged user (Trivy DS-0002) and keep ./data writable for the H2 file # and the generated master key. @@ -41,4 +51,10 @@ USER path ENV PORT=8080 EXPOSE 8080 + +# Readiness includes drain state: a shutting-down pod reports 503, which is what a rolling +# update should see. Uses the same endpoint the Helm chart probes. +HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ + CMD curl -fsS http://127.0.0.1:8080/actuator/health || exit 1 + ENTRYPOINT ["java", "-jar", "/app/app.jar"] diff --git a/README.md b/README.md index b1c9ccb..1d72be4 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ export ANTHROPIC_API_KEY=sk-ant-... export GROQ_API_KEY=gsk_... export GATEWAY_ADMIN_TOKEN=pick-something-long -./mvnw spring-boot:run +./mvnw -pl path-gateway spring-boot:run ``` First boot seeds four providers and a starter route table. To also mint a demo key @@ -97,6 +97,7 @@ print(client.chat.completions.create( | Document | Contents | |---|---| +| [Spring AI adapter](docs/SPRING-AI.md) | `path-spring-ai`: ChatModel and EmbeddingModel over the gateway | | [API reference](docs/API.md) | Every endpoint, request/response shapes, error codes, idempotency, cache, embeddings | | [Architecture](docs/ARCHITECTURE.md) | Request flow, providers, routing, what is metered | | [Configuration](docs/CONFIGURATION.md) | Every environment variable, timeouts, credentials, Postgres | @@ -104,6 +105,9 @@ print(client.chat.completions.create( | [Limits](docs/LIMITS.md) | Design decisions and known gaps | | [Testing](docs/TESTING.md) | Unit tests, the 99-check smoke suite, the mock upstream | | [Why Path](docs/WHY-PATH.md) | Honest comparison and a 60-second demo | +| [SSO and 2FA](docs/SSO.md) | Google/GitHub/enterprise SSO, passkeys and 2FA via an authenticating proxy | +| [Admin UI](docs/ADMIN-UI.md) | The web console lives in `an1ndra/path-admin`; auth, hosting and CORS | +| [Deploy on Dokploy](deploy/dokploy/README.md) | Two apps, compose or plain application, health checks, CORS | | [Publishing](docs/PUBLISHING.md) | Maven Central setup, signing and the release profile | | [Roadmap](docs/ROADMAP.md) | Delivered work and the open stories | | [Changelog](CHANGELOG.md) | Release notes, also used as the GitHub release body | diff --git a/changelog/v0.2.3.md b/changelog/v0.2.3.md new file mode 100644 index 0000000..1e609de --- /dev/null +++ b/changelog/v0.2.3.md @@ -0,0 +1,113 @@ +# v0.2.3 + +The largest release since v0.1.0: a Spring AI adapter module, native Gemini on AI Studio and +Vertex, a daily digest, shared circuit state for clusters, and the alerting and security +work that had accumulated on `Develop` since v0.2.2. + +Everything is additive for existing deployments. Two optional switches are worth a look: +`GATEWAY_SHARED_STATE_BACKEND=valkey` for clusters (see Changed) and the opt-in Postgres +partitioning (see Database). + +## Added + +### Spring AI adapter + +- **`path-spring-ai`**, published as `io.github.an1ndra:path-spring-ai`. A `ChatModel` and + `EmbeddingModel` over the gateway: messages, options, tool definitions and tool responses, + non-streaming tool calls, SSE content streaming, and `Idempotency-Key` when a caller + retries. `X-Path-Cache`, `X-Gateway-Provider` and `X-Gateway-Cost-Usd` land in the + `ChatResponse` metadata; a gateway error surfaces as `PathException` with the status and + the gateway's code. See [`docs/SPRING-AI.md`](docs/SPRING-AI.md). +- The build is now a reactor: `path` (the gateway, unchanged coordinates) and + `path-spring-ai` are sibling modules under a `path-parent` POM. The gateway does not + depend on Spring AI, and the container image still contains only the gateway. + +### Providers + +- **Native Gemini** for AI Studio and Vertex (`GEMINI` and `VERTEX` provider types) on + `generateContent`. `thinkingConfig` (or `reasoning_effort`) and `safetySettings` are + honoured instead of dropped; thinking tokens are billed as output and broken out as + `reasoning_tokens`; `cachedContentTokenCount` is priced as cache reads. A blocked prompt + is 400 `content_blocked` naming the safety category, and a parameter Gemini cannot honour + is 422 `unsupported_parameter` — the same contract the Anthropic adapter keeps. +- **Vertex credentials**: a service-account JSON in the encrypted credential column, an + RS256 assertion exchanged for a cached OAuth token, and `project`/`location` on the + provider row. + +### Metering and reporting + +- **Daily digest email** at `GATEWAY_DIGEST_CRON` (default 08:00): yesterday's requests with + the day-over-day delta, spend, tokens with the cache hit ratio, failures with the top + error code, the slowest model's p95, top spend and any parked provider. Silent without a + mail channel; nothing is sent for an empty day. `POST /admin/digest/test` renders it on + demand. +- **Bucketed time series** on `GET /admin/stats?bucket=1h|5m|1m|1d`: per-bucket requests, + cost, failures and p95 latency from one query, zero-filled and capped at 500 points. + Omitting `bucket` returns the previous response unchanged. +- **Archive slice**: `/admin/usage/export` accepts `from`/`to`, so the exact range a + retention purge will delete can be captured for finance. +- **Semantic cache hits are counted from usage rows** (`cache_tier` column) instead of a + process-local counter, so the number survives a restart and matches across replicas. + +### Operations + +- **Prometheus rules** in `deploy/prometheus/path-rules.yml`: fallback rate, cache hit ratio + against its 7-day average, spend rate, 5xx rate and providers parked for 15 minutes, each + with a `runbook` annotation. The Helm chart renders them with `prometheusRule.enabled`. +- **New metrics**: `path_errors_total{provider,class,code}`, `path_fallbacks_total`, + `path_provider_circuit_open` and `path_alerts_sent_total{channel,outcome}`. +- **Scopes in `/admin/settings`**: `subject` and `scopes` (plus `can_write`) for both token + and proxy auth, so a console can hide write actions instead of discovering a 403. +- **Email alerting**: `GATEWAY_ALERT_EMAIL_TO` plus `SPRING_MAIL_*` sends every alert by + mail; `POST /admin/alerts/test` reports which channels are configured; `GET/PUT + /admin/mail-settings` moves the SMTP settings into the console (password encrypted and + write-only). +- **SSO proxy auth** for the admin API: an authenticating proxy can inject an identity + header instead of a pasted token. Read [`docs/SSO.md`](docs/SSO.md) first — it is only + safe when the gateway is unreachable except through that proxy. +- **Dokploy deployment assets**, including a Traefik router and a container health check. + +## Changed + +- **Webhook alerts retry** up to three times with jittered exponential backoff, only on 429, + 5xx and connect/timeout, honouring `Retry-After`. Both webhook and email run on bounded + queues; a full queue drops and counts instead of blocking the request or growing without + limit. +- **Provider error-rate alert**: a one-minute check over the routing EWMA fires + `provider.error_rate_high` when a provider's rate stays above + `GATEWAY_ALERT_ERROR_RATE` (0.25) with enough samples, and `provider.error_rate_normal` + when it recovers. This catches the provider that fails 30% of the time without ever + tripping the consecutive-failure breaker. +- **Cluster state**: `GATEWAY_SHARED_STATE_BACKEND=valkey` shares circuit-breaker and + in-flight state across replicas. Exactly one replica claims the half-open probe, an OPEN + entry expires even if the replica that opened it dies, and a killed pod's concurrency + permits age out. Memory remains the default and is unchanged. +- **Release images are scanned before they are named.** The workflow pushes only + `sha-`, scans it, and promotes that digest to the version and `latest` with + `imagetools`; a failed scan no longer burns a version number. +- **Seeding the example catalogue is opt-out** (`GATEWAY_SEED_DEFAULTS=false`) for a real + deployment. +- **CI** runs on `ubuntu-26.04` with `azure/setup-helm` v5.1.0 (Node 24). + +## Fixed + +- **Security (external pentest):** three findings closed, including a timing oracle on API + key rejection and an admin limiter that could be bypassed via the origin. API-key + guessing is budgeted and rejected responses are never cached, so a wrong key cannot be + turned into a cache oracle. +- **Security:** a provider API key is never written into the audit log; the proxy-auth + startup warning now actually fires. +- Health no longer reports DOWN when no SMTP host is configured. +- The end-to-end suite authenticates its Prometheus check instead of failing with a 401. +- The Dokploy compose no longer uses `env_file`, which broke the deployment. + +## Database + +- `V19__usage_cache_tier.sql` adds `usage_records.cache_tier` (`exact`, `semantic`, or null). +- `V20__provider_vertex.sql` adds `providers.project` and `providers.location` and widens + `api_key_cipher` to hold a service-account JSON. + +Both run automatically on startup. The partitioning script in +[`deploy/postgres/partitions.sql`](deploy/postgres/partitions.sql) is separate and opt-in: +it rewrites `usage_records`, so it is documented as a maintenance-window step after a +`pg_dump`. diff --git a/deploy/dokploy/README.md b/deploy/dokploy/README.md new file mode 100644 index 0000000..6969c97 --- /dev/null +++ b/deploy/dokploy/README.md @@ -0,0 +1,133 @@ +# Deploy Path on Dokploy + +Two applications, because the console is a separate repository: + +| App | Repository | Dockerfile | Port | Health | +|---|---|---|---|---| +| gateway | `an1ndra/path` | `Dockerfile` (repo root) | 8080 | `/actuator/health` | +| console | `an1ndra/path-admin` | `Dockerfile` (repo root) | 8080 | `/healthz` | + +Either deploy the gateway as a **Compose** stack (`compose.yml` here, brings Postgres and +optional Valkey) or as a plain **Application** pointing at the Dockerfile with an external +database. The console is always a plain Application. + +## 1. Gateway + +### Compose route (recommended: includes Postgres) + +1. Dokploy → **Create → Compose**. +2. Source: GitHub → `an1ndra/path`, branch `main`. +3. Compose path: `deploy/dokploy/compose.yml`. +4. **Environment** (the compose file references all of these): + +```bash +POSTGRES_PASSWORD= +GATEWAY_MASTER_KEY= # keep it: it decrypts stored provider keys +GATEWAY_ADMIN_TOKEN= # only for token auth; ignore with SSO +GATEWAY_CORS_ORIGINS=https://admin.example.com # omit if same hostname as the console +``` + +5. Domain: point `gw.example.com` at port **8080**, enable HTTPS. +6. Deploy. First boot logs `Gateway ready. Admin API: enabled` and, if `-P release` was + used, seeds four providers. + +### Application route (external Postgres) + +Create → **Application** → repository `an1ndra/path`, build type **Dockerfile**, port +**8080**, then: + +```bash +SPRING_PROFILES_ACTIVE=postgres +DB_URL=jdbc:postgresql://:5432/path +DB_USER=path +DB_PASSWORD=<...> +GATEWAY_MASTER_KEY= +GATEWAY_API_DOCS=false +``` + +Mount a volume at `/app/data` if you'd rather keep the H2 database and the master key on +disk than use Postgres. + +## 2. Console + +1. Dokploy → **Create → Application**, repository `an1ndra/path-admin`, branch `main`. +2. Build: **Dockerfile** (node build stage → nginx). Port **8080**. Health: **`/healthz`**. +3. Environment — one variable: + +```bash +# Separate hostnames: +GATEWAY_URL=https://gw.example.com + +# Or leave it EMPTY and route /admin-ui to this container on the gateway's hostname, +# which removes CORS entirely. +GATEWAY_URL= +``` + +4. Domain: `admin.example.com` (or a path prefix on the gateway domain) and HTTPS. + +## 3. Wiring, and the one setting people forget + +If the console is on a **different hostname** from the gateway, the gateway must allow it: + +```bash +GATEWAY_CORS_ORIGINS=https://admin.example.com +``` + +Exact origin, no trailing slash, no wildcard. `X-Admin-Token` is a custom header, so the +browser sends a preflight; `PUT` is allowed as of 0.2.3, which the console's settings screen +needs. + +Same-hostname routing on Dokploy/Traefik: + +``` +Host(`path.example.com`) && PathPrefix(`/admin-ui`) → console:8080 (strip prefix) +Host(`path.example.com`) → gateway:8080 +``` + +## 4. Before it is public + +- Put the console behind **Cloudflare Access**, Authentik or a VPN. A pasted admin token has + no second factor; see `docs/SSO.md` for the proxy-auth mode that fixes that with + `GATEWAY_PROXY_AUTH=true`. +- `GATEWAY_API_DOCS=false` in production (the compose file sets it). +- Keep `GATEWAY_MASTER_KEY` in your password manager. Losing it means re-entering every + provider credential. +- `stop_grace_period` must exceed `GATEWAY_DRAIN_TIMEOUT`, or a redeploy truncates live SSE + streams (the compose file uses 40s against 25s). + +## Origin exposure (do not skip this) + +The origin answers on its own public IP, so anyone can reach it directly and skip +Cloudflare — including its bot challenge and WAF rules. That is not hypothetical: an +external pentest used it to bypass rate limiting, and it makes every client-IP header +(`CF-Connecting-IP`, `X-Forwarded-For`) forgeable, which defeats per-address limits. + +The compose file therefore attaches the `cf-only` Traefik middleware (Cloudflare's +published ranges) to the gateway router. Two things follow: + +- The middleware itself lives in `/etc/dokploy/traefik/dynamic/middlewares.yml`. Set it up + once in **Traefik File System**; the compose label references `cf-only@file`. +- The console is a Dokploy *application*, so its router file is generated by Dokploy. After + any domain change there, re-add `- cf-only` to the `websecure` router's middlewares, or + the console is reachable directly again. + +Verify with both halves — through Cloudflare and straight at the IP: + +```bash +curl -fsS -o /dev/null -w '%{http_code}\n' https://gw.example.com/actuator/health # 200 +curl -k -sS -o /dev/null -w '%{http_code}\n' --resolve gw.example.com:443: \ + https://gw.example.com/actuator/health # 403 +``` + +If you are not behind Cloudflare, use your CDN's ranges, or drop the middleware and accept +that per-address limits can be spoofed. + +## 5. Smoke test after deploy + +```bash +curl -fsS https://gw.example.com/actuator/health # {"status":"UP"} +curl -fsS https://admin.example.com/healthz # ok +curl -fsS -H "X-Admin-Token: $TOKEN" https://gw.example.com/admin/settings +``` + +Then open the console, sign in, and **Settings → Alert email → Send test email**. diff --git a/deploy/dokploy/compose.yml b/deploy/dokploy/compose.yml new file mode 100644 index 0000000..d37417c --- /dev/null +++ b/deploy/dokploy/compose.yml @@ -0,0 +1,95 @@ +# Dokploy compose stack: Postgres + the gateway. The admin console is its own Dokploy +# application (separate repository) and talks to this one over HTTP. +# +# In Dokploy: Compose → this repository → compose path deploy/dokploy/compose.yml, then set +# the environment variables below in the UI. Nothing here contains a secret. + +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-path} + POSTGRES_USER: ${POSTGRES_USER:-path} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-path} -d ${POSTGRES_DB:-path}"] + interval: 5s + timeout: 3s + retries: 20 + + # Optional: shared rate limits and a shared circuit view across replicas. + # Enable by setting GATEWAY_RATE_LIMIT_BACKEND=valkey and uncommenting `profiles`. + valkey: + image: valkey/valkey:8-alpine + restart: unless-stopped + profiles: ["valkey"] + command: ["valkey-server", "--save", "", "--appendonly", "no"] + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + + gateway: + build: + context: ../.. + dockerfile: Dockerfile + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + environment: + SPRING_PROFILES_ACTIVE: postgres + DB_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-path} + DB_USER: ${POSTGRES_USER:-path} + DB_PASSWORD: ${POSTGRES_PASSWORD} + # 32-byte base64. Generate once and keep it: it decrypts stored provider keys, so + # losing it means re-entering every provider credential. + PORT: "8080" + GATEWAY_MASTER_KEY: ${GATEWAY_MASTER_KEY:?set GATEWAY_MASTER_KEY (openssl rand -base64 32)} + # Only needed when the console is on another hostname. Same hostname → leave unset. + GATEWAY_CORS_ORIGINS: ${GATEWAY_CORS_ORIGINS:-} + GATEWAY_ADMIN_TOKEN: ${GATEWAY_ADMIN_TOKEN:-} + GATEWAY_PROXY_AUTH: ${GATEWAY_PROXY_AUTH:-false} + GATEWAY_PROXY_AUTH_HEADER: ${GATEWAY_PROXY_AUTH_HEADER:-X-Forwarded-User} + GATEWAY_RATE_LIMIT_BACKEND: ${GATEWAY_RATE_LIMIT_BACKEND:-memory} + VALKEY_HOST: ${VALKEY_HOST:-valkey} + GATEWAY_USAGE_RETENTION: ${GATEWAY_USAGE_RETENTION:-180d} + GATEWAY_DRAIN_TIMEOUT: ${GATEWAY_DRAIN_TIMEOUT:-25s} + GATEWAY_ALERT_WEBHOOK: ${GATEWAY_ALERT_WEBHOOK:-} + GATEWAY_ALERT_EMAIL_TO: ${GATEWAY_ALERT_EMAIL_TO:-} + SPRING_MAIL_HOST: ${SPRING_MAIL_HOST:-} + SPRING_MAIL_PORT: ${SPRING_MAIL_PORT:-} + SPRING_MAIL_USERNAME: ${SPRING_MAIL_USERNAME:-} + SPRING_MAIL_PASSWORD: ${SPRING_MAIL_PASSWORD:-} + GATEWAY_API_DOCS: "false" + # Provider keys are added through the console (Settings → Providers) or as + # OPENAI_API_KEY / ANTHROPIC_API_KEY / GROQ_API_KEY / GEMINI_API_KEY. + ports: + - "8080:8080" + # Routing is declared here rather than left to the Dokploy UI, so a redeploy always + # produces the router. TLS uses the server's default certificate (Cloudflare Origin + # certificate when Cloudflare is in front), which is what the proxy validates against. + labels: + traefik.enable: "true" + traefik.docker.network: "dokploy-network" + traefik.http.routers.path-gateway.rule: "Host(`path.hacksubset.com`)" + traefik.http.routers.path-gateway.entrypoints: "websecure" + traefik.http.routers.path-gateway.tls: "true" + # Only Cloudflare's edge may reach this router. Without it the origin answers on its + # own public IP, so any client can skip Cloudflare (and its bot challenge) entirely and + # forge client-IP headers, which defeats every per-address rate limit. + traefik.http.routers.path-gateway.middlewares: "cf-only@file" + traefik.http.routers.path-gateway.service: "path-gateway" + traefik.http.services.path-gateway.loadbalancer.server.port: "8080" + # More than the drain timeout, or a redeploy truncates live streams. + stop_grace_period: 40s + volumes: + - pathdata:/app/data + +volumes: + pgdata: + pathdata: diff --git a/deploy/helm/path/README.md b/deploy/helm/path/README.md index 27ef8cf..2d9c8b0 100644 --- a/deploy/helm/path/README.md +++ b/deploy/helm/path/README.md @@ -32,6 +32,9 @@ curl -s -H "X-Admin-Token: $GATEWAY_ADMIN_TOKEN" localhost:8080/actuator/health/ safe without it. - `autoscaling` — CPU is a poor signal for a gateway. If you have the Prometheus adapter, scale on in-flight requests instead. +- `prometheusRule.enabled` — renders the fallback/cache/spend/error/parked alerts from + `deploy/prometheus/path-rules.yml` as a `PrometheusRule`. Needs the Prometheus Operator + CRDs; `runbookBase` sets where the annotations point. ## What this chart does not do diff --git a/deploy/helm/path/templates/deployment.yaml b/deploy/helm/path/templates/deployment.yaml index a6bb0e6..ef39268 100644 --- a/deploy/helm/path/templates/deployment.yaml +++ b/deploy/helm/path/templates/deployment.yaml @@ -61,6 +61,10 @@ spec: {{- if .Values.valkey.enabled }} - name: GATEWAY_RATE_LIMIT_BACKEND value: valkey + # Same explicit choice for circuit-breaker and in-flight state; without it a + # cluster keeps one copy of each per replica. + - name: GATEWAY_SHARED_STATE_BACKEND + value: valkey - name: VALKEY_HOST value: {{ printf "%s-valkey" (include "path.fullname" .) | quote }} {{- end }} diff --git a/deploy/helm/path/templates/prometheusRule.yaml b/deploy/helm/path/templates/prometheusRule.yaml new file mode 100644 index 0000000..7c8e61f --- /dev/null +++ b/deploy/helm/path/templates/prometheusRule.yaml @@ -0,0 +1,90 @@ +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "path.fullname" . }} + labels: + {{- include "path.labels" . | nindent 4 }} + {{- with .Values.prometheusRule.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: path-recording + interval: 30s + rules: + - record: path:cache_hit_ratio:rate30m + expr: | + sum(rate(path_cache_hits_total[30m])) + / + clamp_min(sum(rate(path_requests_total[30m])), 0.0001) + - name: path-alerts + rules: + - alert: PathHighFallbackRate + expr: | + sum(rate(path_fallbacks_total[5m])) + / + clamp_min(sum(rate(path_requests_total[5m])), 0.0001) + > 0.20 + for: 10m + labels: + severity: warning + annotations: + summary: "More than 20% of requests are falling back to another route" + description: >- + A route is failing and every request is being retried on its fallback. Spend + and latency are rising even though requests still succeed. + runbook: {{ .Values.prometheusRule.runbookBase | default "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" | quote }} + - alert: PathCacheHitRatioLow + expr: | + path:cache_hit_ratio:rate30m + < 0.5 * avg_over_time(path:cache_hit_ratio:rate30m[7d]) + for: 30m + labels: + severity: warning + annotations: + summary: "The response-cache hit ratio is below half its 7-day average" + description: >- + Cache hits collapsed against their own baseline: a changed prompt prefix, a + disabled cache_enabled route or a new client with temperature > 0. + runbook: {{ .Values.prometheusRule.runbookBase | default "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" | quote }} + - alert: PathSpendRateHigh + expr: | + sum(increase(path_cost_usd_total[1h])) * 24 + > + 2 * sum(increase(path_cost_usd_total[1d] offset 1d)) + for: 15m + labels: + severity: warning + annotations: + summary: "The current spend rate is on track to exceed twice yesterday" + description: >- + The last hour, extrapolated to a day, is more than double the whole of the + previous day. + runbook: {{ .Values.prometheusRule.runbookBase | default "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" | quote }} + - alert: PathErrorRateHigh + expr: | + sum(rate(path_errors_total{class="5xx"}[5m])) + / + clamp_min(sum(rate(path_requests_total[5m])), 0.0001) + > 0.02 + for: 10m + labels: + severity: critical + annotations: + summary: "More than 2% of requests are failing with a 5xx" + description: >- + Fallback is exhausted or unavailable; usage rows carry error_code and + error_message for the failing calls. + runbook: {{ .Values.prometheusRule.runbookBase | default "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" | quote }} + - alert: PathProviderParked + expr: min_over_time(path_provider_circuit_open[15m]) == 1 + labels: + severity: critical + annotations: + summary: "Provider {{ `{{ $labels.provider_id }}` }} has had its circuit open for 15 minutes" + description: >- + The breaker has been open continuously, so every route on this provider is + being skipped. + runbook: {{ .Values.prometheusRule.runbookBase | default "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" | quote }} +{{- end }} diff --git a/deploy/helm/path/values.yaml b/deploy/helm/path/values.yaml index fe8de34..56ca048 100644 --- a/deploy/helm/path/values.yaml +++ b/deploy/helm/path/values.yaml @@ -60,8 +60,8 @@ postgres: persistence: size: 10Gi -# Optional in-chart Valkey (Redis protocol), needed for shared rate limiting across -# replicas and for a shared circuit breaker. +# Optional in-chart Valkey (Redis protocol). Enabling it also turns on the shared-state +# backend, so rate limits, circuit breaker state and in-flight caps are cluster-wide. valkey: enabled: false image: valkey/valkey:8-alpine @@ -115,3 +115,11 @@ serviceMonitor: enabled: false interval: 30s labels: {} + +# Prometheus Operator PrometheusRule with the recording and alerting rules from +# deploy/prometheus/path-rules.yml. Off by default: it needs the operator CRDs. +prometheusRule: + enabled: false + labels: {} + # Annotations point here; change it when you fork or mirror the repository. + runbookBase: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" diff --git a/deploy/postgres/partitions.sql b/deploy/postgres/partitions.sql new file mode 100644 index 0000000..64fb611 --- /dev/null +++ b/deploy/postgres/partitions.sql @@ -0,0 +1,100 @@ +-- Convert usage_records to monthly range partitions. +-- +-- This is an OPT-IN, ONE-TIME operation, not a Flyway migration: it rewrites the table, +-- so it must run on purpose, in a maintenance window, against the database Path uses. +-- Nothing in the application runs this file. See docs/OPERATIONS.md +-- ("Retention, archiving and partitioning") for the full procedure, including the CSV +-- archive that should be taken first. +-- +-- Requirements: PostgreSQL 13 or newer. Run as the schema owner with the gateway stopped. +-- +-- psql "$DB_URL" -v ON_ERROR_STOP=1 -f deploy/postgres/partitions.sql +-- +-- Afterwards the app keeps working unchanged: the columns, identity and indexes are the +-- same, so Hibernate's ddl-auto=validate still passes. + +begin; + +-- 1. Keep the old table under a name that makes its purpose obvious. +alter table usage_records rename to usage_records_unpartitioned; + +-- 2. Recreate it as a partitioned parent. Like the original, but the primary key has to +-- include the partition key (PostgreSQL requires it), and indexes are added after the +-- data copy instead of being copied by LIKE. +create table usage_records ( + like usage_records_unpartitioned including defaults including identity +) partition by range (created_at); + +-- 3. Monthly partitions, one per calendar month in [from_month, to_month]. Idempotent, so +-- the same function is what a monthly cron calls later. +create or replace function path_ensure_usage_partitions(from_month date, to_month date) +returns void +language plpgsql +as $$ +declare + month_start date := date_trunc('month', from_month)::date; + month_end date := date_trunc('month', to_month)::date; + part_end date; + part_name text; +begin + while month_start <= month_end loop + part_end := (month_start + interval '1 month')::date; + part_name := 'usage_records_' || to_char(month_start, 'YYYY_MM'); + if to_regclass(part_name) is null then + execute format( + 'create table %I partition of usage_records for values from (%L) to (%L)', + part_name, month_start, part_end); + end if; + month_start := part_end; + end loop; +end; +$$; + +-- 4. Cover the whole range the old table holds, so the copy never needs the default +-- partition (a partition cannot be added for a month whose rows already sit in it). +select path_ensure_usage_partitions( + coalesce((select date_trunc('month', min(created_at))::date from usage_records_unpartitioned), + date_trunc('month', now())::date), + greatest((select date_trunc('month', max(created_at))::date from usage_records_unpartitioned), + (date_trunc('month', now()) + interval '3 months')::date)); + +-- 5. A DEFAULT partition catches rows written outside the prepared months, so an insert +-- never fails. If a cron forgets to run, this is what absorbs the gap. +create table usage_records_default partition of usage_records default; + +-- 6. Move the rows. One statement on purpose: it is transactional and runs in the +-- maintenance window where nobody is writing. For a very large history, split this by +-- year into separate transactions, but keep the gateway stopped so nothing races. +insert into usage_records select * from usage_records_unpartitioned; + +-- 7. The identity sequence on the new table starts at 1 after a LIKE, which would collide +-- with the copied ids. Move it past the highest id. +select setval(pg_get_serial_sequence('usage_records', 'id'), + coalesce((select max(id) from usage_records), 0) + 1, false); + +-- 8. Same indexes the application expects. On a partitioned parent PostgreSQL creates one +-- per child automatically. +create index ix_usage_created on usage_records (created_at); +create index ix_usage_client on usage_records (client_key_id); +create index ix_usage_model on usage_records (public_model_id); +alter table usage_records add constraint usage_records_pkey primary key (id, created_at); + +drop table usage_records_unpartitioned; + +commit; + +-- Optional: schedule the monthly top-up with pg_cron (the extension is not required). +-- create extension if not exists pg_cron; +-- select cron.schedule('path-usage-partitions', '0 3 1 * *', $$ +-- select path_ensure_usage_partitions( +-- date_trunc('month', now())::date, +-- (date_trunc('month', now()) + interval '3 months')::date) +-- $$); +-- +-- Optional: drop a month older than the retention once it has been archived. Detaching is +-- cheap; dropping the table releases the space without a row-by-row delete. +-- alter table usage_records detach partition usage_records_2025_01; +-- drop table usage_records_2025_01; +-- +-- Retention (GATEWAY_USAGE_RETENTION) still deletes rows older than its window on a +-- schedule. Partitioning keeps that delete cheap and lets a whole month be detached instead. diff --git a/deploy/prometheus/path-rules.yml b/deploy/prometheus/path-rules.yml new file mode 100644 index 0000000..137f026 --- /dev/null +++ b/deploy/prometheus/path-rules.yml @@ -0,0 +1,100 @@ +# Alerting and recording rules for Path. +# +# Load with: +# promtool check rules deploy/prometheus/path-rules.yml +# kubectl apply -f deploy/prometheus/path-rules.yml +# +# Or let the Helm chart render them: --set prometheusRule.enabled=true (the chart +# expects the Prometheus Operator CRDs). +# +# Every metric used here comes from /actuator/prometheus. The runbook links point at +# the section of docs/OPERATIONS.md that explains what the alert means and what to do. + +groups: + - name: path-recording + interval: 30s + rules: + # Requests served from the response cache as a fraction of all requests. Recorded so + # the alert can compare today's ratio with its own 7-day average. + - record: path:cache_hit_ratio:rate30m + expr: | + sum(rate(path_cache_hits_total[30m])) + / + clamp_min(sum(rate(path_requests_total[30m])), 0.0001) + + - name: path-alerts + rules: + - alert: PathHighFallbackRate + expr: | + sum(rate(path_fallbacks_total[5m])) + / + clamp_min(sum(rate(path_requests_total[5m])), 0.0001) + > 0.20 + for: 10m + labels: + severity: warning + annotations: + summary: "More than 20% of requests are falling back to another route" + description: >- + A route is failing and every request is being retried on its fallback. Spend and + latency are rising even though requests still succeed. + runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" + + - alert: PathCacheHitRatioLow + expr: | + path:cache_hit_ratio:rate30m + < 0.5 * avg_over_time(path:cache_hit_ratio:rate30m[7d]) + for: 30m + labels: + severity: warning + annotations: + summary: "The response-cache hit ratio is below half its 7-day average" + description: >- + Cache hits collapsed against their own baseline. A changed prompt prefix, a + disabled cache_enabled route or a new client with temperature > 0 are the usual + causes, and all of them raise spend. + runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" + + - alert: PathSpendRateHigh + expr: | + sum(increase(path_cost_usd_total[1h])) * 24 + > + 2 * sum(increase(path_cost_usd_total[1d] offset 1d)) + for: 15m + labels: + severity: warning + annotations: + summary: "The current spend rate is on track to exceed twice yesterday" + description: >- + The last hour, extrapolated to a day, is more than double the whole of the + previous day. Check the top models and keys in /admin/stats before the invoice + does it for you. + runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" + + - alert: PathErrorRateHigh + expr: | + sum(rate(path_errors_total{class="5xx"}[5m])) + / + clamp_min(sum(rate(path_requests_total[5m])), 0.0001) + > 0.02 + for: 10m + labels: + severity: critical + annotations: + summary: "More than 2% of requests are failing with a 5xx" + description: >- + Fallback is exhausted or unavailable. The usage rows carry error_code and + error_message for the failing calls, and /actuator/health/providers shows which + providers are parked. + runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" + + - alert: PathProviderParked + expr: min_over_time(path_provider_circuit_open[15m]) == 1 + labels: + severity: critical + annotations: + summary: "Provider {{ $labels.provider_id }} has had its circuit open for 15 minutes" + description: >- + The breaker has been open continuously, so every route on this provider is being + skipped. Requests only succeed while a fallback exists. + runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts" diff --git a/docs/ADMIN-UI.md b/docs/ADMIN-UI.md new file mode 100644 index 0000000..d5816dd --- /dev/null +++ b/docs/ADMIN-UI.md @@ -0,0 +1,66 @@ +# Admin UI + +There is no UI in this repository: the gateway is a single Java process and stays that way. +The admin console lives in its own repository, [an1ndra/path-admin](https://github.com/an1ndra/path-admin) +(Vite + React, served as static files). + +## What it covers + +Read and write against this API: dashboard (spend, cache, latency, provider circuits, top +models, recent failures), providers (create/edit with a provider API key, probe, delete), +routes (create, enable/disable, delete), client keys (create with budgets and allowlists, +top up, revoke). + +## Auth + +The console logs in with an **admin token** (`X-Admin-Token`), kept in `sessionStorage`. +There are no user accounts, passwords or cookies. Mint per-person tokens so actions stay +attributable: + +```bash +curl -X POST http://localhost:8080/admin/tokens \ + -H "X-Admin-Token: $GATEWAY_ADMIN_TOKEN" -H 'Content-Type: application/json' \ + -d '{"name":"anindra-laptop","scopes":"read,write"}' +``` + +`scopes` is `read` or `read,write`. Revoke with `POST /admin/tokens/{id}/revoke`. +Every admin write is recorded in `GET /admin/audit`. + +`GET /admin/settings` reports the caller's own scopes so the console can adapt instead of +discovering a restriction through a failed write: + +```json +{"auth_mode": "token", "subject": "anindra-laptop", "scopes": ["read"], "can_write": false} +``` + +`subject` is the token name, or the SSO identity in proxy mode. The console hides (or +disables with a tooltip) every create, edit, delete and revoke control when `scopes` does +not contain `write`, and shows a "read-only token" badge. Neither value is a credential. + +## Single sign-on + +SSO, 2FA and passkeys are delegated to an authenticating proxy — Authentik, Keycloak, +oauth2-proxy or Cloudflare Access — which injects an identity header the gateway trusts. +See [`SSO.md`](SSO.md); the console detects that mode and skips its token screen. + +## Hosting it + +Two shapes, and the choice decides whether you need CORS: + +| Shape | Gateway setting | +|---|---| +| **Same hostname** — proxy `/admin-ui/*` to the UI and everything else to the gateway | nothing | +| **Separate hostname** — `admin.example.com` → UI, `gw.example.com` → gateway | `GATEWAY_CORS_ORIGINS=https://admin.example.com` | + +The console needs `X-Admin-Token` to pass through the preflight; the default CORS mapping +already allows `*` headers and the admin methods, and it must **not** enable credentials +because the token is a header, not a cookie. + +Do not expose `/admin/**` to the internet without something in front of it — Cloudflare +Access, a VPN, or an IP allowlist. The admin auth limiter (10 failures / 5 min) is a +backstop, not an access control. + +## Related + +- [`PUBLISHING.md`](PUBLISHING.md) — Maven Central +- [`OPERATIONS.md`](OPERATIONS.md) — admin endpoints, metrics, migrations diff --git a/docs/API.md b/docs/API.md index 1e51dee..9b38892 100644 --- a/docs/API.md +++ b/docs/API.md @@ -33,9 +33,7 @@ Every failure uses the OpenAI envelope: | Code | HTTP | Meaning | |---|---|---| -| `missing_api_key` | 401 | no `Authorization` header | -| `invalid_api_key` | 401 | unknown or malformed key | -| `key_disabled` | 401 | key revoked or disabled | +| `invalid_api_key` | 401 | no `Authorization` header, unknown key, or disabled key — deliberately one code, so the response cannot be used to test whether a key exists | | `invalid_admin_token` | 401 | bad `X-Admin-Token` | | `forbidden` | 403 | admin API disabled (no token configured) | | `model_not_found` | 404 | no enabled route for that model id | @@ -354,7 +352,10 @@ ids land at `0.00` and are flagged in `description`. | Method | Path | Purpose | |---|---|---| | GET | `/admin/stats?window=24h` | totals, percentiles, cache, providers/models | +| GET | `/admin/stats?window=24h&bucket=1h` | the same, plus a bucketed time series | | GET | `/admin/usage?window=24h&limit=100&keyId=` | raw usage rows | +| POST | `/admin/digest/test?date=2026-11-19` | send yesterday's summary now and return its text | +| GET | `/admin/usage/export?from=2026-01-01&to=2026-02-01` | CSV for an explicit date slice (archive before a purge) | ```json { @@ -363,7 +364,7 @@ ids land at `0.00` and are flagged in `description`. "ttft_ms": {"p50": 1, "p95": 5}, "tokens_per_second": {"p50": 7000.0}, "cache": {"prompt_tokens": 246, "cached_input_tokens": 84, - "hit_ratio": 0.3415, "savings_usd": 0.00002061}, + "hit_ratio": 0.3415, "savings_usd": 0.00002061, "semantic_hits": 3}, "fallback_requests": 3, "estimated_usage_requests": 0, "by_provider": {"openai": {"requests": 12, "cost_usd": 0.0002}}, @@ -371,8 +372,45 @@ ids land at `0.00` and are flagged in `description`. } ``` +`bucket` is optional and accepts `1m`, `5m`, `1h` or `1d`; anything else is a 400 that +names the accepted values. When present the response also carries: + +```json +{ + "bucket": "1h", + "series": [ + {"from": "2026-11-17T10:00:00Z", "requests": 412, "cost_usd": 0.71, + "failures": 3, "p95_latency_ms": 812} + ] +} +``` + +Every bucket in the window is present, zeroes included, and the window is clamped to at +most 500 points. The series is one query over `usage_records` for the whole window, +bucketed in Java (`AdminReportingService`), not one query per point; `p95_latency_ms` +covers successful calls only. Omitting `bucket` leaves the response exactly as before. + --- +### Gemini extensions + +On a route whose provider type is `GEMINI` or `VERTEX`, the chat body accepts two +Gemini-native fields in addition to the OpenAI shape: + +```json +{"model": "gemini-2.5-pro", + "messages": [{"role": "user", "content": "..."}], + "thinkingConfig": {"thinkingBudget": 2048, "includeThoughts": true}, + "safetySettings": [{"category": "HARM_CATEGORY_HATE_SPEECH", + "threshold": "BLOCK_ONLY_HIGH"}]} +``` + +`reasoning_effort` (`minimal`, `low`, `medium`, `high`) is accepted as a shortcut for a +thinking budget. Thinking tokens are returned in `usage.completion_tokens` (and broken out +as `completion_tokens_details.reasoning_tokens`) because Gemini bills them as output. A +blocked request returns 400 `content_blocked` with the safety category; a parameter the +adapter cannot honour returns 422 `unsupported_parameter`. + ## Metering fields Each usage row records: @@ -387,6 +425,7 @@ Each usage row records: | `input_cost_usd`, `cached_input_cost_usd`, `cache_write_cost_usd`, `output_cost_usd` | itemised | | `cost_usd` | total, 8 decimals | | `cached_savings_usd` | what cached tokens would have cost at the input rate | +| `cache_tier` | `exact` or `semantic` when a cache served the call, null otherwise; how `/admin/stats` counts semantic hits | | `ttft_ms` | time to first streamed token | | `tokens_per_second` | generation throughput (post-TTFT window) | | `upstream_request_id` | provider request id, for support | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 786a447..963d1d5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -25,6 +25,8 @@ Two wire formats cover every provider worth proxying: |---|---|---| | `OPENAI` | chat-completions | OpenAI, Groq, Google Gemini (compatible endpoint), Together, Fireworks, DeepSeek, vLLM, LM Studio, Ollama's `/v1`, any compatible endpoint | | `ANTHROPIC` | Messages API | Anthropic (native, with prompt caching and tool calls) | +| `GEMINI` | generateContent | Google AI Studio (native: thinking budget, safety settings, cached content) | +| `VERTEX` | generateContent | Vertex AI (same wire format, service-account OAuth2, project/location in the path) | Any other provider is a configuration change, not a code change: register it with `POST /admin/providers`, give it a base URL and a wire type, then add routes. A new wire diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 0c272e1..41075d4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -9,7 +9,8 @@ except the provider credentials you actually use. |---|---|---| | `PORT` | `8080` | HTTP port | | `GATEWAY_ADMIN_TOKEN` | empty (admin off) | admin shared secret | -| `GATEWAY_PUBLIC_MODELS` | `true` | allow keyless `/v1/models` | +| `GATEWAY_PUBLIC_MODELS` | `false` | allow keyless `/v1/models` (it names every model and alias) | +| `GATEWAY_CLIENT_IP_HEADER` | empty | header holding the real client address, e.g. `CF-Connecting-IP`; empty ignores `X-Forwarded-For` | | `GATEWAY_CORS_ORIGINS` | `*` | comma-separated origin patterns | | `GATEWAY_MAX_REQUEST_BYTES` | `1048576` | request bodies larger than this get 413 | | `GATEWAY_DRAIN_TIMEOUT` | `30s` | how long shutdown waits for in-flight streams | @@ -19,6 +20,7 @@ except the provider credentials you actually use. | Variable | Default | Meaning | |---|---|---| | `GATEWAY_BOOTSTRAP_DEMO_KEY` | `false` | create a demo key when the table is empty (ignored in `prod`) | +| `GATEWAY_SEED_DEFAULTS` | `true` | seed the example provider/route catalogue on boot; set `false` on a real deployment | | `GATEWAY_DEFAULT_RPM` | `120` | default requests/minute for new keys | | `GATEWAY_DEFAULT_CREDIT_USD` | unlimited | default credit limit for new keys | | `GATEWAY_DEFAULT_MAX_CONCURRENT` | `8` | in-flight requests per key (0 = unlimited) | @@ -50,14 +52,29 @@ last two with `request_timeout_ms` and `stream_idle_timeout_ms`. | `GATEWAY_ALERT_WEBHOOK` | empty | Slack-compatible webhook for alerts | | `GATEWAY_ALERT_COOLDOWN` | `1h` | minimum gap between alerts of the same event | | `GATEWAY_ALERT_BUDGET_THRESHOLD` | `0.8` | alert when a key has less than this fraction left | +| `GATEWAY_SHARED_STATE_BACKEND` | `memory` | `valkey` shares circuit-breaker and in-flight state across replicas | +| `GATEWAY_SHARED_STATE_PREFIX` | `path` | key prefix for that state, same convention as the rate limiter | +| `GATEWAY_CONCURRENCY_LEASE` | `300s` | how long a Valkey in-flight permit survives a dead pod; must exceed the slowest request | +| `GATEWAY_BREAKER_PROBE_LEASE` | `10s` | how long the single HALF_OPEN probe may hold its claim | +| `GATEWAY_ALERT_ERROR_RATE` | `0.25` | provider EWMA error rate above which `provider.error_rate_high` fires; `0` disables the check | +| `GATEWAY_ALERT_MIN_SAMPLES` | `20` | samples a provider needs before its error rate is trusted (a cold start is not an incident) | | `GATEWAY_SEMANTIC_MODEL` | blank (off) | public model used to embed prompts for the semantic cache | | `GATEWAY_SEMANTIC_MAX_ENTRIES` | `200` | candidates compared per key and model | +| `GATEWAY_ALERT_EMAIL_TO` (or the console) | blank (off) | comma-separated recipients for alert email | +| `GATEWAY_ALERT_EMAIL_FROM` | `path-gateway@localhost` | From address | +| `GATEWAY_DIGEST` | `true` | send one daily summary email (needs the alert email channel; off without it) | +| `GATEWAY_DIGEST_CRON` | `0 0 8 * * *` | when to send it, in the gateway's time zone | +| `SPRING_MAIL_HOST` / `_PORT` / `_USERNAME` / `_PASSWORD` | — | SMTP server; without `SPRING_MAIL_HOST` email alerting stays off | | `GATEWAY_GUARDRAILS_LOG_REDACTION` | `true` | redact PII before a prompt is written to the prompt log | | `GATEWAY_GUARDRAILS_REDACT` | `false` | redact the prompt the provider receives | | `GATEWAY_GUARDRAILS_MODERATION` | `false` | pre-flight moderation call (OpenAI-shaped providers) | | `GATEWAY_GUARDRAILS_MODERATION_MODEL` | `omni-moderation-latest` | model used for that call | -| `GATEWAY_ADMIN_AUTH_FAILURES` | `10` | failed admin auth attempts before 429s | +| `GATEWAY_ADMIN_AUTH_FAILURES` | `10` | failed admin auth attempts per address before 429s | +| `GATEWAY_ADMIN_AUTH_GLOBAL_FAILURES` | `0` (off) | failures across all addresses in the window; only for an origin that cannot be reached except through a trusted proxy | | `GATEWAY_ADMIN_AUTH_WINDOW` | `5m` | window for those failures | +| `GATEWAY_CLIENT_AUTH_FAILURES` | `60` | rejected API keys per address per window before 429s (stops the key-validity oracle being usable as a guessing channel) | +| `GATEWAY_CLIENT_AUTH_WINDOW` | `1m` | window for those rejections | +| `GATEWAY_CLIENT_AUTH_GLOBAL_FAILURES` | `0` (off) | rejections across all addresses in the window | | `GATEWAY_API_DOCS` | `true` (false in `prod`) | serve `/v3/api-docs` | | `GATEWAY_PROMPT_LOGGING` | `false` | store redacted request/response bodies | | `GATEWAY_PROMPT_LOG_RETENTION` | `7d` | how long prompt logs are kept | @@ -114,8 +131,31 @@ curl -s -X POST localhost:8080/admin/routes -H "$A" -H 'Content-Type: applicatio `POST /admin/models/sync` imports Gemini's list (image and embedding ids are filtered out) but leaves unknown prices at 0.00, so set them from Google's pricing page before -sending traffic. The native `generateContent` protocol is not used; the compatible layer -covers chat, streaming, tools, images and embeddings. +sending traffic. + +### Gemini native (AI Studio and Vertex) + +The `OPENAI`-type row above uses Google's compatible endpoint. A `GEMINI` or `VERTEX` +provider speaks native `generateContent` instead, which is what carries `thinkingConfig`, +`safetySettings` and cached-content accounting: + +```bash +# AI Studio: an API key, like any other provider +curl -s -X POST localhost:8080/admin/providers -H "$A" -H 'Content-Type: application/json' \ + -d '{"slug":"gemini-native","type":"GEMINI", + "base_url":"https://generativelanguage.googleapis.com/v1beta","api_key":"AIza..."}' + +# Vertex: a service-account JSON as the credential, plus project and location +curl -s -X POST localhost:8080/admin/providers -H "$A" -H 'Content-Type: application/json' \ + -d "{\"slug\":\"vertex\",\"type\":\"VERTEX\",\"project\":\"my-project\", + \"location\":\"us-central1\",\"api_key\":$(jq -Rs . < service-account.json)}" +``` + +`base_url` may be omitted for `VERTEX`: it defaults to +`https://-aiplatform.googleapis.com`. The service-account JSON is encrypted with +the same master key as every other credential and is never returned by the admin API. +Vertex has no model-listing endpoint, so `/admin/models/sync` skips it and its probe says +so; add its routes by hand. ## Database @@ -130,7 +170,7 @@ covers chat, streaming, tools, images and embeddings. export SPRING_PROFILES_ACTIVE=postgres export DB_URL=jdbc:postgresql://localhost:5432/aigateway export DB_USER=aigateway DB_PASSWORD=aigateway -./mvnw spring-boot:run +./mvnw -pl path-gateway spring-boot:run ``` or `docker compose up --build`. diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 6779ba8..984bdc2 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -78,7 +78,7 @@ - **Circuit breaker per provider.** Five consecutive failures park a provider for 30s; routes on a parked provider are skipped in microseconds. An upstream 429 with `Retry-After` parks it for exactly that long. State appears as `circuit_state` on - `/admin/providers`. + `/admin/providers`, and only one replica at a time may send the half-open probe. - **Timeouts are per provider.** `request_timeout_ms` and `stream_idle_timeout_ms` override the global defaults. - **Shutdown drains instead of truncating.** Readiness goes DOWN, new requests get 503, @@ -140,13 +140,34 @@ **Per-node by design** -- **Circuit state and the in-flight limiter live in the JVM.** Breaking load across - instances makes each breaker see only its own failures, and the concurrency cap is per - instance. The RPM limiter and budget reservations *are* shared (Valkey and the - database), so money and rate limits stay correct in a cluster. +- **Circuit state and the in-flight limiter default to the JVM.** Without + `GATEWAY_SHARED_STATE_BACKEND=valkey`, each breaker sees only its own failures and the + concurrency cap is per instance. With `valkey`, state is shared: parking a provider on + one replica parks it everywhere, exactly one replica claims the HALF_OPEN probe, an OPEN + entry expires even if the replica that opened it dies, and a killed pod's concurrency + permits expire after `GATEWAY_CONCURRENCY_LEASE` instead of blocking the key. The RPM + limiter and budget reservations are shared as before, so money and rate limits stay + correct in a cluster either way. Memory remains the default because the Valkey path is a + network call on the hot path. - **Fuzzing is time-boxed.** `ChatRequestFuzzTest` runs Jazzer for 10s per build; deeper fuzzing is a manual `jazzer` run rather than a long CI job. +**Gemini surfaces** + +| Surface | Thinking | Safety settings | Cached content | Tools | Images | +|---|---|---|---|---|---| +| `OPENAI` type, compatible endpoint | dropped | dropped | priced as plain input | yes | yes | +| `GEMINI` type, AI Studio native | `thinkingConfig` or `reasoning_effort` | passed through | `cachedContentTokenCount` priced | yes | no (422) | +| `VERTEX` type, native | same as AI Studio | same | same | yes | no (422) | + +A native Gemini route rejects an OpenAI parameter it cannot honour (`frequency_penalty`, +`presence_penalty`, `seed`, `logit_bias`, `logprobs`, `n > 1`, a non-JSON +`response_format`, `user`) with 422 `unsupported_parameter`, the same contract as +Anthropic. A prompt or candidate blocked by safety returns 400 `content_blocked` naming the +category. Thinking tokens are billed as output: `completion_tokens` includes +`thoughtsTokenCount`, and `reasoning_tokens` keeps the breakdown. Streamed tool-call deltas +are not reassembled; the native stream carries text. + **Provider-inherent** - **Embeddings are OpenAI-compatible only.** Anthropic publishes no embeddings endpoint, diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index a44b5b0..9c32522 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -97,12 +97,39 @@ digest is written to the workflow summary so it can be pinned. ## Observability +- **Email alerts.** Set `SPRING_MAIL_*` and `GATEWAY_ALERT_EMAIL_TO` and every alert also + arrives by email (`[Path] ` with the payload as `key: value` lines). Webhook and + email are independent channels: either, both, or neither. Email is fire-and-forget on a + virtual thread, and a mailbox failure only logs a warning. +- **Webhook delivery is retried.** A 429, a 5xx, a connect failure or a timeout is retried + up to three times with exponential backoff (250ms, 1s, 4s) and jitter, honouring + `Retry-After` when the endpoint sends one (capped at 30s so one endpoint cannot pin a + worker). A 4xx is never retried: it is a permanent rejection. + `path_alerts_sent_total{channel="webhook",outcome}` (tagged `delivered`, `retried`, + `dropped`) makes the failure rate visible. +- **A dead channel cannot grow without bound.** Both webhook and email run on two workers + with a 64-entry queue. When a queue is full the alert is dropped and counted as + `outcome="dropped"` on that channel instead of blocking the request that raised it. + Email is not retried: a real SMTP server has its own queue, and a retry here could + duplicate a message it already accepted. `channel="email"` and `channel="digest"` carry `delivered` and + `dropped` only. +- **Mail settings are editable from the console** (`GET/PUT /admin/mail-settings`): host, + port, username, password, from and recipients. They win over `SPRING_MAIL_*`, which stays + as the config-file route; `source` in the response says which is in force. The password is + AES-GCM encrypted, only ever written, and `-` clears it. `POST /admin/alerts/test` sends + the test through whatever is configured. +- **`POST /admin/alerts/test`** sends a test alert and reports which channels are + configured, so "do alerts work?" is answered before you need them. - **Provider health** at `/actuator/health/providers`: circuit state and retry-after per provider. Overall health stays UP while fallback exists; only a total outage returns OUT_OF_SERVICE. - **Metrics** at `/actuator/prometheus`: `path_requests_total` (model, provider, status, streamed), `path_latency_seconds`, `path_tokens_total` (prompt/completion/cached), - `path_cost_usd_total`, `path_cache_hits_total`. + `path_cost_usd_total`, `path_cache_hits_total` (model, tier exact/semantic), + `path_fallbacks_total`, + `path_provider_circuit_open` (provider_id; 1 while the breaker is open), + `path_errors_total` (provider, class 4xx/5xx/other, gateway error code) and + `path_alerts_sent_total` (channel, outcome). - **Traces** via Micrometer + OpenTelemetry. Off by default (`GATEWAY_TRACE_SAMPLE=0`); set a sampling probability and `OTEL_EXPORTER_OTLP_ENDPOINT` to export. - **Correlation** — every response carries `X-Request-Id`, echoed into the MDC and into @@ -112,6 +139,77 @@ digest is written to the workflow summary so it can be pinned. - **OpenAPI** — springdoc publishes `/v3/api-docs`; CI uploads the generated spec as a build artifact so drift is visible. +## Cluster state + +Circuit-breaker and in-flight counters default to memory, one node each. Set +`GATEWAY_SHARED_STATE_BACKEND=valkey` (same Redis connection as the rate limiter) to share +them: + +- parking a provider on one replica blocks it on the others within one probe interval +- exactly one replica sends the HALF_OPEN probe; the claim also expires after + `GATEWAY_BREAKER_PROBE_LEASE`, so a dead prober cannot wedge a provider +- an OPEN entry expires on its own after its retry-after, even if the opener is gone +- in-flight permits are leased (`GATEWAY_CONCURRENCY_LEASE`), so a killed pod's permits + age out instead of blocking a key forever + +`/actuator/health/providers` and `/admin/providers` read the same store, so with Valkey the +circuit state they show is the cluster's, not the answering pod's. + +## Retention, archiving and partitioning + +`GATEWAY_USAGE_RETENTION` (default `180d`, `0` keeps everything) drives a nightly purge at +03:30 that deletes rows older than the window. It runs whether or not you archive, so an +archive for finance has to be taken before the cutoff passes. + +**Archiving.** The CSV export takes an explicit slice, which is the only safe way to +capture the rows a purge is about to remove: + +```bash +# one month at a time, oldest first +curl -s 'localhost:8080/admin/usage/export?from=2026-01-01&to=2026-02-01&limit=50000' \ + -H "$A" -o usage-2026-01.csv +``` + +`from` and `to` accept an ISO-8601 instant or a plain date (UTC midnight); `window` still +works and ignores them when they are absent. The export is row-per-request, so raise +`limit` (max 50,000) or split the month if a slice overflows. + +**Partitioning (PostgreSQL, opt-in).** [`deploy/postgres/partitions.sql`](../deploy/postgres/partitions.sql) +converts `usage_records` to monthly range partitions: rename, recreate the parent with the +same columns and identity, create one partition per month from the oldest row through three +months ahead, move the rows, restore the indexes and a primary key that includes +`created_at`, and leave a DEFAULT partition for anything outside the prepared months. + +It is **not** a Flyway migration because it rewrites the table. Run it on purpose, with the +gateway stopped, after a `pg_dump`: + +```bash +pg_dump "$DB_URL" -Fc -f path-before-partitions.dump +psql "$DB_URL" -v ON_ERROR_STOP=1 -f deploy/postgres/partitions.sql +``` + +Afterwards, keep the calendar ahead of the data. Either schedule the provided function in +pg_cron: + +```sql +select cron.schedule('path-usage-partitions', '0 3 1 * *', + $$select path_ensure_usage_partitions( + date_trunc('month', now())::date, + (date_trunc('month', now()) + interval '3 months')::date)$$); +``` + +or call `select path_ensure_usage_partitions(...)` from your own job. Once a month has been +archived, detaching and dropping it is cheaper than the row-by-row retention delete: + +```sql +alter table usage_records detach partition usage_records_2025_01; +drop table usage_records_2025_01; +``` + +The application is unchanged by the conversion: columns, identity and indexes stay the +same, so Hibernate's `ddl-auto=validate` still passes. Partitioning is what keeps +`/admin/stats` and `/admin/usage` fast once the history is older than a few months. + ## Schema migrations Flyway owns the schema; Hibernate runs with `ddl-auto: validate` and fails fast when an @@ -146,9 +244,54 @@ Preview the notes locally with `python3 tools/changelog.py vX.Y.Z`. ## Alerts Set `GATEWAY_ALERT_WEBHOOK` to a Slack-compatible URL to receive -`provider.circuit_open`, `provider.rate_limited` and `key.budget_low`, at most once per -event per cooldown. Payloads are JSON with `source`, `event`, `at` and event-specific -fields. +`provider.circuit_open`, `provider.rate_limited`, `key.budget_low`, `project.budget_low`, +`project.budget_exhausted`, `provider.error_rate_high`, `provider.error_rate_normal` and +`alert.test`, at most once per event per cooldown. Payloads are JSON with `source`, +`event`, `at` and event-specific fields. + +Webhook sends are retried (429/5xx/connect/timeout, three attempts, exponential backoff) +and both channels are bounded by a small queue; see "Observability" for the exact +behaviour and the `path_alerts_sent_total` counter. Losing an alert is therefore visible, +not silent. Email is not retried, deliberately. + +**Provider error rate.** The circuit breaker reacts to consecutive failures, so a provider +failing 30% of calls with an occasional success is never parked and requests are quietly +retried onto a fallback. A check runs every minute over the routing EWMA and fires +`provider.error_rate_high` when the count-weighted rate per provider exceeds +`GATEWAY_ALERT_ERROR_RATE` (default 0.25) with at least `GATEWAY_ALERT_MIN_SAMPLES` +(default 20) samples recorded. The payload carries `provider`, `error_rate`, `samples`, +`threshold` and the EWMA `latency_ms`. One alert per provider per hour; when the rate falls +below half the threshold, `provider.error_rate_normal` closes the incident. A provider with +too few samples is ignored: a cold start is not an incident. + +**Daily digest.** One email a day (`GATEWAY_DIGEST_CRON`, default 08:00 in the gateway's +time zone) summarises yesterday: requests with the day-over-day delta, spend, tokens with +the cache hit ratio and savings, failures with the top error code, the slowest model's p95, +the top three models by spend, and any provider whose circuit is open at send time. It +needs the same mail settings as alert email and is silently off without them; a day with no +requests sends nothing, and a mailbox failure only logs a warning. Disable it with +`GATEWAY_DIGEST=false`. `POST /admin/digest/test?date=2026-11-19` sends it on demand and +returns the rendered text (and the reason when it did not send), so the schedule can be +trusted before the first morning. + +### Prometheus rules + +[`deploy/prometheus/path-rules.yml`](../deploy/prometheus/path-rules.yml) ships the +recording and alerting rules; `helm template ... --set prometheusRule.enabled=true` +renders the same rules as a `PrometheusRule` for the Prometheus Operator. CI validates +them with `promtool check rules`. + +| Alert | Fires when | What to do | +|---|---|---| +| `PathHighFallbackRate` | fallback requests are over 20% of traffic for 10m | one route is failing; check `path_requests_total{status="error"}` and the route's probe | +| `PathCacheHitRatioLow` | the 30m cache hit ratio is under half its 7-day average for 30m | a prompt prefix changed, caching was disabled on a route, or a client uses `temperature > 0` | +| `PathSpendRateHigh` | the last hour, extrapolated, is above 2× the whole previous day | check top models and keys at `/admin/stats` | +| `PathErrorRateHigh` | more than 2% of requests fail with a 5xx for 10m | fallback is exhausted; read `error_code` in `/admin/usage` | +| `PathProviderParked` | `path_provider_circuit_open` was 1 for the whole last 15m | a provider has been unreachable since then; every route on it is skipped | + +Every rule has a `for` duration so a single bad minute does not page. The cache rule uses +the recorded `path:cache_hit_ratio:rate30m`, because comparing a ratio with its own average +needs a series to average. ## Audit diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index d43f54f..ba95af6 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -73,7 +73,7 @@ gpg --armor --export-secret-keys > path-signing-key.asc ```bash # 1. Confirm what will be uploaded (nothing leaves the machine). ./mvnw -P release -DskipTests verify -ls target/path-*.jar target/*.asc +ls path-gateway/target/path-*.jar path-gateway/target/*.asc path-spring-ai/target/path-spring-ai-*.jar # 2. Upload to the Portal. autoPublish=false, so you review before release. MAVEN_GPG_PASSPHRASE='...' ./mvnw -P release -DskipTests deploy @@ -93,7 +93,7 @@ path-0.1.0.pom The runnable fat jar is **not** published; the GitHub release and container image carry it. `-P release` skips `spring-boot:repackage` for that reason — which also means -`target/path-0.1.0.jar` is the **plain library** after a release build and `java -jar` on +`path-gateway/target/path-0.2.2.jar` is the **plain library** after a release build and `java -jar` on it fails with "no main manifest attribute". Run a normal `./mvnw package` to get the runnable jar back. @@ -108,9 +108,27 @@ Add three repository secrets and one variable: | secret | `CENTRAL_USERNAME` / `CENTRAL_PASSWORD` | the Portal token | | variable | `PUBLISH_TO_CENTRAL` | `true` to enable the step | +The build is a reactor: `io.github.an1ndra:path-parent` (pom), `path` (the gateway) and +`path-spring-ai` (the adapter). A release publishes all three, because a consumer resolving +the adapter needs its parent POM. + The release workflow then publishes on each `v*` tag. With the variable unset the step is skipped, so a fork never tries to publish. +## Container image order + +The image is gated before it is named. The workflow: + +1. builds and pushes **only** `ghcr.io/an1ndra/path:sha-`; +2. runs Trivy against that tag; +3. promotes the exact digest with + `docker buildx imagetools create -t ghcr.io/an1ndra/path: -t ghcr.io/an1ndra/path:latest ghcr.io/an1ndra/path:sha-`; +4. writes the promoted digest to the workflow summary. + +A failing scan therefore burns no version number and leaves no version tag a deployment +could pull; the `sha-` tag stays for traceability. See issue #80 for the two releases that +were lost to the old order. + ## Consumers ```xml diff --git a/docs/SPRING-AI.md b/docs/SPRING-AI.md new file mode 100644 index 0000000..5006a43 --- /dev/null +++ b/docs/SPRING-AI.md @@ -0,0 +1,114 @@ +# Use from Spring AI + +`path-spring-ai` is a drop-in Spring AI `ChatModel` and `EmbeddingModel` over the gateway's +OpenAI-compatible surface. A Java application that already uses Spring AI changes one bean +and gains Path's routing, fallback, budgets and cache metering without touching its prompts. + +The module is a sibling of the gateway in this repository and is the only one that depends +on Spring AI; the gateway never depends on it. + +## Dependency + +```xml + + io.github.an1ndra + path-spring-ai + 0.2.3 + +``` + +It targets Spring AI 1.1.x (the line for Spring Boot 3.5). Spring AI 2.x needs Spring Boot 4 +and is not supported yet. + +## Chat + +```java +@Bean +ChatModel chatModel() { + return PathChatModel.builder() + .baseUrl("http://path:8080") + .apiKey(System.getenv("PATH_KEY")) + .model("gpt-4o-mini") // used when the Prompt names none + .idempotency(true) // send Idempotency-Key so a retry is not charged twice + .build(); +} +``` + +`ChatClient` works on top of it unchanged: + +```java +String answer = ChatClient.builder(chatModel) + .build() + .prompt() + .user("Summarise this incident in one line") + .call() + .content(); +``` + +### What is mapped + +| Spring AI | Gateway request | +|---|---| +| `SystemMessage`, `UserMessage`, `AssistantMessage` | `messages[]` with the matching role | +| `AssistantMessage.getToolCalls()` | `tool_calls[]` on the assistant turn | +| `ToolResponseMessage` | one `role: "tool"` message per response | +| `ToolCallingChatOptions.getToolCallbacks()` | `tools[].function` (`name`, `description`, `parameters`) | +| `ChatOptions` temperature, maxTokens, topP, penalties, stop | the matching OpenAI fields | + +### What comes back + +`ChatResponseMetadata` carries `id`, `model` and `Usage`, plus three Path-specific values +copied from the gateway's response headers: + +| Metadata key | Header | Meaning | +|---|---|---| +| `path_cache` | `X-Path-Cache` | `hit`, `miss`, `semantic` or `off` | +| `path_provider` | `X-Gateway-Provider` | the provider slug that served the call | +| `path_cost_usd` | `X-Gateway-Cost-Usd` | what the call cost | + +`Generation.getMetadata().getFinishReason()` is the upstream finish reason. A gateway error +is thrown as `PathException`, which exposes `status()` and the gateway's `code()` (for example +`unsupported_parameter` with a 422). + +## Streaming + +`stream(Prompt)` POSTs with `stream: true` and forwards each SSE content delta as a +`ChatResponse` chunk; `ChatClient` aggregates them into the final message. Streamed +tool-call deltas are **not** reassembled: a streaming request that returns tool calls will +show the tool call in the non-streaming `call()` path only. Use `call()` when a conversation +uses tools. + +## Embeddings + +```java +@Bean +EmbeddingModel embeddingModel() { + return PathEmbeddingModel.builder() + .baseUrl("http://path:8080") + .apiKey(System.getenv("PATH_KEY")) + .model("text-embedding-3-small") + .dimensions(1536) + .build(); +} +``` + +`/v1/embeddings` is served for OpenAI-shaped providers only; anything else fails with the +gateway's own error. `dimensions()` returns the configured value (the gateway does not +report one). + +## Building and testing + +```bash +./mvnw -pl path-spring-ai -am package # the adapter and its parent +./mvnw verify # both modules +``` + +The tests run against a stub HTTP server and need no gateway and no provider key. + +## Caveats + +- Images and other non-text user content are not translated; only text is forwarded. +- There is no automatic retry in the adapter. `idempotency(true)` makes a caller-side retry + safe; the gateway's own routing/fallback already handles provider failures. +- A `ChatOptions` model id is a Path public model id (a route or an alias such as `cheap`), + not necessarily an upstream model name. diff --git a/docs/SSO.md b/docs/SSO.md new file mode 100644 index 0000000..83a440c --- /dev/null +++ b/docs/SSO.md @@ -0,0 +1,100 @@ +# SSO, 2FA and passkeys + +Path does not implement an identity provider, and it should not. Google/GitHub/enterprise +SSO, TOTP 2FA and passkeys are all **delegated to a reverse proxy that authenticates the +human** and then tells the gateway who it is: + +``` +browser → IdP (Authentik, Keycloak, oauth2-proxy, Cloudflare Access) + → proxy injects X-Forwarded-User: anindra@example.com + → gateway trusts that header for /admin/** +``` + +That gives you passkeys and 2FA in an afternoon, from software that already does them +properly, instead of a home-grown WebAuthn/TOTP implementation in an LLM gateway. + +> **The one rule:** `GATEWAY_PROXY_AUTH=true` means `/admin/**` accepts anything behind the +> proxy without a token. If the gateway is also reachable directly, anyone can send the +> header themselves and own the gateway. Bind it to loopback or a private network, or make +> the proxy the only ingress. + +## Gateway configuration + +```yaml +GATEWAY_PROXY_AUTH: "true" +GATEWAY_PROXY_AUTH_HEADER: X-Forwarded-User # or Cf-Access-Authenticated-User-Email +GATEWAY_PROXY_AUTH_SCOPES: read,write # use "read" for a viewer group +``` + +On startup the log says, once: + +``` +WARN Admin auth is DELEGATED to a reverse proxy (X-Forwarded-User): the gateway trusts + that header for /admin/**. It must not be reachable except through the proxy. +``` + +`GET /admin/settings` reports `"auth_mode": "proxy"`, and the admin console uses that to +skip its token screen — a signed-in browser lands straight on the dashboard. + +Every admin write is audited against the identity the proxy supplied, so +`GET /admin/audit` answers "who changed this" with an email, not "shared-admin". + +## Authentik (recommended, self-hosted with Dokploy) + +1. Authentik → **Providers → Create → Proxy**, mode *Forward auth (single application)*. +2. **Applications → Create**, bind the provider, attach a policy (your user or group). +3. In Dokploy, add a Traefik middleware on the UI *and* the gateway: + +```yaml +# Traefik labels on the gateway service +- "traefik.http.middlewares.path-auth.forwardauth.address=http://authentik:9000/outpost.goauthentik.io/auth/traefik" +- "traefik.http.middlewares.path-auth.forwardauth.authResponseHeaders=X-Forwarded-User" +- "traefik.http.middlewares.path-auth.forwardauth.trustForwardHeader=true" +- "traefik.http.routers.path.middlewares=path-auth" +``` + +Authentik's flow can require **passkeys/WebAuthn** and **TOTP**, set per stage. Nothing in +Path changes. + +## Keycloak / any OIDC provider + +Same shape with `oauth2-proxy` in forward-auth mode: + +```bash +oauth2-proxy --provider=oidc \ + --oidc-issuer-url=https://keycloak.example.com/realms/main \ + --email-domain=example.com \ + --set-xauthrequest --pass-user-headers \ + --upstream=http://path:8080/ +``` + +`--pass-user-headers` sets `X-Forwarded-User`; Keycloak's authentication flows supply 2FA +and WebAuthn. + +## Cloudflare Access + +If the gateway is behind Cloudflare Tunnel, Access can authenticate with Google/GitHub/ +SAML and inject: + +``` +Cf-Access-Authenticated-User-Email: anindra@example.com +``` + +Set `GATEWAY_PROXY_AUTH_HEADER=Cf-Access-Authenticated-User-Email`. Access supports +one-time PIN and SSO; passkeys come from the identity provider behind it. + +**Do not** stop here: a request that reaches the origin without Cloudflare can spoof that +header. Cloudflare Tunnel with `cloudflared` (no public origin) is what makes it safe, and +optionally verify the `Cf-Access-Jwt-Assertion` signature at the gateway. + +## What the console does + +| Auth mode | Login screen | Sign out | +|---|---|---| +| `token` (default) | paste an admin token | clears it from sessionStorage | +| `proxy` | skipped — the proxy already authenticated the browser | handled by the IdP / proxy | + +## What is still token-only + +`/v1/**` (client keys) and Maven Central publishing. SSO covers the human-facing admin +surface, not machine credentials. diff --git a/docs/TESTING.md b/docs/TESTING.md index 5a20ddf..2ddf15e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -30,7 +30,7 @@ admin surface and metering. ```bash python3 tools/mock-upstream.py 9099 & -GATEWAY_ADMIN_TOKEN=test-admin java -jar target/path-0.1.0.jar & +GATEWAY_ADMIN_TOKEN=test-admin java -jar path-gateway/target/path-0.2.2.jar & python3 tools/smoke_test.py ``` diff --git a/path-gateway/pom.xml b/path-gateway/pom.xml new file mode 100644 index 0000000..fe8c2a8 --- /dev/null +++ b/path-gateway/pom.xml @@ -0,0 +1,136 @@ + + + 4.0.0 + + + io.github.an1ndra + path-parent + 0.2.3 + ../pom.xml + + + path + path + Provider-agnostic LLM gateway: routing, fallback, credits and cache-aware token metering for OpenAI, Anthropic and any OpenAI-compatible API + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-data-redis + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-starter-mail + + + + org.springframework.boot + spring-boot-starter-actuator + + + + org.projectlombok + lombok + true + + + + com.knuddels + jtokkit + 1.1.0 + + + + org.flywaydb + flyway-core + + + + io.micrometer + micrometer-registry-prometheus + + + io.micrometer + micrometer-tracing-bridge-otel + + + io.opentelemetry + opentelemetry-exporter-otlp + + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.8.17 + + + org.flywaydb + flyway-database-postgresql + runtime + + + + com.h2database + h2 + runtime + + + org.postgresql + postgresql + runtime + + + + org.springframework.boot + spring-boot-starter-test + test + + + com.code-intelligence + jazzer-junit + 0.30.0 + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.cyclonedx + cyclonedx-maven-plugin + 2.9.3 + + + package + + makeAggregateBom + + + + + json + bom + false + + + + + diff --git a/src/main/java/com/anindra/path/AiGatewayApplication.java b/path-gateway/src/main/java/com/anindra/path/AiGatewayApplication.java similarity index 100% rename from src/main/java/com/anindra/path/AiGatewayApplication.java rename to path-gateway/src/main/java/com/anindra/path/AiGatewayApplication.java diff --git a/src/main/java/com/anindra/path/admin/AdminController.java b/path-gateway/src/main/java/com/anindra/path/admin/AdminController.java similarity index 68% rename from src/main/java/com/anindra/path/admin/AdminController.java rename to path-gateway/src/main/java/com/anindra/path/admin/AdminController.java index ad40107..61197fc 100644 --- a/src/main/java/com/anindra/path/admin/AdminController.java +++ b/path-gateway/src/main/java/com/anindra/path/admin/AdminController.java @@ -22,8 +22,10 @@ public class AdminController { private final PromptLogService promptLogs; private final TenancyService tenancy; private final AliasService aliasService; + private final AlertService alerts; + private final MailSettingsService mailSettingsService; + private final DailyDigestService digest; private final com.anindra.path.config.GatewayProperties props; - private final com.anindra.path.gateway.SemanticCacheService semantic; // ------------------------------------------------------------- providers @@ -124,14 +126,94 @@ public Map rotateMasterKey(@RequestBody Map body return result; } + public record MailSettingsInput(String host, Integer port, String username, String password, + String from, String to, Boolean enabled) { + } + + @GetMapping("/mail-settings") + public Map mailSettings() { + return mailSettingsService.view(); + } + + @PutMapping("/mail-settings") + public Map updateMailSettings(@RequestBody MailSettingsInput input) { + var updated = mailSettingsService.save(input.host(), input.port(), input.username(), + input.password(), input.from(), input.to(), input.enabled()); + // The password is write-only, so the audit payload carries the other fields only. + audit.record("mail-settings.update", "mail-settings", java.util.Map.of( + "host", String.valueOf(updated.get("host")), + "to", String.valueOf(updated.get("to")), + "enabled", String.valueOf(updated.get("configured")))); + return updated; + } + + @PostMapping("/alerts/test") + public Map testAlert() { + Map payload = new java.util.LinkedHashMap<>(); + payload.put("message", "This is a test alert from Path; if you can read it, alerting works."); + payload.put("severity", "info"); + alerts.notify("alert.test", payload); + Map out = new java.util.LinkedHashMap<>(); + out.put("sent", true); + out.put("email", alerts.emailEnabled()); + out.put("webhook", alerts.webhookEnabled()); + if (!alerts.emailEnabled() && !alerts.webhookEnabled()) { + out.put("note", "Neither channel is configured: set GATEWAY_ALERT_EMAIL_TO (+ spring.mail.*) " + + "or GATEWAY_ALERT_WEBHOOK."); + } + audit.record("alert.test", "alert", null); + return out; + } + + /** + * Sends yesterday's digest now, and returns the rendered text either way, so an + * operator can check the mailbox and the numbers before trusting the 08:00 job. + */ + @PostMapping("/digest/test") + public Map digestTest(@RequestParam(required = false) String date) { + java.time.LocalDate day; + try { + day = date == null || date.isBlank() + ? java.time.LocalDate.now().minusDays(1) + : java.time.LocalDate.parse(date); + } catch (java.time.format.DateTimeParseException e) { + throw com.anindra.path.api.ApiException.badRequest( + "date must be an ISO date such as 2026-11-19", "date"); + } + DailyDigestService.Result result = digest.send(day); + Map out = new java.util.LinkedHashMap<>(); + out.put("date", day.toString()); + out.put("sent", result.sent()); + if (result.reason() != null) { + out.put("reason", result.reason()); + } + out.put("subject", result.subject()); + out.put("text", result.text()); + audit.record("digest.test", day.toString(), null); + return out; + } + @GetMapping("/settings") - public Map settings() { + public Map settings(jakarta.servlet.http.HttpServletRequest request) { Map guardrails = new java.util.LinkedHashMap<>(); guardrails.put("prompt_log_redaction", props.getGuardrails().isPromptLogRedaction()); guardrails.put("redact_before_send", props.getGuardrails().isRedactBeforeSend()); guardrails.put("moderation", props.getGuardrails().isModeration()); + // The principal is set by AuthFilter for every /admin call. Reporting the scopes lets + // the console render read-only instead of letting the operator discover it via a 403. + // A token name and its scopes are not credentials, so this is safe to expose. + TenancyService.Principal principal = request.getAttribute( + com.anindra.path.config.AuthFilter.ADMIN_PRINCIPAL_ATTR) instanceof TenancyService.Principal p + ? p : null; + List scopes = principal == null ? List.of() + : principal.scopes().stream().sorted().toList(); + Map out = new java.util.LinkedHashMap<>(); + out.put("auth_mode", props.getProxyAuth().isEnabled() ? "proxy" : "token"); + out.put("subject", principal == null ? null : principal.name()); + out.put("scopes", scopes); + out.put("can_write", scopes.contains("write")); out.put("prompt_logging", props.getPromptLogging().isEnabled()); out.put("guardrails", guardrails); out.put("rate_limit_backend", props.getRateLimit().getBackend()); @@ -246,15 +328,9 @@ public Map revokeToken(@PathVariable Long id) { @GetMapping("/stats") public Map stats(@RequestParam(defaultValue = "24h") String window, - @RequestParam(required = false) Long projectId) { - Map out = reporting.stats(window, projectId); - Object cache = out.get("cache"); - if (cache instanceof Map existing) { - @SuppressWarnings("unchecked") - Map typed = (Map) existing; - typed.put("semantic_hits", semantic.hitCount()); - } - return out; + @RequestParam(required = false) Long projectId, + @RequestParam(required = false) String bucket) { + return reporting.stats(window, projectId, bucket); } @GetMapping("/usage") @@ -282,18 +358,43 @@ public org.springframework.http.ResponseEntity exportUsage( @RequestParam(required = false) Long projectId, @RequestParam(required = false) String endUser, @RequestParam(defaultValue = "30d") String window, + @RequestParam(required = false) String from, + @RequestParam(required = false) String to, @RequestParam(defaultValue = "10000") int limit) { java.time.Duration parsed = com.anindra.path.api.Durations.cap( com.anindra.path.api.Durations.parse(window, java.time.Duration.ofDays(30)), java.time.Duration.ofDays(365)); - java.time.Instant to = java.time.Instant.now(); - java.time.Instant from = to.minus(parsed); - String csv = reporting.usageCsv(keyId, projectId, endUser, from, to, limit); + // from/to make the slice explicit, which is what an archive before a retention purge + // needs; window alone always ends at "now". + java.time.Instant toInstant = to == null || to.isBlank() + ? java.time.Instant.now() : instant(to, "to"); + java.time.Instant fromInstant = from == null || from.isBlank() + ? toInstant.minus(parsed) : instant(from, "from"); + if (fromInstant.isAfter(toInstant)) { + throw com.anindra.path.api.ApiException.badRequest( + "'from' must be before 'to'.", "from"); + } + String csv = reporting.usageCsv(keyId, projectId, endUser, fromInstant, toInstant, limit); return org.springframework.http.ResponseEntity.ok() - .header("Content-Disposition", "attachment; filename=\"usage-" + to + ".csv\"") + .header("Content-Disposition", "attachment; filename=\"usage-" + toInstant + ".csv\"") .body(csv); } + /** ISO-8601 instant, or a plain date taken as UTC midnight. */ + private static java.time.Instant instant(String value, String param) { + try { + return java.time.Instant.parse(value.trim()); + } catch (java.time.format.DateTimeParseException ignored) { + try { + return java.time.LocalDate.parse(value.trim()) + .atStartOfDay(java.time.ZoneOffset.UTC).toInstant(); + } catch (java.time.format.DateTimeParseException e) { + throw com.anindra.path.api.ApiException.badRequest( + "'" + param + "' must be an ISO-8601 instant or date, e.g. 2026-11-19.", param); + } + } + } + @GetMapping("/audit") public List> auditLog(@RequestParam(required = false) String action, @RequestParam(defaultValue = "100") int limit) { diff --git a/src/main/java/com/anindra/path/admin/AdminReportingService.java b/path-gateway/src/main/java/com/anindra/path/admin/AdminReportingService.java similarity index 73% rename from src/main/java/com/anindra/path/admin/AdminReportingService.java rename to path-gateway/src/main/java/com/anindra/path/admin/AdminReportingService.java index 6a3ff35..006e711 100644 --- a/src/main/java/com/anindra/path/admin/AdminReportingService.java +++ b/path-gateway/src/main/java/com/anindra/path/admin/AdminReportingService.java @@ -65,6 +65,7 @@ public Map usage(Long keyId, Long projectId, String endUser, Str row.put("total_tokens", r.getTotalTokens()); row.put("cached_input_tokens", r.getCachedInputTokens()); row.put("cache_write_tokens", r.getCacheWriteTokens()); + row.put("cache_tier", r.getCacheTier()); row.put("reasoning_tokens", r.getReasoningTokens()); row.put("cost_usd", r.getCostUsd()); row.put("cost_breakdown", Map.of( @@ -165,11 +166,115 @@ private static String escape(String value) { } public Map stats(String window, Long projectId) { + return stats(window, projectId, null); + } + + /** + * Aggregate stats, plus an optional bucketed series when {@code bucket} is given. + * Buckets are filled with zeroes so a chart never has holes, and the window is clamped + * to at most {@link #MAX_POINTS} buckets. + */ + public Map stats(String window, Long projectId, String bucket) { Duration parsed = Durations.cap(Durations.parse(window, Duration.ofHours(24)), Duration.ofDays(365)); + if (bucket != null && !bucket.isBlank() && !BUCKETS.containsKey(bucket)) { + throw com.anindra.path.api.ApiException.badRequest( + "bucket must be one of " + ACCEPTED_BUCKETS, "bucket"); + } + Map out = projectId == null + ? adminService.stats(parsed) + : projectStats(projectId, parsed); + if (bucket != null && !bucket.isBlank()) { + out.put("bucket", bucket); + out.put("series", series(projectId, parsed, bucket)); + } + return out; + } + + /** Accepted bucket sizes; the value is the width in seconds. */ + static final Map BUCKETS = Map.of( + "1m", 60L, + "5m", 300L, + "1h", 3_600L, + "1d", 86_400L); + private static final String ACCEPTED_BUCKETS = "1m, 5m, 1h, 1d"; + + /** A window never renders more than this many points. */ + static final int MAX_POINTS = 500; + + /** + * One query for the whole window, bucketed in Java. The alternative — one query per + * bucket — multiplies database round-trips by the number of points, and PostgreSQL-only + * date functions would break the H2 test suite. + */ + private List> series(Long projectId, Duration window, String bucket) { + Long width = BUCKETS.get(bucket); + if (width == null) { + throw com.anindra.path.api.ApiException.badRequest( + "bucket must be one of " + ACCEPTED_BUCKETS, "bucket"); + } + + long nowSeconds = Instant.now().getEpochSecond(); + // Align to the bucket grid so the newest point is the bucket containing "now". + long end = (nowSeconds / width + 1) * width; + int points = (int) Math.min(MAX_POINTS, Math.max(1, ceilDiv(window.getSeconds(), width))); + long start = end - (long) points * width; + + List rows; if (projectId == null) { - return adminService.stats(parsed); + rows = usage.findSeriesSource(Instant.ofEpochSecond(start), Instant.ofEpochSecond(end)); + } else { + List keyIds = projectKeyIds(projectId); + rows = keyIds.isEmpty() ? List.of() + : usage.findSeriesSourceForKeys(keyIds, + Instant.ofEpochSecond(start), Instant.ofEpochSecond(end)); + } + + long[] requests = new long[points]; + long[] failures = new long[points]; + java.math.BigDecimal[] cost = new java.math.BigDecimal[points]; + List> latencies = new ArrayList<>(points); + for (int i = 0; i < points; i++) { + cost[i] = java.math.BigDecimal.ZERO; + latencies.add(new ArrayList<>()); + } + + for (Object[] row : rows) { + Instant createdAt = (Instant) row[0]; + UsageRecord.Status status = (UsageRecord.Status) row[1]; + java.math.BigDecimal rowCost = (java.math.BigDecimal) row[2]; + Long latency = (Long) row[3]; + int index = (int) ((createdAt.getEpochSecond() - start) / width); + if (index < 0 || index >= points) { + continue; + } + requests[index]++; + if (status == UsageRecord.Status.ERROR) { + failures[index]++; + } else if (latency != null) { + latencies.get(index).add(latency); + } + if (rowCost != null) { + cost[index] = cost[index].add(rowCost); + } + } + + List> out = new ArrayList<>(points); + for (int i = 0; i < points; i++) { + List bucketLatencies = latencies.get(i); + bucketLatencies.sort(null); + Map point = new LinkedHashMap<>(); + point.put("from", Instant.ofEpochSecond(start + (long) i * width).toString()); + point.put("requests", requests[i]); + point.put("cost_usd", cost[i]); + point.put("failures", failures[i]); + point.put("p95_latency_ms", percentile(bucketLatencies, 0.95)); + out.add(point); } - return projectStats(projectId, parsed); + return out; + } + + private static long ceilDiv(long value, long divisor) { + return (value + divisor - 1) / divisor; } /** Aggregates in Java from the project's rows; bounded so a huge project cannot OOM. */ @@ -215,6 +320,8 @@ private Map projectStats(Long projectId, Duration window) { .divide(java.math.BigDecimal.valueOf(promptTokens), 4, java.math.RoundingMode.HALF_UP).doubleValue()); cache.put("savings_usd", savings); + cache.put("semantic_hits", rows.stream() + .filter(r -> "semantic".equals(r.getCacheTier())).count()); out.put("cache", cache); out.put("fallback_requests", rows.stream().filter(r -> r.getAttempts() > 1).count()); out.put("estimated_usage_requests", rows.stream().filter(UsageRecord::isUsageEstimated).count()); diff --git a/src/main/java/com/anindra/path/admin/AdminService.java b/path-gateway/src/main/java/com/anindra/path/admin/AdminService.java similarity index 93% rename from src/main/java/com/anindra/path/admin/AdminService.java rename to path-gateway/src/main/java/com/anindra/path/admin/AdminService.java index 8405e96..248e025 100644 --- a/src/main/java/com/anindra/path/admin/AdminService.java +++ b/path-gateway/src/main/java/com/anindra/path/admin/AdminService.java @@ -39,6 +39,8 @@ public class AdminService { public record ProviderInput(String slug, @JsonAlias("displayName") String displayName, String type, @JsonAlias("baseUrl") String baseUrl, @JsonAlias("apiKey") String apiKey, @JsonAlias("apiVersion") String apiVersion, + @JsonAlias("project") String project, + @JsonAlias("location") String location, @JsonAlias("requestTimeoutMs") Integer requestTimeoutMs, @JsonAlias("streamIdleTimeoutMs") Integer streamIdleTimeoutMs, Boolean enabled) { @@ -110,7 +112,9 @@ public Map createProvider(ProviderInput in) { p.setDisplayName(in.displayName() == null || in.displayName().isBlank() ? in.slug().trim() : in.displayName().trim()); p.setType(parseType(in.type())); - p.setBaseUrl(requireBaseUrl(in.baseUrl())); + p.setProject(blankToNull(in.project())); + p.setLocation(blankToNull(in.location())); + p.setBaseUrl(vertexBaseUrl(p, in.baseUrl())); p.setApiVersion(in.apiVersion() == null || in.apiVersion().isBlank() ? null : in.apiVersion().trim()); p.setEnabled(in.enabled() == null || in.enabled()); p.setRequestTimeoutMs(in.requestTimeoutMs()); @@ -135,6 +139,12 @@ public Map updateProvider(Long id, ProviderInput in) { if (in.baseUrl() != null && !in.baseUrl().isBlank()) { p.setBaseUrl(requireBaseUrl(in.baseUrl())); } + if (in.project() != null) { + p.setProject(blankToNull(in.project())); + } + if (in.location() != null) { + p.setLocation(blankToNull(in.location())); + } if (in.apiVersion() != null) { // Blank clears it: the provider goes back to the plain OpenAI shape. p.setApiVersion(in.apiVersion().isBlank() ? null : in.apiVersion().trim()); @@ -172,6 +182,8 @@ private Map providerView(Provider p) { view.put("type", p.getType().name()); view.put("base_url", p.getBaseUrl()); view.put("api_version", p.getApiVersion()); + view.put("project", p.getProject()); + view.put("location", p.getLocation()); view.put("enabled", p.isEnabled()); view.put("credential_source", credentials.source(p)); view.put("credential_configured", credentials.apiKey(p) != null); @@ -192,8 +204,31 @@ private static ProviderType parseType(String raw) { try { return ProviderType.valueOf(raw.trim().toUpperCase()); } catch (IllegalArgumentException e) { - throw ApiException.badRequest("type must be OPENAI or ANTHROPIC", "type"); + throw ApiException.badRequest( + "type must be OPENAI, ANTHROPIC, GEMINI or VERTEX", "type"); + } + } + + private static String blankToNull(String value) { + return value == null || value.isBlank() ? null : value.trim(); + } + + /** + * Vertex needs a region-derived host. An explicit baseUrl wins (a proxy or a test), so + * only fill it in when the caller left it blank. + */ + private static String vertexBaseUrl(Provider p, String requested) { + if (requested != null && !requested.isBlank()) { + return requireBaseUrl(requested); + } + if (p.getType() == ProviderType.VERTEX) { + if (p.getLocation() == null) { + throw ApiException.badRequest( + "location is required for a VERTEX provider, e.g. us-central1", "location"); + } + return "https://" + p.getLocation().toLowerCase() + "-aiplatform.googleapis.com"; } + throw ApiException.badRequest("base_url is required", "base_url"); } private static String requireBaseUrl(String baseUrl) { diff --git a/path-gateway/src/main/java/com/anindra/path/admin/AlertService.java b/path-gateway/src/main/java/com/anindra/path/admin/AlertService.java new file mode 100644 index 0000000..cc9b63f --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/admin/AlertService.java @@ -0,0 +1,339 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Operational alerts (Slack-compatible JSON) with a per-event cooldown. + * + *

Webhook delivery is retried with exponential backoff and jitter, because a single + * fire-and-forget POST silently loses an alert during a 500, a 429 or a two-second network + * blip. Only 429, 5xx and connect/timeout failures are retried; a 4xx is a permanent + * rejection and retrying it would just repeat the same mistake. + * + *

Both channels run on small bounded executors. If an endpoint is down and the queue + * fills, new alerts are dropped (and counted) rather than piling up behind a dead endpoint. + * A blank webhook URL makes the webhook channel a no-op; email is independent. Email is not + * retried: a real SMTP server has its own queue, and a retry here could duplicate a message + * it already accepted. + */ +@Service +@Slf4j +public class AlertService { + + /** One initial attempt plus two retries. */ + private static final int MAX_ATTEMPTS = 3; + /** Exponential backoff before the next attempt: 250ms, 1s, then (unused) 4s. */ + private static final long[] BACKOFF_MS = {250, 1_000, 4_000}; + /** A Retry-After larger than this is capped so one endpoint cannot pin a worker. */ + private static final long MAX_RETRY_AFTER_MS = 30_000; + private static final int WORKERS = 2; + private static final int QUEUE_CAPACITY = 64; + + private final GatewayProperties props; + private final ObjectMapper mapper; + /** Console-configured (DB) mail settings, falling back to SPRING_MAIL_*. */ + private final MailSettingsService mailSettings; + private final MeterRegistry metrics; + private final HttpClient http = HttpClient.newHttpClient(); + private final Map lastSent = new ConcurrentHashMap<>(); + private final Map sentOnce = new ConcurrentHashMap<>(); + private final ThreadPoolExecutor webhookExecutor; + private final ThreadPoolExecutor emailExecutor; + + @org.springframework.beans.factory.annotation.Autowired + public AlertService(GatewayProperties props, ObjectMapper mapper, + MailSettingsService mailSettings, MeterRegistry metrics) { + this(props, mapper, mailSettings, metrics, WORKERS, QUEUE_CAPACITY); + } + + /** Test seam: a tiny executor makes the dropped-alert path reachable. */ + AlertService(GatewayProperties props, ObjectMapper mapper, MailSettingsService mailSettings, + MeterRegistry metrics, int workers, int queueCapacity) { + this.props = props; + this.mapper = mapper; + this.mailSettings = mailSettings; + this.metrics = metrics; + this.webhookExecutor = new ThreadPoolExecutor(workers, workers, 30, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(Math.max(1, queueCapacity)), + runnable -> { + Thread thread = new Thread(runnable, "alert-webhook"); + thread.setDaemon(true); + return thread; + }, + (runnable, executor) -> { + count("webhook", "dropped"); + log.warn("Alert webhook queue is full; dropping an alert instead of blocking the caller"); + }); + this.webhookExecutor.allowCoreThreadTimeOut(true); + this.emailExecutor = new ThreadPoolExecutor(workers, workers, 30, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(Math.max(1, queueCapacity)), + runnable -> { + Thread thread = new Thread(runnable, "alert-email"); + thread.setDaemon(true); + return thread; + }, + (runnable, executor) -> { + count("email", "dropped"); + log.warn("Alert email queue is full; dropping an alert instead of blocking the caller"); + }); + this.emailExecutor.allowCoreThreadTimeOut(true); + } + + /** + * Fires at most once per {@code dedupeKey}, regardless of the cooldown. Used for budget + * thresholds, where the key carries the period (project, window, day or month) so a + * busy project cannot emit thousands of identical events. + */ + public void notifyOnce(String dedupeKey, String event, Map payload) { + String url = props.getAlerts().getWebhookUrl(); + if ((url == null || url.isBlank()) && !emailEnabled()) { + return; + } + if (sentOnce.putIfAbsent(dedupeKey, System.currentTimeMillis()) != null) { + return; + } + notify(event, payload); + } + + @org.springframework.scheduling.annotation.Scheduled(fixedDelay = 3_600_000) + void forgetOldDedupeKeys() { + long cutoff = System.currentTimeMillis() - java.time.Duration.ofDays(2).toMillis(); + sentOnce.entrySet().removeIf(e -> e.getValue() < cutoff); + } + + public void notify(String event, Map payload) { + String url = props.getAlerts().getWebhookUrl(); + boolean wantsEmail = emailEnabled(); + if ((url == null || url.isBlank()) && !wantsEmail) { + return; + } + long now = System.currentTimeMillis(); + long cooldownMs = props.getAlerts().getCooldown().toMillis(); + Long previous = lastSent.putIfAbsent(event, now); + if (previous != null) { + if (now - previous < cooldownMs) { + return; + } + lastSent.put(event, now); + } + + Map body = new LinkedHashMap<>(); + body.put("source", "path"); + body.put("event", event); + body.put("at", Instant.now().toString()); + body.putAll(payload); + + if (url != null && !url.isBlank()) { + enqueueWebhook(url, body); + } + if (wantsEmail) { + enqueueEmail(body); + } + } + + /** True when recipients are configured and a sender can be built. */ + public boolean emailEnabled() { + return mailSettings.emailEnabled(); + } + + public boolean webhookEnabled() { + return props.getAlerts().getWebhookUrl() != null + && !props.getAlerts().getWebhookUrl().isBlank(); + } + + private void enqueueWebhook(String url, Map body) { + try { + webhookExecutor.execute(() -> send(url, body)); + } catch (java.util.concurrent.RejectedExecutionException e) { + // The handler above counts the drop, but it is not reached while the pool is + // shutting down (for example during a drain). + count("webhook", "dropped"); + log.warn("Alert webhook executor rejected {} while shutting down", body.get("event")); + } + } + + private void enqueueEmail(Map body) { + try { + emailExecutor.execute(() -> email(body)); + } catch (java.util.concurrent.RejectedExecutionException e) { + count("email", "dropped"); + log.warn("Alert email executor rejected {} while shutting down", body.get("event")); + } + } + + private void email(Map body) { + JavaMailSender sender = mailSettings.sender().orElse(null); + if (sender == null) { + count("email", "dropped"); + return; + } + try { + SimpleMailMessage message = new SimpleMailMessage(); + message.setFrom(mailSettings.fromAddress()); + message.setTo(mailSettings.recipients().split("\\s*,\\s*")); + message.setSubject("[Path] " + body.get("event")); + message.setText(text(body)); + sender.send(message); + count("email", "delivered"); + } catch (Exception e) { + // An unreachable mailbox must never affect the request that triggered the alert. + count("email", "dropped"); + log.warn("Alert email failed for {}: {}", body.get("event"), e.getMessage()); + } + } + + /** Plain text body: the event, the time, then one {@code key: value} line per field. */ + static String text(Map body) { + StringBuilder sb = new StringBuilder(); + sb.append("Path gateway alert\n\n"); + sb.append("event: ").append(body.get("event")).append('\n'); + sb.append("at: ").append(body.get("at")).append('\n'); + for (Map.Entry entry : body.entrySet()) { + if (entry.getKey().equals("source") || entry.getKey().equals("event") + || entry.getKey().equals("at")) { + continue; + } + sb.append(entry.getKey()).append(": ").append(entry.getValue()).append('\n'); + } + return sb.toString(); + } + + /** + * Delivers one alert, retrying a transient failure up to {@link #MAX_ATTEMPTS} times. + * Counting happens once per outcome for the whole alert: {@code delivered} when a 2xx + * arrives, {@code retried} once per retry, {@code dropped} when it is given up on. + */ + private void send(String url, Map body) { + String json; + URI uri; + try { + json = mapper.writeValueAsString(body); + uri = URI.create(url); + } catch (Exception e) { + count("webhook", "dropped"); + log.warn("Alert webhook payload is not serializable for {}: {}", body.get("event"), e.getMessage()); + return; + } + + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + long retryAfterMs = -1; + try { + HttpRequest request = HttpRequest.newBuilder(uri) + .timeout(java.time.Duration.ofSeconds(5)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(json)) + .build(); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.discarding()); + int status = response.statusCode(); + if (status >= 200 && status < 300) { + count("webhook", "delivered"); + return; + } + if (!retryable(status)) { + count("webhook", "dropped"); + log.warn("Alert webhook rejected {} with {}; not retrying a permanent failure", + body.get("event"), status); + return; + } + retryAfterMs = retryAfterMs(response); + if (attempt == MAX_ATTEMPTS) { + count("webhook", "dropped"); + log.warn("Alert webhook failed for {} after {} attempts (last status {})", + body.get("event"), attempt, status); + return; + } + log.warn("Alert webhook for {} returned {}; retrying (attempt {}/{})", + body.get("event"), status, attempt + 1, MAX_ATTEMPTS); + } catch (IOException e) { + // Connect failures, timeouts and a dropped connection are all transient. + if (attempt == MAX_ATTEMPTS) { + count("webhook", "dropped"); + log.warn("Alert webhook failed for {} after {} attempts: {}", + body.get("event"), attempt, e.getMessage()); + return; + } + log.warn("Alert webhook for {} failed: {}; retrying (attempt {}/{})", + body.get("event"), e.getMessage(), attempt + 1, MAX_ATTEMPTS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + count("webhook", "dropped"); + log.warn("Alert webhook for {} was interrupted", body.get("event")); + return; + } catch (Exception e) { + count("webhook", "dropped"); + log.warn("Alert webhook failed for {}: {}", body.get("event"), e.getMessage()); + return; + } + + count("webhook", "retried"); + long delay = retryAfterMs >= 0 ? Math.min(retryAfterMs, MAX_RETRY_AFTER_MS) + : jittered(attempt); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + count("webhook", "dropped"); + return; + } + } + count("webhook", "dropped"); + } + + /** Only rate limits and server-side failures are worth another attempt. */ + private static boolean retryable(int status) { + return status == 429 || status >= 500; + } + + /** 250ms, 1s (then 4s) with jitter in the upper half of that window. */ + private static long jittered(int attempt) { + long base = BACKOFF_MS[Math.min(attempt - 1, BACKOFF_MS.length - 1)]; + return base / 2 + ThreadLocalRandom.current().nextLong(base / 2 + 1); + } + + /** Seconds or an HTTP date, per RFC 9110. Returns -1 when absent or unparseable. */ + private static long retryAfterMs(HttpResponse response) { + String value = response.headers().firstValue("Retry-After").orElse(null); + if (value == null || value.isBlank()) { + return -1; + } + String trimmed = value.trim(); + try { + return Math.max(0, Long.parseLong(trimmed) * 1_000); + } catch (NumberFormatException ignored) { + // Fall through to the HTTP-date form. + } + try { + ZonedDateTime at = ZonedDateTime.parse(trimmed, DateTimeFormatter.RFC_1123_DATE_TIME); + return Math.max(0, at.toInstant().toEpochMilli() - System.currentTimeMillis()); + } catch (DateTimeParseException ignored) { + return -1; + } + } + + private void count(String channel, String outcome) { + metrics.counter("path.alerts.sent", "channel", channel, "outcome", outcome).increment(); + } +} diff --git a/src/main/java/com/anindra/path/admin/AliasService.java b/path-gateway/src/main/java/com/anindra/path/admin/AliasService.java similarity index 100% rename from src/main/java/com/anindra/path/admin/AliasService.java rename to path-gateway/src/main/java/com/anindra/path/admin/AliasService.java diff --git a/path-gateway/src/main/java/com/anindra/path/admin/AuditService.java b/path-gateway/src/main/java/com/anindra/path/admin/AuditService.java new file mode 100644 index 0000000..8b1dc9f --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/admin/AuditService.java @@ -0,0 +1,120 @@ +package com.anindra.path.admin; + +import com.anindra.path.domain.AdminAudit; +import com.anindra.path.repo.AdminAuditRepository; +import com.anindra.path.config.AuthFilter; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Records every mutating admin action; read back through /admin/audit. */ +@Service +@RequiredArgsConstructor +@Slf4j +public class AuditService { + + private static final int MAX_DETAILS = 4000; + + private final AdminAuditRepository audit; + private final ObjectMapper mapper; + + public void record(String action, String target, Object details) { + try { + AdminAudit entry = new AdminAudit(); + entry.setActor(currentActor()); + entry.setAction(action); + entry.setTarget(target); + String json = details == null ? null : mapper.writeValueAsString(sanitise(details)); + entry.setDetails(json != null && json.length() > MAX_DETAILS + ? json.substring(0, MAX_DETAILS) : json); + audit.save(entry); + } catch (Exception e) { + // Auditing must never break the operation it describes. + log.warn("Could not record audit entry {}: {}", action, e.getMessage()); + } + } + + /** + * Audit details must never contain a credential. Admin inputs carry them (a provider's + * apiKey, a rotated master key), and the audit table is readable through the API, so a + * plaintext key there would defeat the encryption at rest. + */ + static Object sanitise(Object details) { + var tree = MAPPER.valueToTree(details); + scrub(tree); + return tree; + } + + private static void scrub(com.fasterxml.jackson.databind.JsonNode node) { + if (node instanceof com.fasterxml.jackson.databind.node.ObjectNode object) { + java.util.List names = new java.util.ArrayList<>(); + object.fieldNames().forEachRemaining(names::add); + for (String name : names) { + if (SENSITIVE.matcher(name).find()) { + object.put(name, "***"); + } else { + scrub(object.get(name)); + } + } + } else if (node.isArray()) { + for (com.fasterxml.jackson.databind.JsonNode child : node) { + scrub(child); + } + } + } + + private static final java.util.regex.Pattern SENSITIVE = java.util.regex.Pattern.compile( + "(?i)(api[-_]?key|password|secret|token|authorization|new_key|private)"); + + private static final com.fasterxml.jackson.databind.ObjectMapper MAPPER = + new com.fasterxml.jackson.databind.ObjectMapper(); + + /** + * The identity behind the request: the proxy-authenticated SSO user, the name of the + * admin token, or "shared-admin" when neither is available. Attribution is the point of + * the audit log, so this is resolved per request rather than per service instance. + */ + private static String currentActor() { + try { + var attributes = org.springframework.web.context.request.RequestContextHolder + .getRequestAttributes(); + if (attributes instanceof org.springframework.web.context.request.ServletRequestAttributes servlet) { + Object principal = servlet.getRequest().getAttribute(AuthFilter.ADMIN_PRINCIPAL_ATTR); + if (principal instanceof com.anindra.path.admin.TenancyService.Principal p + && p.name() != null && !p.name().isBlank()) { + return p.name(); + } + } + } catch (Exception ignored) { + // outside a request (scheduled work, tests): fall through + } + return "shared-admin"; + } + + public List> list(String action, int limit) { + int capped = Math.max(1, Math.min(limit, 500)); + List rows = action == null || action.isBlank() + ? audit.findAllByOrderByCreatedAtDesc(PageRequest.of(0, capped)) + : audit.findByActionOrderByCreatedAtDesc(action, PageRequest.of(0, capped)); + + List> out = new ArrayList<>(); + for (AdminAudit row : rows) { + Map item = new LinkedHashMap<>(); + item.put("id", row.getId()); + item.put("at", row.getCreatedAt().toString()); + item.put("actor", row.getActor()); + item.put("action", row.getAction()); + item.put("target", row.getTarget()); + item.put("details", row.getDetails()); + out.add(item); + } + return out; + } +} diff --git a/path-gateway/src/main/java/com/anindra/path/admin/DailyDigestService.java b/path-gateway/src/main/java/com/anindra/path/admin/DailyDigestService.java new file mode 100644 index 0000000..e5a129e --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/admin/DailyDigestService.java @@ -0,0 +1,289 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.UsageRecord; +import com.anindra.path.gateway.ProviderCircuitBreaker; +import com.anindra.path.repo.ProviderRepository; +import com.anindra.path.repo.UsageRecordRepository; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.PageRequest; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * One email a day describing yesterday: requests, spend, tokens, failures, the slowest + * model, the biggest spenders and any provider still parked. + * + *

Threshold alerts only fire when something is already wrong, and nobody opens a + * dashboard every morning; a slow spend drift is otherwise noticed on the invoice. The + * digest reuses the alert mail channel and is silently off when no recipients are + * configured. + * + *

"Yesterday" is the previous calendar day in the gateway's time zone, not the last 24 + * hours, so a digest at 08:00 covers the day an operator thinks of as yesterday. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class DailyDigestService { + + /** A successful-call sample used for the per-model p95; bounded like the stats view. */ + private static final int LATENCY_SAMPLE = 50_000; + private static final int TOP_MODELS = 3; + private static final DateTimeFormatter DAY = DateTimeFormatter.ISO_LOCAL_DATE; + + private final UsageRecordRepository usage; + private final MailSettingsService mail; + private final GatewayProperties props; + private final ProviderCircuitBreaker breaker; + private final ProviderRepository providers; + private final MeterRegistry metrics; + + /** Yesterday's numbers, enough to render the email. */ + public record Day(long requests, long failures, BigDecimal cost, long tokens, + long promptTokens, long cachedTokens, BigDecimal savings, + String topErrorCode, long topErrorCount, + String slowestModel, long slowestP95Ms, + List topModels) { + } + + public record ModelSpend(String model, BigDecimal cost) { + } + + /** What the send did, so the admin endpoint can explain itself. */ + public record Result(boolean sent, String reason, String subject, String text) { + } + + @Scheduled(cron = "${gateway.digest.cron:0 0 8 * * *}") + public void scheduled() { + if (!props.getDigest().isEnabled()) { + return; + } + send(LocalDate.now().minusDays(1)); + } + + /** + * Builds and sends the digest for {@code day}. Sends nothing when disabled, when no + * mailbox is configured, or when the day had no traffic; the caller gets the reason. + */ + public Result send(LocalDate day) { + if (!props.getDigest().isEnabled()) { + return new Result(false, "disabled", null, null); + } + ZoneId zone = ZoneId.systemDefault(); + Instant from = day.atStartOfDay(zone).toInstant(); + Instant to = day.plusDays(1).atStartOfDay(zone).toInstant(); + + Day today = collect(from, to); + if (today.requests() == 0) { + return new Result(false, "no_traffic", null, null); + } + Day previous = collect(from.minusSeconds(86_400), from); + String subject = subject(today); + String text = render(day, zone, today, previous, parkedNow()); + + if (!mail.emailEnabled()) { + return new Result(false, "email_not_configured", subject, text); + } + JavaMailSender sender = mail.sender().orElse(null); + if (sender == null) { + return new Result(false, "email_not_configured", subject, text); + } + try { + SimpleMailMessage message = new SimpleMailMessage(); + message.setFrom(mail.fromAddress()); + message.setTo(mail.recipients().split("\\s*,\\s*")); + message.setSubject(subject); + message.setText(text); + sender.send(message); + metrics.counter("path.alerts.sent", "channel", "digest", "outcome", "delivered").increment(); + return new Result(true, null, subject, text); + } catch (Exception e) { + // A broken mailbox must never stop tomorrow's digest. + metrics.counter("path.alerts.sent", "channel", "digest", "outcome", "dropped").increment(); + log.warn("Daily digest for {} could not be sent: {}", day, e.getMessage()); + return new Result(false, "send_failed", subject, text); + } + } + + // --------------------------------------------------------------- gathering + + private Day collect(Instant from, Instant to) { + long requests = usage.countByCreatedAtBetween(from, to); + long failures = usage.countByStatusAndCreatedAtBetween(UsageRecord.Status.ERROR, from, to); + BigDecimal cost = nz(usage.sumCostBetween(from, to)); + long tokens = nz(usage.sumTokensBetween(from, to)); + long promptTokens = nz(usage.sumPromptTokensBetween(from, to)); + long cachedTokens = nz(usage.sumCachedTokensBetween(from, to)); + BigDecimal savings = nz(usage.sumCachedSavingsBetween(from, to)); + + String topError = null; + long topErrorCount = 0; + List errors = usage.topErrorCodesBetween(from, to, PageRequest.of(0, 1)); + if (!errors.isEmpty()) { + topError = String.valueOf(errors.get(0)[0]); + topErrorCount = ((Number) errors.get(0)[1]).longValue(); + } + + List topModels = new ArrayList<>(); + for (Object[] row : usage.topModelsByCostBetween(from, to, PageRequest.of(0, TOP_MODELS))) { + topModels.add(new ModelSpend(String.valueOf(row[0]), (BigDecimal) row[1])); + } + + String slowestModel = null; + long slowestP95 = 0; + Map> latencies = new LinkedHashMap<>(); + for (Object[] row : usage.successLatenciesBetween(from, to, PageRequest.of(0, LATENCY_SAMPLE))) { + if (row[1] == null) { + continue; + } + latencies.computeIfAbsent(String.valueOf(row[0]), k -> new ArrayList<>()) + .add(((Number) row[1]).longValue()); + } + for (Map.Entry> entry : latencies.entrySet()) { + List sorted = entry.getValue(); + sorted.sort(null); + long p95 = p95(sorted); + if (p95 > slowestP95) { + slowestP95 = p95; + slowestModel = entry.getKey(); + } + } + + return new Day(requests, failures, cost, tokens, promptTokens, cachedTokens, savings, + topError, topErrorCount, slowestModel, slowestP95, topModels); + } + + /** Providers whose circuit is open right now; the breaker keeps no history to count. */ + private List parkedNow() { + List parked = new ArrayList<>(); + Map snapshot = breaker.snapshot(); + for (Provider provider : providers.findAllByOrderBySlugAsc()) { + Object state = snapshot.get(String.valueOf(provider.getId())); + if (state instanceof Map view && "open".equals(view.get("state"))) { + parked.add(provider.getSlug()); + } + } + return parked; + } + + // --------------------------------------------------------------- rendering + + static String subject(Day day) { + return "[Path] yesterday: " + count(day.requests()) + " requests, " + + money(day.cost()) + ", " + percent(failureRatio(day)) + " failed"; + } + + static String render(LocalDate day, ZoneId zone, Day today, Day previous, List parked) { + StringBuilder sb = new StringBuilder(); + sb.append("Path daily digest — ").append(DAY.format(day)) + .append(" (").append(zone.getId()).append(")\n\n"); + sb.append(row("Requests", count(today.requests()) + " (" + requestDelta(today, previous) + ")")); + sb.append(row("Spend", money(today.cost()) + " (" + spendDelta(today, previous) + ")")); + String cache = today.promptTokens() == 0 ? "cache hit n/a" + : "cache hit " + percent(ratio(today.cachedTokens(), today.promptTokens())) + + ", saved " + money(today.savings()); + sb.append(row("Tokens", tokens(today.tokens()) + " " + cache)); + String failures = percent(failureRatio(today)); + if (today.topErrorCode() != null) { + failures += " (top: " + today.topErrorCode() + " ×" + count(today.topErrorCount()) + ")"; + } + sb.append(row("Failures", count(today.failures()) + " " + failures)); + sb.append(row("Slowest", today.slowestModel() == null ? "—" + : today.slowestModel() + " p95 " + seconds(today.slowestP95Ms()))); + List spenders = new ArrayList<>(); + for (ModelSpend model : today.topModels()) { + spenders.add(model.model() + " " + money(model.cost())); + } + sb.append(row("Top spend", spenders.isEmpty() ? "—" : String.join(" · ", spenders))); + if (!parked.isEmpty()) { + sb.append(row("Parked", String.join(", ", parked) + " (circuit open now)")); + } + return sb.toString(); + } + + /** Label left-aligned in 14 columns, the same shape as the alert email. */ + private static String row(String label, String value) { + return String.format(Locale.ROOT, "%-14s%s%n", label, value); + } + + private static String requestDelta(Day today, Day previous) { + if (previous.requests() == 0) { + return "no traffic the day before"; + } + double change = (today.requests() - previous.requests()) * 100.0 / previous.requests(); + return String.format(Locale.ROOT, "%+.0f%% vs the day before", change); + } + + private static String spendDelta(Day today, Day previous) { + BigDecimal change = today.cost().subtract(previous.cost()); + return String.format(Locale.ROOT, "%s%s", change.signum() >= 0 ? "+" : "-", + money(change.abs())); + } + + private static double failureRatio(Day day) { + return day.requests() == 0 ? 0 : (double) day.failures() / day.requests(); + } + + private static double ratio(long part, long whole) { + return whole == 0 ? 0 : (double) part / whole; + } + + private static String count(long value) { + return String.format(Locale.ROOT, "%,d", value); + } + + private static String tokens(long value) { + if (value >= 1_000_000) { + return String.format(Locale.ROOT, "%.1fM", value / 1_000_000.0); + } + if (value >= 10_000) { + return String.format(Locale.ROOT, "%.1fk", value / 1_000.0); + } + return count(value); + } + + private static String money(BigDecimal value) { + return String.format(Locale.ROOT, "$%,.2f", value == null ? BigDecimal.ZERO : value); + } + + private static String percent(double ratio) { + return String.format(Locale.ROOT, "%.1f%%", ratio * 100); + } + + private static String seconds(long millis) { + return String.format(Locale.ROOT, "%.1fs", millis / 1000.0); + } + + private static long p95(List sorted) { + if (sorted.isEmpty()) { + return 0; + } + int index = (int) Math.ceil(0.95 * sorted.size()) - 1; + return sorted.get(Math.max(0, Math.min(index, sorted.size() - 1))); + } + + private static BigDecimal nz(BigDecimal value) { + return value == null ? BigDecimal.ZERO : value; + } + + private static long nz(Long value) { + return value == null ? 0 : value; + } +} diff --git a/path-gateway/src/main/java/com/anindra/path/admin/MailSettingsService.java b/path-gateway/src/main/java/com/anindra/path/admin/MailSettingsService.java new file mode 100644 index 0000000..09795e0 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/admin/MailSettingsService.java @@ -0,0 +1,201 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.MailSettings; +import com.anindra.path.repo.MailSettingsRepository; +import com.anindra.path.secret.SecretCipher; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.mail.javamail.JavaMailSender; +import org.springframework.mail.javamail.JavaMailSenderImpl; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +/** + * Alert-email delivery, configurable from the console or the environment. + * + *

Precedence: an enabled row in {@code mail_settings} wins; otherwise the Spring Mail + * environment ({@code SPRING_MAIL_*}) is used, which is what a config-file deployment + * already does. The SMTP password is encrypted with the master key and is only ever + * written, never read back. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class MailSettingsService { + + private final MailSettingsRepository repository; + private final SecretCipher cipher; + private final GatewayProperties props; + private final org.springframework.core.env.Environment environment; + + /** What the console shows: everything except the password. */ + public Map view() { + MailSettings settings = repository.findById(MailSettings.SINGLETON).orElse(null); + Map out = new LinkedHashMap<>(); + out.put("configured", settings != null && settings.isEnabled()); + out.put("source", source()); + out.put("host", settings == null ? envHost() : settings.getHost()); + out.put("port", settings == null ? envPort() : settings.getPort()); + out.put("username", settings == null ? "" : nullToEmpty(settings.getUsername())); + out.put("from", settings == null ? props.getAlerts().getEmailFrom() : settings.getFromAddress()); + out.put("to", settings == null ? props.getAlerts().getEmailTo() : settings.getToAddress()); + out.put("has_password", settings != null && settings.getPasswordCipher() != null); + out.put("updated_at", settings == null || settings.getUpdatedAt() == null + ? null : settings.getUpdatedAt().toString()); + return out; + } + + private String source() { + MailSettings settings = repository.findById(MailSettings.SINGLETON).orElse(null); + if (settings != null && settings.isEnabled()) { + return "console"; + } + if (envHost() != null && !envHost().isBlank()) { + return "environment"; + } + return "none"; + } + + /** SPRING_MAIL_HOST: the config-file route, kept working alongside the console. */ + private String envHost() { + return environment.getProperty("spring.mail.host", ""); + } + + private Integer envPort() { + String port = environment.getProperty("spring.mail.port"); + return port == null || port.isBlank() ? null : Integer.valueOf(port); + } + + @Transactional + public Map save(String host, Integer port, String username, String password, + String from, String to, Boolean enabled) { + MailSettings settings = repository.findById(MailSettings.SINGLETON) + .orElseGet(() -> { + MailSettings created = new MailSettings(); + created.setId(MailSettings.SINGLETON); + return created; + }); + + boolean turnOn = enabled == null ? true : enabled; + if (turnOn) { + String effectiveHost = host == null || host.isBlank() ? settings.getHost() : host.trim(); + String effectiveTo = to == null || to.isBlank() ? settings.getToAddress() : to.trim(); + if (effectiveHost == null || effectiveHost.isBlank()) { + throw ApiException.badRequest("'host' is required to enable email alerting.", "host"); + } + if (effectiveTo == null || effectiveTo.isBlank()) { + throw ApiException.badRequest("'to' must list at least one recipient.", "to"); + } + } + + if (host != null) { + settings.setHost(host.trim()); + } + if (port != null) { + if (port <= 0 || port > 65535) { + throw ApiException.badRequest("'port' must be between 1 and 65535.", "port"); + } + settings.setPort(port); + } + if (username != null) { + settings.setUsername(username.isBlank() ? null : username.trim()); + } + // Blank keeps the stored password, so an operator can change the host without + // re-typing the secret; "clear" is explicit. + if (password != null) { + if (password.equals("-")) { + settings.setPasswordCipher(null); + } else if (!password.isBlank()) { + settings.setPasswordCipher(cipher.encrypt(password)); + } + } + if (from != null && !from.isBlank()) { + settings.setFromAddress(from.trim()); + } + if (to != null) { + settings.setToAddress(to.isBlank() ? null : to.trim()); + } + settings.setEnabled(turnOn); + settings.setUpdatedAt(Instant.now()); + repository.save(settings); + return view(); + } + + /** Recipients for alerts, from the console row or the environment. */ + public String recipients() { + MailSettings settings = repository.findById(MailSettings.SINGLETON).orElse(null); + if (settings != null && settings.isEnabled() && settings.getToAddress() != null + && !settings.getToAddress().isBlank()) { + return settings.getToAddress(); + } + return props.getAlerts().getEmailTo(); + } + + public String fromAddress() { + MailSettings settings = repository.findById(MailSettings.SINGLETON).orElse(null); + if (settings != null && settings.isEnabled() && settings.getFromAddress() != null + && !settings.getFromAddress().isBlank()) { + return settings.getFromAddress(); + } + return props.getAlerts().getEmailFrom(); + } + + public boolean emailEnabled() { + String to = recipients(); + return to != null && !to.isBlank() && sender().isPresent(); + } + + /** The sender to use: a console-configured one, or the Spring Mail environment. */ + public Optional sender() { + MailSettings settings = repository.findById(MailSettings.SINGLETON).orElse(null); + if (settings != null && settings.isEnabled() && settings.getHost() != null + && !settings.getHost().isBlank()) { + return Optional.of(build(settings)); + } + return Optional.ofNullable(envSender()); + } + + private JavaMailSender envSender() { + MailSettings probe = new MailSettings(); + probe.setHost(envHost()); + probe.setPort(envPort()); + if (probe.getHost() == null || probe.getHost().isBlank()) { + return null; + } + return build(probe); + } + + private JavaMailSender build(MailSettings settings) { + JavaMailSenderImpl sender = new JavaMailSenderImpl(); + sender.setHost(settings.getHost()); + sender.setPort(settings.getPort() == null ? 587 : settings.getPort()); + if (settings.getUsername() != null && !settings.getUsername().isBlank()) { + sender.setUsername(settings.getUsername()); + String password = settings.getPasswordCipher() == null + ? null : cipher.decrypt(settings.getPasswordCipher()); + sender.setPassword(password); + } + + Properties mailProps = sender.getJavaMailProperties(); + mailProps.put("mail.transport.protocol", "smtp"); + mailProps.put("mail.smtp.auth", String.valueOf( + settings.getUsername() != null && !settings.getUsername().isBlank())); + mailProps.put("mail.smtp.starttls.enable", "true"); + mailProps.put("mail.smtp.connectiontimeout", "5000"); + mailProps.put("mail.smtp.timeout", "5000"); + mailProps.put("mail.smtp.writetimeout", "5000"); + return sender; + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/src/main/java/com/anindra/path/admin/ModelSyncService.java b/path-gateway/src/main/java/com/anindra/path/admin/ModelSyncService.java similarity index 90% rename from src/main/java/com/anindra/path/admin/ModelSyncService.java rename to path-gateway/src/main/java/com/anindra/path/admin/ModelSyncService.java index be0a986..2a5e391 100644 --- a/src/main/java/com/anindra/path/admin/ModelSyncService.java +++ b/path-gateway/src/main/java/com/anindra/path/admin/ModelSyncService.java @@ -156,11 +156,18 @@ private boolean isNonChat(String id) { } private List fetchModelIds(Provider provider, String apiKey) throws Exception { + if (provider.getType() == ProviderType.VERTEX) { + // No model-listing endpoint, and the credential is a service account, not a key. + throw new ApiException(400, "invalid_request_error", "models_unsupported", + "Vertex does not publish a model list; add routes by hand."); + } String url = provider.getBaseUrl() + "/models"; Map headers = new LinkedHashMap<>(); if (provider.getType() == ProviderType.ANTHROPIC) { headers.put("x-api-key", apiKey); headers.put("anthropic-version", "2023-06-01"); + } else if (provider.getType() == ProviderType.GEMINI) { + headers.put("x-goog-api-key", apiKey); } else { headers.put("Authorization", "Bearer " + apiKey); } @@ -180,6 +187,16 @@ private List fetchModelIds(Provider provider, String apiKey) throws Exce } } } + // Gemini lists {models:[{name:"models/gemini-2.5-pro", ...}]}; the id is the tail. + JsonNode gemini = root.path("models"); + if (gemini.isArray()) { + for (JsonNode node : gemini) { + String name = node.path("name").asText(""); + if (!name.isBlank()) { + ids.add(name.startsWith("models/") ? name.substring("models/".length()) : name); + } + } + } return ids; } diff --git a/src/main/java/com/anindra/path/admin/PromptLogService.java b/path-gateway/src/main/java/com/anindra/path/admin/PromptLogService.java similarity index 100% rename from src/main/java/com/anindra/path/admin/PromptLogService.java rename to path-gateway/src/main/java/com/anindra/path/admin/PromptLogService.java diff --git a/src/main/java/com/anindra/path/admin/ProviderProbeService.java b/path-gateway/src/main/java/com/anindra/path/admin/ProviderProbeService.java similarity index 83% rename from src/main/java/com/anindra/path/admin/ProviderProbeService.java rename to path-gateway/src/main/java/com/anindra/path/admin/ProviderProbeService.java index e966706..95bd9ac 100644 --- a/src/main/java/com/anindra/path/admin/ProviderProbeService.java +++ b/path-gateway/src/main/java/com/anindra/path/admin/ProviderProbeService.java @@ -42,10 +42,20 @@ public Map probe(Long providerId) { return out; } + if (provider.getType() == ProviderType.VERTEX) { + // Vertex has no model-listing endpoint and its credential is a service account, + // not a bearer token; a probe here would only test the wrong thing. + out.put("ok", false); + out.put("note", "Vertex has no /models endpoint; probe a route instead."); + return out; + } + Map headers = new LinkedHashMap<>(); if (provider.getType() == ProviderType.ANTHROPIC) { headers.put("x-api-key", apiKey); headers.put("anthropic-version", "2023-06-01"); + } else if (provider.getType() == ProviderType.GEMINI) { + headers.put("x-goog-api-key", apiKey); } else { headers.putAll(com.anindra.path.gateway.ProviderUrls.headers(provider, apiKey, false)); } @@ -59,7 +69,9 @@ public Map probe(Long providerId) { out.put("status", response.status()); out.put("latency_ms", latency); if (response.ok()) { - JsonNode data = mapper.readTree(response.body()).path("data"); + JsonNode body = mapper.readTree(response.body()); + // Gemini calls the list "models"; the OpenAI shape calls it "data". + JsonNode data = body.path("data").isArray() ? body.path("data") : body.path("models"); out.put("model_count", data.isArray() ? data.size() : 0); } else { out.put("error", abbreviate(response.body())); diff --git a/src/main/java/com/anindra/path/admin/TenancyService.java b/path-gateway/src/main/java/com/anindra/path/admin/TenancyService.java similarity index 100% rename from src/main/java/com/anindra/path/admin/TenancyService.java rename to path-gateway/src/main/java/com/anindra/path/admin/TenancyService.java diff --git a/src/main/java/com/anindra/path/api/ApiException.java b/path-gateway/src/main/java/com/anindra/path/api/ApiException.java similarity index 100% rename from src/main/java/com/anindra/path/api/ApiException.java rename to path-gateway/src/main/java/com/anindra/path/api/ApiException.java diff --git a/src/main/java/com/anindra/path/api/CapturingResponseWrapper.java b/path-gateway/src/main/java/com/anindra/path/api/CapturingResponseWrapper.java similarity index 100% rename from src/main/java/com/anindra/path/api/CapturingResponseWrapper.java rename to path-gateway/src/main/java/com/anindra/path/api/CapturingResponseWrapper.java diff --git a/src/main/java/com/anindra/path/api/ChatCompletionsController.java b/path-gateway/src/main/java/com/anindra/path/api/ChatCompletionsController.java similarity index 96% rename from src/main/java/com/anindra/path/api/ChatCompletionsController.java rename to path-gateway/src/main/java/com/anindra/path/api/ChatCompletionsController.java index 19121db..ac35b15 100644 --- a/src/main/java/com/anindra/path/api/ChatCompletionsController.java +++ b/path-gateway/src/main/java/com/anindra/path/api/ChatCompletionsController.java @@ -67,7 +67,7 @@ public ResponseEntity chatCompletions( if (key == null) { throw ApiException.unauthorized( - "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + "Invalid or missing API key. Send a current key as 'Authorization: Bearer sk-gw-...'.", "invalid_api_key"); } drain.begin(); @@ -225,7 +225,7 @@ public ResponseEntity embeddings( HttpServletRequest request) { if (key == null) { throw ApiException.unauthorized( - "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + "Invalid or missing API key. Send a current key as 'Authorization: Bearer sk-gw-...'.", "invalid_api_key"); } String idempotencyKey = normalizeIdempotencyKey(request.getHeader(IDEMPOTENCY_HEADER)); @@ -281,7 +281,7 @@ public Map tokenCount( HttpServletRequest request) { if (key == null) { throw ApiException.unauthorized( - "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + "Invalid or missing API key. Send a current key as 'Authorization: Bearer sk-gw-...'.", "invalid_api_key"); } drain.begin(); try { @@ -345,7 +345,7 @@ public Map credits( private static void requireKey(ClientKey key) { if (key == null) { throw ApiException.unauthorized( - "Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); + "Invalid or missing API key. Send a current key as 'Authorization: Bearer sk-gw-...'.", "invalid_api_key"); } } } diff --git a/src/main/java/com/anindra/path/api/Durations.java b/path-gateway/src/main/java/com/anindra/path/api/Durations.java similarity index 100% rename from src/main/java/com/anindra/path/api/Durations.java rename to path-gateway/src/main/java/com/anindra/path/api/Durations.java diff --git a/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java b/path-gateway/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java similarity index 100% rename from src/main/java/com/anindra/path/api/GlobalExceptionHandler.java rename to path-gateway/src/main/java/com/anindra/path/api/GlobalExceptionHandler.java diff --git a/path-gateway/src/main/java/com/anindra/path/config/AdminAuthLimiter.java b/path-gateway/src/main/java/com/anindra/path/config/AdminAuthLimiter.java new file mode 100644 index 0000000..6e8c1d0 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/config/AdminAuthLimiter.java @@ -0,0 +1,42 @@ +package com.anindra.path.config; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Slows down guessing of the admin token: too many failed attempts from one address + * within the window return 429 instead of another comparison. + * + * The address must come from a trustworthy source — see {@code AuthFilter.clientAddress}. + * A per-address cap is only as good as the address: behind a proxy the peer is an edge + * node, and any client-supplied forwarding header can be rotated at will. + */ +@Component +public class AdminAuthLimiter { + + private final FailureWindow window; + + public AdminAuthLimiter(GatewayProperties props) { + this.window = new FailureWindow( + props.getAdminAuth().getMaxFailures(), + props.getAdminAuth().getWindow(), + props.getAdminAuth().getGlobalMaxFailures()); + } + + public long retryAfterSeconds(String address) { + return window.retryAfterSeconds(address); + } + + public void onFailure(String address) { + window.onFailure(address); + } + + public void onSuccess(String address) { + window.onSuccess(address); + } + + @Scheduled(fixedDelay = 300_000) + void evictStale() { + window.evictStale(); + } +} diff --git a/src/main/java/com/anindra/path/config/AuthFilter.java b/path-gateway/src/main/java/com/anindra/path/config/AuthFilter.java similarity index 63% rename from src/main/java/com/anindra/path/config/AuthFilter.java rename to path-gateway/src/main/java/com/anindra/path/config/AuthFilter.java index 10c654c..ed603a3 100644 --- a/src/main/java/com/anindra/path/config/AuthFilter.java +++ b/path-gateway/src/main/java/com/anindra/path/config/AuthFilter.java @@ -25,7 +25,7 @@ * Single entry gate: * /v1/** -> client API key (except GET /v1/models when public-models=true) * /admin/** -> X-Admin-Token - * /actuator/** -> open + * /actuator/health -> open (status only); every other actuator endpoint needs the admin token */ @Component @Order(Ordered.HIGHEST_PRECEDENCE + 10) @@ -41,6 +41,7 @@ public class AuthFilter extends OncePerRequestFilter { private final RateLimitStore rateLimiter; private final com.anindra.path.admin.TenancyService tenancy; private final AdminAuthLimiter adminAuthLimiter; + private final ClientAuthLimiter clientAuthLimiter; private final GatewayProperties props; private final com.anindra.path.repo.ProjectRepository projects; private final ObjectMapper mapper; @@ -48,7 +49,7 @@ public class AuthFilter extends OncePerRequestFilter { @Override protected boolean shouldNotFilter(HttpServletRequest request) { String path = request.getRequestURI(); - return !path.startsWith("/v1") && !path.startsWith("/admin"); + return !path.startsWith("/v1") && !path.startsWith("/admin") && !path.startsWith("/actuator"); } @Override @@ -65,8 +66,10 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse try { if (path.startsWith("/admin")) { requireAdmin(request); + } else if (path.startsWith("/actuator")) { + requireActuatorAccess(request, path); } else if (!isPublicModelsRequest(request)) { - ClientKey key = resolveClientKey(request); + ClientKey key = authenticateClientKey(request); int retryAfter = rateLimiter.acquire(key.getId(), key.getRpmLimit()); if (retryAfter > 0) { throw ApiException.rateLimited( @@ -109,6 +112,29 @@ private void enforceProjectLimit(HttpServletRequest request, ClientKey key) { } } + /** Fails closed: no identity header means no admin access, whatever else is sent. */ + private void requireProxyIdentity(HttpServletRequest request) { + String header = props.getProxyAuth().getHeader(); + String identity = request.getHeader(header); + if (identity == null || identity.isBlank()) { + throw ApiException.unauthorized( + "Admin access requires a signed-in session; the proxy did not send " + header + ".", + "proxy_identity_missing"); + } + java.util.Set scopes = java.util.Arrays.stream( + props.getProxyAuth().getScopes().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(java.util.stream.Collectors.toSet()); + request.setAttribute(ADMIN_PRINCIPAL_ATTR, + new com.anindra.path.admin.TenancyService.Principal(identity.trim(), scopes)); + boolean mutating = !"GET".equalsIgnoreCase(request.getMethod()) + && !"HEAD".equalsIgnoreCase(request.getMethod()); + if (mutating && !scopes.contains("write")) { + throw ApiException.forbidden("This identity has read-only scope."); + } + } + private boolean isPublicModelsRequest(HttpServletRequest request) { return props.isPublicModels() && "GET".equalsIgnoreCase(request.getMethod()) @@ -116,6 +142,44 @@ private boolean isPublicModelsRequest(HttpServletRequest request) { || request.getRequestURI().startsWith("/v1/models/")); } + /** + * Only the liveness probe is public, and it reports status alone. /actuator/health/providers, + * /actuator/metrics and /actuator/prometheus expose the framework version, datasource + * internals, repository names and provider topology, so they are gated like /admin. + */ + private void requireActuatorAccess(HttpServletRequest request, String path) { + boolean liveness = "GET".equalsIgnoreCase(request.getMethod()) + && "/actuator/health".equals(path); + if (!liveness) { + requireAdmin(request); + } + } + + /** + * Resolves the client key, metering failures per address first. + * + * The 429 is checked before the key lookup so a guessing campaign never reaches the + * database, and it is returned for a wrong key exactly as for a flood of them — the + * response must not reveal whether a guess was at least well-formed. + */ + private ClientKey authenticateClientKey(HttpServletRequest request) { + String address = clientAddress(request); + long retryAfter = clientAuthLimiter.retryAfterSeconds(address); + if (retryAfter > 0) { + throw ApiException.rateLimited( + "Too many rejected API keys from this address. Retry in " + retryAfter + "s.", + (int) retryAfter); + } + try { + ClientKey key = resolveClientKey(request); + clientAuthLimiter.onSuccess(address); + return key; + } catch (ApiException e) { + clientAuthLimiter.onFailure(address); + throw e; + } + } + private ClientKey resolveClientKey(HttpServletRequest request) { String header = request.getHeader(HttpHeaders.AUTHORIZATION); String token = null; @@ -128,6 +192,15 @@ private ClientKey resolveClientKey(HttpServletRequest request) { } private void requireAdmin(HttpServletRequest request) { + // SSO at the edge: an authenticating proxy (Authentik, Keycloak, oauth2-proxy, Cloudflare + // Access) resolves the human and injects an identity header. The gateway then trusts it + // instead of a pasted token. This is only safe when the gateway cannot be reached except + // through that proxy — docs/SSO.md says so in bold, and the startup log repeats it. + if (props.getProxyAuth().isEnabled()) { + requireProxyIdentity(request); + return; + } + String expected = props.getAdminToken(); String provided = request.getHeader("X-Admin-Token"); if (provided == null || provided.isBlank()) { @@ -164,10 +237,20 @@ private void requireAdmin(HttpServletRequest request) { } } - private static String clientAddress(HttpServletRequest request) { - String forwarded = request.getHeader("X-Forwarded-For"); - if (forwarded != null && !forwarded.isBlank()) { - return forwarded.split(",")[0].trim(); + /** + * The address the limiter counts against. + * + * Deliberately does NOT read X-Forwarded-For: a client can prepend any value, and the + * first hop was being trusted, so rotating it reset the counter on every request. When + * a trusted edge is in front (Cloudflare), configure client-ip-header and it is used. + */ + private String clientAddress(HttpServletRequest request) { + String header = props.getClientIpHeader(); + if (header != null && !header.isBlank()) { + String value = request.getHeader(header.trim()); + if (value != null && !value.isBlank()) { + return value.split(",")[0].trim(); + } } return request.getRemoteAddr(); } diff --git a/path-gateway/src/main/java/com/anindra/path/config/ClientAuthLimiter.java b/path-gateway/src/main/java/com/anindra/path/config/ClientAuthLimiter.java new file mode 100644 index 0000000..595690c --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/config/ClientAuthLimiter.java @@ -0,0 +1,43 @@ +package com.anindra.path.config; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Caps how many API keys one address may have rejected per window. + * + * Without it the public data plane is an unlimited credential oracle: every request does a + * key lookup and answers 401, so a client can evaluate guesses as fast as the gateway will + * serve them — measurable in the hundreds per second, and enough traffic to knock a proxy + * over. The comparison is cheap, so the cost of the attack is close to zero; the limit is + * what gives it a budget. + */ +@Component +public class ClientAuthLimiter { + + private final FailureWindow window; + + public ClientAuthLimiter(GatewayProperties props) { + this.window = new FailureWindow( + props.getClientAuth().getMaxFailures(), + props.getClientAuth().getWindow(), + props.getClientAuth().getGlobalMaxFailures()); + } + + public long retryAfterSeconds(String address) { + return window.retryAfterSeconds(address); + } + + public void onFailure(String address) { + window.onFailure(address); + } + + public void onSuccess(String address) { + window.onSuccess(address); + } + + @Scheduled(fixedDelay = 300_000) + void evictStale() { + window.evictStale(); + } +} diff --git a/path-gateway/src/main/java/com/anindra/path/config/FailureWindow.java b/path-gateway/src/main/java/com/anindra/path/config/FailureWindow.java new file mode 100644 index 0000000..ab0a0df --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/config/FailureWindow.java @@ -0,0 +1,92 @@ +package com.anindra.path.config; + +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Sliding-window failure counter, shared by the admin-token and client-key auth gates. + * + * Two counters: per address, and (optionally) process-wide. The per-address one is the real + * control — it limits how many credentials a single source can have evaluated. The global + * one is off unless configured, because a fail-closed global cap is a denial-of-service + * against the operator: one hostile source spending the budget locks everyone out. + */ +class FailureWindow { + + private record Entry(long windowStart, int failures) { + } + + private final Map entries = new ConcurrentHashMap<>(); + private final Object globalLock = new Object(); + private long globalWindowStart = System.currentTimeMillis(); + private int globalFailures; + + private final int maxFailures; + private final Duration window; + private final int globalMaxFailures; + + FailureWindow(int maxFailures, Duration window, int globalMaxFailures) { + this.maxFailures = maxFailures; + this.window = window; + this.globalMaxFailures = globalMaxFailures; + } + + long retryAfterSeconds(String address) { + long now = System.currentTimeMillis(); + long windowMs = window.toMillis(); + + synchronized (globalLock) { + rollGlobalWindow(now, windowMs); + if (globalMaxFailures > 0 && globalFailures >= globalMaxFailures) { + return remainingSeconds(now, globalWindowStart, windowMs); + } + } + + Entry entry = entries.get(address); + if (entry == null) { + return 0; + } + if (now - entry.windowStart() >= windowMs || entry.failures() < maxFailures) { + return 0; + } + return remainingSeconds(now, entry.windowStart(), windowMs); + } + + void onFailure(String address) { + long now = System.currentTimeMillis(); + long windowMs = window.toMillis(); + + synchronized (globalLock) { + rollGlobalWindow(now, windowMs); + globalFailures++; + } + + entries.compute(address, (key, current) -> { + if (current == null || now - current.windowStart() >= windowMs) { + return new Entry(now, 1); + } + return new Entry(current.windowStart(), current.failures() + 1); + }); + } + + void onSuccess(String address) { + entries.remove(address); + } + + void evictStale() { + long cutoff = System.currentTimeMillis() - window.toMillis(); + entries.entrySet().removeIf(e -> e.getValue().windowStart() < cutoff); + } + + private void rollGlobalWindow(long now, long windowMs) { + if (now - globalWindowStart >= windowMs) { + globalWindowStart = now; + globalFailures = 0; + } + } + + private static long remainingSeconds(long now, long windowStart, long windowMs) { + return Math.max(1, (windowMs - (now - windowStart)) / 1000); + } +} diff --git a/src/main/java/com/anindra/path/config/GatewayProperties.java b/path-gateway/src/main/java/com/anindra/path/config/GatewayProperties.java similarity index 57% rename from src/main/java/com/anindra/path/config/GatewayProperties.java rename to path-gateway/src/main/java/com/anindra/path/config/GatewayProperties.java index d23d0cf..ec2a109 100644 --- a/src/main/java/com/anindra/path/config/GatewayProperties.java +++ b/path-gateway/src/main/java/com/anindra/path/config/GatewayProperties.java @@ -22,6 +22,14 @@ public class GatewayProperties { /** When true, GET /v1/models works without an API key. */ private boolean publicModels = true; + /** + * Header carrying the real client address, e.g. "CF-Connecting-IP" behind Cloudflare. + * Empty means use the socket address and ignore forwarding headers entirely: honouring + * a client-supplied X-Forwarded-For lets an attacker rotate the value and defeat every + * per-address limit, including the admin auth limiter. + */ + private String clientIpHeader = ""; + /** Comma separated origin patterns, or * for any. */ private String corsOrigins = "*"; @@ -36,6 +44,12 @@ public class GatewayProperties { /** Create a demo client key on boot when the key table is empty. */ private boolean bootstrapDemoKey = false; + /** + * Seed the example provider/route catalogue on boot. A real deployment wants this off: + * otherwise four providers nobody configured reappear on every restart. + */ + private boolean seedDefaults = true; + private int defaultRpmLimit = 120; /** Concurrent in-flight requests allowed per key unless the key overrides it. */ @@ -64,6 +78,34 @@ public class GatewayProperties { private PromptLogging promptLogging = new PromptLogging(); private RateLimit rateLimit = new RateLimit(); private AdminAuth adminAuth = new AdminAuth(); + private ClientAuth clientAuth = new ClientAuth(); + private ProxyAuth proxyAuth = new ProxyAuth(); + private SharedState sharedState = new SharedState(); + private Alert alert = new Alert(); + private Digest digest = new Digest(); + + /** Daily spend digest. Off when no email channel is configured, whatever this says. */ + @Getter + @Setter + public static class Digest { + /** GATEWAY_DIGEST=false turns the daily email off without touching the cron. */ + private boolean enabled = true; + /** Spring cron; the gateway's time zone, five fields by default. */ + private String cron = "0 0 8 * * *"; + } + + /** + * Threshold alerts that are not tied to a single call (provider error rate). + * Env: GATEWAY_ALERT_ERROR_RATE, GATEWAY_ALERT_MIN_SAMPLES. + */ + @Getter + @Setter + public static class Alert { + /** EWMA error rate above which a provider is reported. 0 disables the check. */ + private double errorRate = 0.25; + /** Samples a provider needs before its rate is trusted (a cold start is not an incident). */ + private int minSamples = 20; + } @Getter @Setter @@ -74,6 +116,30 @@ public static class Breaker { private Duration openDuration = Duration.ofSeconds(30); /** Routes tried per request, including the first. */ private int maxAttempts = 3; + /** How long a HALF_OPEN probe may hold the claim before another replica may try. */ + private Duration probeLease = Duration.ofSeconds(10); + } + + /** Where circuit and concurrency state live: memory (per JVM) or Valkey (shared). */ + @Getter + @Setter + public static class SharedState { + private String backend = "memory"; + /** Key namespace, the same convention as the rate limiter. */ + private String prefix = "path"; + /** A permit's lease; longer than the slowest request or the cap can be exceeded. */ + private Duration concurrencyLease = Duration.ofMinutes(5); + } + + @Getter + @Setter + public static class ProxyAuth { + /** Trust an identity header injected by an authenticating reverse proxy (SSO at the edge). */ + private boolean enabled = false; + /** Header the proxy sets, e.g. X-Forwarded-User or Cf-Access-Authenticated-User-Email. */ + private String header = "X-Forwarded-User"; + /** Scopes granted to a proxy-authenticated identity. */ + private String scopes = "read,write"; } @Getter @@ -82,6 +148,29 @@ public static class AdminAuth { /** Failed admin auth attempts allowed per window before 429s start. */ private int maxFailures = 10; private Duration window = Duration.ofMinutes(5); + /** + * Failures across ALL addresses in the window, or 0 to disable. + * + * Off by default on purpose: a fail-closed global cap is a denial-of-service against + * the operator (one scanner spending 200 guesses locks the console for the whole + * window). The per-address cap is the real control once the client address is + * trustworthy — which it is only when the origin is not reachable except through the + * trusted proxy. Enable this for a deployment that cannot make that guarantee. + */ + private int globalMaxFailures = 0; + } + + @Getter + @Setter + public static class ClientAuth { + /** + * Rejected API keys allowed per address per window before 429s start. Generous + * enough that a legitimate client with a typo or a rotated key never notices. + */ + private int maxFailures = 60; + private Duration window = Duration.ofMinutes(1); + /** Process-wide cap; 0 disables. Same trade-off as the admin one. */ + private int globalMaxFailures = 0; } @Getter @@ -104,8 +193,12 @@ public static class PromptLogging { @Getter @Setter public static class Alerts { - /** Slack-compatible webhook; blank disables alerting. */ + /** Slack-compatible webhook; blank disables webhook alerting. */ private String webhookUrl = ""; + + /** Comma-separated recipients. Email alerting also needs spring.mail.host. */ + private String emailTo = ""; + private String emailFrom = "path-gateway@localhost"; private Duration cooldown = Duration.ofHours(1); /** Warn when a key has less than this fraction of its credit left. */ private double budgetThreshold = 0.8; diff --git a/path-gateway/src/main/java/com/anindra/path/config/GeminiConfig.java b/path-gateway/src/main/java/com/anindra/path/config/GeminiConfig.java new file mode 100644 index 0000000..3873aab --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/config/GeminiConfig.java @@ -0,0 +1,27 @@ +package com.anindra.path.config; + +import com.anindra.path.domain.ProviderType; +import com.anindra.path.gateway.GeminiAdapter; +import com.anindra.path.gateway.ProviderAdapter; +import com.anindra.path.gateway.VertexTokenService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Two adapters over one implementation: AI Studio and Vertex differ only in URL and auth, + * and the provider type is what selects one at request time. + */ +@Configuration +public class GeminiConfig { + + @Bean + public ProviderAdapter geminiAdapter(ObjectMapper mapper, VertexTokenService tokens) { + return new GeminiAdapter(mapper, ProviderType.GEMINI, tokens); + } + + @Bean + public ProviderAdapter vertexAdapter(ObjectMapper mapper, VertexTokenService tokens) { + return new GeminiAdapter(mapper, ProviderType.VERTEX, tokens); + } +} diff --git a/src/main/java/com/anindra/path/config/LimitedInputStream.java b/path-gateway/src/main/java/com/anindra/path/config/LimitedInputStream.java similarity index 100% rename from src/main/java/com/anindra/path/config/LimitedInputStream.java rename to path-gateway/src/main/java/com/anindra/path/config/LimitedInputStream.java diff --git a/path-gateway/src/main/java/com/anindra/path/config/NoStoreFilter.java b/path-gateway/src/main/java/com/anindra/path/config/NoStoreFilter.java new file mode 100644 index 0000000..410daa8 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/config/NoStoreFilter.java @@ -0,0 +1,49 @@ +package com.anindra.path.config; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; + +/** + * Marks every API response as uncacheable. + * + * A CDN decides what to cache largely by URL, not by what the endpoint is: append a static + * extension to an authenticated URL and Cloudflare will happily store the 200 for hours and + * serve it to anonymous clients. That is how a pentest kept reading pre-hardening Actuator + * data ("cache ghosts") long after the origin started returning 401. Explicit + * {@code no-store} removes the ambiguity at the source, so the edge has nothing to keep + * whatever the path looks like. + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 5) +public class NoStoreFilter extends OncePerRequestFilter { + + private static final String[] PROTECTED_PREFIXES = {"/admin", "/v1", "/actuator"}; + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + String path = request.getRequestURI(); + for (String prefix : PROTECTED_PREFIXES) { + if (path.startsWith(prefix)) { + return false; + } + } + return true; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + // Set before the chain runs so it survives error and 401/429 paths too. + response.setHeader("Cache-Control", "no-store, no-cache, must-revalidate, private"); + response.setHeader("Pragma", "no-cache"); + chain.doFilter(request, response); + } +} diff --git a/src/main/java/com/anindra/path/config/PayloadTooLargeException.java b/path-gateway/src/main/java/com/anindra/path/config/PayloadTooLargeException.java similarity index 100% rename from src/main/java/com/anindra/path/config/PayloadTooLargeException.java rename to path-gateway/src/main/java/com/anindra/path/config/PayloadTooLargeException.java diff --git a/src/main/java/com/anindra/path/config/RateLimitConfig.java b/path-gateway/src/main/java/com/anindra/path/config/RateLimitConfig.java similarity index 100% rename from src/main/java/com/anindra/path/config/RateLimitConfig.java rename to path-gateway/src/main/java/com/anindra/path/config/RateLimitConfig.java diff --git a/src/main/java/com/anindra/path/config/RequestIdFilter.java b/path-gateway/src/main/java/com/anindra/path/config/RequestIdFilter.java similarity index 100% rename from src/main/java/com/anindra/path/config/RequestIdFilter.java rename to path-gateway/src/main/java/com/anindra/path/config/RequestIdFilter.java diff --git a/src/main/java/com/anindra/path/config/RequestSizeFilter.java b/path-gateway/src/main/java/com/anindra/path/config/RequestSizeFilter.java similarity index 100% rename from src/main/java/com/anindra/path/config/RequestSizeFilter.java rename to path-gateway/src/main/java/com/anindra/path/config/RequestSizeFilter.java diff --git a/path-gateway/src/main/java/com/anindra/path/config/SharedStateConfig.java b/path-gateway/src/main/java/com/anindra/path/config/SharedStateConfig.java new file mode 100644 index 0000000..5de6a0c --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/config/SharedStateConfig.java @@ -0,0 +1,52 @@ +package com.anindra.path.config; + +import com.anindra.path.gateway.CircuitBreakerStore; +import com.anindra.path.gateway.ConcurrencyStore; +import com.anindra.path.gateway.InMemoryCircuitBreakerStore; +import com.anindra.path.gateway.InMemoryConcurrencyStore; +import com.anindra.path.gateway.ValkeyCircuitBreakerStore; +import com.anindra.path.gateway.ValkeyConcurrencyStore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * Chooses the backend for circuit-breaker and concurrency state. + * + *

Memory is the default, deliberately: putting Valkey on the hot path must be an + * operator's explicit choice. Setting {@code GATEWAY_SHARED_STATE_BACKEND=valkey} reuses the + * existing Redis connection and key prefix conventions used by the rate limiter. + */ +@Configuration +public class SharedStateConfig { + + @Bean + @ConditionalOnProperty(name = "gateway.shared-state.backend", havingValue = "memory", + matchIfMissing = true) + public CircuitBreakerStore inMemoryCircuitBreakerStore() { + return new InMemoryCircuitBreakerStore(); + } + + @Bean + @ConditionalOnProperty(name = "gateway.shared-state.backend", havingValue = "valkey") + public CircuitBreakerStore valkeyCircuitBreakerStore(StringRedisTemplate redis, + GatewayProperties props) { + return new ValkeyCircuitBreakerStore(redis, props.getSharedState().getPrefix()); + } + + @Bean + @ConditionalOnProperty(name = "gateway.shared-state.backend", havingValue = "memory", + matchIfMissing = true) + public ConcurrencyStore inMemoryConcurrencyStore() { + return new InMemoryConcurrencyStore(); + } + + @Bean + @ConditionalOnProperty(name = "gateway.shared-state.backend", havingValue = "valkey") + public ConcurrencyStore valkeyConcurrencyStore(StringRedisTemplate redis, + GatewayProperties props) { + return new ValkeyConcurrencyStore(redis, props.getSharedState().getPrefix(), + props.getSharedState().getConcurrencyLease()); + } +} diff --git a/src/main/java/com/anindra/path/config/WebConfig.java b/path-gateway/src/main/java/com/anindra/path/config/WebConfig.java similarity index 92% rename from src/main/java/com/anindra/path/config/WebConfig.java rename to path-gateway/src/main/java/com/anindra/path/config/WebConfig.java index 6e1736e..155440c 100644 --- a/src/main/java/com/anindra/path/config/WebConfig.java +++ b/path-gateway/src/main/java/com/anindra/path/config/WebConfig.java @@ -18,7 +18,7 @@ public void addCorsMappings(CorsRegistry registry) { : props.getCorsOrigins().split("\\s*,\\s*"); registry.addMapping("/**") .allowedOriginPatterns(origins) - .allowedMethods("GET", "POST", "PATCH", "DELETE", "OPTIONS") + .allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS") .allowedHeaders("*") .exposedHeaders("X-Gateway-Provider", "X-Gateway-Upstream-Model", "X-Gateway-Cost-Usd", "X-Gateway-Latency-Ms", "Retry-After") diff --git a/src/main/java/com/anindra/path/domain/AdminAudit.java b/path-gateway/src/main/java/com/anindra/path/domain/AdminAudit.java similarity index 100% rename from src/main/java/com/anindra/path/domain/AdminAudit.java rename to path-gateway/src/main/java/com/anindra/path/domain/AdminAudit.java diff --git a/src/main/java/com/anindra/path/domain/AdminToken.java b/path-gateway/src/main/java/com/anindra/path/domain/AdminToken.java similarity index 100% rename from src/main/java/com/anindra/path/domain/AdminToken.java rename to path-gateway/src/main/java/com/anindra/path/domain/AdminToken.java diff --git a/src/main/java/com/anindra/path/domain/ClientKey.java b/path-gateway/src/main/java/com/anindra/path/domain/ClientKey.java similarity index 100% rename from src/main/java/com/anindra/path/domain/ClientKey.java rename to path-gateway/src/main/java/com/anindra/path/domain/ClientKey.java diff --git a/src/main/java/com/anindra/path/domain/EndUserReservation.java b/path-gateway/src/main/java/com/anindra/path/domain/EndUserReservation.java similarity index 100% rename from src/main/java/com/anindra/path/domain/EndUserReservation.java rename to path-gateway/src/main/java/com/anindra/path/domain/EndUserReservation.java diff --git a/src/main/java/com/anindra/path/domain/IdempotencyRecord.java b/path-gateway/src/main/java/com/anindra/path/domain/IdempotencyRecord.java similarity index 100% rename from src/main/java/com/anindra/path/domain/IdempotencyRecord.java rename to path-gateway/src/main/java/com/anindra/path/domain/IdempotencyRecord.java diff --git a/path-gateway/src/main/java/com/anindra/path/domain/MailSettings.java b/path-gateway/src/main/java/com/anindra/path/domain/MailSettings.java new file mode 100644 index 0000000..88e38e6 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/domain/MailSettings.java @@ -0,0 +1,43 @@ +package com.anindra.path.domain; + +import jakarta.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +/** Singleton row: how alert email is delivered. */ +@Entity +@Table(name = "mail_settings") +@Getter +@Setter +@NoArgsConstructor +public class MailSettings { + + /** Always 1: there is one gateway, so there is one mail configuration. */ + public static final Long SINGLETON = 1L; + + @Id + private Long id = SINGLETON; + + private boolean enabled; + + private String host; + + private Integer port; + + private String username; + + /** AES-GCM ciphertext; never returned by the API. */ + @Column(length = 1024) + private String passwordCipher; + + private String fromAddress; + + /** Comma separated recipients. */ + @Column(length = 1024) + private String toAddress; + + private Instant updatedAt; +} diff --git a/src/main/java/com/anindra/path/domain/ModelAlias.java b/path-gateway/src/main/java/com/anindra/path/domain/ModelAlias.java similarity index 100% rename from src/main/java/com/anindra/path/domain/ModelAlias.java rename to path-gateway/src/main/java/com/anindra/path/domain/ModelAlias.java diff --git a/src/main/java/com/anindra/path/domain/ModelRoute.java b/path-gateway/src/main/java/com/anindra/path/domain/ModelRoute.java similarity index 100% rename from src/main/java/com/anindra/path/domain/ModelRoute.java rename to path-gateway/src/main/java/com/anindra/path/domain/ModelRoute.java diff --git a/src/main/java/com/anindra/path/domain/Project.java b/path-gateway/src/main/java/com/anindra/path/domain/Project.java similarity index 100% rename from src/main/java/com/anindra/path/domain/Project.java rename to path-gateway/src/main/java/com/anindra/path/domain/Project.java diff --git a/src/main/java/com/anindra/path/domain/PromptLog.java b/path-gateway/src/main/java/com/anindra/path/domain/PromptLog.java similarity index 100% rename from src/main/java/com/anindra/path/domain/PromptLog.java rename to path-gateway/src/main/java/com/anindra/path/domain/PromptLog.java diff --git a/src/main/java/com/anindra/path/domain/Provider.java b/path-gateway/src/main/java/com/anindra/path/domain/Provider.java similarity index 87% rename from src/main/java/com/anindra/path/domain/Provider.java rename to path-gateway/src/main/java/com/anindra/path/domain/Provider.java index 384b5ac..7cc8e41 100644 --- a/src/main/java/com/anindra/path/domain/Provider.java +++ b/path-gateway/src/main/java/com/anindra/path/domain/Provider.java @@ -44,9 +44,17 @@ public class Provider { private String apiVersion; /** AES-GCM encrypted upstream credential. Null falls back to the environment. */ - @Column(length = 2048) + @Column(length = 4096) private String apiKeyCipher; + /** Vertex only: the GCP project that owns the model. */ + @Column(length = 128) + private String project; + + /** Vertex only: the region, e.g. us-central1. */ + @Column(length = 64) + private String location; + @Column(nullable = false) private boolean enabled = true; diff --git a/src/main/java/com/anindra/path/domain/ProviderType.java b/path-gateway/src/main/java/com/anindra/path/domain/ProviderType.java similarity index 51% rename from src/main/java/com/anindra/path/domain/ProviderType.java rename to path-gateway/src/main/java/com/anindra/path/domain/ProviderType.java index e7cc2c6..99c4d8a 100644 --- a/src/main/java/com/anindra/path/domain/ProviderType.java +++ b/path-gateway/src/main/java/com/anindra/path/domain/ProviderType.java @@ -4,5 +4,9 @@ public enum ProviderType { /** OpenAI chat-completions wire format (OpenAI, Groq, Together, vLLM, ...). */ OPENAI, /** Anthropic Messages API wire format. */ - ANTHROPIC + ANTHROPIC, + /** Google Gemini native generateContent (AI Studio). */ + GEMINI, + /** Vertex AI generateContent; credentials are a service-account JSON, not an API key. */ + VERTEX } diff --git a/src/main/java/com/anindra/path/domain/ResponseCacheEntry.java b/path-gateway/src/main/java/com/anindra/path/domain/ResponseCacheEntry.java similarity index 100% rename from src/main/java/com/anindra/path/domain/ResponseCacheEntry.java rename to path-gateway/src/main/java/com/anindra/path/domain/ResponseCacheEntry.java diff --git a/src/main/java/com/anindra/path/domain/UsageRecord.java b/path-gateway/src/main/java/com/anindra/path/domain/UsageRecord.java similarity index 95% rename from src/main/java/com/anindra/path/domain/UsageRecord.java rename to path-gateway/src/main/java/com/anindra/path/domain/UsageRecord.java index fed4732..80524b2 100644 --- a/src/main/java/com/anindra/path/domain/UsageRecord.java +++ b/path-gateway/src/main/java/com/anindra/path/domain/UsageRecord.java @@ -78,6 +78,10 @@ public enum Status {SUCCESS, ERROR} @Column(precision = 18, scale = 8) private BigDecimal cachedSavingsUsd = BigDecimal.ZERO; + /** Cache tier that served the call: "exact" or "semantic"; null when it was not a hit. */ + @Column(length = 16) + private String cacheTier; + @Column(precision = 18, scale = 8) private BigDecimal costUsd = BigDecimal.ZERO; diff --git a/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java b/path-gateway/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/AnthropicAdapter.java rename to path-gateway/src/main/java/com/anindra/path/gateway/AnthropicAdapter.java diff --git a/src/main/java/com/anindra/path/gateway/ApiKeyService.java b/path-gateway/src/main/java/com/anindra/path/gateway/ApiKeyService.java similarity index 84% rename from src/main/java/com/anindra/path/gateway/ApiKeyService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ApiKeyService.java index 160de96..fd734a3 100644 --- a/src/main/java/com/anindra/path/gateway/ApiKeyService.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ApiKeyService.java @@ -62,17 +62,27 @@ public Optional findByRawKey(String raw) { /** Resolve a bearer token, enforcing enabled/revoked state. */ public ClientKey authenticate(String raw) { - if (raw == null || raw.isBlank()) { - throw ApiException.unauthorized("Missing API key. Send it as 'Authorization: Bearer sk-gw-...'.", "missing_api_key"); - } - ClientKey key = findByRawKey(raw.trim()) - .orElseThrow(() -> ApiException.unauthorized("Invalid API key.", "invalid_api_key")); + // One error for every rejection. Distinct codes for missing/invalid/disabled turned + // this into a key-validity oracle: an attacker could tell a well-formed-but-unknown + // key from a revoked one, and 193k guesses were evaluated against it at ~474/s. + // + // The lookup runs even for a missing key, so the work is the same shape whatever was + // sent. Skipping it made "no header" measurably faster than "wrong key", which is a + // timing oracle of its own even though the bodies are now identical. + ClientKey key = findByRawKey(raw == null ? "" : raw.trim()) + .orElseThrow(ApiKeyService::invalidKey); if (!key.isEnabled() || key.getRevokedAt() != null) { - throw ApiException.unauthorized("This API key has been disabled.", "key_disabled"); + throw invalidKey(); } return key; } + private static ApiException invalidKey() { + return ApiException.unauthorized( + "Invalid or missing API key. Send a current key as 'Authorization: Bearer sk-gw-...'.", + "invalid_api_key"); + } + /** Deduct spend with a row lock so parallel requests cannot lose updates. */ @Transactional(propagation = Propagation.REQUIRES_NEW) public void charge(Long keyId, BigDecimal costUsd) { diff --git a/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java b/path-gateway/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ChatRequestSpec.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ChatRequestSpec.java diff --git a/src/main/java/com/anindra/path/gateway/ChatService.java b/path-gateway/src/main/java/com/anindra/path/gateway/ChatService.java similarity index 98% rename from src/main/java/com/anindra/path/gateway/ChatService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ChatService.java index 1de73d3..ee63847 100644 --- a/src/main/java/com/anindra/path/gateway/ChatService.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ChatService.java @@ -80,7 +80,7 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { var hit = cache.lookup(cacheKey); if (hit.isPresent()) { ResponseCacheService.Entry entry = hit.get(); - usage.recordCacheHit(key, spec, chain.get(0), entry, false); + usage.recordCacheHit(key, spec, chain.get(0), entry, false, "exact"); try { return new Result(mapper.readTree(entry.body()), entry.providerSlug(), chain.get(0).route().getUpstreamModelId(), BigDecimal.ZERO, @@ -98,7 +98,7 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { var near = semantic.lookup(key.getId(), spec.publicModel(), promptVector, threshold); if (near.isPresent()) { ResponseCacheService.Entry entry = entryOf(near.get().entry()); - usage.recordCacheHit(key, spec, chain.get(0), entry, false); + usage.recordCacheHit(key, spec, chain.get(0), entry, false, "semantic"); try { return new Result(mapper.readTree(entry.body()), entry.providerSlug(), chain.get(0).route().getUpstreamModelId(), BigDecimal.ZERO, @@ -110,7 +110,7 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { } } - long available = chain.stream().filter(r -> breaker.isAvailable(r.provider().getId())).count(); + long available = chain.stream().filter(r -> !breaker.isParked(r.provider().getId())).count(); int maxAttempts = Math.min(chain.size(), Math.max(1, props.getBreaker().getMaxAttempts())); for (int i = 0; i < chain.size() && attempts < maxAttempts; i++) { @@ -163,7 +163,7 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { lastError = toApiException(failure); long latency = elapsedMs(started); usage.recordError(key, spec, rr.providerSlug(), rr.route().getUpstreamModelId(), - latency, false, attempts, lastError.code(), lastError.getMessage()); + latency, false, attempts, lastError.status(), lastError.code(), lastError.getMessage()); throw lastError; } @@ -222,7 +222,7 @@ public Result complete(ChatRequestSpec spec, ClientKey key) { lastError = new ApiException(lastError.status(), lastError.type(), lastError.code(), lastError.getMessage() + " (routes tried: " + String.join("; ", failures) + ")"); } - usage.recordError(key, spec, null, null, elapsedMs(started), false, attempts, + usage.recordError(key, spec, null, null, elapsedMs(started), false, attempts, lastError.status(), lastError.code(), lastError.getMessage()); throw lastError; } finally { diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/CircuitBreakerStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/CircuitBreakerStore.java new file mode 100644 index 0000000..77298ff --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/CircuitBreakerStore.java @@ -0,0 +1,47 @@ +package com.anindra.path.gateway; + +import java.util.Map; + +/** + * Where a provider's circuit state lives. + * + *

Memory is the default and is exactly the v1 behaviour: state is per JVM and a restart + * clears it. Valkey makes the state shared, so parking a provider on one replica parks it + * on the others within one probe interval, and an OPEN entry expires on its own even if the + * replica that opened it dies. + * + *

All mutating calls must be atomic: the whole point is that exactly one replica wins the + * OPEN → HALF_OPEN probe. + */ +public interface CircuitBreakerStore { + + enum State {CLOSED, OPEN, HALF_OPEN} + + record Snapshot(State state, int failures, long openUntilMs) { + public static Snapshot closed() { + return new Snapshot(State.CLOSED, 0, 0); + } + } + + /** The current state; a provider that was never seen reads CLOSED. */ + Snapshot load(Long providerId); + + /** One more failure. Returns the new state when this failure opened the circuit, else null. */ + Snapshot recordFailure(Long providerId, int threshold, long nowMs, long openDurationMs); + + /** Opens the circuit unconditionally (an upstream 429 with Retry-After). */ + void open(Long providerId, int failures, long openUntilMs, long ttlMs); + + /** Claims the single HALF_OPEN probe when the OPEN window has elapsed. */ + boolean tryProbe(Long providerId, long nowMs, long probeLeaseMs); + + /** The provider answered: back to CLOSED. */ + void close(Long providerId); + + /** Every provider this store has state for, for the admin views. */ + Map snapshot(); + + /** Drops state that has been idle; a no-op for a store that expires keys itself. */ + default void evictIdle() { + } +} diff --git a/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java b/path-gateway/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java similarity index 70% rename from src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java index f66cb5a..97c11a9 100644 --- a/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ConcurrencyLimiter.java @@ -1,25 +1,33 @@ package com.anindra.path.gateway; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Semaphore; - /** * Caps concurrent in-flight requests per client key. * * The rate limiter bounds requests per minute; this bounds requests at an instant, which * is what stops one key from opening thousands of streams and starving the pool. + * + *

The counting lives in a {@link ConcurrencyStore}: semaphores in this JVM by default, + * a leased sorted set in Valkey when {@code gateway.shared-state.backend=valkey}. With the + * shared store, a killed pod's permits expire instead of holding a key forever. */ @Component public class ConcurrencyLimiter { - private record Entry(int limit, Semaphore semaphore) { + private final ConcurrencyStore store; + + /** Kept so a unit test (or a non-Spring caller) gets the old behaviour. */ + public ConcurrencyLimiter() { + this(new InMemoryConcurrencyStore()); } - private final Map entries = new ConcurrentHashMap<>(); + @Autowired + public ConcurrencyLimiter(ConcurrencyStore store) { + this.store = store; + } public static String keyBucket(Long keyId) { return "key:" + keyId; @@ -65,14 +73,7 @@ public boolean tryAcquire(Long keyId, int limit) { /** @return true when a permit was taken; callers must release the same bucket. */ public boolean tryAcquire(String bucket, int limit) { - if (limit <= 0) { - return true; - } - Entry entry = entries.compute(bucket, (id, current) -> - current == null || current.limit() != limit - ? new Entry(limit, new Semaphore(limit)) - : current); - return entry.semaphore().tryAcquire(); + return store.tryAcquire(bucket, limit); } public void release(Long keyId) { @@ -80,10 +81,7 @@ public void release(Long keyId) { } public void release(String bucket) { - Entry entry = entries.get(bucket); - if (entry != null && entry.semaphore().availablePermits() < entry.limit()) { - entry.semaphore().release(); - } + store.release(bucket); } public int inFlight(Long keyId) { @@ -91,13 +89,11 @@ public int inFlight(Long keyId) { } public int inFlight(String bucket) { - Entry entry = entries.get(bucket); - return entry == null ? 0 : entry.limit() - entry.semaphore().availablePermits(); + return store.inFlight(bucket); } @Scheduled(fixedDelay = 600_000) void evictIdle() { - entries.entrySet().removeIf(e -> - e.getValue().semaphore().availablePermits() == e.getValue().limit()); + store.evictIdle(); } } diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/ConcurrencyStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/ConcurrencyStore.java new file mode 100644 index 0000000..cff128b --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ConcurrencyStore.java @@ -0,0 +1,22 @@ +package com.anindra.path.gateway; + +/** + * Where per-bucket in-flight permits live. + * + *

Memory is the default: a semaphore per bucket, per JVM, so a cluster's effective cap is + * {@code replicas × limit}. Valkey shares the count and gives every permit a lease, so a + * pod that is killed mid-request does not hold a permit forever. + */ +public interface ConcurrencyStore { + + /** @return true when a permit was taken; callers must release the same bucket. */ + boolean tryAcquire(String bucket, int limit); + + void release(String bucket); + + int inFlight(String bucket); + + /** Drops idle bookkeeping; a no-op for a store that expires entries itself. */ + default void evictIdle() { + } +} diff --git a/src/main/java/com/anindra/path/gateway/CreditGuard.java b/path-gateway/src/main/java/com/anindra/path/gateway/CreditGuard.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/CreditGuard.java rename to path-gateway/src/main/java/com/anindra/path/gateway/CreditGuard.java diff --git a/src/main/java/com/anindra/path/gateway/DrainState.java b/path-gateway/src/main/java/com/anindra/path/gateway/DrainState.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/DrainState.java rename to path-gateway/src/main/java/com/anindra/path/gateway/DrainState.java diff --git a/src/main/java/com/anindra/path/gateway/EmbeddingsService.java b/path-gateway/src/main/java/com/anindra/path/gateway/EmbeddingsService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/EmbeddingsService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/EmbeddingsService.java diff --git a/src/main/java/com/anindra/path/gateway/EndUserBudget.java b/path-gateway/src/main/java/com/anindra/path/gateway/EndUserBudget.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/EndUserBudget.java rename to path-gateway/src/main/java/com/anindra/path/gateway/EndUserBudget.java diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/GeminiAdapter.java b/path-gateway/src/main/java/com/anindra/path/gateway/GeminiAdapter.java new file mode 100644 index 0000000..4700f9c --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/GeminiAdapter.java @@ -0,0 +1,557 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * Gemini native {@code generateContent}, for AI Studio and Vertex AI. + * + *

The OpenAI-compatible Gemini endpoint covers chat and streaming but silently drops the + * parts enterprises buy Gemini for: {@code thinkingConfig}, {@code safetySettings}, + * {@code systemInstruction} semantics and cached-content pricing. This adapter translates + * them properly instead, and rejects an OpenAI parameter it cannot honour with 422 + * {@code unsupported_parameter} rather than dropping it. + * + *

The two surfaces share the wire format; only the URL and auth differ. AI Studio uses + * {@code x-goog-api-key}; Vertex uses a service-account access token and puts project and + * location in the path. + */ +public class GeminiAdapter implements ProviderAdapter { + + /** + * OpenAI parameters with no Gemini equivalent. Dropping them would change behaviour, so + * they fail loudly, the same contract the Anthropic adapter keeps. + */ + private static final List UNSUPPORTED = List.of( + "logit_bias", "frequency_penalty", "presence_penalty", "seed", + "logprobs", "top_logprobs", "modalities", "audio", "prediction", "user"); + + private final ObjectMapper mapper; + private final ProviderType type; + private final VertexTokenService vertexTokens; + + public GeminiAdapter(ObjectMapper mapper, ProviderType type, VertexTokenService vertexTokens) { + this.mapper = mapper; + this.type = type; + this.vertexTokens = vertexTokens; + } + + @Override + public ProviderType type() { + return type; + } + + private boolean vertex() { + return type == ProviderType.VERTEX; + } + + // ---------------------------------------------------------------- request + + @Override + public UpstreamRequest build(ChatRequestSpec spec, ModelRoute route, Provider provider, + String apiKey, boolean stream) { + JsonNode body = spec.body(); + validate(body); + + ObjectNode out = mapper.createObjectNode(); + String system = spec.systemText(); + if (system != null && !system.isBlank()) { + out.putObject("systemInstruction").putArray("parts").addObject().put("text", system); + } + out.set("contents", contents(body.path("messages"))); + + ObjectNode generation = out.putObject("generationConfig"); + copyNumber(body, generation, "temperature", "temperature"); + copyNumber(body, generation, "top_p", "topP"); + copyNumber(body, generation, "top_k", "topK"); + Integer max = spec.maxTokens(); + if (max != null) { + generation.put("maxOutputTokens", max); + } + JsonNode stop = body.get("stop"); + if (stop != null) { + ArrayNode sequences = generation.putArray("stopSequences"); + if (stop.isTextual()) { + sequences.add(stop.asText()); + } else if (stop.isArray()) { + stop.forEach(value -> { + if (value.isTextual()) { + sequences.add(value.asText()); + } + }); + } + } + JsonNode responseFormat = body.get("response_format"); + if (responseFormat != null && "json_object".equals(responseFormat.path("type").asText())) { + generation.put("responseMimeType", "application/json"); + } + thinkingConfig(body, generation); + safetySettings(body, out); + tools(body, out); + + Map headers = new LinkedHashMap<>(); + headers.put("Content-Type", "application/json"); + String url = url(provider, route.getUpstreamModelId(), stream, apiKey, headers); + return new UpstreamRequest(url, headers, out.toString()); + } + + private String url(Provider provider, String upstreamModel, boolean stream, + String credential, Map headers) { + String method = stream ? "streamGenerateContent?alt=sse" : "generateContent"; + String base = trimSlash(provider.getBaseUrl()); + if (vertex()) { + String token = vertexTokens.accessToken(provider, credential); + headers.put("Authorization", "Bearer " + token); + if (provider.getProject() == null || provider.getLocation() == null) { + throw new UpstreamException(400, "vertex_project_missing", + "A Vertex provider needs a project and a location."); + } + return base + "/v1/projects/" + provider.getProject() + + "/locations/" + provider.getLocation() + + "/publishers/google/models/" + upstreamModel + ":" + method; + } + headers.put("x-goog-api-key", credential); + return base + "/models/" + upstreamModel + ":" + method; + } + + private void validate(JsonNode body) { + for (String parameter : UNSUPPORTED) { + if (body.hasNonNull(parameter)) { + throw unsupported(parameter); + } + } + if (body.hasNonNull("n") && body.get("n").asInt(1) != 1) { + throw unsupported("n"); + } + JsonNode format = body.get("response_format"); + if (format != null && !"text".equals(format.path("type").asText("text")) + && !"json_object".equals(format.path("type").asText())) { + throw unsupported("response_format"); + } + } + + private static ApiException unsupported(String parameter) { + return new ApiException(422, "invalid_request_error", "unsupported_parameter", + "Parameter '" + parameter + "' is not supported by the Gemini wire format.", + parameter); + } + + /** OpenAI messages to Gemini contents, with tool calls and tool responses. */ + private ArrayNode contents(JsonNode messages) { + ArrayNode contents = mapper.createArrayNode(); + Map toolNamesById = new HashMap<>(); + for (JsonNode message : messages) { + String role = message.path("role").asText("user"); + if ("system".equals(role)) { + continue; // spec.systemText() already collected these + } + if ("tool".equals(role)) { + ObjectNode part = mapper.createObjectNode(); + ObjectNode response = part.putObject("functionResponse"); + response.put("name", toolNamesById.getOrDefault( + message.path("tool_call_id").asText(), "tool")); + response.putObject("response").put("result", message.path("content").asText()); + append(contents, "user", part); + continue; + } + if ("assistant".equals(role)) { + ObjectNode turn = mapper.createObjectNode(); + turn.put("role", "model"); + ArrayNode parts = turn.putArray("parts"); + String text = textOf(message.get("content")); + if (text != null && !text.isEmpty()) { + parts.addObject().put("text", text); + } + for (JsonNode call : message.path("tool_calls")) { + JsonNode function = call.path("function"); + toolNamesById.put(call.path("id").asText(), function.path("name").asText()); + ObjectNode part = parts.addObject(); + ObjectNode invocation = part.putObject("functionCall"); + invocation.put("name", function.path("name").asText()); + invocation.set("args", parseArguments(function.path("arguments").asText())); + } + if (!parts.isEmpty()) { + contents.add(turn); + } + continue; + } + ObjectNode part = mapper.createObjectNode(); + part.put("text", String.valueOf(textOf(message.get("content")))); + append(contents, "user", part); + } + return contents; + } + + private void append(ArrayNode contents, String role, ObjectNode part) { + if (!contents.isEmpty() + && role.equals(contents.get(contents.size() - 1).path("role").asText())) { + ((ArrayNode) contents.get(contents.size() - 1).path("parts")).add(part); + return; + } + ObjectNode turn = contents.addObject(); + turn.put("role", role); + turn.putArray("parts").add(part); + } + + /** OpenAI content is a string or an array of text parts; images are not translated. */ + private String textOf(JsonNode content) { + if (content == null || content.isNull()) { + return null; + } + if (content.isTextual()) { + return content.asText(); + } + StringBuilder text = new StringBuilder(); + if (content.isArray()) { + for (JsonNode part : content) { + if ("text".equals(part.path("type").asText())) { + text.append(part.path("text").asText()); + } else if (part.hasNonNull("image_url")) { + throw new ApiException(422, "invalid_request_error", "unsupported_parameter", + "Image content is not translated by the Gemini adapter yet.", + "messages"); + } + } + } + return text.toString(); + } + + private JsonNode parseArguments(String arguments) { + try { + return mapper.readTree(arguments == null || arguments.isBlank() ? "{}" : arguments); + } catch (Exception e) { + return mapper.createObjectNode(); + } + } + + private void thinkingConfig(JsonNode body, ObjectNode generation) { + JsonNode config = body.get("thinkingConfig"); + if (config == null) { + config = body.get("thinking_config"); + } + ObjectNode out = generation.putObject("thinkingConfig"); + if (config != null && config.isObject()) { + Long budget = budget(config.get("thinkingBudget"), config.get("thinking_budget")); + if (budget != null) { + out.put("thinkingBudget", budget); + } + JsonNode thoughts = config.has("includeThoughts") + ? config.get("includeThoughts") : config.get("include_thoughts"); + if (thoughts != null && thoughts.isBoolean()) { + out.put("includeThoughts", thoughts.asBoolean()); + } + if (out.isEmpty()) { + generation.remove("thinkingConfig"); + } + return; + } + // reasoning_effort is the OpenAI-shaped knob; map it to a budget. + String effort = body.path("reasoning_effort").asText(null); + if (effort != null) { + out.put("thinkingBudget", switch (effort) { + case "minimal" -> 128; + case "low" -> 512; + case "medium" -> 2048; + case "high" -> 8192; + default -> throw unsupported("reasoning_effort"); + }); + return; + } + generation.remove("thinkingConfig"); + } + + private static Long budget(JsonNode camel, JsonNode snake) { + JsonNode node = camel != null ? camel : snake; + return node == null || !node.isNumber() ? null : node.asLong(); + } + + private void safetySettings(JsonNode body, ObjectNode out) { + JsonNode settings = body.has("safetySettings") ? body.get("safetySettings") + : body.get("safety_settings"); + if (settings == null) { + return; + } + if (!settings.isArray()) { + throw new ApiException(422, "invalid_request_error", "unsupported_parameter", + "safetySettings must be an array of {category, threshold}.", "safetySettings"); + } + for (JsonNode setting : settings) { + String category = setting.path("category").asText(null); + String threshold = setting.path("threshold").asText(null); + if (category == null || threshold == null) { + throw new ApiException(422, "invalid_request_error", "unsupported_parameter", + "Each safetySettings entry needs a category and a threshold.", + "safetySettings"); + } + } + out.set("safetySettings", settings); + } + + private void tools(JsonNode body, ObjectNode out) { + JsonNode tools = body.get("tools"); + if (tools == null || !tools.isArray() || tools.isEmpty()) { + return; + } + ArrayNode declarations = out.putArray("tools").addObject().putArray("functionDeclarations"); + for (JsonNode tool : tools) { + JsonNode function = tool.path("function"); + ObjectNode declaration = declarations.addObject(); + declaration.put("name", function.path("name").asText()); + if (function.hasNonNull("description")) { + declaration.put("description", function.path("description").asText()); + } + declaration.set("parameters", function.path("parameters").isObject() + ? function.get("parameters") : mapper.createObjectNode().put("type", "object")); + } + } + + private static void copyNumber(JsonNode from, ObjectNode to, String source, String target) { + if (from.hasNonNull(source) && from.get(source).isNumber()) { + to.set(target, from.get(source)); + } + } + + // --------------------------------------------------------------- response + + @Override + public JsonNode parseFullResponse(JsonNode upstream, ChatRequestSpec spec, ModelRoute route) { + ObjectNode out = mapper.createObjectNode(); + out.put("id", "chatcmpl-" + UUID.randomUUID().toString().replace("-", "")); + out.put("object", "chat.completion"); + out.put("created", Instant.now().getEpochSecond()); + out.put("model", spec.publicModel()); + + JsonNode promptFeedback = upstream.path("promptFeedback"); + if (promptFeedback.hasNonNull("blockReason")) { + throw blocked(promptFeedback.path("blockReason").asText(), promptFeedback.path("safetyRatings")); + } + + JsonNode candidate = upstream.path("candidates").path(0); + if (candidate.isMissingNode() || candidate.isNull()) { + throw new UpstreamException(400, "content_blocked", + "Gemini returned no candidate and no content."); + } + String finishReason = mapFinish(candidate.path("finishReason").asText(null)); + JsonNode parts = candidate.path("content").path("parts"); + StringBuilder text = new StringBuilder(); + ArrayNode toolCalls = mapper.createArrayNode(); + for (JsonNode part : parts) { + if (part.path("thought").asBoolean(false)) { + continue; // reasoning stays in the token counts, not in the answer + } + if (part.hasNonNull("text")) { + text.append(part.get("text").asText()); + } + if (part.has("functionCall")) { + JsonNode call = part.get("functionCall"); + ObjectNode entry = toolCalls.addObject(); + entry.put("id", "call_" + UUID.randomUUID().toString().replace("-", "").substring(0, 12)); + entry.put("type", "function"); + ObjectNode function = entry.putObject("function"); + function.put("name", call.path("name").asText()); + function.put("arguments", call.path("args").toString()); + } + } + if ("content_filter".equals(finishReason) && text.isEmpty() && toolCalls.isEmpty()) { + throw blocked(finishReason, candidate.path("safetyRatings")); + } + + ObjectNode choice = out.putArray("choices").addObject(); + choice.put("index", 0); + ObjectNode message = choice.putObject("message"); + message.put("role", "assistant"); + message.put("content", text.isEmpty() && !toolCalls.isEmpty() ? null : text.toString()); + if (!toolCalls.isEmpty()) { + message.set("tool_calls", toolCalls); + } + choice.put("finish_reason", finishReason); + + out.set("usage", usageNode(upstream.path("usageMetadata"), tokenUsage(upstream))); + return out; + } + + private ApiException blocked(String reason, JsonNode ratings) { + String category = reason; + for (JsonNode rating : ratings) { + if (rating.path("blocked").asBoolean(false) + || "HIGH".equals(rating.path("probability").asText())) { + category = rating.path("category").asText(reason); + break; + } + } + return new ApiException(400, "invalid_request_error", "content_blocked", + "Gemini blocked this request (" + category + ").", "safetySettings"); + } + + @Override + public TokenUsage parseUsage(JsonNode upstream) { + return tokenUsage(upstream); + } + + /** + * Gemini counts thinking separately in {@code thoughtsTokenCount}, but bills it as + * output. Completion therefore includes it; {@code reasoningTokens} keeps the breakdown. + */ + private static TokenUsage tokenUsage(JsonNode upstream) { + JsonNode usage = upstream.path("usageMetadata"); + if (!usage.isObject()) { + return TokenUsage.EMPTY; + } + int prompt = usage.path("promptTokenCount").asInt(0); + int candidates = usage.path("candidatesTokenCount").asInt(0); + int thoughts = usage.path("thoughtsTokenCount").asInt(0); + int cached = usage.path("cachedContentTokenCount").asInt(0); + return new TokenUsage(prompt, candidates + thoughts, cached, 0, thoughts, 0, 0, false); + } + + private ObjectNode usageNode(JsonNode usage, TokenUsage tokens) { + ObjectNode out = mapper.createObjectNode(); + if (!tokens.reported()) { + return out; + } + out.put("prompt_tokens", tokens.prompt()); + out.put("completion_tokens", tokens.completion()); + out.put("total_tokens", tokens.total()); + out.putObject("prompt_tokens_details").put("cached_tokens", tokens.cached()); + out.putObject("completion_tokens_details").put("reasoning_tokens", tokens.reasoning()); + return out; + } + + private static String mapFinish(String finish) { + if (finish == null) { + return "stop"; + } + return switch (finish) { + case "STOP" -> "stop"; + case "MAX_TOKENS" -> "length"; + case "SAFETY", "RECITATION", "BLOCKLIST", "PROHIBITED_CONTENT", "SPII" -> "content_filter"; + default -> "stop"; + }; + } + + @Override + public StreamTranslator newStreamTranslator(ChatRequestSpec spec, ModelRoute route) { + return new Translator(spec); + } + + @Override + public BigDecimal priceInput(ModelRoute route) { + return route.getInputPricePerMillion() == null ? BigDecimal.ZERO : route.getInputPricePerMillion(); + } + + @Override + public BigDecimal priceOutput(ModelRoute route) { + return route.getOutputPricePerMillion() == null ? BigDecimal.ZERO : route.getOutputPricePerMillion(); + } + + private static String trimSlash(String value) { + return value.endsWith("/") ? value.substring(0, value.length() - 1) : value; + } + + /** Gemini SSE events to OpenAI chunks. Thought parts are emitted as content, not hidden. */ + private final class Translator implements StreamTranslator { + + private final ChatRequestSpec spec; + private final long created = Instant.now().getEpochSecond(); + private TokenUsage usage = TokenUsage.EMPTY; + private boolean done; + private boolean finishSent; + + private Translator(ChatRequestSpec spec) { + this.spec = spec; + } + + @Override + public List translate(String line) { + if (line == null || line.isBlank() || line.startsWith(":") + || !line.startsWith("data:")) { + return List.of(); + } + String payload = line.substring(5).trim(); + if (payload.isEmpty() || payload.equals("[DONE]")) { + return List.of(); + } + JsonNode event; + try { + event = mapper.readTree(payload); + } catch (Exception e) { + return List.of(); + } + if (event.path("promptFeedback").hasNonNull("blockReason")) { + throw blocked(event.path("promptFeedback").path("blockReason").asText(), + event.path("promptFeedback").path("safetyRatings")); + } + if (event.path("usageMetadata").isObject()) { + usage = tokenUsage(event); + } + JsonNode candidate = event.path("candidates").path(0); + if (candidate.isMissingNode()) { + return List.of(); + } + StringBuilder text = new StringBuilder(); + for (JsonNode part : candidate.path("content").path("parts")) { + if (part.hasNonNull("text") && !part.path("thought").asBoolean(false)) { + text.append(part.get("text").asText()); + } + } + String finish = candidate.hasNonNull("finishReason") + ? mapFinish(candidate.get("finishReason").asText()) : null; + List out = new ArrayList<>(1); + if (finish != null) { + out.add(chunk("", finish)); + done = true; + finishSent = true; + } else if (!text.isEmpty()) { + out.add(chunk(text.toString(), null)); + } + return out; + } + + private ObjectNode chunk(String content, String finishReason) { + ObjectNode chunk = mapper.createObjectNode(); + chunk.put("id", "chatcmpl-gemini"); + chunk.put("object", "chat.completion.chunk"); + chunk.put("created", created); + chunk.put("model", spec.publicModel()); + ObjectNode choice = chunk.putArray("choices").addObject(); + choice.put("index", 0); + ObjectNode delta = choice.putObject("delta"); + if (!content.isEmpty()) { + delta.put("content", content); + } else { + delta.put("role", "assistant"); + } + if (finishReason == null) { + choice.putNull("finish_reason"); + } else { + choice.put("finish_reason", finishReason); + } + return chunk; + } + + @Override + public boolean done() { + return done; + } + + @Override + public TokenUsage usage() { + return finishSent ? usage : TokenUsage.EMPTY; + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/IdempotencyService.java b/path-gateway/src/main/java/com/anindra/path/gateway/IdempotencyService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/IdempotencyService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/IdempotencyService.java diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryCircuitBreakerStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryCircuitBreakerStore.java new file mode 100644 index 0000000..33b11a3 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryCircuitBreakerStore.java @@ -0,0 +1,102 @@ +package com.anindra.path.gateway; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * The default backend: one map in this JVM, the same behaviour the breaker had before the + * store existed. A restart clears it, and two replicas know nothing about each other. + */ +public class InMemoryCircuitBreakerStore implements CircuitBreakerStore { + + private static final class Entry { + State state = State.CLOSED; + int failures; + long openUntilMs; + } + + private final Map entries = new ConcurrentHashMap<>(); + + @Override + public Snapshot load(Long providerId) { + return snapshotOf(entries.get(providerId)); + } + + @Override + public Snapshot recordFailure(Long providerId, int threshold, long nowMs, long openDurationMs) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + entry.failures++; + if (entry.state == State.HALF_OPEN || entry.failures >= Math.max(1, threshold)) { + entry.state = State.OPEN; + entry.openUntilMs = nowMs + Math.max(1, openDurationMs); + return snapshotOf(entry); + } + return null; + } + } + + @Override + public void open(Long providerId, int failures, long openUntilMs, long ttlMs) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + entry.state = State.OPEN; + entry.failures = Math.max(entry.failures, failures); + entry.openUntilMs = openUntilMs; + } + } + + @Override + public boolean tryProbe(Long providerId, long nowMs, long probeLeaseMs) { + Entry entry = entries.get(providerId); + if (entry == null) { + return false; + } + synchronized (entry) { + if (entry.state != State.OPEN || nowMs < entry.openUntilMs) { + return false; + } + // Exactly one caller sees the transition: everyone else finds HALF_OPEN. + entry.state = State.HALF_OPEN; + return true; + } + } + + @Override + public void close(Long providerId) { + Entry entry = entries.computeIfAbsent(providerId, k -> new Entry()); + synchronized (entry) { + entry.state = State.CLOSED; + entry.failures = 0; + entry.openUntilMs = 0; + } + } + + @Override + public Map snapshot() { + Map out = new LinkedHashMap<>(); + entries.forEach((providerId, entry) -> { + synchronized (entry) { + out.put(providerId, snapshotOf(entry)); + } + }); + return out; + } + + @Override + public void evictIdle() { + // Bounded by the number of providers, which is small; kept as a hook like the + // routing metrics so growth is not unbounded by accident. + if (entries.size() > 10_000) { + entries.clear(); + } + } + + private static Snapshot snapshotOf(Entry entry) { + if (entry == null) { + return Snapshot.closed(); + } + return new Snapshot(entry.state, entry.failures, entry.openUntilMs); + } +} diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryConcurrencyStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryConcurrencyStore.java new file mode 100644 index 0000000..e8f444a --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryConcurrencyStore.java @@ -0,0 +1,46 @@ +package com.anindra.path.gateway; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; + +/** Semaphores in this JVM: the pre-Valkey behaviour, unchanged. */ +public class InMemoryConcurrencyStore implements ConcurrencyStore { + + private record Entry(int limit, Semaphore semaphore) { + } + + private final Map entries = new ConcurrentHashMap<>(); + + @Override + public boolean tryAcquire(String bucket, int limit) { + if (limit <= 0) { + return true; + } + Entry entry = entries.compute(bucket, (id, current) -> + current == null || current.limit() != limit + ? new Entry(limit, new Semaphore(limit)) + : current); + return entry.semaphore().tryAcquire(); + } + + @Override + public void release(String bucket) { + Entry entry = entries.get(bucket); + if (entry != null && entry.semaphore().availablePermits() < entry.limit()) { + entry.semaphore().release(); + } + } + + @Override + public int inFlight(String bucket) { + Entry entry = entries.get(bucket); + return entry == null ? 0 : entry.limit() - entry.semaphore().availablePermits(); + } + + @Override + public void evictIdle() { + entries.entrySet().removeIf(e -> + e.getValue().semaphore().availablePermits() == e.getValue().limit()); + } +} diff --git a/src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java rename to path-gateway/src/main/java/com/anindra/path/gateway/InMemoryRateLimitStore.java diff --git a/src/main/java/com/anindra/path/gateway/LineStream.java b/path-gateway/src/main/java/com/anindra/path/gateway/LineStream.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/LineStream.java rename to path-gateway/src/main/java/com/anindra/path/gateway/LineStream.java diff --git a/src/main/java/com/anindra/path/gateway/ModelAliasService.java b/path-gateway/src/main/java/com/anindra/path/gateway/ModelAliasService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ModelAliasService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ModelAliasService.java diff --git a/src/main/java/com/anindra/path/gateway/ModelCatalogService.java b/path-gateway/src/main/java/com/anindra/path/gateway/ModelCatalogService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ModelCatalogService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ModelCatalogService.java diff --git a/src/main/java/com/anindra/path/gateway/ModerationService.java b/path-gateway/src/main/java/com/anindra/path/gateway/ModerationService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ModerationService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ModerationService.java diff --git a/src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java b/path-gateway/src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java rename to path-gateway/src/main/java/com/anindra/path/gateway/OpenAiCompatibleAdapter.java diff --git a/src/main/java/com/anindra/path/gateway/PiiRedactor.java b/path-gateway/src/main/java/com/anindra/path/gateway/PiiRedactor.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/PiiRedactor.java rename to path-gateway/src/main/java/com/anindra/path/gateway/PiiRedactor.java diff --git a/src/main/java/com/anindra/path/gateway/PromptSanitizer.java b/path-gateway/src/main/java/com/anindra/path/gateway/PromptSanitizer.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/PromptSanitizer.java rename to path-gateway/src/main/java/com/anindra/path/gateway/PromptSanitizer.java diff --git a/src/main/java/com/anindra/path/gateway/ProviderAdapter.java b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderAdapter.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ProviderAdapter.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ProviderAdapter.java diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/ProviderCircuitBreaker.java b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderCircuitBreaker.java new file mode 100644 index 0000000..3b166ac --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderCircuitBreaker.java @@ -0,0 +1,135 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Per-provider circuit breaker. + * + * A provider that keeps failing is parked for {@code open-duration}, during which routes + * on it are skipped in microseconds instead of costing every request a connect timeout. + * After the window a single probe request is allowed; success closes the circuit, failure + * reopens it. An upstream 429 with {@code Retry-After} parks the provider for exactly that long. + * + *

State lives in a {@link CircuitBreakerStore}: memory by default, Valkey when + * {@code gateway.shared-state.backend=valkey}. The interface is what makes "exactly one + * replica probes" possible at all. + */ +@Component +@RequiredArgsConstructor +public class ProviderCircuitBreaker { + + public enum State {CLOSED, OPEN, HALF_OPEN} + + private final GatewayProperties props; + private final com.anindra.path.admin.AlertService alerts; + private final MeterRegistry metrics; + private final CircuitBreakerStore store; + + /** Providers whose Prometheus gauge is registered; the gauge reads the store on scrape. */ + private final java.util.Set gauged = ConcurrentHashMap.newKeySet(); + + private void registerGauge(Long providerId) { + if (gauged.add(providerId)) { + metrics.gauge("path.provider.circuit.open", + java.util.List.of(io.micrometer.core.instrument.Tag.of( + "provider_id", String.valueOf(providerId))), + providerId, + id -> store.load(id).state() == CircuitBreakerStore.State.OPEN ? 1.0 : 0.0); + } + } + + /** Read-only projection used by the admin API. */ + public Map snapshot() { + Map out = new LinkedHashMap<>(); + store.snapshot().forEach((providerId, entry) -> out.put(String.valueOf(providerId), Map.of( + "state", entry.state().name().toLowerCase(), + "failures", entry.failures(), + "retry_after_seconds", retryAfterSeconds(entry, System.currentTimeMillis())))); + return out; + } + + public State state(Long providerId) { + registerGauge(providerId); + return stateOf(store.load(providerId)); + } + + public long retryAfterSeconds(Long providerId) { + return retryAfterSeconds(store.load(providerId), System.currentTimeMillis()); + } + + /** True when the provider is parked right now, without consuming the half-open probe. */ + public boolean isParked(Long providerId) { + CircuitBreakerStore.Snapshot entry = store.load(providerId); + return entry.state() == CircuitBreakerStore.State.OPEN + && System.currentTimeMillis() < entry.openUntilMs(); + } + + /** + * True when a request may be sent to this provider right now. When an OPEN window has + * elapsed, the first caller claims the single HALF_OPEN probe; everyone else is told no + * until that probe reports back or its lease expires. + */ + public boolean isAvailable(Long providerId) { + registerGauge(providerId); + CircuitBreakerStore.Snapshot entry = store.load(providerId); + if (entry.state() == CircuitBreakerStore.State.CLOSED) { + return true; + } + if (entry.state() == CircuitBreakerStore.State.HALF_OPEN) { + return false; + } + long now = System.currentTimeMillis(); + if (now < entry.openUntilMs()) { + return false; + } + return store.tryProbe(providerId, now, props.getBreaker().getProbeLease().toMillis()); + } + + public void onSuccess(Long providerId) { + store.close(providerId); + } + + public void onFailure(Long providerId) { + long now = System.currentTimeMillis(); + CircuitBreakerStore.Snapshot opened = store.recordFailure(providerId, + props.getBreaker().getFailureThreshold(), now, + props.getBreaker().getOpenDuration().toMillis()); + if (opened != null) { + alerts.notify("provider.circuit_open", Map.of( + "provider_id", providerId, + "failures", opened.failures(), + "retry_after_seconds", retryAfterSeconds(opened, now))); + } + } + + /** Upstream asked us to back off; park the provider for exactly that long. */ + public void onRateLimited(Long providerId, Duration retryAfter) { + Duration duration = retryAfter == null ? props.getBreaker().getOpenDuration() : retryAfter; + long now = System.currentTimeMillis(); + long openFor = Math.max(1, duration.toMillis()); + CircuitBreakerStore.Snapshot current = store.load(providerId); + store.open(providerId, current.failures(), now + openFor, openFor + 5_000); + alerts.notify("provider.rate_limited", Map.of( + "provider_id", providerId, + "retry_after_seconds", openFor / 1000)); + } + + private static State stateOf(CircuitBreakerStore.Snapshot entry) { + return State.valueOf(entry.state().name()); + } + + private static long retryAfterSeconds(CircuitBreakerStore.Snapshot entry, long now) { + if (entry.state() != CircuitBreakerStore.State.OPEN) { + return 0; + } + return Math.max(0, (entry.openUntilMs() - now) / 1000); + } +} diff --git a/src/main/java/com/anindra/path/gateway/ProviderCredentials.java b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderCredentials.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ProviderCredentials.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ProviderCredentials.java diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/ProviderErrorRateWatcher.java b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderErrorRateWatcher.java new file mode 100644 index 0000000..0ed8013 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderErrorRateWatcher.java @@ -0,0 +1,126 @@ +package com.anindra.path.gateway; + +import com.anindra.path.admin.AlertService; +import com.anindra.path.config.GatewayProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Watches the routing EWMA and reports a provider whose error rate stays high while its + * circuit is closed. + * + *

The breaker only opens after consecutive failures, so a provider failing 30% of the + * time with an occasional success never trips it: every affected request is quietly retried + * onto a fallback, and cost and latency rise with nobody watching. This check runs once a + * minute and reuses the EWMA the route view already shows. + * + *

One alert per provider per hour, and one recovery per incident. The threshold is + * {@code gateway.alert.error-rate} (default 0.25) and the rate is ignored until the + * provider has {@code gateway.alert.min-samples} recorded samples (default 20). The count + * is the provider's sample total, not a decaying window; the EWMA itself is what makes the + * rate recent. + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class ProviderErrorRateWatcher { + + private final RoutingMetrics routing; + private final AlertService alerts; + private final GatewayProperties props; + + /** Providers currently in the "high" state, so recovery fires once. */ + private final java.util.Set high = ConcurrentHashMap.newKeySet(); + /** Provider -> the hour its last high alert was sent for. */ + private final Map alertedHour = new ConcurrentHashMap<>(); + + @Scheduled(fixedDelay = 60_000) + public void check() { + double threshold = props.getAlert().getErrorRate(); + if (threshold <= 0) { + return; + } + long minSamples = Math.max(1, props.getAlert().getMinSamples()); + + Map byProvider = aggregate(routing.snapshot()); + long hour = Instant.now().getEpochSecond() / 3_600; + + byProvider.forEach((provider, aggregate) -> { + if (aggregate.samples < minSamples) { + // A cold start is not an incident. + return; + } + double rate = aggregate.errorRate(); + if (rate > threshold) { + high.add(provider); + Long lastHour = alertedHour.get(provider); + if (lastHour == null || lastHour != hour) { + alertedHour.put(provider, hour); + alerts.notifyOnce("provider.error_rate_high:" + provider + ":" + hour, + "provider.error_rate_high", payload(provider, aggregate, threshold)); + } + } else if (rate < threshold / 2 && high.remove(provider)) { + alertedHour.remove(provider); + alerts.notifyOnce("provider.error_rate_normal:" + provider + ":" + hour, + "provider.error_rate_normal", payload(provider, aggregate, threshold)); + } + }); + } + + private static Map payload(String provider, Aggregate aggregate, double threshold) { + Map body = new LinkedHashMap<>(); + body.put("provider", provider); + body.put("error_rate", round(aggregate.errorRate())); + body.put("samples", aggregate.samples); + body.put("threshold", threshold); + if (!Double.isNaN(aggregate.latencyMs())) { + body.put("latency_ms", Math.round(aggregate.latencyMs())); + } + return body; + } + + private static double round(double value) { + return Math.round(value * 10_000.0) / 10_000.0; + } + + /** Count-weighted mean across the provider's models. */ + static Map aggregate(List snapshot) { + Map out = new LinkedHashMap<>(); + for (RoutingMetrics.Snapshot row : snapshot) { + if (row.providerSlug() == null || row.count() <= 0) { + continue; + } + Aggregate aggregate = out.computeIfAbsent(row.providerSlug(), k -> new Aggregate()); + aggregate.errorSum += row.errorRate() * row.count(); + if (!Double.isNaN(row.latencyMs()) && row.latencyMs() > 0) { + aggregate.latencySum += row.latencyMs() * row.count(); + aggregate.latencyWeight += row.count(); + } + aggregate.samples += row.count(); + } + return out; + } + + static final class Aggregate { + double errorSum; + double latencySum; + double latencyWeight; + long samples; + + double errorRate() { + return samples == 0 ? 0 : errorSum / samples; + } + + double latencyMs() { + return latencyWeight == 0 ? Double.NaN : latencySum / latencyWeight; + } + } +} diff --git a/src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ProviderHealthIndicator.java diff --git a/src/main/java/com/anindra/path/gateway/ProviderUrls.java b/path-gateway/src/main/java/com/anindra/path/gateway/ProviderUrls.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ProviderUrls.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ProviderUrls.java diff --git a/src/main/java/com/anindra/path/gateway/RateLimitStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/RateLimitStore.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/RateLimitStore.java rename to path-gateway/src/main/java/com/anindra/path/gateway/RateLimitStore.java diff --git a/src/main/java/com/anindra/path/gateway/ResponseCacheService.java b/path-gateway/src/main/java/com/anindra/path/gateway/ResponseCacheService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ResponseCacheService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ResponseCacheService.java diff --git a/src/main/java/com/anindra/path/gateway/RouteResolver.java b/path-gateway/src/main/java/com/anindra/path/gateway/RouteResolver.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/RouteResolver.java rename to path-gateway/src/main/java/com/anindra/path/gateway/RouteResolver.java diff --git a/src/main/java/com/anindra/path/gateway/RoutingMetrics.java b/path-gateway/src/main/java/com/anindra/path/gateway/RoutingMetrics.java similarity index 82% rename from src/main/java/com/anindra/path/gateway/RoutingMetrics.java rename to path-gateway/src/main/java/com/anindra/path/gateway/RoutingMetrics.java index 90dfe50..914dbb9 100644 --- a/src/main/java/com/anindra/path/gateway/RoutingMetrics.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/RoutingMetrics.java @@ -80,6 +80,25 @@ public long sampleCount(String providerSlug, String publicModel) { return sample == null ? 0 : sample.count; } + /** One (provider, model) row as the alert watcher sees it. */ + public record Snapshot(String providerSlug, String publicModel, double errorRate, + double latencyMs, long count) { + } + + /** A copy of every current sample, for the scheduled error-rate check. */ + public java.util.List snapshot() { + java.util.List out = new java.util.ArrayList<>(); + samples.forEach((key, sample) -> { + int split = key.indexOf('|'); + if (split < 0) { + return; + } + out.add(new Snapshot(key.substring(0, split), key.substring(split + 1), + sample.errorRate, sample.latencyMs, sample.count)); + }); + return out; + } + public void reset() { samples.clear(); } diff --git a/src/main/java/com/anindra/path/gateway/SemanticCacheService.java b/path-gateway/src/main/java/com/anindra/path/gateway/SemanticCacheService.java similarity index 96% rename from src/main/java/com/anindra/path/gateway/SemanticCacheService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/SemanticCacheService.java index 54e19c9..35ffaab 100644 --- a/src/main/java/com/anindra/path/gateway/SemanticCacheService.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/SemanticCacheService.java @@ -16,7 +16,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.LongAdder; /** * Second cache tier: a near-duplicate prompt may reuse a stored completion. @@ -42,17 +41,12 @@ public record Hit(ResponseCacheEntry entry, double similarity) { private final EmbeddingsService embeddings; private final GatewayProperties props; private final ObjectMapper mapper; - private final LongAdder hits = new LongAdder(); public boolean enabled() { String model = props.getCache().getSemanticModel(); return model != null && !model.isBlank(); } - public long hitCount() { - return hits.sum(); - } - /** * @return the prompt vector, or null when the tier is off or embedding failed. A * failure here must never fail the request: the gateway simply behaves as if @@ -115,7 +109,6 @@ public Optional lookup(Long clientKeyId, String publicModel, double[] vecto if (bestEntry == null) { return Optional.empty(); } - hits.increment(); return Optional.of(new Hit(bestEntry, best)); } diff --git a/src/main/java/com/anindra/path/gateway/StreamService.java b/path-gateway/src/main/java/com/anindra/path/gateway/StreamService.java similarity index 97% rename from src/main/java/com/anindra/path/gateway/StreamService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/StreamService.java index beaea4b..bcd4fb5 100644 --- a/src/main/java/com/anindra/path/gateway/StreamService.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/StreamService.java @@ -97,7 +97,7 @@ public boolean stream(HttpServletResponse response, ChatRequestSpec spec, Client OutputStream cached = response.getOutputStream(); cached.write(hit.get().body().getBytes(StandardCharsets.UTF_8)); cached.flush(); - usage.recordCacheHit(key, spec, chain.get(0), hit.get(), true); + usage.recordCacheHit(key, spec, chain.get(0), hit.get(), true, "exact"); return true; } } @@ -118,7 +118,7 @@ public boolean stream(HttpServletResponse response, ChatRequestSpec spec, Client String requestId = null; String finishReason = null; int generatedChars = 0; - long available = chain.stream().filter(r -> breaker.isAvailable(r.provider().getId())).count(); + long available = chain.stream().filter(r -> !breaker.isParked(r.provider().getId())).count(); int maxAttempts = Math.min(chain.size(), Math.max(1, props.getBreaker().getMaxAttempts())); for (int i = 0; i < chain.size() && attempts < maxAttempts; i++) { @@ -243,7 +243,7 @@ public boolean stream(HttpServletResponse response, ChatRequestSpec spec, Client continue; } usage.recordError(key, spec, provider.getSlug(), rr.route().getUpstreamModelId(), - ChatService.elapsedMs(started), true, attempts, lastError.code(), lastError.getMessage()); + ChatService.elapsedMs(started), true, attempts, lastError.status(), lastError.code(), lastError.getMessage()); emitError(out, lastError); return false; @@ -259,7 +259,7 @@ public boolean stream(HttpServletResponse response, ChatRequestSpec spec, Client continue; } usage.recordError(key, spec, provider.getSlug(), rr.route().getUpstreamModelId(), - ChatService.elapsedMs(started), true, attempts, lastError.code(), lastError.getMessage()); + ChatService.elapsedMs(started), true, attempts, lastError.status(), lastError.code(), lastError.getMessage()); emitError(out, lastError); return false; @@ -279,7 +279,7 @@ public boolean stream(HttpServletResponse response, ChatRequestSpec spec, Client "Stream to " + provider.getSlug() + " broke: " + e.getMessage(), "upstream_stream_error"); usage.recordError(key, spec, provider.getSlug(), rr.route().getUpstreamModelId(), - ChatService.elapsedMs(started), true, attempts, lastError.code(), lastError.getMessage()); + ChatService.elapsedMs(started), true, attempts, lastError.status(), lastError.code(), lastError.getMessage()); try { emitError(out, lastError); } catch (IOException clientGone) { @@ -304,7 +304,7 @@ public boolean stream(HttpServletResponse response, ChatRequestSpec spec, Client lastError = new ApiException(lastError.status(), lastError.type(), lastError.code(), lastError.getMessage() + " (routes tried: " + String.join("; ", failures) + ")"); } - usage.recordError(key, spec, null, null, ChatService.elapsedMs(started), true, attempts, + usage.recordError(key, spec, null, null, ChatService.elapsedMs(started), true, attempts, lastError.status(), lastError.code(), lastError.getMessage()); emitError(out, lastError); return false; diff --git a/src/main/java/com/anindra/path/gateway/StreamTranslator.java b/path-gateway/src/main/java/com/anindra/path/gateway/StreamTranslator.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/StreamTranslator.java rename to path-gateway/src/main/java/com/anindra/path/gateway/StreamTranslator.java diff --git a/src/main/java/com/anindra/path/gateway/TokenCountService.java b/path-gateway/src/main/java/com/anindra/path/gateway/TokenCountService.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/TokenCountService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/TokenCountService.java diff --git a/src/main/java/com/anindra/path/gateway/TokenCounter.java b/path-gateway/src/main/java/com/anindra/path/gateway/TokenCounter.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/TokenCounter.java rename to path-gateway/src/main/java/com/anindra/path/gateway/TokenCounter.java diff --git a/src/main/java/com/anindra/path/gateway/TokenUsage.java b/path-gateway/src/main/java/com/anindra/path/gateway/TokenUsage.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/TokenUsage.java rename to path-gateway/src/main/java/com/anindra/path/gateway/TokenUsage.java diff --git a/src/main/java/com/anindra/path/gateway/UpstreamClient.java b/path-gateway/src/main/java/com/anindra/path/gateway/UpstreamClient.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/UpstreamClient.java rename to path-gateway/src/main/java/com/anindra/path/gateway/UpstreamClient.java diff --git a/src/main/java/com/anindra/path/gateway/UpstreamException.java b/path-gateway/src/main/java/com/anindra/path/gateway/UpstreamException.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/UpstreamException.java rename to path-gateway/src/main/java/com/anindra/path/gateway/UpstreamException.java diff --git a/src/main/java/com/anindra/path/gateway/UpstreamRequest.java b/path-gateway/src/main/java/com/anindra/path/gateway/UpstreamRequest.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/UpstreamRequest.java rename to path-gateway/src/main/java/com/anindra/path/gateway/UpstreamRequest.java diff --git a/src/main/java/com/anindra/path/gateway/UsageService.java b/path-gateway/src/main/java/com/anindra/path/gateway/UsageService.java similarity index 95% rename from src/main/java/com/anindra/path/gateway/UsageService.java rename to path-gateway/src/main/java/com/anindra/path/gateway/UsageService.java index 3cf1d6e..9230fe4 100644 --- a/src/main/java/com/anindra/path/gateway/UsageService.java +++ b/path-gateway/src/main/java/com/anindra/path/gateway/UsageService.java @@ -136,7 +136,7 @@ public BigDecimal recordSuccess(ClientKey key, ChatRequestSpec spec, RouteResolv /** Records a cache hit: no upstream call, no charge, savings carried for reporting. */ @Transactional public void recordCacheHit(ClientKey key, ChatRequestSpec spec, RouteResolver.ResolvedRoute resolved, - ResponseCacheService.Entry entry, boolean streamed) { + ResponseCacheService.Entry entry, boolean streamed, String tier) { UsageRecord record = new UsageRecord(); record.setClientKeyId(key.getId()); record.setClientKeyName(key.getName()); @@ -153,16 +153,18 @@ public void recordCacheHit(ClientKey key, ChatRequestSpec spec, RouteResolver.Re record.setStreamed(streamed); record.setAttempts(0); record.setFinishReason("cache"); + record.setCacheTier(tier); record.setStatus(UsageRecord.Status.SUCCESS); usage.save(record); metrics.counter("path.cache.hits", - "model", String.valueOf(record.getPublicModelId())).increment(); + "model", String.valueOf(record.getPublicModelId()), + "tier", tier == null ? "unknown" : tier).increment(); recordMetrics(record); } @Transactional public void recordError(ClientKey key, ChatRequestSpec spec, String providerSlug, String upstreamModel, - long latencyMs, boolean streamed, int attempts, String code, String message) { + long latencyMs, boolean streamed, int attempts, int status, String code, String message) { UsageRecord record = new UsageRecord(); if (key != null) { record.setClientKeyId(key.getId()); @@ -178,6 +180,12 @@ public void recordError(ClientKey key, ChatRequestSpec spec, String providerSlug record.setErrorCode(code); record.setErrorMessage(message != null && message.length() > 500 ? message.substring(0, 500) : message); usage.save(record); + // Kept separate from path_requests_total so an alert can tell a 4xx from a 5xx + // without a high-cardinality label on the request counter. + metrics.counter("path.errors", + "provider", String.valueOf(providerSlug), + "class", status >= 500 ? "5xx" : status >= 400 ? "4xx" : "other", + "code", code == null ? "unknown" : code).increment(); recordMetrics(record); } @@ -195,6 +203,11 @@ private void recordMetrics(UsageRecord record) { metrics.counter("path.tokens", "type", "prompt").increment(record.getPromptTokens()); metrics.counter("path.tokens", "type", "completion").increment(record.getCompletionTokens()); metrics.counter("path.tokens", "type", "cached").increment(record.getCachedInputTokens()); + if (record.getAttempts() > 1) { + // A request served after at least one failed route: the fallback-rate rule in + // deploy/prometheus/path-rules.yml divides this by path_requests_total. + metrics.counter("path.fallbacks", "provider", provider).increment(); + } if (record.getCostUsd() != null) { metrics.counter("path.cost.usd", "model", model, "provider", provider).increment(record.getCostUsd().doubleValue()); @@ -391,6 +404,7 @@ public Map adminStats(Duration window) { cache.put("cached_input_tokens", cachedTokens); cache.put("hit_ratio", ratio(cachedTokens, promptTokens)); cache.put("savings_usd", savings); + cache.put("semantic_hits", usage.countByCreatedAtAfterAndCacheTier(from, "semantic")); out.put("cache", cache); out.put("fallback_requests", usage.countByAttemptsGreaterThanAndCreatedAtAfter(1, from)); diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyCircuitBreakerStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyCircuitBreakerStore.java new file mode 100644 index 0000000..151c2a6 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyCircuitBreakerStore.java @@ -0,0 +1,148 @@ +package com.anindra.path.gateway; + +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.ScanOptions; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.core.script.RedisScript; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Circuit state in Valkey, shared by every replica. + * + *

One hash per provider: {@code state}, {@code open_until} (epoch ms) and {@code failures}. + * A provider with no key is CLOSED, so a successful call deletes the key instead of writing + * a tombstone. The key's TTL is the safety net: an OPEN entry expires shortly after its own + * retry-after even if the replica that opened it never comes back, and an abandoned + * HALF_OPEN probe lease expires the same way. + * + *

The scripts are the atomic part. Two replicas reacting to the same expired OPEN window + * both run {@code PATH_PROBE}; the first flips the state to HALF_OPEN and the second sees it + * and loses the claim, so exactly one probe is sent. + */ +public class ValkeyCircuitBreakerStore implements CircuitBreakerStore { + + private static final RedisScript RECORD_FAILURE = new DefaultRedisScript<>( + "local failures = redis.call('HINCRBY', KEYS[1], 'failures', 1) " + + "local state = redis.call('HGET', KEYS[1], 'state') " + + "if state == 'HALF_OPEN' or failures >= tonumber(ARGV[1]) then " + + " local untilMs = tonumber(ARGV[2]) + tonumber(ARGV[3]) " + + " redis.call('HSET', KEYS[1], 'state', 'OPEN', 'open_until', untilMs, 'failures', failures) " + + " redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[4])) " + + " return {1, failures, untilMs} " + + "end " + + "redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[5])) " + + "return {0, failures, 0}", + List.class); + + private static final RedisScript OPEN = new DefaultRedisScript<>( + "redis.call('HSET', KEYS[1], 'state', 'OPEN', 'open_until', ARGV[1], 'failures', ARGV[2]) " + + "redis.call('PEXPIRE', KEYS[1], ARGV[3]) " + + "return 1", + Long.class); + + private static final RedisScript TRY_PROBE = new DefaultRedisScript<>( + "local value = redis.call('HMGET', KEYS[1], 'state', 'open_until') " + + "if not value[1] then return 0 end " + + "if value[1] ~= 'OPEN' then return 0 end " + + "if tonumber(ARGV[1]) < tonumber(value[2]) then return 0 end " + + "redis.call('HSET', KEYS[1], 'state', 'HALF_OPEN') " + + "redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[2])) " + + "return 1", + Long.class); + + /** Failures kept for an hour when the circuit did not open, so a slow drift still counts. */ + private static final long FAILURE_TTL_MS = 3_600_000; + + private final StringRedisTemplate redis; + private final String prefix; + + public ValkeyCircuitBreakerStore(StringRedisTemplate redis, String prefix) { + this.redis = redis; + this.prefix = prefix; + } + + private String key(Long providerId) { + return prefix + ":breaker:" + providerId; + } + + @Override + public Snapshot load(Long providerId) { + List values = redis.opsForHash().multiGet(key(providerId), + List.of("state", "open_until", "failures")); + if (values == null || values.get(0) == null) { + return Snapshot.closed(); + } + return new Snapshot(State.valueOf(String.valueOf(values.get(0))), + values.get(2) == null ? 0 : Integer.parseInt(String.valueOf(values.get(2))), + values.get(1) == null ? 0 : Long.parseLong(String.valueOf(values.get(1)))); + } + + @Override + public Snapshot recordFailure(Long providerId, int threshold, long nowMs, long openDurationMs) { + long openFor = Math.max(1, openDurationMs); + List result = redis.execute(RECORD_FAILURE, List.of(key(providerId)), + String.valueOf(Math.max(1, threshold)), + String.valueOf(nowMs), + String.valueOf(openFor), + String.valueOf(openFor + 5_000), + String.valueOf(FAILURE_TTL_MS)); + if (result == null || result.size() < 3 || !"1".equals(String.valueOf(result.get(0)))) { + return null; + } + return new Snapshot(State.OPEN, + ((Number) result.get(1)).intValue(), + ((Number) result.get(2)).longValue()); + } + + @Override + public void open(Long providerId, int failures, long openUntilMs, long ttlMs) { + redis.execute(OPEN, List.of(key(providerId)), + String.valueOf(openUntilMs), + String.valueOf(failures), + String.valueOf(Math.max(1, ttlMs))); + } + + @Override + public boolean tryProbe(Long providerId, long nowMs, long probeLeaseMs) { + Long claimed = redis.execute(TRY_PROBE, List.of(key(providerId)), + String.valueOf(nowMs), + String.valueOf(Math.max(1, probeLeaseMs))); + return claimed != null && claimed == 1L; + } + + @Override + public void close(Long providerId) { + redis.delete(key(providerId)); + } + + @Override + public Map snapshot() { + Map out = new LinkedHashMap<>(); + ScanOptions options = ScanOptions.scanOptions() + .match(prefix + ":breaker:*") + .count(100) + .build(); + try (Cursor cursor = redis.scan(options)) { + while (cursor.hasNext()) { + String key = cursor.next(); + Long providerId = parseProviderId(key); + if (providerId != null) { + out.put(providerId, load(providerId)); + } + } + } + return out; + } + + private Long parseProviderId(String key) { + try { + return Long.valueOf(key.substring(key.lastIndexOf(':') + 1)); + } catch (NumberFormatException e) { + return null; + } + } +} diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyConcurrencyStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyConcurrencyStore.java new file mode 100644 index 0000000..aaa0bca --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyConcurrencyStore.java @@ -0,0 +1,82 @@ +package com.anindra.path.gateway; + +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.core.script.RedisScript; + +import java.time.Duration; +import java.util.List; +import java.util.UUID; + +/** + * In-flight permits in Valkey, one sorted set per bucket. + * + *

Each member is a permit with its lease expiry as the score. Acquiring prunes expired + * members, counts the rest and adds a new one only when the bucket is under its limit, all + * inside one script so two replicas cannot both see {@code limit-1}. Releasing pops one + * member: permits are fungible, so which one goes does not matter, only that the count drops + * by one. A killed pod's permits simply age out of the set, which is the whole point of the + * lease. + */ +public class ValkeyConcurrencyStore implements ConcurrencyStore { + + private static final RedisScript ACQUIRE = new DefaultRedisScript<>( + "redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) " + + "if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[3]) then return 0 end " + + "redis.call('ZADD', KEYS[1], ARGV[2], ARGV[4]) " + + "redis.call('PEXPIREAT', KEYS[1], tonumber(ARGV[2]) + 1000) " + + "return 1", + Long.class); + + private static final RedisScript RELEASE = new DefaultRedisScript<>( + "redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) " + + "local popped = redis.call('ZPOPMIN', KEYS[1]) " + + "if #popped > 0 then return 1 end " + + "return 0", + Long.class); + + private static final RedisScript IN_FLIGHT = new DefaultRedisScript<>( + "redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) " + + "return redis.call('ZCARD', KEYS[1])", + Long.class); + + private final StringRedisTemplate redis; + private final String prefix; + private final long leaseMs; + + public ValkeyConcurrencyStore(StringRedisTemplate redis, String prefix, Duration lease) { + this.redis = redis; + this.prefix = prefix; + this.leaseMs = Math.max(1, lease.toMillis()); + } + + private String key(String bucket) { + return prefix + ":conc:" + bucket; + } + + @Override + public boolean tryAcquire(String bucket, int limit) { + if (limit <= 0) { + return true; + } + long now = System.currentTimeMillis(); + Long taken = redis.execute(ACQUIRE, List.of(key(bucket)), + String.valueOf(now), + String.valueOf(now + leaseMs), + String.valueOf(limit), + UUID.randomUUID().toString()); + return taken != null && taken == 1L; + } + + @Override + public void release(String bucket) { + redis.execute(RELEASE, List.of(key(bucket)), String.valueOf(System.currentTimeMillis())); + } + + @Override + public int inFlight(String bucket) { + Long count = redis.execute(IN_FLIGHT, List.of(key(bucket)), + String.valueOf(System.currentTimeMillis())); + return count == null ? 0 : count.intValue(); + } +} diff --git a/src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java b/path-gateway/src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java similarity index 100% rename from src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java rename to path-gateway/src/main/java/com/anindra/path/gateway/ValkeyRateLimitStore.java diff --git a/path-gateway/src/main/java/com/anindra/path/gateway/VertexTokenService.java b/path-gateway/src/main/java/com/anindra/path/gateway/VertexTokenService.java new file mode 100644 index 0000000..aeb826b --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/gateway/VertexTokenService.java @@ -0,0 +1,144 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.Provider; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.Signature; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Duration; +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * OAuth2 access tokens for Vertex AI, from a service-account JSON. + * + *

The credential is the JSON key file Google issues (client_email, private_key, + * token_uri). It is stored in the provider's encrypted credential column like any other + * secret, so it is AES-GCM at rest and never shown by the admin API. + * + *

The exchange is the standard JWT-bearer flow: sign a short assertion with the + * service-account private key, POST it to the token endpoint, keep the access token until + * shortly before it expires. Tokens are cached per provider, so a request does not pay for + * an OAuth round-trip. + */ +@Component +@Slf4j +public class VertexTokenService { + + private static final String SCOPE = "https://www.googleapis.com/auth/cloud-platform"; + + private record Cached(String token, long expiresAtMs) { + } + + private final com.fasterxml.jackson.databind.ObjectMapper mapper; + private final Map cache = new ConcurrentHashMap<>(); + private final HttpClient http = HttpClient.newHttpClient(); + + public VertexTokenService(com.fasterxml.jackson.databind.ObjectMapper mapper) { + this.mapper = mapper; + } + + /** + * @param serviceAccountJson the decrypted provider credential + * @return an access token, from the cache when it is still valid + */ + public String accessToken(Provider provider, String serviceAccountJson) { + Cached cached = cache.get(provider.getId()); + if (cached != null && cached.expiresAtMs() > System.currentTimeMillis() + 60_000) { + return cached.token(); + } + Cached fresh = exchange(serviceAccountJson); + cache.put(provider.getId(), fresh); + return fresh.token(); + } + + private Cached exchange(String serviceAccountJson) { + com.fasterxml.jackson.databind.JsonNode account; + try { + account = mapper.readTree(serviceAccountJson); + } catch (Exception e) { + throw new UpstreamException(400, "vertex_credentials_invalid", + "The Vertex credential is not a service-account JSON key."); + } + String email = account.path("client_email").asText(null); + String privateKeyPem = account.path("private_key").asText(null); + String tokenUri = account.path("token_uri").asText("https://oauth2.googleapis.com/token"); + if (email == null || privateKeyPem == null) { + throw new UpstreamException(400, "vertex_credentials_invalid", + "The service-account JSON needs client_email and private_key."); + } + + String assertion = jwt(email, privateKeyPem, tokenUri); + String form = "grant_type=" + URLEncoder.encode( + "urn:ietf:params:oauth:grant-type:jwt-bearer", StandardCharsets.UTF_8) + + "&assertion=" + URLEncoder.encode(assertion, StandardCharsets.UTF_8); + HttpRequest request = HttpRequest.newBuilder(URI.create(tokenUri)) + .timeout(Duration.ofSeconds(15)) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString(form)) + .build(); + try { + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() / 100 != 2) { + throw new UpstreamException(response.statusCode(), "vertex_auth_failed", + "Google rejected the service-account assertion: " + response.body()); + } + com.fasterxml.jackson.databind.JsonNode body = mapper.readTree(response.body()); + String token = body.path("access_token").asText(null); + if (token == null) { + throw new UpstreamException(502, "vertex_auth_failed", + "The token endpoint returned no access_token."); + } + long ttl = body.path("expires_in").asLong(3600); + return new Cached(token, System.currentTimeMillis() + ttl * 1000); + } catch (java.io.IOException e) { + throw new UpstreamException(502, "vertex_auth_failed", + "Could not reach the Google token endpoint: " + e.getMessage()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UpstreamException(502, "vertex_auth_failed", "Token exchange interrupted."); + } + } + + /** RS256 JWT with the claims Google's JWT-bearer flow expects. */ + String jwt(String email, String privateKeyPem, String audience) { + long now = System.currentTimeMillis() / 1000; + String header = base64Url("{\"alg\":\"RS256\",\"typ\":\"JWT\"}".getBytes(StandardCharsets.UTF_8)); + String claims = base64Url(("{\"iss\":\"" + email + "\",\"scope\":\"" + SCOPE + + "\",\"aud\":\"" + audience + "\",\"iat\":" + now + ",\"exp\":" + (now + 3600) + + "}").getBytes(StandardCharsets.UTF_8)); + String signingInput = header + "." + claims; + try { + Signature signature = Signature.getInstance("SHA256withRSA"); + signature.initSign(privateKey(privateKeyPem)); + signature.update(signingInput.getBytes(StandardCharsets.US_ASCII)); + return signingInput + "." + base64Url(signature.sign()); + } catch (Exception e) { + throw new UpstreamException(400, "vertex_credentials_invalid", + "Could not sign the service-account assertion: " + e.getMessage()); + } + } + + private static PrivateKey privateKey(String pem) throws Exception { + String body = pem.replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "") + .replaceAll("\\s", ""); + byte[] der = Base64.getDecoder().decode(body); + return KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(der)); + } + + private static String base64Url(byte[] value) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(value); + } +} diff --git a/src/main/java/com/anindra/path/repo/AdminAuditRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/AdminAuditRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/AdminAuditRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/AdminAuditRepository.java diff --git a/src/main/java/com/anindra/path/repo/AdminTokenRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/AdminTokenRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/AdminTokenRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/AdminTokenRepository.java diff --git a/src/main/java/com/anindra/path/repo/ClientKeyRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/ClientKeyRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/ClientKeyRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/ClientKeyRepository.java diff --git a/src/main/java/com/anindra/path/repo/EndUserReservationRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/EndUserReservationRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/EndUserReservationRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/EndUserReservationRepository.java diff --git a/src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/IdempotencyRecordRepository.java diff --git a/path-gateway/src/main/java/com/anindra/path/repo/MailSettingsRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/MailSettingsRepository.java new file mode 100644 index 0000000..f8ad197 --- /dev/null +++ b/path-gateway/src/main/java/com/anindra/path/repo/MailSettingsRepository.java @@ -0,0 +1,7 @@ +package com.anindra.path.repo; + +import com.anindra.path.domain.MailSettings; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface MailSettingsRepository extends JpaRepository { +} diff --git a/src/main/java/com/anindra/path/repo/ModelAliasRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/ModelAliasRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/ModelAliasRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/ModelAliasRepository.java diff --git a/src/main/java/com/anindra/path/repo/ModelRouteRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/ModelRouteRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/ModelRouteRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/ModelRouteRepository.java diff --git a/src/main/java/com/anindra/path/repo/ProjectRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/ProjectRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/ProjectRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/ProjectRepository.java diff --git a/src/main/java/com/anindra/path/repo/PromptLogRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/PromptLogRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/PromptLogRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/PromptLogRepository.java diff --git a/src/main/java/com/anindra/path/repo/ProviderRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/ProviderRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/ProviderRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/ProviderRepository.java diff --git a/src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java similarity index 100% rename from src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/ResponseCacheEntryRepository.java diff --git a/src/main/java/com/anindra/path/repo/UsageRecordRepository.java b/path-gateway/src/main/java/com/anindra/path/repo/UsageRecordRepository.java similarity index 65% rename from src/main/java/com/anindra/path/repo/UsageRecordRepository.java rename to path-gateway/src/main/java/com/anindra/path/repo/UsageRecordRepository.java index 55183ac..3c5b9c6 100644 --- a/src/main/java/com/anindra/path/repo/UsageRecordRepository.java +++ b/path-gateway/src/main/java/com/anindra/path/repo/UsageRecordRepository.java @@ -25,6 +25,52 @@ List findByClientKeyIdInAndCreatedAtBetweenOrderByCreatedAtAsc( long countByCreatedAtAfter(Instant from); + long countByCreatedAtBetween(Instant from, Instant to); + + long countByStatusAndCreatedAtBetween(UsageRecord.Status status, Instant from, Instant to); + + @Query("select coalesce(sum(u.costUsd), 0) from UsageRecord u " + + "where u.createdAt >= :from and u.createdAt < :to") + BigDecimal sumCostBetween(Instant from, Instant to); + + @Query("select coalesce(sum(u.totalTokens), 0) from UsageRecord u " + + "where u.createdAt >= :from and u.createdAt < :to") + Long sumTokensBetween(Instant from, Instant to); + + @Query("select coalesce(sum(u.promptTokens), 0) from UsageRecord u " + + "where u.createdAt >= :from and u.createdAt < :to") + Long sumPromptTokensBetween(Instant from, Instant to); + + @Query("select coalesce(sum(u.cachedInputTokens), 0) from UsageRecord u " + + "where u.createdAt >= :from and u.createdAt < :to") + Long sumCachedTokensBetween(Instant from, Instant to); + + @Query("select coalesce(sum(u.cachedSavingsUsd), 0) from UsageRecord u " + + "where u.createdAt >= :from and u.createdAt < :to") + BigDecimal sumCachedSavingsBetween(Instant from, Instant to); + + @Query("select u.errorCode, count(u) from UsageRecord u " + + "where u.status = com.anindra.path.domain.UsageRecord.Status.ERROR " + + "and u.errorCode is not null and u.createdAt >= :from and u.createdAt < :to " + + "group by u.errorCode order by count(u) desc") + List topErrorCodesBetween(Instant from, Instant to, Pageable pageable); + + @Query("select u.publicModelId, coalesce(sum(u.costUsd), 0) from UsageRecord u " + + "where u.publicModelId is not null and u.createdAt >= :from and u.createdAt < :to " + + "group by u.publicModelId order by sum(u.costUsd) desc") + List topModelsByCostBetween(Instant from, Instant to, Pageable pageable); + + @Query("select u.publicModelId, u.latencyMs from UsageRecord u " + + "where u.status = com.anindra.path.domain.UsageRecord.Status.SUCCESS " + + "and u.publicModelId is not null and u.createdAt >= :from and u.createdAt < :to " + + "order by u.createdAt desc") + List successLatenciesBetween(Instant from, Instant to, Pageable pageable); + + long countByCreatedAtAfterAndCacheTier(Instant from, String cacheTier); + + long countByClientKeyIdInAndCreatedAtAfterAndCacheTier( + java.util.Collection keyIds, Instant from, String cacheTier); + long deleteByCreatedAtBefore(Instant cutoff); long countByClientKeyIdAndCreatedAtAfter(Long clientKeyId, Instant from); @@ -64,6 +110,19 @@ List findByClientKeyIdAndEndUserOrderByCreatedAtDesc( @Query("select u.latencyMs from UsageRecord u where u.createdAt >= :from and u.status = com.anindra.path.domain.UsageRecord.Status.SUCCESS") List findLatenciesSince(Instant from); + /** + * Every row a time-series bucket needs, in one query. The caller groups by time in Java + * so the same JPQL works on H2 and Postgres without database-specific date arithmetic. + */ + @Query("select u.createdAt, u.status, u.costUsd, u.latencyMs from UsageRecord u " + + "where u.createdAt >= :from and u.createdAt < :to") + List findSeriesSource(Instant from, Instant to); + + @Query("select u.createdAt, u.status, u.costUsd, u.latencyMs from UsageRecord u " + + "where u.clientKeyId in :keyIds and u.createdAt >= :from and u.createdAt < :to") + List findSeriesSourceForKeys(java.util.Collection keyIds, Instant from, Instant to); + + @Query("select u.providerSlug, count(u), coalesce(sum(u.costUsd), 0) from UsageRecord u where u.createdAt >= :from group by u.providerSlug") List groupByProviderSince(Instant from); diff --git a/src/main/java/com/anindra/path/secret/SecretCipher.java b/path-gateway/src/main/java/com/anindra/path/secret/SecretCipher.java similarity index 100% rename from src/main/java/com/anindra/path/secret/SecretCipher.java rename to path-gateway/src/main/java/com/anindra/path/secret/SecretCipher.java diff --git a/src/main/java/com/anindra/path/seed/DataSeeder.java b/path-gateway/src/main/java/com/anindra/path/seed/DataSeeder.java similarity index 90% rename from src/main/java/com/anindra/path/seed/DataSeeder.java rename to path-gateway/src/main/java/com/anindra/path/seed/DataSeeder.java index 39dcd0c..29ee9bc 100644 --- a/src/main/java/com/anindra/path/seed/DataSeeder.java +++ b/path-gateway/src/main/java/com/anindra/path/seed/DataSeeder.java @@ -41,9 +41,13 @@ public class DataSeeder implements ApplicationRunner { @Transactional public void run(ApplicationArguments args) { boolean fresh = providers.count() == 0; - Map bySlug = ensureProviders(); - if (fresh && routes.count() == 0) { - seedRoutes(bySlug); + if (props.isSeedDefaults()) { + Map bySlug = ensureProviders(); + if (fresh && routes.count() == 0) { + seedRoutes(bySlug); + } + } else { + log.info("Gateway seed-defaults is off: not creating the example provider/route catalogue."); } if (projectsRepository.count() == 0) { projectsRepository.save(new com.anindra.path.domain.Project("default", "Default project")); @@ -55,6 +59,11 @@ public void run(ApplicationArguments args) { } else if (prod && props.isBootstrapDemoKey()) { log.warn("Ignoring gateway.bootstrap-demo-key in the prod profile; no demo key created."); } + if (props.getProxyAuth().isEnabled()) { + log.warn("Admin auth is DELEGATED to a reverse proxy ({}): the gateway trusts that " + + "header for /admin/**. It must not be reachable except through the proxy.", + props.getProxyAuth().getHeader()); + } seedState.markReady(); log.info("Gateway ready. Admin API: {} | Public API: /v1/chat/completions, /v1/models", props.getAdminToken() == null || props.getAdminToken().isBlank() diff --git a/src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java b/path-gateway/src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java similarity index 100% rename from src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java rename to path-gateway/src/main/java/com/anindra/path/seed/SeedReadinessIndicator.java diff --git a/src/main/java/com/anindra/path/seed/SeedState.java b/path-gateway/src/main/java/com/anindra/path/seed/SeedState.java similarity index 100% rename from src/main/java/com/anindra/path/seed/SeedState.java rename to path-gateway/src/main/java/com/anindra/path/seed/SeedState.java diff --git a/src/main/resources/application-postgres.yml b/path-gateway/src/main/resources/application-postgres.yml similarity index 100% rename from src/main/resources/application-postgres.yml rename to path-gateway/src/main/resources/application-postgres.yml diff --git a/src/main/resources/application-prod.yml b/path-gateway/src/main/resources/application-prod.yml similarity index 100% rename from src/main/resources/application-prod.yml rename to path-gateway/src/main/resources/application-prod.yml diff --git a/src/main/resources/application.yml b/path-gateway/src/main/resources/application.yml similarity index 66% rename from src/main/resources/application.yml rename to path-gateway/src/main/resources/application.yml index 518bf7b..25bf5e1 100644 --- a/src/main/resources/application.yml +++ b/path-gateway/src/main/resources/application.yml @@ -43,7 +43,11 @@ gateway: # Empty means the admin API is disabled (403). Set GATEWAY_ADMIN_TOKEN to enable it. admin-token: ${GATEWAY_ADMIN_TOKEN:} # Allow GET /v1/models without an API key. - public-models: ${GATEWAY_PUBLIC_MODELS:true} + public-models: ${GATEWAY_PUBLIC_MODELS:false} + # Header that carries the real client address when a trusted edge sits in front + # (e.g. CF-Connecting-IP behind Cloudflare). Empty = use the socket address and ignore + # X-Forwarded-For, which a client can forge to dodge per-address limits. + client-ip-header: ${GATEWAY_CLIENT_IP_HEADER:} cors-origins: ${GATEWAY_CORS_ORIGINS:*} connect-timeout: 10s request-timeout: 180s @@ -52,6 +56,9 @@ gateway: master-key: ${GATEWAY_MASTER_KEY:} master-key-path: ${GATEWAY_MASTER_KEY_PATH:./data/master.key} bootstrap-demo-key: ${GATEWAY_BOOTSTRAP_DEMO_KEY:false} + # Seed the example provider/route catalogue on boot. Turn off for a real deployment, + # otherwise four providers you never configured appear in the console on every start. + seed-defaults: ${GATEWAY_SEED_DEFAULTS:true} default-rpm-limit: ${GATEWAY_DEFAULT_RPM:120} default-credit-limit-usd: ${GATEWAY_DEFAULT_CREDIT_USD:} default-max-concurrent: ${GATEWAY_DEFAULT_MAX_CONCURRENT:8} @@ -62,13 +69,32 @@ gateway: failure-threshold: ${GATEWAY_BREAKER_FAILURES:5} open-duration: ${GATEWAY_BREAKER_OPEN:30s} max-attempts: ${GATEWAY_MAX_ATTEMPTS:3} + # How long a HALF_OPEN probe holds the claim before another replica may retry. + probe-lease: ${GATEWAY_BREAKER_PROBE_LEASE:10s} + # Circuit-breaker and in-flight state. "valkey" shares them across replicas; memory is + # per JVM and a restart clears it. + shared-state: + backend: ${GATEWAY_SHARED_STATE_BACKEND:memory} + prefix: ${GATEWAY_SHARED_STATE_PREFIX:path} + concurrency-lease: ${GATEWAY_CONCURRENCY_LEASE:300s} cache: semantic-model: ${GATEWAY_SEMANTIC_MODEL:} semantic-max-entries: ${GATEWAY_SEMANTIC_MAX_ENTRIES:200} max-body-bytes: ${GATEWAY_CACHE_MAX_BYTES:2097152} + proxy-auth: + # Set to true only when the gateway is unreachable except through the proxy that sets + # this header. See docs/SSO.md. + enabled: ${GATEWAY_PROXY_AUTH:false} + header: ${GATEWAY_PROXY_AUTH_HEADER:X-Forwarded-User} + scopes: ${GATEWAY_PROXY_AUTH_SCOPES:read,write} admin-auth: max-failures: ${GATEWAY_ADMIN_AUTH_FAILURES:10} window: ${GATEWAY_ADMIN_AUTH_WINDOW:5m} + global-max-failures: ${GATEWAY_ADMIN_AUTH_GLOBAL_FAILURES:0} + client-auth: + max-failures: ${GATEWAY_CLIENT_AUTH_FAILURES:60} + window: ${GATEWAY_CLIENT_AUTH_WINDOW:1m} + global-max-failures: ${GATEWAY_CLIENT_AUTH_GLOBAL_FAILURES:0} guardrails: prompt-log-redaction: ${GATEWAY_GUARDRAILS_LOG_REDACTION:true} redact-before-send: ${GATEWAY_GUARDRAILS_REDACT:false} @@ -82,9 +108,15 @@ gateway: retention: ${GATEWAY_PROMPT_LOG_RETENTION:7d} max-body-bytes: ${GATEWAY_PROMPT_LOG_MAX_BYTES:65536} alerts: + email-to: ${GATEWAY_ALERT_EMAIL_TO:} + email-from: ${GATEWAY_ALERT_EMAIL_FROM:path-gateway@localhost} webhook-url: ${GATEWAY_ALERT_WEBHOOK:} cooldown: ${GATEWAY_ALERT_COOLDOWN:1h} budget-threshold: ${GATEWAY_ALERT_BUDGET_THRESHOLD:0.8} + digest: + # One email a day summarising yesterday. Only sends when an email channel is configured. + enabled: ${GATEWAY_DIGEST:true} + cron: "${GATEWAY_DIGEST_CRON:0 0 8 * * *}" usage-retention: ${GATEWAY_USAGE_RETENTION:180d} idempotency: ttl: ${GATEWAY_IDEMPOTENCY_TTL:24h} @@ -120,6 +152,11 @@ management: health: redis: enabled: false + # SMTP is configured from the console (Settings -> alert email), not from Spring + # properties, so the stock mail indicator would probe an empty host and report DOWN, + # marking the container unhealthy for a channel that is optional. + mail: + enabled: false tracing: sampling: probability: ${GATEWAY_TRACE_SAMPLE:0} diff --git a/src/main/resources/db/migration/V10__project_budgets.sql b/path-gateway/src/main/resources/db/migration/V10__project_budgets.sql similarity index 100% rename from src/main/resources/db/migration/V10__project_budgets.sql rename to path-gateway/src/main/resources/db/migration/V10__project_budgets.sql diff --git a/src/main/resources/db/migration/V11__project_limits.sql b/path-gateway/src/main/resources/db/migration/V11__project_limits.sql similarity index 100% rename from src/main/resources/db/migration/V11__project_limits.sql rename to path-gateway/src/main/resources/db/migration/V11__project_limits.sql diff --git a/src/main/resources/db/migration/V12__model_aliases.sql b/path-gateway/src/main/resources/db/migration/V12__model_aliases.sql similarity index 100% rename from src/main/resources/db/migration/V12__model_aliases.sql rename to path-gateway/src/main/resources/db/migration/V12__model_aliases.sql diff --git a/src/main/resources/db/migration/V13__route_strategy.sql b/path-gateway/src/main/resources/db/migration/V13__route_strategy.sql similarity index 100% rename from src/main/resources/db/migration/V13__route_strategy.sql rename to path-gateway/src/main/resources/db/migration/V13__route_strategy.sql diff --git a/src/main/resources/db/migration/V14__end_user_attribution.sql b/path-gateway/src/main/resources/db/migration/V14__end_user_attribution.sql similarity index 100% rename from src/main/resources/db/migration/V14__end_user_attribution.sql rename to path-gateway/src/main/resources/db/migration/V14__end_user_attribution.sql diff --git a/src/main/resources/db/migration/V15__end_user_reservations.sql b/path-gateway/src/main/resources/db/migration/V15__end_user_reservations.sql similarity index 100% rename from src/main/resources/db/migration/V15__end_user_reservations.sql rename to path-gateway/src/main/resources/db/migration/V15__end_user_reservations.sql diff --git a/src/main/resources/db/migration/V16__provider_api_version.sql b/path-gateway/src/main/resources/db/migration/V16__provider_api_version.sql similarity index 100% rename from src/main/resources/db/migration/V16__provider_api_version.sql rename to path-gateway/src/main/resources/db/migration/V16__provider_api_version.sql diff --git a/src/main/resources/db/migration/V17__semantic_cache.sql b/path-gateway/src/main/resources/db/migration/V17__semantic_cache.sql similarity index 100% rename from src/main/resources/db/migration/V17__semantic_cache.sql rename to path-gateway/src/main/resources/db/migration/V17__semantic_cache.sql diff --git a/path-gateway/src/main/resources/db/migration/V18__mail_settings.sql b/path-gateway/src/main/resources/db/migration/V18__mail_settings.sql new file mode 100644 index 0000000..27e67e6 --- /dev/null +++ b/path-gateway/src/main/resources/db/migration/V18__mail_settings.sql @@ -0,0 +1,16 @@ +-- Alert email configuration editable from the admin console (issue: UI email config). +-- One row; the SMTP password is encrypted with the same AES-GCM master key as provider +-- credentials and is never returned by the API. + +create table mail_settings ( + id bigint not null, + enabled boolean not null default false, + host varchar(255), + port integer, + username varchar(255), + password_cipher varchar(1024), + from_address varchar(255), + to_address varchar(1024), + updated_at timestamp(6) with time zone, + primary key (id) +); diff --git a/path-gateway/src/main/resources/db/migration/V19__usage_cache_tier.sql b/path-gateway/src/main/resources/db/migration/V19__usage_cache_tier.sql new file mode 100644 index 0000000..f7974c4 --- /dev/null +++ b/path-gateway/src/main/resources/db/migration/V19__usage_cache_tier.sql @@ -0,0 +1,5 @@ +-- Which cache tier served a call. Null means the request was not a cache hit. +-- "exact" is the response cache, "semantic" the near-duplicate tier. Keeping it on the row +-- makes /admin/stats survive a restart and agree across replicas, which an in-process +-- counter cannot do. +alter table usage_records add column cache_tier varchar(16); diff --git a/src/main/resources/db/migration/V1__baseline.sql b/path-gateway/src/main/resources/db/migration/V1__baseline.sql similarity index 100% rename from src/main/resources/db/migration/V1__baseline.sql rename to path-gateway/src/main/resources/db/migration/V1__baseline.sql diff --git a/path-gateway/src/main/resources/db/migration/V20__provider_vertex.sql b/path-gateway/src/main/resources/db/migration/V20__provider_vertex.sql new file mode 100644 index 0000000..f015723 --- /dev/null +++ b/path-gateway/src/main/resources/db/migration/V20__provider_vertex.sql @@ -0,0 +1,7 @@ +-- Vertex AI needs a project and a region; AI Studio and the OpenAI-shaped providers leave +-- both null. The service-account JSON lives in the existing encrypted credential column. +-- A service-account JSON is a few kilobytes; the old 2048 limit truncated its ciphertext. +alter table providers alter column api_key_cipher set data type varchar(4096); + +alter table providers add column project varchar(128); +alter table providers add column location varchar(64); diff --git a/src/main/resources/db/migration/V2__response_cache.sql b/path-gateway/src/main/resources/db/migration/V2__response_cache.sql similarity index 100% rename from src/main/resources/db/migration/V2__response_cache.sql rename to path-gateway/src/main/resources/db/migration/V2__response_cache.sql diff --git a/src/main/resources/db/migration/V3__budgets_and_timeouts.sql b/path-gateway/src/main/resources/db/migration/V3__budgets_and_timeouts.sql similarity index 100% rename from src/main/resources/db/migration/V3__budgets_and_timeouts.sql rename to path-gateway/src/main/resources/db/migration/V3__budgets_and_timeouts.sql diff --git a/src/main/resources/db/migration/V4__routing_weights.sql b/path-gateway/src/main/resources/db/migration/V4__routing_weights.sql similarity index 100% rename from src/main/resources/db/migration/V4__routing_weights.sql rename to path-gateway/src/main/resources/db/migration/V4__routing_weights.sql diff --git a/src/main/resources/db/migration/V5__audit_and_alerts.sql b/path-gateway/src/main/resources/db/migration/V5__audit_and_alerts.sql similarity index 100% rename from src/main/resources/db/migration/V5__audit_and_alerts.sql rename to path-gateway/src/main/resources/db/migration/V5__audit_and_alerts.sql diff --git a/src/main/resources/db/migration/V6__prompt_logs.sql b/path-gateway/src/main/resources/db/migration/V6__prompt_logs.sql similarity index 100% rename from src/main/resources/db/migration/V6__prompt_logs.sql rename to path-gateway/src/main/resources/db/migration/V6__prompt_logs.sql diff --git a/src/main/resources/db/migration/V7__key_model_allowlist.sql b/path-gateway/src/main/resources/db/migration/V7__key_model_allowlist.sql similarity index 100% rename from src/main/resources/db/migration/V7__key_model_allowlist.sql rename to path-gateway/src/main/resources/db/migration/V7__key_model_allowlist.sql diff --git a/src/main/resources/db/migration/V8__budget_reservations.sql b/path-gateway/src/main/resources/db/migration/V8__budget_reservations.sql similarity index 100% rename from src/main/resources/db/migration/V8__budget_reservations.sql rename to path-gateway/src/main/resources/db/migration/V8__budget_reservations.sql diff --git a/src/main/resources/db/migration/V9__projects_and_admin_tokens.sql b/path-gateway/src/main/resources/db/migration/V9__projects_and_admin_tokens.sql similarity index 100% rename from src/main/resources/db/migration/V9__projects_and_admin_tokens.sql rename to path-gateway/src/main/resources/db/migration/V9__projects_and_admin_tokens.sql diff --git a/src/test/java/com/anindra/path/AnthropicAdapterTest.java b/path-gateway/src/test/java/com/anindra/path/AnthropicAdapterTest.java similarity index 100% rename from src/test/java/com/anindra/path/AnthropicAdapterTest.java rename to path-gateway/src/test/java/com/anindra/path/AnthropicAdapterTest.java diff --git a/src/test/java/com/anindra/path/GatewayApiTest.java b/path-gateway/src/test/java/com/anindra/path/GatewayApiTest.java similarity index 66% rename from src/test/java/com/anindra/path/GatewayApiTest.java rename to path-gateway/src/test/java/com/anindra/path/GatewayApiTest.java index c56702b..5ab9d41 100644 --- a/src/test/java/com/anindra/path/GatewayApiTest.java +++ b/path-gateway/src/test/java/com/anindra/path/GatewayApiTest.java @@ -10,6 +10,7 @@ import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; +import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -46,8 +47,8 @@ void setUp() { } @Test - void catalogueIsPublicAndSeeded() throws Exception { - mvc.perform(get("/v1/models")) + void catalogueIsSeededAndServedToAnAuthenticatedCaller() throws Exception { + mvc.perform(get("/v1/models").header("Authorization", "Bearer " + key)) .andExpect(status().isOk()) .andExpect(jsonPath("$.object").value("list")) .andExpect(jsonPath("$.data[?(@.id == 'gpt-4o-mini')]").exists()) @@ -55,14 +56,39 @@ void catalogueIsPublicAndSeeded() throws Exception { } @Test - void modelsRequireNoKeyButChatDoes() throws Exception { + void catalogueRequiresAKeyByDefault() throws Exception { + // The catalogue names every model and alias, so it is gated by default; a deployment + // that wants it keyless sets gateway.public-models=true. + mvc.perform(get("/v1/models")) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.error.code").value("invalid_api_key")); + } + + @Test + void onlyLivenessIsPublicOnActuator() throws Exception { + mvc.perform(get("/actuator/health")) + .andExpect(status().isOk()); + + mvc.perform(get("/actuator/metrics")) + .andExpect(status().isUnauthorized()); + mvc.perform(get("/actuator/prometheus")) + .andExpect(status().isUnauthorized()); + mvc.perform(get("/actuator/health/providers")) + .andExpect(status().isUnauthorized()); + + mvc.perform(get("/actuator/metrics").header("X-Admin-Token", "test-admin")) + .andExpect(status().isOk()); + } + + @Test + void chatRequiresAKey() throws Exception { mvc.perform(post("/v1/chat/completions") .contentType(MediaType.APPLICATION_JSON) .content(""" {"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}]} """)) .andExpect(status().isUnauthorized()) - .andExpect(jsonPath("$.error.code").value("missing_api_key")); + .andExpect(jsonPath("$.error.code").value("invalid_api_key")); } @Test @@ -122,8 +148,35 @@ void openApiSpecIsPublished() throws Exception { } @Test - void everyResponseCarriesARequestId() throws Exception { + void everyKeyRejectionIsByteForByteIdentical() throws Exception { + String missing = mvc.perform(get("/v1/models")) + .andExpect(status().isUnauthorized()) + .andReturn().getResponse().getContentAsString(); + String invalid = mvc.perform(get("/v1/models") + .header("Authorization", "Bearer sk-gw-nope")) + .andExpect(status().isUnauthorized()) + .andReturn().getResponse().getContentAsString(); + + // An attacker must not be able to tell "no key" from "wrong key" from the body. + assertThat(invalid).isEqualTo(missing); + } + + @Test + void apiResponsesAreNotCacheable() throws Exception { + // A CDN caches by URL, so "…/actuator/health/x.css" can be stored and served to + // anonymous clients unless the origin says otherwise. + mvc.perform(get("/actuator/health")) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .header().string("Cache-Control", org.hamcrest.Matchers.containsString("no-store"))); + mvc.perform(get("/v1/models")) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .header().string("Cache-Control", org.hamcrest.Matchers.containsString("no-store"))); + } + + @Test + void everyResponseCarriesARequestId() throws Exception { + mvc.perform(get("/actuator/health")) .andExpect(status().isOk()) .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers .header().exists("X-Request-Id")); @@ -150,6 +203,25 @@ void readOnlyAdminTokensCannotMutate() throws Exception { .andExpect(status().isOk()); } + @Test + void adminSettingsReportsTheTokenScopesAndSubject() throws Exception { + // A read-scoped token must be able to learn that it is read-only, so the console can + // hide the write controls instead of failing with a 403 after the click. + String readOnly = tenancy.createToken("settings-viewer", "read").plaintext(); + mvc.perform(get("/admin/settings").header("X-Admin-Token", readOnly)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.subject").value("settings-viewer")) + .andExpect(jsonPath("$.scopes.length()").value(1)) + .andExpect(jsonPath("$.scopes[0]").value("read")) + .andExpect(jsonPath("$.can_write").value(false)); + + mvc.perform(get("/admin/settings").header("X-Admin-Token", "test-admin")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.subject").value("bootstrap")) + .andExpect(jsonPath("$.scopes[?(@ == 'write')]").exists()) + .andExpect(jsonPath("$.can_write").value(true)); + } + @Test void adminRequiresToken() throws Exception { mvc.perform(get("/admin/providers")) diff --git a/path-gateway/src/test/java/com/anindra/path/ProxyAuthReadOnlyTest.java b/path-gateway/src/test/java/com/anindra/path/ProxyAuthReadOnlyTest.java new file mode 100644 index 0000000..d6c9a6c --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/ProxyAuthReadOnlyTest.java @@ -0,0 +1,57 @@ +package com.anindra.path; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** GATEWAY_PROXY_AUTH_SCOPES=read: the identity can look but not change anything. */ +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:proxyauthreadonly;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.admin-token=test-admin", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key", + "gateway.proxy-auth.enabled=true", + "gateway.proxy-auth.scopes=read" +}) +@AutoConfigureMockMvc +class ProxyAuthReadOnlyTest { + + @Autowired + private MockMvc mvc; + + @Test + void reportsTheProxyIdentityAndItsScopes() throws Exception { + mvc.perform(get("/admin/settings").header("X-Forwarded-User", "viewer@example.com")) + .andExpect(status().isOk()) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .jsonPath("$.auth_mode").value("proxy")) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .jsonPath("$.subject").value("viewer@example.com")) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .jsonPath("$.scopes[0]").value("read")) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .jsonPath("$.can_write").value(false)); + } + + @Test + void canRead() throws Exception { + mvc.perform(get("/admin/providers").header("X-Forwarded-User", "viewer@example.com")) + .andExpect(status().isOk()); + } + + @Test + void cannotWrite() throws Exception { + mvc.perform(post("/admin/providers") + .header("X-Forwarded-User", "viewer@example.com") + .contentType("application/json") + .content("{\"slug\":\"nope\",\"type\":\"openai\",\"base_url\":\"https://example.test/v1\"}")) + .andExpect(status().isForbidden()); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/ProxyAuthTest.java b/path-gateway/src/test/java/com/anindra/path/ProxyAuthTest.java new file mode 100644 index 0000000..27c58c7 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/ProxyAuthTest.java @@ -0,0 +1,68 @@ +package com.anindra.path; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * SSO at the edge: the proxy authenticates the human and injects an identity header. The + * gateway must then (a) refuse everything without that header, (b) attribute writes to the + * identity, and (c) not require the pasted admin token at all. + */ +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:proxyauth;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.admin-token=test-admin", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key", + "gateway.proxy-auth.enabled=true", + "gateway.proxy-auth.header=X-Forwarded-User" +}) +@AutoConfigureMockMvc +class ProxyAuthTest { + + @Autowired + private MockMvc mvc; + + @Test + void adminIsClosedWithoutTheProxyHeader() throws Exception { + mvc.perform(get("/admin/providers")) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.error.code").value("proxy_identity_missing")); + } + + @Test + void thePastedTokenIsNotEnoughWhenProxyAuthIsOn() throws Exception { + mvc.perform(get("/admin/providers").header("X-Admin-Token", "test-admin")) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.error.code").value("proxy_identity_missing")); + } + + @Test + void aSignedInIdentityGetsIn() throws Exception { + mvc.perform(get("/admin/settings").header("X-Forwarded-User", "anindra@example.com")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.auth_mode").value("proxy")); + } + + @Test + void writesAreAuditedAgainstTheIdentity() throws Exception { + mvc.perform(post("/admin/providers") + .header("X-Forwarded-User", "anindra@example.com") + .contentType("application/json") + .content("{\"slug\":\"sso-test\",\"type\":\"openai\",\"base_url\":\"https://example.test/v1\"}")) + .andExpect(status().isOk()); + + mvc.perform(get("/admin/audit").header("X-Forwarded-User", "anindra@example.com")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[0].actor").value("anindra@example.com")); + } + +} diff --git a/path-gateway/src/test/java/com/anindra/path/PublicModelsTest.java b/path-gateway/src/test/java/com/anindra/path/PublicModelsTest.java new file mode 100644 index 0000000..1730027 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/PublicModelsTest.java @@ -0,0 +1,38 @@ +package com.anindra.path; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * The catalogue is gated by default; an operator can make it keyless again. Both directions + * need coverage, because flipping the default is exactly the kind of change that regresses + * silently. + */ +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:publicmodels;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.admin-token=test-admin", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key", + "gateway.public-models=true" +}) +@AutoConfigureMockMvc +class PublicModelsTest { + + @Autowired + private MockMvc mvc; + + @Test + void catalogueIsKeylessWhenExplicitlyEnabled() throws Exception { + mvc.perform(get("/v1/models")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.object").value("list")); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/SemanticHitStatsTest.java b/path-gateway/src/test/java/com/anindra/path/SemanticHitStatsTest.java new file mode 100644 index 0000000..06d3330 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/SemanticHitStatsTest.java @@ -0,0 +1,107 @@ +package com.anindra.path; + +import com.anindra.path.admin.AdminReportingService; +import com.anindra.path.admin.TenancyService; +import com.anindra.path.domain.ClientKey; +import com.anindra.path.domain.UsageRecord; +import com.anindra.path.gateway.ApiKeyService; +import com.anindra.path.repo.ClientKeyRepository; +import com.anindra.path.repo.UsageRecordRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Semantic hits are read from the usage rows, so they survive a restart and are identical + * on every replica. An in-process counter could do neither. + */ +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:semantichits;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.admin-token=test-admin", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class SemanticHitStatsTest { + + @Autowired + private UsageRecordRepository usage; + + @Autowired + private AdminReportingService reporting; + + @Autowired + private ClientKeyRepository keys; + + @Autowired + private ApiKeyService apiKeys; + + @Autowired + private TenancyService tenancy; + + private Long keyId; + + @BeforeEach + void setUp() { + usage.deleteAll(); + keys.deleteAll(); + keyId = apiKeys.create("semantic-stats", null, 1000).key().getId(); + } + + private void record(Long clientKeyId, String tier) { + UsageRecord record = new UsageRecord(); + record.setClientKeyId(clientKeyId); + record.setPublicModelId("gpt-4o-mini"); + record.setProviderSlug("openai"); + record.setStatus(UsageRecord.Status.SUCCESS); + record.setFinishReason("cache"); + record.setCacheTier(tier); + usage.save(record); + } + + @SuppressWarnings("unchecked") + private Map cache(Map stats) { + return (Map) stats.get("cache"); + } + + @Test + void semanticHitsComeFromTheWindowRatherThanMemory() { + record(keyId, "semantic"); + record(keyId, "semantic"); + record(keyId, "exact"); + record(keyId, null); + + Map stats = reporting.stats("24h", null, null); + + assertThat(cache(stats).get("semantic_hits")).isEqualTo(2L); + } + + @Test + void aProjectCountsOnlyItsOwnSemanticHits() { + Long projectId = ((Number) tenancy.createProject("semantic-project", "Semantic project", + null, null, null, null, null).get("id")).longValue(); + ClientKey member = keys.findById(keyId).orElseThrow(); + member.setProjectId(projectId); + keys.save(member); + ClientKey other = apiKeys.create("other-project-key", null, 1000).key(); + record(member.getId(), "semantic"); + record(other.getId(), "semantic"); + record(other.getId(), "semantic"); + + Map stats = reporting.stats("24h", projectId, null); + + assertThat(cache(stats).get("semantic_hits")).isEqualTo(1L); + } + + @Test + void anEmptyWindowReportsZeroRatherThanNothing() { + Map stats = reporting.stats("24h", null, null); + + assertThat(cache(stats).get("semantic_hits")).isEqualTo(0L); + } +} diff --git a/src/test/java/com/anindra/path/UsageCostTest.java b/path-gateway/src/test/java/com/anindra/path/UsageCostTest.java similarity index 100% rename from src/test/java/com/anindra/path/UsageCostTest.java rename to path-gateway/src/test/java/com/anindra/path/UsageCostTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/UsageExportRangeTest.java b/path-gateway/src/test/java/com/anindra/path/UsageExportRangeTest.java new file mode 100644 index 0000000..37a5190 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/UsageExportRangeTest.java @@ -0,0 +1,80 @@ +package com.anindra.path; + +import com.anindra.path.domain.UsageRecord; +import com.anindra.path.repo.UsageRecordRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.MockMvc; + +import java.time.Instant; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** The archive slice must be selectable, because retention deletes by the same dates. */ +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:usageexport;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.admin-token=test-admin", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +@AutoConfigureMockMvc +class UsageExportRangeTest { + + @Autowired + private MockMvc mvc; + + @Autowired + private UsageRecordRepository usage; + + @BeforeEach + void seedTwoMonths() { + usage.deleteAll(); + usage.save(record("january-model", Instant.parse("2026-01-15T10:00:00Z"))); + usage.save(record("march-model", Instant.parse("2026-03-15T10:00:00Z"))); + } + + private static UsageRecord record(String model, Instant at) { + UsageRecord record = new UsageRecord(); + record.setCreatedAt(at); + record.setPublicModelId(model); + record.setProviderSlug("openai"); + record.setStatus(UsageRecord.Status.SUCCESS); + return record; + } + + @Test + void fromAndToSelectExactlyTheSliceToArchive() throws Exception { + mvc.perform(get("/admin/usage/export") + .header("X-Admin-Token", "test-admin") + .param("from", "2026-01-01") + .param("to", "2026-02-01")) + .andExpect(status().isOk()) + .andExpect(content().string(org.hamcrest.Matchers.containsString("january-model"))) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("march-model")))); + } + + @Test + void anUnparseableDateIsA400NamingTheParameter() throws Exception { + mvc.perform(get("/admin/usage/export") + .header("X-Admin-Token", "test-admin") + .param("from", "last-tuesday")) + .andExpect(status().isBadRequest()) + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"param\":\"from\""))); + } + + @Test + void fromAfterToIsRejected() throws Exception { + mvc.perform(get("/admin/usage/export") + .header("X-Admin-Token", "test-admin") + .param("from", "2026-03-01") + .param("to", "2026-02-01")) + .andExpect(status().isBadRequest()); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/admin/AdminStatsSeriesTest.java b/path-gateway/src/test/java/com/anindra/path/admin/AdminStatsSeriesTest.java new file mode 100644 index 0000000..bdb8198 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/admin/AdminStatsSeriesTest.java @@ -0,0 +1,134 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.domain.UsageRecord; +import com.anindra.path.repo.ClientKeyRepository; +import com.anindra.path.repo.UsageRecordRepository; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class AdminStatsSeriesTest { + + private final UsageRecordRepository usage = Mockito.mock(UsageRecordRepository.class); + private final AdminService adminService = Mockito.mock(AdminService.class); + private final ClientKeyRepository keys = Mockito.mock(ClientKeyRepository.class); + private final AdminReportingService reporting = new AdminReportingService(usage, adminService, keys); + + @SuppressWarnings("unchecked") + private static Map emptyStats() { + Map out = new LinkedHashMap<>(); + out.put("requests", 0); + return out; + } + + private static Object[] row(Instant at, UsageRecord.Status status, String cost, Long latencyMs) { + return new Object[]{at, status, new BigDecimal(cost), latencyMs}; + } + + private static List rows(Object[]... rows) { + return List.of(rows); + } + + @Test + void a24hWindowInHourlyBucketsHas24PointsIncludingEmptyOnes() { + when(adminService.stats(any())).thenReturn(emptyStats()); + Instant now = Instant.now(); + when(usage.findSeriesSource(any(), any())).thenReturn(rows( + row(now.minusSeconds(30), UsageRecord.Status.SUCCESS, "0.25", 100L), + row(now.minusSeconds(3_600), UsageRecord.Status.SUCCESS, "0.50", 300L), + row(now.minusSeconds(3_600), UsageRecord.Status.ERROR, "0", 0L))); + + Map out = reporting.stats("24h", null, "1h"); + + List> series = (List>) out.get("series"); + assertThat(series).hasSize(24); + long requests = series.stream().mapToLong(p -> ((Number) p.get("requests")).longValue()).sum(); + long failures = series.stream().mapToLong(p -> ((Number) p.get("failures")).longValue()).sum(); + BigDecimal cost = series.stream() + .map(p -> (BigDecimal) p.get("cost_usd")) + .reduce(BigDecimal.ZERO, BigDecimal::add); + assertThat(requests).isEqualTo(3); + assertThat(failures).isEqualTo(1); + assertThat(cost).isEqualByComparingTo("0.75"); + // Empty buckets are present and zeroed rather than omitted. + assertThat(series).anyMatch(p -> ((Number) p.get("requests")).longValue() == 0 + && ((BigDecimal) p.get("cost_usd")).signum() == 0); + assertThat(series).allMatch(p -> p.containsKey("from") && p.containsKey("p95_latency_ms")); + // The newest point is the one containing "now". + assertThat(((Number) series.get(series.size() - 1).get("requests")).longValue()).isEqualTo(1L); + } + + @Test + void theWholeWindowIsReadInOneQuery() { + when(adminService.stats(any())).thenReturn(emptyStats()); + Instant now = Instant.now(); + when(usage.findSeriesSource(any(), any())).thenReturn(rows( + row(now, UsageRecord.Status.SUCCESS, "0.1", 10L))); + + reporting.stats("6h", null, "5m"); + + verify(usage, times(1)).findSeriesSource(any(), any()); + } + + @Test + void aProjectSeriesReadsItsKeysInOneQuery() { + when(adminService.stats(any())).thenReturn(emptyStats()); + com.anindra.path.domain.ClientKey key = Mockito.mock(com.anindra.path.domain.ClientKey.class); + when(key.getId()).thenReturn(1L); + when(keys.findByProjectIdOrderByCreatedAtDesc(7L)).thenReturn(List.of(key)); + when(usage.findByClientKeyIdInAndCreatedAtAfterOrderByCreatedAtDesc(anyCollection(), any(), any())) + .thenReturn(List.of()); + when(usage.findSeriesSourceForKeys(anyCollection(), any(), any())).thenReturn(List.of()); + + Map out = reporting.stats("24h", 7L, "1h"); + + assertThat((List) out.get("series")).hasSize(24); + verify(usage, times(1)).findSeriesSourceForKeys(anyCollection(), any(), any()); + } + + @Test + void anInvalidBucketIsA400ListingTheAcceptedValues() { + assertThatThrownBy(() -> reporting.stats("24h", null, "3h")) + .isInstanceOf(ApiException.class) + .satisfies(e -> { + ApiException api = (ApiException) e; + assertThat(api.status()).isEqualTo(400); + assertThat(api.getMessage()).contains("1m").contains("5m").contains("1h").contains("1d"); + }); + } + + @Test + void omittingTheBucketKeepsTodaysResponseUnchanged() { + when(adminService.stats(any())).thenReturn(emptyStats()); + + Map out = reporting.stats("24h", null); + + assertThat(out).doesNotContainKeys("series", "bucket"); + assertThat(out.get("requests")).isEqualTo(0); + } + + @Test + void aHugeWindowIsClampedTo500Points() { + when(adminService.stats(any())).thenReturn(emptyStats()); + when(usage.findSeriesSource(any(), any())).thenReturn(new ArrayList<>()); + + Map out = reporting.stats("365d", null, "1h"); + + assertThat((List) out.get("series")).hasSize(500); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/admin/AlertEmailTest.java b/path-gateway/src/test/java/com/anindra/path/admin/AlertEmailTest.java new file mode 100644 index 0000000..6f8fea8 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/admin/AlertEmailTest.java @@ -0,0 +1,105 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; + +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class AlertEmailTest { + + private final JavaMailSender sender = mock(JavaMailSender.class); + private final MailSettingsService mail = mock(MailSettingsService.class); + private final AlertService alerts = + new AlertService(new GatewayProperties(), new ObjectMapper(), mail, + new io.micrometer.core.instrument.simple.SimpleMeterRegistry()); + + private void configure(String recipients) { + when(mail.recipients()).thenReturn(recipients); + when(mail.fromAddress()).thenReturn("path@example.com"); + when(mail.emailEnabled()).thenReturn(!recipients.isBlank()); + when(mail.sender()).thenReturn(Optional.of(sender)); + } + + @Test + void sendsOneEmailPerAlert() { + configure("ops@example.com"); + + alerts.notify("key.budget_low", Map.of("key_id", 1, "remaining_usd", "0.42")); + + ArgumentCaptor message = ArgumentCaptor.forClass(SimpleMailMessage.class); + verify(sender, timeout(2000)).send(message.capture()); + assertThat(message.getValue().getSubject()).isEqualTo("[Path] key.budget_low"); + assertThat(message.getValue().getTo()).containsExactly("ops@example.com"); + assertThat(message.getValue().getText()).contains("key.budget_low").contains("remaining_usd: 0.42"); + } + + @Test + void multipleRecipientsAreSplit() { + configure("a@example.com, b@example.com"); + + alerts.notify("provider.circuit_open", Map.of("provider", "openai")); + + ArgumentCaptor message = ArgumentCaptor.forClass(SimpleMailMessage.class); + verify(sender, timeout(2000)).send(message.capture()); + assertThat(message.getValue().getTo()).containsExactlyInAnyOrder("a@example.com", "b@example.com"); + } + + @Test + void noRecipientsMeansNoMail() { + configure(""); + + assertThat(alerts.emailEnabled()).isFalse(); + alerts.notify("anything", Map.of()); + + verifyNoInteractions(sender); + verify(mail, org.mockito.Mockito.never()).sender(); + } + + @Test + void aMissingSenderIsANoOp() { + when(mail.recipients()).thenReturn("ops@example.com"); + when(mail.emailEnabled()).thenReturn(false); + when(mail.sender()).thenReturn(Optional.empty()); + + alerts.notify("anything", Map.of()); + + verifyNoInteractions(sender); + } + + @Test + void bodyListsEveryField() { + String text = AlertService.text(Map.of( + "source", "path", + "event", "project.budget_exhausted", + "at", "2026-11-17T10:00:00Z", + "project", "acme", + "spent_usd", "10.00")); + + assertThat(text).contains("project.budget_exhausted").contains("project: acme").contains("spent_usd: 10.00"); + assertThat(text).doesNotContain("source: path"); + } + + @Test + void mailboxFailuresDoNotEscape() { + configure("ops@example.com"); + org.mockito.Mockito.doThrow(new org.springframework.mail.MailSendException("smtp down")) + .when(sender).send(any(SimpleMailMessage.class)); + + alerts.notify("provider.circuit_open", Map.of("provider", "openai")); + + verify(sender, timeout(2000)).send(any(SimpleMailMessage.class)); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/admin/AlertServiceTest.java b/path-gateway/src/test/java/com/anindra/path/admin/AlertServiceTest.java new file mode 100644 index 0000000..6460862 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/admin/AlertServiceTest.java @@ -0,0 +1,250 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntFunction; + +import static org.assertj.core.api.Assertions.assertThat; + +class AlertServiceTest { + + private HttpServer server; + private final AtomicInteger attempts = new AtomicInteger(); + private final AtomicInteger deliveries = new AtomicInteger(); + private final GatewayProperties props = new GatewayProperties(); + private final SimpleMeterRegistry metrics = new SimpleMeterRegistry(); + /** Given a 1-based attempt number, choose the status to answer with. */ + private IntFunction statusForAttempt = attempt -> 200; + private AlertService alerts; + + @BeforeEach + void startWebhook() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/hook", this::handle); + server.start(); + props.getAlerts().setWebhookUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/hook"); + props.getAlerts().setCooldown(java.time.Duration.ZERO); + alerts = new AlertService(props, new ObjectMapper(), + org.mockito.Mockito.mock(MailSettingsService.class), metrics); + } + + private void handle(HttpExchange exchange) throws IOException { + int attempt = attempts.incrementAndGet(); + int status = statusForAttempt.apply(attempt); + if (status == 429) { + exchange.getResponseHeaders().add("Retry-After", "0"); + } + if (status >= 200 && status < 300) { + deliveries.incrementAndGet(); + } + exchange.sendResponseHeaders(status, -1); + exchange.close(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + @Test + void notifyOnceDeliversOneEventPerKey() throws Exception { + for (int i = 0; i < 5; i++) { + alerts.notifyOnce("project.budget_low:1:daily@2026-11-17", "project.budget_low", + Map.of("project", "acme")); + } + // A different period is a different alert. + alerts.notifyOnce("project.budget_low:1:daily@2026-11-18", "project.budget_low", + Map.of("project", "acme")); + + assertThat(awaitDeliveries(2)).isEqualTo(2); + } + + @Test + void noWebhookConfiguredIsANoOp() { + props.getAlerts().setWebhookUrl(""); + alerts.notifyOnce("k", "event", Map.of()); + + assertThat(deliveries.get()).isZero(); + assertThat(attempts.get()).isZero(); + } + + @Test + void retriesA500ThenDeliversExactlyOnce() throws Exception { + statusForAttempt = attempt -> attempt == 1 ? 500 : 200; + + alerts.notify("provider.circuit_open", Map.of("provider", "openai")); + + assertThat(awaitDeliveries(1)).isEqualTo(1); + assertThat(attempts.get()).isEqualTo(2); + assertThat(counter("delivered")).isEqualTo(1.0); + assertThat(counter("retried")).isEqualTo(1.0); + assertThat(counter("dropped")).isZero(); + } + + @Test + void doesNotRetryA400() throws Exception { + statusForAttempt = attempt -> 400; + + alerts.notify("provider.circuit_open", Map.of("provider", "openai")); + + assertThat(awaitAttempts(1)).isEqualTo(1); + // Give an incorrect retry time to show up before asserting the count. + Thread.sleep(300); + assertThat(attempts.get()).isEqualTo(1); + assertThat(deliveries.get()).isZero(); + assertThat(counter("dropped")).isEqualTo(1.0); + assertThat(counter("retried")).isZero(); + } + + @Test + void honoursRetryAfterOn429() throws Exception { + statusForAttempt = attempt -> attempt == 1 ? 429 : 200; + + alerts.notify("key.budget_low", Map.of("key_id", 1)); + + assertThat(awaitDeliveries(1)).isEqualTo(1); + assertThat(attempts.get()).isEqualTo(2); + assertThat(counter("delivered")).isEqualTo(1.0); + } + + @Test + void givesUpAfterThreeAttemptsAndCountsOneDrop() throws Exception { + statusForAttempt = attempt -> 503; + + alerts.notify("provider.circuit_open", Map.of("provider", "openai")); + + assertThat(awaitAttempts(3)).isEqualTo(3); + Thread.sleep(300); + assertThat(attempts.get()).isEqualTo(3); + assertThat(counter("dropped")).isEqualTo(1.0); + assertThat(counter("retried")).isEqualTo(2.0); + } + + @Test + void aDeadEndpointCannotGrowTheQueueWithoutBound() throws Exception { + // One worker, one queued task: anything beyond that must be dropped, not block. + alerts = new AlertService(props, new ObjectMapper(), + org.mockito.Mockito.mock(MailSettingsService.class), metrics, 1, 1); + CountDownLatch release = new CountDownLatch(1); + // Block the only worker by making the first response slow. + server.stop(0); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/hook", exchange -> { + try { + release.await(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + deliveries.incrementAndGet(); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + props.getAlerts().setWebhookUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/hook"); + + for (int i = 0; i < 5; i++) { + alerts.notify("event." + i, Map.of("i", i)); + } + release.countDown(); + + assertThat(counter("dropped")).isGreaterThanOrEqualTo(2.0); + assertThat(awaitDeliveries(1)).isGreaterThanOrEqualTo(1); + } + + @Test + void aSlowMailboxCannotGrowTheEmailQueueWithoutBound() throws Exception { + props.getAlerts().setWebhookUrl(""); + MailSettingsService mail = org.mockito.Mockito.mock(MailSettingsService.class); + org.springframework.mail.javamail.JavaMailSender sender = + org.mockito.Mockito.mock(org.springframework.mail.javamail.JavaMailSender.class); + org.mockito.Mockito.when(mail.emailEnabled()).thenReturn(true); + org.mockito.Mockito.when(mail.sender()).thenReturn(java.util.Optional.of(sender)); + org.mockito.Mockito.when(mail.recipients()).thenReturn("ops@example.com"); + org.mockito.Mockito.when(mail.fromAddress()).thenReturn("path@example.com"); + CountDownLatch release = new CountDownLatch(1); + org.mockito.Mockito.doAnswer(invocation -> { + release.await(2, TimeUnit.SECONDS); + return null; + }).when(sender).send(org.mockito.ArgumentMatchers.any(org.springframework.mail.SimpleMailMessage.class)); + alerts = new AlertService(props, new ObjectMapper(), mail, metrics, 1, 1); + + for (int i = 0; i < 5; i++) { + alerts.notify("email.event." + i, Map.of("i", i)); + } + release.countDown(); + + assertThat(metrics.counter("path.alerts.sent", "channel", "email", + "outcome", "dropped").count()).isGreaterThanOrEqualTo(2.0); + assertThat(awaitEmailDelivered(1)).isGreaterThanOrEqualTo(1.0); + } + + @Test + void aDeliveredEmailIsCountedOnItsOwnChannel() throws Exception { + props.getAlerts().setWebhookUrl(""); + MailSettingsService mail = org.mockito.Mockito.mock(MailSettingsService.class); + org.springframework.mail.javamail.JavaMailSender sender = + org.mockito.Mockito.mock(org.springframework.mail.javamail.JavaMailSender.class); + org.mockito.Mockito.when(mail.emailEnabled()).thenReturn(true); + org.mockito.Mockito.when(mail.sender()).thenReturn(java.util.Optional.of(sender)); + org.mockito.Mockito.when(mail.recipients()).thenReturn("ops@example.com"); + org.mockito.Mockito.when(mail.fromAddress()).thenReturn("path@example.com"); + alerts = new AlertService(props, new ObjectMapper(), mail, metrics); + + alerts.notify("key.budget_low", Map.of("key_id", 1)); + + assertThat(awaitEmailDelivered(1)).isEqualTo(1.0); + assertThat(metrics.counter("path.alerts.sent", "channel", "email", + "outcome", "delivered").count()).isEqualTo(1.0); + } + + private double awaitEmailDelivered(double expected) throws InterruptedException { + for (int i = 0; i < 60; i++) { + double value = metrics.counter("path.alerts.sent", "channel", "email", + "outcome", "delivered").count(); + if (value >= expected) { + return value; + } + Thread.sleep(50); + } + return metrics.counter("path.alerts.sent", "channel", "email", "outcome", "delivered").count(); + } + + private double counter(String outcome) { + return metrics.counter("path.alerts.sent", "channel", "webhook", "outcome", outcome).count(); + } + + private int awaitDeliveries(int expected) throws InterruptedException { + for (int i = 0; i < 120; i++) { + if (deliveries.get() >= expected) { + // give an over-delivery a chance to show up + Thread.sleep(150); + return deliveries.get(); + } + Thread.sleep(50); + } + return deliveries.get(); + } + + private int awaitAttempts(int expected) throws InterruptedException { + for (int i = 0; i < 120; i++) { + if (attempts.get() >= expected) { + return attempts.get(); + } + Thread.sleep(50); + } + return attempts.get(); + } +} diff --git a/src/test/java/com/anindra/path/admin/AliasServiceTest.java b/path-gateway/src/test/java/com/anindra/path/admin/AliasServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/admin/AliasServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/admin/AliasServiceTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/admin/AuditRedactionTest.java b/path-gateway/src/test/java/com/anindra/path/admin/AuditRedactionTest.java new file mode 100644 index 0000000..2883673 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/admin/AuditRedactionTest.java @@ -0,0 +1,38 @@ +package com.anindra.path.admin; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** The audit table is readable through /admin/audit, so it must never hold a credential. */ +class AuditRedactionTest { + + record ProviderInput(String slug, String baseUrl, String apiKey) { + } + + @Test + void providerApiKeysAreRedacted() { + String json = String.valueOf(AuditService.sanitise( + new ProviderInput("openai", "https://api.openai.com/v1", "sk-super-secret-value"))); + + assertThat(json).doesNotContain("sk-super-secret-value"); + assertThat(json).contains("\"apiKey\":\"***\""); + assertThat(json).contains("openai"); + } + + @Test + void nestedAndAlternateNamesAreRedacted() { + String json = String.valueOf(AuditService.sanitise(Map.of( + "provider", Map.of("api_key", "sk-nested", "base_url", "https://x"), + "new_key", "base64-master-key", + "password", "hunter2", + "note", "kept"))); + + assertThat(json).doesNotContain("sk-nested") + .doesNotContain("base64-master-key") + .doesNotContain("hunter2"); + assertThat(json).contains("kept").contains("https://x"); + } +} diff --git a/src/test/java/com/anindra/path/admin/AuditServiceTest.java b/path-gateway/src/test/java/com/anindra/path/admin/AuditServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/admin/AuditServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/admin/AuditServiceTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/admin/DailyDigestServiceTest.java b/path-gateway/src/test/java/com/anindra/path/admin/DailyDigestServiceTest.java new file mode 100644 index 0000000..3cf0a14 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/admin/DailyDigestServiceTest.java @@ -0,0 +1,158 @@ +package com.anindra.path.admin; + +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.gateway.ProviderCircuitBreaker; +import com.anindra.path.repo.ProviderRepository; +import com.anindra.path.repo.UsageRecordRepository; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.mail.MailSendException; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class DailyDigestServiceTest { + + private final UsageRecordRepository usage = Mockito.mock(UsageRecordRepository.class); + private final MailSettingsService mail = Mockito.mock(MailSettingsService.class); + private final ProviderCircuitBreaker breaker = Mockito.mock(ProviderCircuitBreaker.class); + private final ProviderRepository providers = Mockito.mock(ProviderRepository.class); + private final GatewayProperties props = new GatewayProperties(); + private final SimpleMeterRegistry registry = new SimpleMeterRegistry(); + private JavaMailSender sender; + private final DailyDigestService digest = new DailyDigestService( + usage, mail, props, breaker, providers, registry); + + private static DailyDigestService.Day day(long requests, long failures, String cost, + long tokens, long prompt, long cached, String savings, + String topError, long topErrorCount, + String slowest, long slowestP95, + List models) { + return new DailyDigestService.Day(requests, failures, new BigDecimal(cost), tokens, + prompt, cached, new BigDecimal(savings), topError, topErrorCount, + slowest, slowestP95, models); + } + + @Test + void theRenderedDigestMatchesTheDocumentedShape() { + DailyDigestService.Day today = day(12_431, 149, "18.42", 4_100_000, 1_000, 380, "2.90", + "upstream_timeout", 120, "claude-sonnet-4.5", 3_400, + List.of(new DailyDigestService.ModelSpend("gpt-4o", new BigDecimal("9.10")), + new DailyDigestService.ModelSpend("claude-sonnet-4.5", new BigDecimal("7.80")))); + DailyDigestService.Day yesterday = day(11_500, 90, "17.32", 3_800_000, 900, 300, "2.10", + "upstream_timeout", 70, "gpt-4o", 3_100, + List.of(new DailyDigestService.ModelSpend("gpt-4o", new BigDecimal("8.00")))); + + assertThat(DailyDigestService.subject(today)) + .isEqualTo("[Path] yesterday: 12,431 requests, $18.42, 1.2% failed"); + + String text = DailyDigestService.render(LocalDate.of(2026, 11, 19), + ZoneId.of("Asia/Kolkata"), today, yesterday, List.of("openai")); + + assertThat(text) + .contains("Path daily digest — 2026-11-19 (Asia/Kolkata)") + .contains("Requests 12,431 (+8% vs the day before)") + .contains("Spend $18.42 (+$1.10)") + .contains("Tokens 4.1M cache hit 38.0%, saved $2.90") + .contains("Failures 149 1.2% (top: upstream_timeout ×120)") + .contains("Slowest claude-sonnet-4.5 p95 3.4s") + .contains("Top spend gpt-4o $9.10 · claude-sonnet-4.5 $7.80") + .contains("Parked openai (circuit open now)"); + } + + private void configureMailbox() { + sender = Mockito.mock(JavaMailSender.class); + when(mail.emailEnabled()).thenReturn(true); + when(mail.sender()).thenReturn(Optional.of(sender)); + when(mail.recipients()).thenReturn("ops@example.com"); + when(mail.fromAddress()).thenReturn("path@example.com"); + when(breaker.snapshot()).thenReturn(Map.of()); + when(providers.findAllByOrderBySlugAsc()).thenReturn(List.of()); + } + + @Test + void sendsOneEmailForYesterday() { + configureMailbox(); + when(usage.countByCreatedAtBetween(any(), any())).thenReturn(5L, 4L); + when(usage.sumCostBetween(any(), any())).thenReturn(new BigDecimal("1.25")); + when(usage.sumTokensBetween(any(), any())).thenReturn(900L); + + DailyDigestService.Result result = digest.send(LocalDate.of(2026, 11, 19)); + + assertThat(result.sent()).isTrue(); + assertThat(result.reason()).isNull(); + ArgumentCaptor message = ArgumentCaptor.forClass(SimpleMailMessage.class); + verify(sender).send(message.capture()); + assertThat(message.getValue().getSubject()) + .isEqualTo("[Path] yesterday: 5 requests, $1.25, 0.0% failed"); + assertThat(message.getValue().getTo()).containsExactly("ops@example.com"); + assertThat(message.getValue().getText()).contains("Path daily digest — 2026-11-19"); + assertThat(registry.counter("path.alerts.sent", "channel", "digest", + "outcome", "delivered").count()).isEqualTo(1.0); + } + + @Test + void aDayWithNoTrafficSendsNothingRatherThanAWallOfZeroes() { + configureMailbox(); + when(usage.countByCreatedAtBetween(any(), any())).thenReturn(0L); + + DailyDigestService.Result result = digest.send(LocalDate.of(2026, 11, 19)); + + assertThat(result.sent()).isFalse(); + assertThat(result.reason()).isEqualTo("no_traffic"); + verify(mail, never()).sender(); + } + + @Test + void aDisabledDigestTouchesNothing() { + props.getDigest().setEnabled(false); + + DailyDigestService.Result result = digest.send(LocalDate.of(2026, 11, 19)); + + assertThat(result.sent()).isFalse(); + assertThat(result.reason()).isEqualTo("disabled"); + verifyNoInteractions(usage, mail, breaker, providers); + } + + @Test + void noMailboxMeansNoSendButTheTextIsStillAvailable() { + when(mail.emailEnabled()).thenReturn(false); + when(usage.countByCreatedAtBetween(any(), any())).thenReturn(3L, 1L); + + DailyDigestService.Result result = digest.send(LocalDate.of(2026, 11, 19)); + + assertThat(result.sent()).isFalse(); + assertThat(result.reason()).isEqualTo("email_not_configured"); + assertThat(result.text()).contains("Path daily digest"); + } + + @Test + void aBrokenMailboxLogsAndDoesNotThrow() { + configureMailbox(); + when(usage.countByCreatedAtBetween(any(), any())).thenReturn(3L, 1L); + Mockito.doThrow(new MailSendException("smtp down")) + .when(sender).send(any(SimpleMailMessage.class)); + + DailyDigestService.Result result = digest.send(LocalDate.of(2026, 11, 19)); + + assertThat(result.sent()).isFalse(); + assertThat(result.reason()).isEqualTo("send_failed"); + assertThat(registry.counter("path.alerts.sent", "channel", "digest", + "outcome", "dropped").count()).isEqualTo(1.0); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/admin/MailSettingsTest.java b/path-gateway/src/test/java/com/anindra/path/admin/MailSettingsTest.java new file mode 100644 index 0000000..93cbc9d --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/admin/MailSettingsTest.java @@ -0,0 +1,103 @@ +package com.anindra.path.admin; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.repo.MailSettingsRepository; +import com.anindra.path.secret.SecretCipher; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:mailsettings;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.jpa.hibernate.ddl-auto=validate", + "gateway.bootstrap-demo-key=false", + "gateway.master-key-path=./target/test-master.key" +}) +class MailSettingsTest { + + @Autowired + private MailSettingsService mail; + + @Autowired + private MailSettingsRepository repository; + + @Autowired + private SecretCipher cipher; + + @Autowired + private GatewayProperties props; + + @Autowired + private Environment environment; + + @BeforeEach + void clean() { + repository.deleteAll(); + } + + @Test + void savingSettingsEncryptsThePasswordAndNeverReturnsIt() { + Map view = mail.save("smtp.example.com", 587, "user", "hunter2", + "path@example.com", "ops@example.com", true); + + assertThat(view.get("host")).isEqualTo("smtp.example.com"); + assertThat(view.get("configured")).isEqualTo(true); + assertThat(view.get("has_password")).isEqualTo(true); + assertThat(view.toString()).doesNotContain("hunter2"); + assertThat(repository.findById(1L).orElseThrow().getPasswordCipher()) + .doesNotContain("hunter2") + .startsWith("v1:"); + } + + @Test + void aBlankPasswordKeepsTheStoredOne() { + mail.save("smtp.example.com", 587, "user", "hunter2", "a@b.c", "ops@example.com", true); + String before = repository.findById(1L).orElseThrow().getPasswordCipher(); + + mail.save("smtp2.example.com", null, null, "", null, null, true); + + assertThat(repository.findById(1L).orElseThrow().getPasswordCipher()).isEqualTo(before); + assertThat(mail.view().get("host")).isEqualTo("smtp2.example.com"); + } + + @Test + void enablingRequiresHostAndRecipients() { + assertThatThrownBy(() -> mail.save("", null, null, null, null, "ops@example.com", true)) + .isInstanceOf(ApiException.class) + .hasMessageContaining("host"); + assertThatThrownBy(() -> mail.save("smtp.example.com", null, null, null, null, "", true)) + .isInstanceOf(ApiException.class) + .hasMessageContaining("recipient"); + } + + @Test + void disablingTurnsAlertingOffWithoutLosingTheConfiguration() { + mail.save("smtp.example.com", 587, null, null, "a@b.c", "ops@example.com", true); + mail.save(null, null, null, null, null, null, false); + + assertThat(mail.emailEnabled()).isFalse(); + assertThat(mail.view().get("host")).isEqualTo("smtp.example.com"); + } + + @Test + void theConsoleConfigurationWinsOverTheEnvironment() { + // No row: the environment decides, as before this feature existed. + assertThat(mail.view().get("source")).isNotEqualTo("console"); + + mail.save("smtp.console.test", 2525, null, null, "path@example.com", "ops@example.com", true); + + assertThat(mail.view().get("source")).isEqualTo("console"); + assertThat(mail.recipients()).isEqualTo("ops@example.com"); + assertThat(mail.fromAddress()).isEqualTo("path@example.com"); + assertThat(environment.getProperty("spring.mail.host", "")).isEmpty(); + assertThat(props.getAlerts().getEmailTo()).isEmpty(); + } +} diff --git a/src/test/java/com/anindra/path/admin/PromptLogServiceTest.java b/path-gateway/src/test/java/com/anindra/path/admin/PromptLogServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/admin/PromptLogServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/admin/PromptLogServiceTest.java diff --git a/src/test/java/com/anindra/path/admin/TenancyServiceTest.java b/path-gateway/src/test/java/com/anindra/path/admin/TenancyServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/admin/TenancyServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/admin/TenancyServiceTest.java diff --git a/src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java b/path-gateway/src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java similarity index 100% rename from src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java rename to path-gateway/src/test/java/com/anindra/path/api/GlobalExceptionHandlerTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/config/AdminAuthLimiterTest.java b/path-gateway/src/test/java/com/anindra/path/config/AdminAuthLimiterTest.java new file mode 100644 index 0000000..5b86801 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/config/AdminAuthLimiterTest.java @@ -0,0 +1,90 @@ +package com.anindra.path.config; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class AdminAuthLimiterTest { + + private AdminAuthLimiter limiter() { + GatewayProperties props = new GatewayProperties(); + props.getAdminAuth().setMaxFailures(2); + props.getAdminAuth().setWindow(Duration.ofMinutes(5)); + return new AdminAuthLimiter(props); + } + + @Test + void locksOutAfterTheConfiguredFailures() { + AdminAuthLimiter limiter = limiter(); + + limiter.onFailure("10.0.0.1"); + assertThat(limiter.retryAfterSeconds("10.0.0.1")).isZero(); + + limiter.onFailure("10.0.0.1"); + assertThat(limiter.retryAfterSeconds("10.0.0.1")).isGreaterThan(0); + assertThat(limiter.retryAfterSeconds("10.0.0.2")).isZero(); + } + + @Test + void successClearsTheCounter() { + AdminAuthLimiter limiter = limiter(); + limiter.onFailure("10.0.0.3"); + limiter.onFailure("10.0.0.3"); + assertThat(limiter.retryAfterSeconds("10.0.0.3")).isGreaterThan(0); + + limiter.onSuccess("10.0.0.3"); + assertThat(limiter.retryAfterSeconds("10.0.0.3")).isZero(); + } + + @Test + void aRotatingAddressDoesNotLiftTheCap() { + // The bypass: Cloudflare edge addresses change per request, and a forged + // X-Forwarded-For changes per request by construction. Counting per address alone + // let thousands of guesses through, so the process-wide window has to close first. + GatewayProperties props = new GatewayProperties(); + props.getAdminAuth().setMaxFailures(2); + props.getAdminAuth().setGlobalMaxFailures(5); + props.getAdminAuth().setWindow(Duration.ofMinutes(5)); + AdminAuthLimiter limiter = new AdminAuthLimiter(props); + + for (int i = 0; i < 5; i++) { + limiter.onFailure("203.0.113." + i); + } + + assertThat(limiter.retryAfterSeconds("203.0.113.99")).isGreaterThan(0); + assertThat(limiter.retryAfterSeconds("198.51.100.7")).isGreaterThan(0); + } + + @Test + void aLegitimateAdminIsNotLockedOutByOtherAddresses() { + GatewayProperties props = new GatewayProperties(); + props.getAdminAuth().setMaxFailures(2); + props.getAdminAuth().setGlobalMaxFailures(5); + props.getAdminAuth().setWindow(Duration.ofMinutes(5)); + AdminAuthLimiter limiter = new AdminAuthLimiter(props); + + limiter.onFailure("203.0.113.1"); + assertThat(limiter.retryAfterSeconds("198.51.100.7")).isZero(); + } + + @Test + void theGlobalCapIsOffByDefaultSoAScannerCannotLockTheOperatorOut() { + // A fail-closed global cap is a denial-of-service: one hostile source spending the + // budget in the window locks the console for everybody, including the operator. + GatewayProperties props = new GatewayProperties(); + props.getAdminAuth().setMaxFailures(2); + props.getAdminAuth().setWindow(Duration.ofMinutes(5)); + AdminAuthLimiter limiter = new AdminAuthLimiter(props); + + for (int i = 0; i < 500; i++) { + limiter.onFailure("203.0.113.200"); + } + + assertThat(props.getAdminAuth().getGlobalMaxFailures()).isZero(); + assertThat(limiter.retryAfterSeconds("198.51.100.7")).isZero(); + // The noisy address itself is still stopped. + assertThat(limiter.retryAfterSeconds("203.0.113.200")).isGreaterThan(0); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/config/ClientAuthLimiterTest.java b/path-gateway/src/test/java/com/anindra/path/config/ClientAuthLimiterTest.java new file mode 100644 index 0000000..de67fb8 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/config/ClientAuthLimiterTest.java @@ -0,0 +1,40 @@ +package com.anindra.path.config; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class ClientAuthLimiterTest { + + private ClientAuthLimiter limiter(int maxFailures) { + GatewayProperties props = new GatewayProperties(); + props.getClientAuth().setMaxFailures(maxFailures); + props.getClientAuth().setWindow(Duration.ofMinutes(1)); + return new ClientAuthLimiter(props); + } + + @Test + void stopsASourceAfterTooManyRejectedKeys() { + ClientAuthLimiter limiter = limiter(3); + + for (int i = 0; i < 3; i++) { + limiter.onFailure("203.0.113.5"); + } + + assertThat(limiter.retryAfterSeconds("203.0.113.5")).isGreaterThan(0); + assertThat(limiter.retryAfterSeconds("203.0.113.6")).isZero(); + } + + @Test + void aWorkingKeyClearsTheCounter() { + ClientAuthLimiter limiter = limiter(2); + limiter.onFailure("203.0.113.7"); + limiter.onFailure("203.0.113.7"); + assertThat(limiter.retryAfterSeconds("203.0.113.7")).isGreaterThan(0); + + limiter.onSuccess("203.0.113.7"); + assertThat(limiter.retryAfterSeconds("203.0.113.7")).isZero(); + } +} diff --git a/src/test/java/com/anindra/path/config/LimitedInputStreamTest.java b/path-gateway/src/test/java/com/anindra/path/config/LimitedInputStreamTest.java similarity index 100% rename from src/test/java/com/anindra/path/config/LimitedInputStreamTest.java rename to path-gateway/src/test/java/com/anindra/path/config/LimitedInputStreamTest.java diff --git a/src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/AnthropicTranslationPropertyTest.java diff --git a/src/test/java/com/anindra/path/gateway/BudgetReservationTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/BudgetReservationTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/BudgetReservationTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/BudgetReservationTest.java diff --git a/src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ChatRequestFuzzTest.java diff --git a/src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ConcurrencyLimiterTest.java diff --git a/src/test/java/com/anindra/path/gateway/CreditGuardTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/CreditGuardTest.java similarity index 97% rename from src/test/java/com/anindra/path/gateway/CreditGuardTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/CreditGuardTest.java index ed4ae93..164668d 100644 --- a/src/test/java/com/anindra/path/gateway/CreditGuardTest.java +++ b/path-gateway/src/test/java/com/anindra/path/gateway/CreditGuardTest.java @@ -29,7 +29,7 @@ class CreditGuardTest { mock(com.anindra.path.repo.ProjectRepository.class); private final EndUserBudget endUserBudget = mock(EndUserBudget.class); private final CreditGuard guard = new CreditGuard(apiKeys, endUserBudget, usage, props, - new com.anindra.path.admin.AlertService(props, new ObjectMapper()), projects); + new com.anindra.path.admin.AlertService(props, new ObjectMapper(), org.mockito.Mockito.mock(com.anindra.path.admin.MailSettingsService.class), new io.micrometer.core.instrument.simple.SimpleMeterRegistry()), projects); private final ObjectMapper mapper = new ObjectMapper(); @org.junit.jupiter.api.BeforeEach diff --git a/src/test/java/com/anindra/path/gateway/DrainStateTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/DrainStateTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/DrainStateTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/DrainStateTest.java diff --git a/src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/EndUserBudgetTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/gateway/ErrorMetricsTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ErrorMetricsTest.java new file mode 100644 index 0000000..fa87c9e --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/gateway/ErrorMetricsTest.java @@ -0,0 +1,42 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.repo.UsageRecordRepository; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.assertj.core.api.Assertions.assertThat; + +class ErrorMetricsTest { + + private final SimpleMeterRegistry registry = new SimpleMeterRegistry(); + private final UsageService usage = new UsageService( + Mockito.mock(UsageRecordRepository.class), + Mockito.mock(ApiKeyService.class), + registry, + new GatewayProperties()); + + @Test + void aServerFailureAndAClientFailureLandInDifferentClasses() { + usage.recordError(null, null, "openai", "gpt-4o", 12, false, 1, 500, + "upstream_error", "the provider fell over"); + usage.recordError(null, null, "openai", "gpt-4o", 12, false, 1, 400, + "invalid_request", "the client sent nonsense"); + + assertThat(registry.counter("path.errors", "provider", "openai", + "class", "5xx", "code", "upstream_error").count()).isEqualTo(1.0); + assertThat(registry.counter("path.errors", "provider", "openai", + "class", "4xx", "code", "invalid_request").count()).isEqualTo(1.0); + // The 5xx rule divides the 5xx series by all requests; it must not see the 400. + assertThat(registry.find("path.errors").tag("class", "5xx").counter().count()).isEqualTo(1.0); + } + + @Test + void aMissingCodeIsUnknownRatherThanACardinalityBomb() { + usage.recordError(null, null, "groq", "llama", 5, false, 1, 503, null, "no body"); + + assertThat(registry.counter("path.errors", "provider", "groq", + "class", "5xx", "code", "unknown").count()).isEqualTo(1.0); + } +} diff --git a/path-gateway/src/test/java/com/anindra/path/gateway/GeminiAdapterTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/GeminiAdapterTest.java new file mode 100644 index 0000000..0fdb365 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/gateway/GeminiAdapterTest.java @@ -0,0 +1,206 @@ +package com.anindra.path.gateway; + +import com.anindra.path.api.ApiException; +import com.anindra.path.config.GatewayProperties; +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.anindra.path.repo.UsageRecordRepository; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.math.BigDecimal; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class GeminiAdapterTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final GeminiAdapter adapter = new GeminiAdapter(mapper, ProviderType.GEMINI, + Mockito.mock(VertexTokenService.class)); + + private Provider provider() { + return new Provider("gemini", "Gemini", ProviderType.GEMINI, + "https://generativelanguage.googleapis.com/v1beta"); + } + + private ModelRoute route(String model) { + ModelRoute route = new ModelRoute(); + route.setPublicModelId(model); + route.setUpstreamModelId(model); + route.setInputPricePerMillion(new BigDecimal("1.00")); + route.setOutputPricePerMillion(new BigDecimal("2.00")); + return route; + } + + private ChatRequestSpec spec(String json) throws Exception { + return ChatRequestSpec.parse((ObjectNode) mapper.readTree(json)); + } + + @Test + void translatesThinkingSafetyAndToolsIntoTheNativeShape() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"gemini-2.5-pro", + "messages":[ + {"role":"system","content":"be precise"}, + {"role":"user","content":"weather in Pune?"}], + "temperature":0.3, + "max_tokens":256, + "thinkingConfig":{"thinkingBudget":1024,"includeThoughts":true}, + "safetySettings":[{"category":"HARM_CATEGORY_HATE_SPEECH","threshold":"BLOCK_ONLY_HIGH"}], + "tools":[{"type":"function","function":{"name":"weather","description":"Weather", + "parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]} + """); + + UpstreamRequest request = adapter.build(spec, route("gemini-2.5-pro"), + provider(), "api-key-1", false); + + assertThat(request.url()).isEqualTo( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent"); + assertThat(request.headers()).containsEntry("x-goog-api-key", "api-key-1"); + + JsonNode sent = mapper.readTree(request.body()); + assertThat(sent.path("systemInstruction").path("parts").get(0).path("text").asText()) + .isEqualTo("be precise"); + assertThat(sent.path("contents").get(0).path("role").asText()).isEqualTo("user"); + assertThat(sent.path("contents").get(0).path("parts").get(0).path("text").asText()) + .isEqualTo("weather in Pune?"); + assertThat(sent.path("generationConfig").path("temperature").asDouble()).isEqualTo(0.3); + assertThat(sent.path("generationConfig").path("maxOutputTokens").asInt()).isEqualTo(256); + assertThat(sent.path("generationConfig").path("thinkingConfig").path("thinkingBudget").asInt()) + .isEqualTo(1024); + assertThat(sent.path("generationConfig").path("thinkingConfig").path("includeThoughts").asBoolean()) + .isTrue(); + assertThat(sent.path("safetySettings").get(0).path("category").asText()) + .isEqualTo("HARM_CATEGORY_HATE_SPEECH"); + assertThat(sent.path("tools").get(0).path("functionDeclarations").get(0).path("name").asText()) + .isEqualTo("weather"); + } + + @Test + void reasoningEffortBecomesAThinkingBudget() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"gemini-2.5-pro","messages":[{"role":"user","content":"hi"}], + "reasoning_effort":"high"} + """); + + UpstreamRequest request = adapter.build(spec, route("gemini-2.5-pro"), provider(), "k", false); + + assertThat(mapper.readTree(request.body()) + .path("generationConfig").path("thinkingConfig").path("thinkingBudget").asInt()) + .isEqualTo(8192); + } + + @Test + void aParameterGeminiCannotHonourIs422() throws Exception { + ChatRequestSpec spec = spec(""" + {"model":"gemini-2.5-pro","messages":[{"role":"user","content":"hi"}], + "frequency_penalty":0.5} + """); + + assertThatThrownBy(() -> adapter.build(spec, route("gemini-2.5-pro"), provider(), "k", false)) + .isInstanceOf(ApiException.class) + .satisfies(e -> assertThat(((ApiException) e).code()).isEqualTo("unsupported_parameter")); + } + + @Test + void normalisesTheResponseAndCountsThinkingAsCompletion() throws Exception { + JsonNode nativeResponse = mapper.readTree(""" + {"candidates":[{"content":{"role":"model","parts":[ + {"text":"Let me think","thought":true}, + {"text":"It is sunny"}]},"finishReason":"STOP", + "safetyRatings":[{"category":"HARM_CATEGORY_HATE_SPEECH","probability":"NEGLIGIBLE"}]}], + "usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":5, + "thoughtsTokenCount":7,"cachedContentTokenCount":4,"totalTokenCount":22}} + """); + ChatRequestSpec spec = spec(""" + {"model":"gemini-2.5-pro","messages":[{"role":"user","content":"hi"}]} + """); + + JsonNode normalised = adapter.parseFullResponse(nativeResponse, spec, route("gemini-2.5-pro")); + assertThat(normalised.path("choices").get(0).path("message").path("content").asText()) + .isEqualTo("It is sunny"); + assertThat(normalised.path("choices").get(0).path("finish_reason").asText()).isEqualTo("stop"); + assertThat(normalised.path("usage").path("completion_tokens").asInt()).isEqualTo(12); + assertThat(normalised.path("usage").path("completion_tokens_details").path("reasoning_tokens").asInt()) + .isEqualTo(7); + + TokenUsage usage = adapter.parseUsage(nativeResponse); + assertThat(usage.prompt()).isEqualTo(10); + assertThat(usage.completion()).isEqualTo(12); + assertThat(usage.reasoning()).isEqualTo(7); + assertThat(usage.cached()).isEqualTo(4); + } + + @Test + void thinkingTokensArePricedAsOutput() throws Exception { + JsonNode nativeResponse = mapper.readTree(""" + {"candidates":[{"content":{"parts":[{"text":"ok"}]},"finishReason":"STOP"}], + "usageMetadata":{"promptTokenCount":1000000,"candidatesTokenCount":1000000, + "thoughtsTokenCount":1000000,"totalTokenCount":3000000}} + """); + TokenUsage usage = adapter.parseUsage(nativeResponse); + + UsageService usageService = new UsageService( + Mockito.mock(UsageRecordRepository.class), + Mockito.mock(ApiKeyService.class), + new SimpleMeterRegistry(), + new GatewayProperties()); + UsageService.CostBreakdown cost = usageService.cost(route("gemini-2.5-pro"), usage); + + // prompt 1M at $1/M = $1.00; completion 1M + 1M thoughts at $2/M = $4.00. + assertThat(usage.completion()).isEqualTo(2_000_000); + assertThat(cost.outputUsd()).isEqualByComparingTo("4.00000000"); + assertThat(cost.inputUsd()).isEqualByComparingTo("1.00000000"); + assertThat(cost.totalUsd()).isEqualByComparingTo("5.00000000"); + } + + @Test + void aBlockedPromptIsA400NamingTheCategory() throws Exception { + JsonNode blocked = mapper.readTree(""" + {"promptFeedback":{"blockReason":"SAFETY","safetyRatings":[ + {"category":"HARM_CATEGORY_DANGEROUS_CONTENT","probability":"HIGH","blocked":true}]}} + """); + ChatRequestSpec spec = spec(""" + {"model":"gemini-2.5-pro","messages":[{"role":"user","content":"nh"}]} + """); + + assertThatThrownBy(() -> adapter.parseFullResponse(blocked, spec, route("gemini-2.5-pro"))) + .isInstanceOf(ApiException.class) + .satisfies(e -> { + ApiException api = (ApiException) e; + assertThat(api.status()).isEqualTo(400); + assertThat(api.code()).isEqualTo("content_blocked"); + assertThat(api.getMessage()).contains("HARM_CATEGORY_DANGEROUS_CONTENT"); + }); + } + + @Test + void streamsContentAndKeepsTheUsage() throws Exception { + ChatRequestSpec spec = ChatRequestSpec.parse((ObjectNode) mapper.readTree(""" + {"model":"gemini-2.5-pro","messages":[{"role":"user","content":"hi"}]} + """)); + StreamTranslator translator = adapter.newStreamTranslator(spec, route("gemini-2.5-pro")); + + List first = translator.translate( + "data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"Hel\"}]}}]}"); + List second = translator.translate( + "data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"lo\"}]},\"finishReason\":\"STOP\"}]," + + "\"usageMetadata\":{\"promptTokenCount\":3,\"candidatesTokenCount\":2," + + "\"thoughtsTokenCount\":1,\"totalTokenCount\":6}}"); + + assertThat(first).hasSize(1); + assertThat(first.get(0).path("choices").get(0).path("delta").path("content").asText()) + .isEqualTo("Hel"); + assertThat(second.get(0).path("choices").get(0).path("finish_reason").asText()).isEqualTo("stop"); + assertThat(translator.done()).isTrue(); + assertThat(translator.usage().completion()).isEqualTo(3); + assertThat(translator.usage().reasoning()).isEqualTo(1); + } +} diff --git a/src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/IdempotencyServiceTest.java diff --git a/src/test/java/com/anindra/path/gateway/PiiRedactorTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/PiiRedactorTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/PiiRedactorTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/PiiRedactorTest.java diff --git a/src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ProjectBudgetAlertTest.java diff --git a/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java similarity index 84% rename from src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java index 0dd9e3f..a0895ac 100644 --- a/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java +++ b/path-gateway/src/test/java/com/anindra/path/gateway/ProviderCircuitBreakerTest.java @@ -18,7 +18,12 @@ void setUp() { props.getBreaker().setFailureThreshold(2); props.getBreaker().setOpenDuration(Duration.ofMillis(120)); breaker = new ProviderCircuitBreaker(props, - new com.anindra.path.admin.AlertService(props, new com.fasterxml.jackson.databind.ObjectMapper())); + new com.anindra.path.admin.AlertService(props, + new com.fasterxml.jackson.databind.ObjectMapper(), + org.mockito.Mockito.mock(com.anindra.path.admin.MailSettingsService.class), + new io.micrometer.core.instrument.simple.SimpleMeterRegistry()), + new io.micrometer.core.instrument.simple.SimpleMeterRegistry(), + new InMemoryCircuitBreakerStore()); } @Test diff --git a/path-gateway/src/test/java/com/anindra/path/gateway/ProviderErrorRateWatcherTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ProviderErrorRateWatcherTest.java new file mode 100644 index 0000000..d56e4d1 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/gateway/ProviderErrorRateWatcherTest.java @@ -0,0 +1,120 @@ +package com.anindra.path.gateway; + +import com.anindra.path.admin.AlertService; +import com.anindra.path.config.GatewayProperties; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +class ProviderErrorRateWatcherTest { + + private final RoutingMetrics routing = new RoutingMetrics(); + private final AlertService alerts = Mockito.mock(AlertService.class); + private final GatewayProperties props = new GatewayProperties(); + private final ProviderErrorRateWatcher watcher = + new ProviderErrorRateWatcher(routing, alerts, props); + + private void fail(String provider, String model, int times) { + for (int i = 0; i < times; i++) { + routing.recordFailure(provider, model); + } + } + + private void succeed(String provider, String model, int times) { + for (int i = 0; i < times; i++) { + routing.recordSuccess(provider, model, 120); + } + } + + @Test + void firesOncePerProviderPerHourNoMatterHowManyRequestsFailed() { + fail("openai", "gpt-4o", 200); // many requests, one alert + + watcher.check(); + watcher.check(); + watcher.check(); + + verify(alerts, times(1)).notifyOnce( + startsWith("provider.error_rate_high:openai:"), + eq("provider.error_rate_high"), anyMap()); + } + + @Test + void thePayloadCarriesTheRateAndTheSampleCount() { + fail("openai", "gpt-4o", 25); + + watcher.check(); + + verify(alerts).notifyOnce(any(), eq("provider.error_rate_high"), + argThat(payload -> payload.get("provider").equals("openai") + && ((Number) payload.get("samples")).longValue() >= 25 + && ((Double) payload.get("error_rate")) > 0.25)); + } + + @Test + void doesNotFireBelowTheSampleMinimum() { + fail("openai", "gpt-4o", 10); // 10 < default 20: a cold start is not an incident + + watcher.check(); + + verifyNoInteractions(alerts); + } + + @Test + void oneAlertCoversEveryModelOfTheSameProvider() { + fail("openai", "gpt-4o", 10); + fail("openai", "gpt-4o-mini", 10); + + watcher.check(); + + verify(alerts, times(1)).notifyOnce( + startsWith("provider.error_rate_high:openai:"), + eq("provider.error_rate_high"), anyMap()); + } + + @Test + void recoveryFiresOnceWhenTheRateSettles() { + fail("openai", "gpt-4o", 20); + watcher.check(); + + // 0.8^12 of the original 1.0 is ~0.07, under half the 0.25 threshold. + succeed("openai", "gpt-4o", 12); + watcher.check(); + watcher.check(); + + verify(alerts, times(1)).notifyOnce( + startsWith("provider.error_rate_normal:openai:"), + eq("provider.error_rate_normal"), anyMap()); + } + + @Test + void twoBadProvidersAreTwoAlerts() { + fail("openai", "gpt-4o", 20); + fail("anthropic", "claude", 20); + + watcher.check(); + + verify(alerts, times(1)).notifyOnce(startsWith("provider.error_rate_high:openai:"), + eq("provider.error_rate_high"), anyMap()); + verify(alerts, times(1)).notifyOnce(startsWith("provider.error_rate_high:anthropic:"), + eq("provider.error_rate_high"), anyMap()); + } + + @Test + void aDisabledThresholdTurnsTheCheckOff() { + props.getAlert().setErrorRate(0); + fail("openai", "gpt-4o", 50); + + watcher.check(); + + verifyNoInteractions(alerts); + } +} diff --git a/src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ProviderHealthIndicatorTest.java diff --git a/src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ProviderUrlsTest.java diff --git a/src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ResponseCacheEligibilityTest.java diff --git a/src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/ResponseCacheServiceTest.java diff --git a/src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/RouteResolverAdaptiveTest.java diff --git a/src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/RoutingMetricsTest.java diff --git a/src/test/java/com/anindra/path/gateway/RoutingTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/RoutingTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/RoutingTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/RoutingTest.java diff --git a/src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/SemanticCacheGeometryTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/gateway/SharedCircuitBreakerTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/SharedCircuitBreakerTest.java new file mode 100644 index 0000000..1d0ee66 --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/gateway/SharedCircuitBreakerTest.java @@ -0,0 +1,105 @@ +package com.anindra.path.gateway; + +import com.anindra.path.config.GatewayProperties; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Two replicas, one store: this is the behaviour #74 asked for. The same assertions run + * against the Valkey store in production; memory proves the breaker logic does not depend + * on two JVMs sharing an object. + */ +class SharedCircuitBreakerTest { + + private final GatewayProperties props = new GatewayProperties(); + private final CircuitBreakerStore store = new InMemoryCircuitBreakerStore(); + + private ProviderCircuitBreaker replica() { + props.getBreaker().setFailureThreshold(2); + props.getBreaker().setOpenDuration(Duration.ofMillis(100)); + props.getBreaker().setProbeLease(Duration.ofSeconds(2)); + return new ProviderCircuitBreaker(props, + Mockito.mock(com.anindra.path.admin.AlertService.class), + new SimpleMeterRegistry(), + store); + } + + @Test + void parkingOnOneReplicaIsVisibleOnTheOther() { + ProviderCircuitBreaker one = replica(); + ProviderCircuitBreaker two = replica(); + + one.onFailure(7L); + one.onFailure(7L); + + assertThat(one.state(7L)).isEqualTo(ProviderCircuitBreaker.State.OPEN); + assertThat(two.state(7L)).isEqualTo(ProviderCircuitBreaker.State.OPEN); + assertThat(two.isParked(7L)).isTrue(); + assertThat(two.isAvailable(7L)).isFalse(); + } + + @Test + void exactlyOneReplicaClaimsTheProbe() throws Exception { + ProviderCircuitBreaker one = replica(); + ProviderCircuitBreaker two = replica(); + one.onFailure(7L); + one.onFailure(7L); + Thread.sleep(150); + + boolean first = one.isAvailable(7L); + boolean second = two.isAvailable(7L); + + assertThat(first).isTrue(); + assertThat(second).isFalse(); + assertThat(two.isParked(7L)).isFalse(); // HALF_OPEN is not "parked", it is being probed + } + + @Test + void aSuccessfulProbeClosesTheCircuitForEveryone() throws Exception { + ProviderCircuitBreaker one = replica(); + ProviderCircuitBreaker two = replica(); + one.onFailure(7L); + one.onFailure(7L); + Thread.sleep(150); + + assertThat(one.isAvailable(7L)).isTrue(); + one.onSuccess(7L); + + assertThat(two.state(7L)).isEqualTo(ProviderCircuitBreaker.State.CLOSED); + assertThat(two.isAvailable(7L)).isTrue(); + } + + @Test + void aFailedProbeReopensTheCircuit() throws Exception { + ProviderCircuitBreaker one = replica(); + ProviderCircuitBreaker two = replica(); + one.onFailure(7L); + one.onFailure(7L); + Thread.sleep(150); + + assertThat(one.isAvailable(7L)).isTrue(); + one.onFailure(7L); + + assertThat(two.state(7L)).isEqualTo(ProviderCircuitBreaker.State.OPEN); + } + + @Test + void twoLimitersShareOneCapWhenTheyShareTheStore() { + ConcurrencyStore shared = new InMemoryConcurrencyStore(); + ConcurrencyLimiter one = new ConcurrencyLimiter(shared); + ConcurrencyLimiter two = new ConcurrencyLimiter(shared); + + assertThat(one.tryAcquire(11L, 2)).isTrue(); + assertThat(two.tryAcquire(11L, 2)).isTrue(); + assertThat(one.tryAcquire(11L, 2)).isFalse(); + assertThat(two.inFlight(11L)).isEqualTo(2); + + one.release(11L); + assertThat(two.tryAcquire(11L, 2)).isTrue(); + } +} diff --git a/src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/TokenCountServiceTest.java diff --git a/src/test/java/com/anindra/path/gateway/TokenCounterTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/TokenCounterTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/TokenCounterTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/TokenCounterTest.java diff --git a/src/test/java/com/anindra/path/gateway/TokenEstimateTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/TokenEstimateTest.java similarity index 100% rename from src/test/java/com/anindra/path/gateway/TokenEstimateTest.java rename to path-gateway/src/test/java/com/anindra/path/gateway/TokenEstimateTest.java diff --git a/path-gateway/src/test/java/com/anindra/path/gateway/VertexTokenServiceTest.java b/path-gateway/src/test/java/com/anindra/path/gateway/VertexTokenServiceTest.java new file mode 100644 index 0000000..ba666cc --- /dev/null +++ b/path-gateway/src/test/java/com/anindra/path/gateway/VertexTokenServiceTest.java @@ -0,0 +1,150 @@ +package com.anindra.path.gateway; + +import com.anindra.path.domain.ModelRoute; +import com.anindra.path.domain.Provider; +import com.anindra.path.domain.ProviderType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.Signature; +import java.util.Base64; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Vertex auth end to end against a stub Google: the service-account JSON signs an RS256 + * assertion, the assertion is exchanged for an access token, and the token is reused. + */ +class VertexTokenServiceTest { + + private HttpServer server; + private final ObjectMapper mapper = new ObjectMapper(); + private KeyPair keyPair; + private final AtomicInteger tokenCalls = new AtomicInteger(); + private final AtomicReference authorization = new AtomicReference<>(); + private final AtomicReference assertion = new AtomicReference<>(); + + @BeforeEach + void start() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + keyPair = generator.generateKeyPair(); + + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/token", exchange -> { + tokenCalls.incrementAndGet(); + String form = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + for (String pair : form.split("&")) { + if (pair.startsWith("assertion=")) { + assertion.set(URLDecoder.decode(pair.substring("assertion=".length()), + StandardCharsets.UTF_8)); + } + } + byte[] body = "{\"access_token\":\"ya29.test\",\"expires_in\":3600,\"token_type\":\"Bearer\"}" + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.createContext("/v1/projects/demo/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent", + exchange -> { + authorization.set(exchange.getRequestHeaders().getFirst("Authorization")); + byte[] body = "{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"ok\"}]}}]}" + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + private String serviceAccountJson() { + String pem = "-----BEGIN PRIVATE KEY-----\n" + + Base64.getMimeEncoder(64, "\n".getBytes()).encodeToString(keyPair.getPrivate().getEncoded()) + + "\n-----END PRIVATE KEY-----\n"; + ObjectNode account = mapper.createObjectNode(); + account.put("type", "service_account"); + account.put("client_email", "path@demo.iam.gserviceaccount.com"); + account.put("private_key", pem); + account.put("token_uri", "http://127.0.0.1:" + server.getAddress().getPort() + "/token"); + return account.toString(); + } + + private Provider provider() { + Provider provider = new Provider("vertex", "Vertex", ProviderType.VERTEX, + "http://127.0.0.1:" + server.getAddress().getPort()); + provider.setId(1L); + provider.setProject("demo"); + provider.setLocation("us-central1"); + return provider; + } + + @Test + void signsAnAssertionExchangesItAndCachesTheToken() throws Exception { + VertexTokenService tokens = new VertexTokenService(mapper); + GeminiAdapter adapter = new GeminiAdapter(mapper, ProviderType.VERTEX, tokens); + ModelRoute route = new ModelRoute(); + route.setPublicModelId("gemini-2.5-pro"); + route.setUpstreamModelId("gemini-2.5-pro"); + ChatRequestSpec spec = ChatRequestSpec.parse((ObjectNode) mapper.readTree(""" + {"model":"gemini-2.5-pro","messages":[{"role":"user","content":"hi"}]} + """)); + + UpstreamRequest request = adapter.build(spec, route, provider(), serviceAccountJson(), false); + adapter.build(spec, route, provider(), serviceAccountJson(), false); + + java.net.http.HttpRequest send = java.net.http.HttpRequest.newBuilder( + java.net.URI.create(request.url())) + .header("Content-Type", "application/json") + .header("Authorization", request.headers().get("Authorization")) + .POST(java.net.http.HttpRequest.BodyPublishers.ofString(request.body())) + .build(); + java.net.http.HttpClient.newHttpClient().send(send, + java.net.http.HttpResponse.BodyHandlers.ofString()); + + assertThat(request.url()).isEqualTo("http://127.0.0.1:" + server.getAddress().getPort() + + "/v1/projects/demo/locations/us-central1/publishers/google/models/" + + "gemini-2.5-pro:generateContent"); + assertThat(request.headers()).containsEntry("Authorization", "Bearer ya29.test"); + assertThat(tokenCalls.get()).isEqualTo(1); + assertThat(authorization.get()).isEqualTo("Bearer ya29.test"); + } + + @Test + void theAssertionIsAValidRs256Jwt() throws Exception { + new VertexTokenService(mapper).accessToken(provider(), serviceAccountJson()); + + String[] parts = assertion.get().split("\\."); + assertThat(parts).hasSize(3); + String signingInput = parts[0] + "." + parts[1]; + Signature verifier = Signature.getInstance("SHA256withRSA"); + verifier.initVerify(keyPair.getPublic()); + verifier.update(signingInput.getBytes(StandardCharsets.US_ASCII)); + assertThat(verifier.verify(Base64.getUrlDecoder().decode(parts[2]))).isTrue(); + + var claims = mapper.readTree(Base64.getUrlDecoder().decode(parts[1])); + assertThat(claims.path("iss").asText()).isEqualTo("path@demo.iam.gserviceaccount.com"); + assertThat(claims.path("scope").asText()).isEqualTo("https://www.googleapis.com/auth/cloud-platform"); + assertThat(claims.path("aud").asText()) + .isEqualTo("http://127.0.0.1:" + server.getAddress().getPort() + "/token"); + } +} diff --git a/src/test/java/com/anindra/path/secret/SecretCipherTest.java b/path-gateway/src/test/java/com/anindra/path/secret/SecretCipherTest.java similarity index 100% rename from src/test/java/com/anindra/path/secret/SecretCipherTest.java rename to path-gateway/src/test/java/com/anindra/path/secret/SecretCipherTest.java diff --git a/path-spring-ai/pom.xml b/path-spring-ai/pom.xml new file mode 100644 index 0000000..5577b91 --- /dev/null +++ b/path-spring-ai/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + + io.github.an1ndra + path-parent + 0.2.3 + ../pom.xml + + + path-spring-ai + path-spring-ai + Spring AI ChatModel and EmbeddingModel over a Path gateway + + + + + org.springframework.ai + spring-ai-bom + ${spring-ai.version} + pom + import + + + + + + + org.springframework.ai + spring-ai-client-chat + + + org.springframework.ai + spring-ai-model + + + com.fasterxml.jackson.core + jackson-databind + + + io.projectreactor + reactor-core + + + + org.springframework.boot + spring-boot-starter-test + test + + + io.projectreactor + reactor-test + test + + + diff --git a/path-spring-ai/src/main/java/com/anindra/path/springai/PathChatModel.java b/path-spring-ai/src/main/java/com/anindra/path/springai/PathChatModel.java new file mode 100644 index 0000000..bbe9403 --- /dev/null +++ b/path-spring-ai/src/main/java/com/anindra/path/springai/PathChatModel.java @@ -0,0 +1,362 @@ +package com.anindra.path.springai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.MessageType; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.metadata.ChatResponseMetadata; +import org.springframework.ai.chat.metadata.DefaultUsage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.tool.ToolCallingChatOptions; +import org.springframework.ai.tool.ToolCallback; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * A Spring AI {@link ChatModel} that speaks to a Path gateway. + * + *

{@code
+ * @Bean
+ * ChatModel chatModel() {
+ *     return PathChatModel.builder()
+ *             .baseUrl("http://path:8080")
+ *             .apiKey(System.getenv("PATH_KEY"))
+ *             .idempotency(true)
+ *             .build();
+ * }
+ * }
+ * + *

Streaming assembles the SSE content deltas into {@link ChatResponse} chunks; Spring AI's + * {@code ChatClient} aggregates them. Tool calls work on the non-streaming path, where the + * gateway returns a complete {@code tool_calls} array; streamed tool-call deltas are not + * reassembled here (documented in {@code docs/SPRING-AI.md}). + * + *

Path-specific response headers are copied into the response metadata: + * {@code path_cache}, {@code path_provider} and {@code path_cost_usd}, so a Spring AI caller + * can see what the gateway did without knowing the headers. + */ +public final class PathChatModel implements ChatModel { + + public static final String DEFAULT_MODEL = "gpt-4o-mini"; + + private final PathHttp http; + private final ChatOptions defaultOptions; + + private PathChatModel(PathHttp http, ChatOptions defaultOptions) { + this.http = http; + this.defaultOptions = defaultOptions; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private String baseUrl = "http://localhost:8080"; + private String apiKey; + private String model = DEFAULT_MODEL; + private boolean idempotency; + private Duration timeout = Duration.ofSeconds(180); + private ObjectMapper mapper = new ObjectMapper(); + + public Builder baseUrl(String baseUrl) { + this.baseUrl = baseUrl; + return this; + } + + public Builder apiKey(String apiKey) { + this.apiKey = apiKey; + return this; + } + + /** The public model id sent when the Prompt's options do not name one. */ + public Builder model(String model) { + this.model = model; + return this; + } + + /** Send an {@code Idempotency-Key} so a caller-side retry is not charged twice. */ + public Builder idempotency(boolean idempotency) { + this.idempotency = idempotency; + return this; + } + + public Builder timeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + public Builder objectMapper(ObjectMapper mapper) { + this.mapper = mapper; + return this; + } + + public PathChatModel build() { + ChatOptions options = ChatOptions.builder().model(model).build(); + return new PathChatModel(new PathHttp(baseUrl, apiKey, idempotency, timeout, mapper), options); + } + } + + @Override + public ChatResponse call(Prompt prompt) { + ObjectNode body = body(prompt, false); + HttpResponse response = http.postJson("/v1/chat/completions", body); + PathHttp.requireSuccess(response, response.body()); + return parse(response, http.mapper()); + } + + @Override + public Flux stream(Prompt prompt) { + ObjectNode body = body(prompt, true); + return Flux.create(sink -> { + HttpResponse response = http.postStream("/v1/chat/completions", body); + if (response.statusCode() / 100 != 2) { + PathHttp.requireSuccess(response, readAll(response.body())); + sink.complete(); + return; + } + Thread.ofVirtual().name("path-spring-ai-sse").start(() -> pump(response, sink)); + sink.onCancel(() -> closeQuietly(response.body())); + }); + } + + @Override + public ChatOptions getDefaultOptions() { + return defaultOptions; + } + + // ------------------------------------------------------------------ request + + private ObjectNode body(Prompt prompt, boolean stream) { + ObjectMapper mapper = http.mapper(); + ObjectNode body = mapper.createObjectNode(); + ChatOptions options = prompt.getOptions() == null ? defaultOptions : prompt.getOptions(); + body.put("model", resolveModel(options)); + body.put("stream", stream); + if (options.getTemperature() != null) { + body.put("temperature", options.getTemperature()); + } + if (options.getMaxTokens() != null) { + body.put("max_tokens", options.getMaxTokens()); + } + if (options.getTopP() != null) { + body.put("top_p", options.getTopP()); + } + if (options.getFrequencyPenalty() != null) { + body.put("frequency_penalty", options.getFrequencyPenalty()); + } + if (options.getPresencePenalty() != null) { + body.put("presence_penalty", options.getPresencePenalty()); + } + if (options.getStopSequences() != null && !options.getStopSequences().isEmpty()) { + ArrayNode stop = body.putArray("stop"); + options.getStopSequences().forEach(stop::add); + } + + ArrayNode messages = body.putArray("messages"); + for (Message message : prompt.getInstructions()) { + append(messages, message, mapper); + } + if (options instanceof ToolCallingChatOptions tooling) { + appendTools(body, tooling.getToolCallbacks(), mapper); + } + return body; + } + + private static void append(ArrayNode messages, Message message, ObjectMapper mapper) { + if (message instanceof ToolResponseMessage toolResponses) { + for (ToolResponseMessage.ToolResponse response : toolResponses.getResponses()) { + ObjectNode row = messages.addObject(); + row.put("role", "tool"); + row.put("tool_call_id", response.id()); + row.put("content", response.responseData()); + } + return; + } + ObjectNode row = messages.addObject(); + row.put("role", role(message.getMessageType())); + if (message instanceof AssistantMessage assistant) { + row.put("content", assistant.getText()); + if (assistant.hasToolCalls()) { + ArrayNode calls = row.putArray("tool_calls"); + for (AssistantMessage.ToolCall call : assistant.getToolCalls()) { + ObjectNode entry = calls.addObject(); + entry.put("id", call.id()); + entry.put("type", call.type() == null ? "function" : call.type()); + ObjectNode function = entry.putObject("function"); + function.put("name", call.name()); + function.put("arguments", call.arguments()); + } + } + } else { + row.put("content", message.getText()); + } + } + + private static void appendTools(ObjectNode body, List callbacks, ObjectMapper mapper) { + if (callbacks == null || callbacks.isEmpty()) { + return; + } + ArrayNode tools = body.putArray("tools"); + for (ToolCallback callback : callbacks) { + var definition = callback.getToolDefinition(); + ObjectNode tool = tools.addObject(); + tool.put("type", "function"); + ObjectNode function = tool.putObject("function"); + function.put("name", definition.name()); + if (definition.description() != null) { + function.put("description", definition.description()); + } + try { + function.set("parameters", mapper.readTree(definition.inputSchema())); + } catch (Exception e) { + function.putObject("parameters").put("type", "object"); + } + } + } + + private String resolveModel(ChatOptions options) { + if (options != null && options.getModel() != null && !options.getModel().isBlank()) { + return options.getModel(); + } + return DEFAULT_MODEL; + } + + private static String role(MessageType type) { + return switch (type) { + case SYSTEM -> "system"; + case USER -> "user"; + case ASSISTANT -> "assistant"; + case TOOL -> "tool"; + }; + } + + // ----------------------------------------------------------------- response + + private static ChatResponse parse(HttpResponse response, ObjectMapper mapper) { + try { + return toResponse(mapper.readTree(response.body()), response); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new PathException(response.statusCode(), "invalid_response", + "The gateway returned a body that is not JSON", e); + } + } + + private static ChatResponse toResponse(JsonNode root, HttpResponse response) { + JsonNode choice = root.path("choices").path(0); + JsonNode message = choice.path("message"); + + List calls = new ArrayList<>(); + for (JsonNode call : message.path("tool_calls")) { + JsonNode function = call.path("function"); + calls.add(new AssistantMessage.ToolCall( + call.path("id").asText(null), + call.path("type").asText("function"), + function.path("name").asText(null), + function.path("arguments").asText("{}"))); + } + + AssistantMessage output = AssistantMessage.builder() + .content(message.path("content").isNull() ? null : message.path("content").asText(null)) + .toolCalls(calls) + .build(); + + ChatGenerationMetadata generationMetadata = ChatGenerationMetadata.builder() + .finishReason(choice.path("finish_reason").asText(null)) + .build(); + Generation generation = new Generation(output, generationMetadata); + + ChatResponseMetadata.Builder metadata = ChatResponseMetadata.builder() + .id(root.path("id").asText(null)) + .model(root.path("model").asText(null)); + JsonNode usage = root.path("usage"); + if (usage.isObject()) { + metadata.usage(new DefaultUsage(usage.path("prompt_tokens").asInt(0), + usage.path("completion_tokens").asInt(0), + usage.path("total_tokens").asInt(0))); + } + header(response, "X-Path-Cache").ifPresent(value -> metadata.keyValue("path_cache", value)); + header(response, "X-Gateway-Provider").ifPresent(value -> metadata.keyValue("path_provider", value)); + header(response, "X-Gateway-Cost-Usd").ifPresent(value -> metadata.keyValue("path_cost_usd", value)); + + return new ChatResponse(List.of(generation), metadata.build()); + } + + private static java.util.Optional header(HttpResponse response, String name) { + return response.headers().firstValue(name); + } + + private void pump(HttpResponse response, FluxSink sink) { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(response.body(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.startsWith("data:")) { + continue; + } + String data = line.substring(5).trim(); + if (data.equals("[DONE]")) { + break; + } + JsonNode chunk = http.mapper().readTree(data); + JsonNode choice = chunk.path("choices").path(0); + String delta = choice.path("delta").path("content").asText(""); + String finish = choice.path("finish_reason").asText(null); + if (delta.isEmpty() && finish == null) { + continue; + } + AssistantMessage message = AssistantMessage.builder().content(delta).build(); + ChatGenerationMetadata generationMetadata = ChatGenerationMetadata.builder() + .finishReason(finish) + .build(); + ChatResponseMetadata metadata = ChatResponseMetadata.builder() + .id(chunk.path("id").asText(null)) + .model(chunk.path("model").asText(null)) + .build(); + sink.next(new ChatResponse(List.of(new Generation(message, generationMetadata)), metadata)); + } + sink.complete(); + } catch (Exception e) { + sink.error(e instanceof PathException p ? p : PathException.network(e)); + } + } + + private static String readAll(java.io.InputStream stream) { + try (stream) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + return ""; + } + } + + private static void closeQuietly(java.io.InputStream stream) { + try { + stream.close(); + } catch (IOException ignored) { + // The read loop is already unwinding. + } + } + + /** Convenience for callers that only need one message. */ + public String call(String message) { + return ChatModel.super.call(message); + } +} diff --git a/path-spring-ai/src/main/java/com/anindra/path/springai/PathEmbeddingModel.java b/path-spring-ai/src/main/java/com/anindra/path/springai/PathEmbeddingModel.java new file mode 100644 index 0000000..fdb3e0d --- /dev/null +++ b/path-spring-ai/src/main/java/com/anindra/path/springai/PathEmbeddingModel.java @@ -0,0 +1,154 @@ +package com.anindra.path.springai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.Embedding; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingRequest; +import org.springframework.ai.embedding.EmbeddingResponse; +import org.springframework.ai.embedding.EmbeddingResponseMetadata; + +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * A Spring AI {@link EmbeddingModel} over the gateway's {@code /v1/embeddings} endpoint. + * + *

{@code
+ * @Bean
+ * EmbeddingModel embeddingModel() {
+ *     return PathEmbeddingModel.builder()
+ *             .baseUrl("http://path:8080")
+ *             .apiKey(System.getenv("PATH_KEY"))
+ *             .model("text-embedding-3-small")
+ *             .build();
+ * }
+ * }
+ * + *

The gateway currently serves embeddings for OpenAI-shaped providers only; anything + * else fails with the gateway's own error, which surfaces as a {@link PathException}. + */ +public final class PathEmbeddingModel implements EmbeddingModel { + + public static final String DEFAULT_MODEL = "text-embedding-3-small"; + private static final int DEFAULT_DIMENSIONS = 1536; + + private final PathHttp http; + private final String model; + private final int dimensions; + + private PathEmbeddingModel(PathHttp http, String model, int dimensions) { + this.http = http; + this.model = model; + this.dimensions = dimensions; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private String baseUrl = "http://localhost:8080"; + private String apiKey; + private String model = DEFAULT_MODEL; + private boolean idempotency; + private Duration timeout = Duration.ofSeconds(60); + private int dimensions = DEFAULT_DIMENSIONS; + private ObjectMapper mapper = new ObjectMapper(); + + public Builder baseUrl(String baseUrl) { + this.baseUrl = baseUrl; + return this; + } + + public Builder apiKey(String apiKey) { + this.apiKey = apiKey; + return this; + } + + public Builder model(String model) { + this.model = model; + return this; + } + + public Builder idempotency(boolean idempotency) { + this.idempotency = idempotency; + return this; + } + + public Builder timeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + /** Reported by {@link #dimensions()}; the gateway does not return it. */ + public Builder dimensions(int dimensions) { + this.dimensions = dimensions; + return this; + } + + public Builder objectMapper(ObjectMapper mapper) { + this.mapper = mapper; + return this; + } + + public PathEmbeddingModel build() { + return new PathEmbeddingModel(new PathHttp(baseUrl, apiKey, idempotency, timeout, mapper), + model, dimensions); + } + } + + @Override + public EmbeddingResponse call(EmbeddingRequest request) { + ObjectMapper mapper = http.mapper(); + ObjectNode body = mapper.createObjectNode(); + String requested = request.getOptions() == null ? null : request.getOptions().getModel(); + body.put("model", requested == null || requested.isBlank() ? model : requested); + ArrayNode input = body.putArray("input"); + for (String text : request.getInstructions()) { + input.add(text); + } + Integer requestedDimensions = request.getOptions() == null ? null : request.getOptions().getDimensions(); + if (requestedDimensions != null) { + body.put("dimensions", requestedDimensions); + } + + HttpResponse response = http.postJson("/v1/embeddings", body); + PathHttp.requireSuccess(response, response.body()); + JsonNode root; + try { + root = mapper.readTree(response.body()); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new PathException(response.statusCode(), "invalid_response", + "The gateway returned a body that is not JSON", e); + } + + List embeddings = new ArrayList<>(); + for (JsonNode item : root.path("data")) { + JsonNode vector = item.path("embedding"); + float[] values = new float[vector.size()]; + for (int i = 0; i < vector.size(); i++) { + values[i] = (float) vector.get(i).asDouble(); + } + embeddings.add(new Embedding(values, item.path("index").asInt(embeddings.size()))); + } + return new EmbeddingResponse(embeddings, + new EmbeddingResponseMetadata(root.path("model").asText(model), + (org.springframework.ai.chat.metadata.Usage) null)); + } + + @Override + public float[] embed(Document document) { + return embed(document.getText()); + } + + @Override + public int dimensions() { + return dimensions; + } +} diff --git a/path-spring-ai/src/main/java/com/anindra/path/springai/PathException.java b/path-spring-ai/src/main/java/com/anindra/path/springai/PathException.java new file mode 100644 index 0000000..eba7a75 --- /dev/null +++ b/path-spring-ai/src/main/java/com/anindra/path/springai/PathException.java @@ -0,0 +1,32 @@ +package com.anindra.path.springai; + +/** + * A failure from the gateway, kept separate from Spring AI's own exceptions so a caller can + * see the HTTP status and the gateway's error code. + */ +public class PathException extends RuntimeException { + + private final int status; + private final String code; + + public PathException(int status, String code, String message, Throwable cause) { + super(message, cause); + this.status = status; + this.code = code; + } + + /** HTTP status; 0 when the call never reached the gateway. */ + public int status() { + return status; + } + + /** The gateway's {@code error.code}, or null when the response carried none. */ + public String code() { + return code; + } + + static PathException network(Throwable cause) { + return new PathException(0, "network_error", + "Could not reach the Path gateway: " + cause.getMessage(), cause); + } +} diff --git a/path-spring-ai/src/main/java/com/anindra/path/springai/PathHttp.java b/path-spring-ai/src/main/java/com/anindra/path/springai/PathHttp.java new file mode 100644 index 0000000..521aaf8 --- /dev/null +++ b/path-spring-ai/src/main/java/com/anindra/path/springai/PathHttp.java @@ -0,0 +1,103 @@ +package com.anindra.path.springai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +/** + * The small amount of HTTP shared by the chat and embedding models. + * + *

No Spring AI transport and no client SDK: the gateway speaks the OpenAI wire format, + * and a plain HTTP call keeps unknown fields out of the picture and the dependency surface + * to Jackson only. This module is the only place that knows about Spring AI; the gateway + * does not depend on it. + */ +final class PathHttp { + + private final URI base; + private final String apiKey; + private final boolean idempotency; + private final Duration timeout; + private final HttpClient http; + private final ObjectMapper mapper; + + PathHttp(String baseUrl, String apiKey, boolean idempotency, Duration timeout, ObjectMapper mapper) { + this.base = URI.create(trimSlash(baseUrl)); + this.apiKey = apiKey; + this.idempotency = idempotency; + this.timeout = timeout; + this.mapper = mapper; + this.http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + } + + private static String trimSlash(String url) { + return url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + } + + ObjectMapper mapper() { + return mapper; + } + + HttpResponse postJson(String path, ObjectNode body) { + return send(path, body, HttpResponse.BodyHandlers.ofString()); + } + + HttpResponse postStream(String path, ObjectNode body) { + return send(path, body, HttpResponse.BodyHandlers.ofInputStream()); + } + + private HttpResponse send(String path, ObjectNode body, + HttpResponse.BodyHandler handler) { + HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(base + path)) + .timeout(timeout) + .header("Content-Type", "application/json") + .header("Accept", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body.toString())); + if (apiKey != null && !apiKey.isBlank()) { + request.header("Authorization", "Bearer " + apiKey); + } + if (idempotency) { + // One key per logical call. The gateway replays the stored response instead of + // charging twice, which is what makes a caller-side retry safe. + request.header("Idempotency-Key", java.util.UUID.randomUUID().toString()); + } + try { + return http.send(request.build(), handler); + } catch (IOException e) { + throw PathException.network(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw PathException.network(e); + } + } + + /** Turns a non-2xx answer into an exception that names the gateway's error code. */ + static void requireSuccess(HttpResponse response, String rawBody) { + int status = response.statusCode(); + if (status >= 200 && status < 300) { + return; + } + String code = null; + String message = "Path gateway returned HTTP " + status; + try { + JsonNode error = new ObjectMapper().readTree(rawBody).path("error"); + if (error.isObject()) { + code = error.path("code").asText(null); + String detail = error.path("message").asText(null); + if (detail != null && !detail.isBlank()) { + message = detail; + } + } + } catch (Exception ignored) { + // A non-JSON error body is still an error; the status is enough. + } + throw new PathException(status, code, message, null); + } +} diff --git a/path-spring-ai/src/test/java/com/anindra/path/springai/PathChatModelTest.java b/path-spring-ai/src/test/java/com/anindra/path/springai/PathChatModelTest.java new file mode 100644 index 0000000..9953f60 --- /dev/null +++ b/path-spring-ai/src/test/java/com/anindra/path/springai/PathChatModelTest.java @@ -0,0 +1,236 @@ +package com.anindra.path.springai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.tool.ToolCallingChatOptions; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PathChatModelTest { + + private HttpServer server; + private final ObjectMapper mapper = new ObjectMapper(); + private volatile int status = 200; + private volatile String responseBody = "{}"; + private volatile String contentType = "application/json"; + private volatile java.util.Map responseHeaders = java.util.Map.of(); + private final AtomicReference requestBody = new AtomicReference<>(); + private final AtomicReference requestHeaders = new AtomicReference<>(); + + @BeforeEach + void start() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/chat/completions", this::handle); + server.start(); + } + + private void handle(HttpExchange exchange) throws IOException { + requestBody.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + requestHeaders.set(exchange.getRequestHeaders()); + responseHeaders.forEach((name, value) -> exchange.getResponseHeaders().add(name, value)); + exchange.getResponseHeaders().add("Content-Type", contentType); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + private PathChatModel model(String apiKey, boolean idempotency) { + return PathChatModel.builder() + .baseUrl("http://127.0.0.1:" + server.getAddress().getPort()) + .apiKey(apiKey) + .model("gpt-4o-mini") + .idempotency(idempotency) + .build(); + } + + private static String completion(String content, String finishReason, int prompt, int completion) { + return """ + {"id":"chatcmpl-1","model":"gpt-4o-mini","choices":[{"index":0, + "message":{"role":"assistant","content":%s},"finish_reason":%s}], + "usage":{"prompt_tokens":%d,"completion_tokens":%d,"total_tokens":%d}} + """.formatted(content == null ? "null" : "\"" + content + "\"", + finishReason == null ? "null" : "\"" + finishReason + "\"", + prompt, completion, prompt + completion); + } + + @Test + void parsesContentUsageAndThePathHeaders() { + responseHeaders = java.util.Map.of( + "X-Path-Cache", "hit", + "X-Gateway-Provider", "openai", + "X-Gateway-Cost-Usd", "0.00042"); + responseBody = completion("Hello from Path", "stop", 12, 4); + + ChatResponse response = model("sk-test", false) + .call(new Prompt(List.of(new UserMessage("hi")))); + + assertThat(response.getResult().getOutput().getText()).isEqualTo("Hello from Path"); + assertThat(response.getMetadata().getUsage().getPromptTokens()).isEqualTo(12); + assertThat(response.getMetadata().getUsage().getCompletionTokens()).isEqualTo(4); + assertThat(response.getMetadata().getUsage().getTotalTokens()).isEqualTo(16); + assertThat((Object) response.getMetadata().get("path_cache")).isEqualTo("hit"); + assertThat((Object) response.getMetadata().get("path_provider")).isEqualTo("openai"); + assertThat((Object) response.getMetadata().get("path_cost_usd")).isEqualTo("0.00042"); + assertThat(response.getResult().getMetadata().getFinishReason()).isEqualTo("stop"); + } + + @Test + void sendsMessagesOptionsAuthAndAnIdempotencyKey() throws Exception { + responseBody = completion("ok", "stop", 1, 1); + ChatOptions options = ChatOptions.builder() + .model("claude-3.5-haiku") + .temperature(0.2) + .maxTokens(64) + .build(); + + model("sk-test", true).call(new Prompt(List.of( + new SystemMessage("be terse"), new UserMessage("hi")), options)); + + JsonNode sent = mapper.readTree(requestBody.get()); + assertThat(sent.path("model").asText()).isEqualTo("claude-3.5-haiku"); + assertThat(sent.path("temperature").asDouble()).isEqualTo(0.2); + assertThat(sent.path("max_tokens").asInt()).isEqualTo(64); + assertThat(sent.path("stream").asBoolean()).isFalse(); + assertThat(sent.path("messages").get(0).path("role").asText()).isEqualTo("system"); + assertThat(sent.path("messages").get(1).path("content").asText()).isEqualTo("hi"); + assertThat(requestHeaders.get().getFirst("Authorization")).isEqualTo("Bearer sk-test"); + assertThat(requestHeaders.get().getFirst("Idempotency-Key")).isNotBlank(); + } + + @Test + void toolCallsRoundTrip() throws Exception { + responseBody = """ + {"id":"chatcmpl-2","model":"gpt-4o-mini","choices":[{"index":0, + "message":{"role":"assistant","content":null,"tool_calls":[ + {"id":"call_1","type":"function","function":{"name":"weather","arguments":"{\\"city\\":\\"Pune\\"}"}}]}, + "finish_reason":"tool_calls"}], + "usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}} + """; + + ChatResponse response = model("sk-test", false).call(new Prompt(List.of(new UserMessage("weather?")))); + + AssistantMessage output = response.getResult().getOutput(); + assertThat(output.hasToolCalls()).isTrue(); + assertThat(output.getToolCalls().get(0).id()).isEqualTo("call_1"); + assertThat(output.getToolCalls().get(0).name()).isEqualTo("weather"); + assertThat(output.getToolCalls().get(0).arguments()).contains("Pune"); + + // The follow-up turn sends the tool definition and the tool's answer back. + responseBody = completion("It is sunny", "stop", 20, 3); + ToolCallback callback = new ToolCallback() { + @Override + public ToolDefinition getToolDefinition() { + return ToolDefinition.builder() + .name("weather") + .description("Current weather for a city") + .inputSchema("{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}") + .build(); + } + + @Override + public String call(String input) { + return "sunny"; + } + }; + ToolCallingChatOptions options = ToolCallingChatOptions.builder() + .model("gpt-4o-mini") + .toolCallbacks(List.of(callback)) + .build(); + List conversation = List.of( + new UserMessage("weather?"), + AssistantMessage.builder() + .content(null) + .toolCalls(List.of(new AssistantMessage.ToolCall("call_1", "function", "weather", + "{\"city\":\"Pune\"}"))) + .build(), + ToolResponseMessage.builder() + .responses(List.of(new ToolResponseMessage.ToolResponse("call_1", "weather", "sunny"))) + .build()); + + model("sk-test", false).call(new Prompt(conversation, options)); + + JsonNode sent = mapper.readTree(requestBody.get()); + assertThat(sent.path("tools").get(0).path("function").path("name").asText()).isEqualTo("weather"); + assertThat(sent.path("tools").get(0).path("function").path("parameters").path("type").asText()) + .isEqualTo("object"); + assertThat(sent.path("messages").get(1).path("tool_calls").get(0).path("id").asText()) + .isEqualTo("call_1"); + assertThat(sent.path("messages").get(2).path("role").asText()).isEqualTo("tool"); + assertThat(sent.path("messages").get(2).path("tool_call_id").asText()).isEqualTo("call_1"); + assertThat(sent.path("messages").get(2).path("content").asText()).isEqualTo("sunny"); + } + + @Test + void streamAssemblesContentChunks() throws Exception { + contentType = "text/event-stream"; + responseBody = """ + data: {"id":"c1","model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]} + + data: {"id":"c1","model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]} + + data: {"id":"c1","model":"gpt-4o-mini","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + + """; + + List chunks = model("sk-test", false) + .stream(new Prompt(List.of(new UserMessage("hi")))) + .collectList() + .block(); + + assertThat(chunks).isNotNull(); + String text = chunks.stream().map(c -> c.getResult().getOutput().getText()) + .reduce("", String::concat); + assertThat(text).isEqualTo("Hello"); + assertThat(chunks.get(chunks.size() - 1).getResult().getMetadata().getFinishReason()) + .isEqualTo("stop"); + assertThat(mapper.readTree(requestBody.get()).path("stream").asBoolean()).isTrue(); + } + + @Test + void anErrorEnvelopeBecomesAPathException() { + status = 422; + responseBody = """ + {"error":{"message":"unsupported parameter: seed","type":"invalid_request_error", + "code":"unsupported_parameter"}} + """; + + assertThatThrownBy(() -> model("sk-test", false).call(new Prompt("hi"))) + .isInstanceOf(PathException.class) + .satisfies(e -> { + PathException failure = (PathException) e; + assertThat(failure.status()).isEqualTo(422); + assertThat(failure.code()).isEqualTo("unsupported_parameter"); + assertThat(failure.getMessage()).contains("seed"); + }); + } +} diff --git a/path-spring-ai/src/test/java/com/anindra/path/springai/PathEmbeddingModelTest.java b/path-spring-ai/src/test/java/com/anindra/path/springai/PathEmbeddingModelTest.java new file mode 100644 index 0000000..51a149b --- /dev/null +++ b/path-spring-ai/src/test/java/com/anindra/path/springai/PathEmbeddingModelTest.java @@ -0,0 +1,82 @@ +package com.anindra.path.springai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.ai.embedding.EmbeddingRequest; +import org.springframework.ai.embedding.EmbeddingResponse; +import org.springframework.ai.embedding.EmbeddingOptions; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +class PathEmbeddingModelTest { + + private HttpServer server; + private final ObjectMapper mapper = new ObjectMapper(); + private final AtomicReference requestBody = new AtomicReference<>(); + + @BeforeEach + void start() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/embeddings", exchange -> { + requestBody.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + byte[] body = """ + {"object":"list","model":"text-embedding-3-small", + "data":[{"object":"embedding","index":0,"embedding":[0.5,-0.25]}], + "usage":{"prompt_tokens":3,"total_tokens":3}} + """.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + private PathEmbeddingModel model() { + return PathEmbeddingModel.builder() + .baseUrl("http://127.0.0.1:" + server.getAddress().getPort()) + .apiKey("sk-test") + .model("text-embedding-3-small") + .dimensions(2) + .build(); + } + + @Test + void returnsTheVectorsTheGatewaySent() { + EmbeddingResponse response = model().call(new EmbeddingRequest(List.of("hello"), + EmbeddingOptions.builder().model("text-embedding-3-small").build())); + + assertThat(response.getResults()).hasSize(1); + assertThat(response.getResult().getOutput()).containsExactly(0.5f, -0.25f); + assertThat(response.getMetadata().getModel()).isEqualTo("text-embedding-3-small"); + } + + @Test + void sendsTheInputAndModel() throws Exception { + model().embed("hello"); + + JsonNode sent = mapper.readTree(requestBody.get()); + assertThat(sent.path("model").asText()).isEqualTo("text-embedding-3-small"); + assertThat(sent.path("input").get(0).asText()).isEqualTo("hello"); + } + + @Test + void reportsTheConfiguredDimensions() { + assertThat(model().dimensions()).isEqualTo(2); + } +} diff --git a/pom.xml b/pom.xml index 032475e..1c0f0f4 100644 --- a/pom.xml +++ b/pom.xml @@ -12,10 +12,11 @@ io.github.an1ndra - path - 0.2.2 - path - Provider-agnostic LLM gateway: routing, fallback, credits and cache-aware token metering for OpenAI, Anthropic and any OpenAI-compatible API + path-parent + 0.2.3 + pom + path-parent + Build parent for Path, the provider-agnostic LLM gateway, and its Spring AI adapter https://github.com/an1ndra/path @@ -43,6 +44,7 @@ 21 + 1.1.8 + path-gateway + path-spring-ai +