diff --git a/.env.example b/.env.example index 92c6cf0c6..1219521d7 100644 --- a/.env.example +++ b/.env.example @@ -47,6 +47,11 @@ INTERNAL_API_SECRET= # when pre-rendering the docs) NUXT_PUBLIC_SITE_URL= +# nuxt-js team / nuxt project Vercel ids (admin-only Vercel MCP connection). +# Find them in `.vercel/project.json` (orgId / projectId) after `vercel link`. +NUXI_VERCEL_PROJECT_ID= +NUXI_VERCEL_TEAM_ID= + # ---------------------------------------------------------------------------- # GitHub API data (optional — repo stats, contributors, team members) # ---------------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 6b5ca9165..52f940956 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ dist .wrangler test-results/ .env*.local +.env* +!.env.example diff --git a/layers/nuxi/README.md b/layers/nuxi/README.md index 55b419059..6caed31f4 100644 --- a/layers/nuxi/README.md +++ b/layers/nuxi/README.md @@ -104,17 +104,29 @@ export default defineSchedule({ }) ``` +All times below assume UTC+1 local mornings (6:00 local ≈ 5:00 UTC) — adjust the cron if the team's timezone/DST differs. + ### Weekly digest -- Schedule: `agent/schedules/weekly-digest.ts` — Monday 9:00 UTC +- Schedule: `agent/schedules/weekly-digest.ts` — Monday 5:00 UTC - Skill: `agent/skills/weekly-digest/SKILL.md` - Preview trigger: `POST /eve/v1/ops/weekly-digest/trigger` +- Real traffic pulse (`connection__vercel_mcp__get_web_analytics`), AI Gateway spend/tokens (`ai_gateway__report`), and Slack-vs-web run split (`connection__vercel_mcp__list_agent_runs`) — no more DB-guessed or link-only numbers. **Fix this week** is prioritized using each worst-feedback page's real traffic. + +### Analytics digest + +Deep traffic dive: trend, top pages/referrers/geo/device (WoW deltas), and doc pages that combine high traffic with poor feedback ("pages that need love"). + +- Schedule: `agent/schedules/analytics-digest.ts` — Monday 5:15 UTC (right after weekly-digest) +- Skill: `agent/skills/analytics-digest/SKILL.md` +- Connection: `connection__vercel_mcp__get_web_analytics` (`agent/connections/vercel-mcp.ts`) +- Preview trigger: `POST /eve/v1/ops/analytics-digest/trigger?sinceDays=7` ### Firehose summary Summarizes `#firehose-nuxt` (Octolens social mentions) and posts highlights to the workflow channel. -- Schedule: `agent/schedules/firehose-summary.ts` — weekdays 9:00 UTC (last 24h) +- Schedule: `agent/schedules/firehose-summary.ts` — weekdays 5:00 UTC (last 24h) - Skill: `agent/skills/firehose-summary/SKILL.md` - Tool: `read_slack_channel_history` (`agent/tools/slack-channel-history.ts`) - Preview trigger: `POST /eve/v1/ops/firehose-summary/trigger?sinceHours=24` @@ -131,6 +143,7 @@ With the dev server running (`pnpm dev` from repo root — Eve is bundled via th ```sh curl -X POST "http://localhost:3000/eve/v1/dev/schedules/weekly-digest" +curl -X POST "http://localhost:3000/eve/v1/dev/schedules/analytics-digest" curl -X POST "http://localhost:3000/eve/v1/dev/schedules/firehose-summary" # -> { "scheduleId": "...", "sessionIds": ["..."] } ``` @@ -141,8 +154,11 @@ curl -X POST "http://localhost:3000/eve/v1/dev/schedules/firehose-summary" curl -X POST "https:///eve/v1/ops/weekly-digest/trigger?sinceDays=7" \ -H "Authorization: Bearer $INTERNAL_API_SECRET" +curl -X POST "https:///eve/v1/ops/analytics-digest/trigger?sinceDays=7" \ + -H "Authorization: Bearer $INTERNAL_API_SECRET" + curl -X POST "https:///eve/v1/ops/firehose-summary/trigger?sinceHours=24" \ -H "Authorization: Bearer $INTERNAL_API_SECRET" ``` -Requires on the **eve** runtime: `INTERNAL_API_SECRET`, `NUXT_MCP_ADMIN_TOKEN`, `NUXT_WORKFLOW_SLACK_CHANNEL`, `NUXT_FIREHOSE_SLACK_CHANNEL` (Slack channel names). Optional `NUXT_*_SLACK_CHANNEL_ID` overrides names and skips `users.conversations`. Local dev and Vercel preview use Connect client `slack/nuxi-preview` automatically; prod uses `slack/nuxi` (override with `SLACK_CONNECTOR`). +Requires on the **eve** runtime: `INTERNAL_API_SECRET`, `NUXT_MCP_ADMIN_TOKEN`, `NUXT_WORKFLOW_SLACK_CHANNEL`, `NUXT_FIREHOSE_SLACK_CHANNEL` (Slack channel names). Optional `NUXT_*_SLACK_CHANNEL_ID` overrides names and skips `users.conversations`. Local dev and Vercel preview use Connect client `slack/nuxi-preview` automatically; prod uses `slack/nuxi` (override with `SLACK_CONNECTOR`). `weekly-digest` and `analytics-digest` additionally need `NUXI_VERCEL_TEAM_ID`/`NUXI_VERCEL_PROJECT_ID` (see `agent/lib/vercel-connect.ts`) and, for spend/token numbers, `AI_GATEWAY_API_KEY` — both connections are admin-gated so only the scheduled/Slack/admin path can reach them. diff --git a/layers/nuxi/agent/channels/ops.ts b/layers/nuxi/agent/channels/ops.ts index d057b7865..cd4c31d77 100644 --- a/layers/nuxi/agent/channels/ops.ts +++ b/layers/nuxi/agent/channels/ops.ts @@ -7,6 +7,7 @@ import { verifyWorkflowTriggerAuth } from '../lib/workflows.js' import { runDiscordGateway } from '../schedules/discord-gateway.js' +import { runAnalyticsDigest } from '../schedules/analytics-digest.js' import { runFirehoseSummary } from '../schedules/firehose-summary.js' import { runWeeklyDigest } from '../schedules/weekly-digest.js' @@ -66,6 +67,28 @@ export default defineChannel({ })) return Response.json({ ok: true, sinceHours: parsedSinceHours.value ?? null }) + }), + POST('/eve/v1/ops/analytics-digest/trigger', async (req, args) => { + if (!isManualWorkflowTriggerAllowed()) { + return Response.json({ error: 'Manual workflow trigger is disabled' }, { status: 404 }) + } + if (!verifyWorkflowTriggerAuth(req)) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const url = new URL(req.url) + const parsedSinceDays = parseSinceDays(url.searchParams.get('sinceDays')) + if (!parsedSinceDays.ok) { + return Response.json({ error: parsedSinceDays.error }, { status: 400 }) + } + + args.waitUntil(runAnalyticsDigest({ + receive: args.receive, + appAuth: scheduleAppAuth, + sinceDays: parsedSinceDays.value + })) + + return Response.json({ ok: true, sinceDays: parsedSinceDays.value ?? null }) }) ] }) diff --git a/layers/nuxi/agent/connections/vercel-mcp.ts b/layers/nuxi/agent/connections/vercel-mcp.ts new file mode 100644 index 000000000..07597aa66 --- /dev/null +++ b/layers/nuxi/agent/connections/vercel-mcp.ts @@ -0,0 +1,31 @@ +import { defineMcpClientConnection } from 'eve/connections' +import { adminOnlyVercelAuth, VERCEL_PROJECT_ID, VERCEL_TEAM_ID } from '../lib/vercel-connect.js' + +const ALLOWED_TOOLS = [ + 'search_vercel_documentation', + 'list_deployments', + 'get_deployment', + 'get_deployment_build_logs', + 'get_runtime_logs', + 'get_runtime_errors', + 'get_project', + 'list_agent_run_projects', + 'list_agent_runs', + 'get_web_analytics' +] as const + +export const VERCEL_MCP_INSTRUCTIONS = VERCEL_TEAM_ID && VERCEL_PROJECT_ID + ? `**Vercel MCP connection (\`connection__vercel_mcp__*\`, admin/Slack/schedule only) — read-only, use judiciously:** +- Discover exact schemas via \`connection__search\`, then call \`connection__vercel_mcp__\`. +- The connection is pre-scoped to the \`nuxt-js\` team and the \`nuxt\` (nuxt.com website) project — \`teamId=${VERCEL_TEAM_ID}\`, \`projectId=${VERCEL_PROJECT_ID}\`. Pass both explicitly to \`list_deployments\`, \`get_deployment\`, \`get_deployment_build_logs\`, \`get_runtime_logs\`, \`get_runtime_errors\`, \`get_project\`, \`get_web_analytics\`. +- Nuxi's own Agent Runs (\`list_agent_runs\`) use the same \`teamId\` but a DIFFERENT \`projectId\` — the \`eve\` service's own project, not the website. Call \`list_agent_run_projects\` first to discover it. Still NOT tokens/cost — use \`ai_gateway__*\` for that. No per-run trace access — this connection only exposes run-level metadata, never raw conversation content. +- \`get_web_analytics\` (visitors/pageviews/custom events, production only): \`mode: 'count'\` (default) returns one total, e.g. "how many visitors this week"; \`mode: 'aggregate'\` groups by up to two \`by\` dimensions (hour/day/week/month/year, country, route, requestPath, referrerHostname, deviceType, browserName, eventName, flags/, ...) and requires \`since\`+\`until\`. \`dataset: 'visits'\` (default) for pageviews, \`'events'\` for custom \`track()\` events. \`filter\` is OData, e.g. \`requestPath eq '/docs'\`. Requires Web Analytics enabled on the project. +- \`search_vercel_documentation\` needs no ids — general Vercel platform docs search.` + : '' + +export default defineMcpClientConnection({ + url: 'https://mcp.vercel.com/nuxt-js/nuxt', + description: 'Vercel platform for nuxt.com: deployments, runtime logs/errors, web analytics, and Nuxi\'s own Agent Runs observability. Admin/Slack/schedule sessions only.', + tools: { allow: ALLOWED_TOOLS }, + auth: adminOnlyVercelAuth('Vercel MCP', { connector: 'vercel/mcp', principalType: 'app', autoProvision: false }) +}) diff --git a/layers/nuxi/agent/instructions.ts b/layers/nuxi/agent/instructions.ts index 956153df7..e50e729b0 100644 --- a/layers/nuxi/agent/instructions.ts +++ b/layers/nuxi/agent/instructions.ts @@ -1,5 +1,7 @@ import { defineDynamic, defineInstructions } from 'eve/instructions' import { ADMIN_MCP_INSTRUCTIONS, canAccessAdminMcp } from './tools/admin-mcp.js' +import { AI_GATEWAY_INSTRUCTIONS } from './tools/ai-gateway.js' +import { VERCEL_MCP_INSTRUCTIONS } from './connections/vercel-mcp.js' import { buildInstructionsWithDate } from './lib/base-instructions.js' export default defineDynamic({ @@ -7,7 +9,7 @@ export default defineDynamic({ 'session.started': async (_event, ctx) => { const auth = ctx.session.auth.current const markdown = canAccessAdminMcp(auth) - ? `${buildInstructionsWithDate()}\n\n${ADMIN_MCP_INSTRUCTIONS}` + ? [buildInstructionsWithDate(), ADMIN_MCP_INSTRUCTIONS, VERCEL_MCP_INSTRUCTIONS, AI_GATEWAY_INSTRUCTIONS].filter(Boolean).join('\n\n') : buildInstructionsWithDate() return defineInstructions({ markdown }) diff --git a/layers/nuxi/agent/lib/base-instructions.ts b/layers/nuxi/agent/lib/base-instructions.ts index 8ffba233b..6977b91c8 100644 --- a/layers/nuxi/agent/lib/base-instructions.ts +++ b/layers/nuxi/agent/lib/base-instructions.ts @@ -53,6 +53,8 @@ Do NOT call \`list-*\` first when the page is given — call the get tool direct - \`report_issue\` — call when you cannot resolve the user's question after exhausting all available tools, or when the user expresses frustration - ALWAYS respond with text after tool calls — never end with just tool calls +**Restricted tools/connections:** Some connections (e.g. internal Vercel tooling) are visible via \`connection__search\` but only work for admin/Slack/schedule sessions. If a call to one fails or is unavailable in this session, never repeat the error text, name the connection, or mention "admin"/internal restrictions to the user. Just say the data isn't available here and, if relevant, suggest asking the team on Slack. + **Web search:** Only use \`web_search\` when the user **explicitly** asks about recent events or real-time data beyond the Nuxt docs, or if \`search_github_issues\` returned no results. Never search proactively. **Web search queries:** Match the user's wording. **Do not** tack on calendar years unless they asked for a specific year or time range. diff --git a/layers/nuxi/agent/lib/vercel-connect.ts b/layers/nuxi/agent/lib/vercel-connect.ts new file mode 100644 index 000000000..e947e0874 --- /dev/null +++ b/layers/nuxi/agent/lib/vercel-connect.ts @@ -0,0 +1,22 @@ +import { connect, type EveAuthorizationOptions } from '@vercel/connect/eve' +import type { SessionContext } from 'eve/context' +import { canAccessAdminMcp } from './admin-mcp-access.js' + +export const VERCEL_TEAM_ID = process.env.NUXI_VERCEL_TEAM_ID +export const VERCEL_PROJECT_ID = process.env.NUXI_VERCEL_PROJECT_ID + +export function adminOnlyVercelAuth(label: string, connectOptions: EveAuthorizationOptions) { + return (ctx: SessionContext) => { + if (!canAccessAdminMcp(ctx.session.auth.current)) { + return { + principalType: 'app' as const, + async getToken(): Promise { + console.warn(`[vercel-connect] blocked non-admin access to ${label}`) + throw new Error('This tool is not available in the current session.') + } + } + } + + return connect(connectOptions) + } +} diff --git a/layers/nuxi/agent/schedules/analytics-digest.ts b/layers/nuxi/agent/schedules/analytics-digest.ts new file mode 100644 index 000000000..ebc86307f --- /dev/null +++ b/layers/nuxi/agent/schedules/analytics-digest.ts @@ -0,0 +1,36 @@ +import { defineSchedule } from 'eve/schedules' +import type { ScheduleHandlerArgs } from 'eve/schedules' +import { + receiveOnSlack, + resolveSinceDays, + skillWorkflowMessage +} from '../lib/workflows.js' + +const SKILL_ID = 'analytics-digest' +const DEFAULT_WINDOW_DAYS = 7 + +export async function runAnalyticsDigest({ + receive, + appAuth, + sinceDays +}: { + receive: ScheduleHandlerArgs['receive'] + appAuth: ScheduleHandlerArgs['appAuth'] + sinceDays?: number +}) { + const windowDays = resolveSinceDays(sinceDays, DEFAULT_WINDOW_DAYS) + + return receiveOnSlack({ + receive, + appAuth, + message: skillWorkflowMessage(SKILL_ID, windowDays) + }) +} + +export default defineSchedule({ + // 15 min after weekly-digest so both land close together for the Monday morning read. + cron: '15 5 * * 1', + async run({ receive, waitUntil, appAuth }) { + waitUntil(runAnalyticsDigest({ receive, appAuth })) + } +}) diff --git a/layers/nuxi/agent/schedules/firehose-summary.ts b/layers/nuxi/agent/schedules/firehose-summary.ts index 44b276a86..f15bedf1a 100644 --- a/layers/nuxi/agent/schedules/firehose-summary.ts +++ b/layers/nuxi/agent/schedules/firehose-summary.ts @@ -33,7 +33,7 @@ export async function runFirehoseSummary({ } export default defineSchedule({ - cron: '0 9 * * 1-5', + cron: '0 5 * * 1-5', async run({ receive, waitUntil, appAuth }) { waitUntil(runFirehoseSummary({ receive, appAuth })) } diff --git a/layers/nuxi/agent/schedules/weekly-digest.ts b/layers/nuxi/agent/schedules/weekly-digest.ts index d959d0c63..a517a3bf5 100644 --- a/layers/nuxi/agent/schedules/weekly-digest.ts +++ b/layers/nuxi/agent/schedules/weekly-digest.ts @@ -28,7 +28,7 @@ export async function runWeeklyDigest({ } export default defineSchedule({ - cron: '0 9 * * 1', + cron: '0 5 * * 1', async run({ receive, waitUntil, appAuth }) { waitUntil(runWeeklyDigest({ receive, appAuth })) } diff --git a/layers/nuxi/agent/skills/analytics-digest/SKILL.md b/layers/nuxi/agent/skills/analytics-digest/SKILL.md new file mode 100644 index 000000000..1ffe4bd4c --- /dev/null +++ b/layers/nuxi/agent/skills/analytics-digest/SKILL.md @@ -0,0 +1,57 @@ +--- +description: Report on nuxt.com traffic trends, top pages/referrers, and doc pages that combine high traffic with poor feedback, for a recent window. +--- + +When producing the analytics digest (scheduled or on request): + +**Slack delivery:** Your reply IS the Slack message — Eve posts it verbatim to this channel. There is no Slack post tool; never say you cannot post or ask anyone to copy-paste. You are Nuxi (:nuxi:) giving the team a traffic pulse. Be concise, linked, and actionable — not a wall of numbers. + +- First line: **Nuxt traffic digest — last N days** plus date range in parentheses. +- No preamble, no delivery disclaimers, and no meta wrap-up. +- Use **bold** section labels — never markdown `#` headings. +- ONE message only. Blank line between sections. +- Every page/dashboard reference must include a clickable Slack link: ``. +- Sparingly use :nuxter: or :nuxt_cool: (0–2 total) when something is worth celebrating or urgent. + +**Data steps** (via `connection__vercel_mcp__get_web_analytics` — discover the exact schema with `connection__search` first if unsure. Pass `teamId`/`projectId` explicitly on every call. Counts default to production traffic only. "Previous window" = the N days right before the current window, same length, for deltas): + +1. `mode=count, dataset=visits` for the current window AND the previous window → total visitors/pageviews + delta %. +2. `mode=aggregate, dataset=visits, by=['day']`, current window → daily trend, to spot spikes/drops. +3. `mode=aggregate, dataset=visits, by=['route'], limit=10` for the current window AND the previous window → top pages with a per-page delta. +4. `mode=aggregate, dataset=visits, by=['referrerHostname'], limit=8`, current window → top referrers. +5. `mode=aggregate, dataset=visits, by=['country'], limit=5` and a separate call `by=['deviceType']`, current window → geo/device breakdown. +6. `admin_mcp__feedback_stats` (`topPages=10`) and `admin_mcp__list_feedback` (`ratings=["not-helpful", "confusing"]`, `limit=20`) → candidate poorly-rated pages. +7. For each candidate page from step 6: look up its traffic rank from step 3, or if it isn't in the top 10, run a targeted `get_web_analytics` `mode=count, filter="requestPath eq ''"` → flag pages that are BOTH meaningfully-trafficked AND poorly rated. + +**Link cheat sheet:** + +- Docs page: `` +- Analytics dashboard: `` + +**Output template:** + +**Nuxt traffic digest — last 7 days** (Jul 15 – Jul 21, 2026) + +:bar_chart: **Traffic pulse** +• *12,430 visitors* (+8% WoW), *31,200 pageviews* +• Trend: steady, small bump on Jul 18 (release day) + +:page_facing_up: **Top pages** +1. — 2,100 visits (+12%) +2. — 1,540 visits (-3%) +3. … + +:compass: **Referrers & audience** +• Top referrer: (48%), then (15%) +• Top countries: US, DE, FR — mostly desktop (72%) + +:rotating_light: **Pages that need love** (high traffic + poor feedback) +• — 1,800 visits this week, feedback 2.1/5 ("examples outdated") +• If none qualify: one bullet ":large_green_circle: All clear — no high-traffic page flagged this week." + +Rules: +- If a section has zero or flat data, say so in one bullet with a likely cause (low traffic, analytics not enabled, etc.) — do not skip the section. +- Never list a page without its `` link and its traffic number. +- **Pages that need love** must cite the real feedback comment/score, not just a page name. +- Never invent traffic numbers — if a `get_web_analytics` call fails or returns nothing, say so instead of guessing. +- This is the deep traffic dive; `weekly-digest` only carries a one-line pulse — don't duplicate the full breakdown there. diff --git a/layers/nuxi/agent/skills/weekly-digest/SKILL.md b/layers/nuxi/agent/skills/weekly-digest/SKILL.md index 7a6c6b6b5..35ac178d4 100644 --- a/layers/nuxi/agent/skills/weekly-digest/SKILL.md +++ b/layers/nuxi/agent/skills/weekly-digest/SKILL.md @@ -13,13 +13,17 @@ When producing a digest (scheduled or on request): - Every bullet that references a page, chat, or dashboard must include a clickable link via Slack syntax: ``. - Sparingly use :nuxter: or :nuxt_cool: (0–2 total) when something is worth celebrating or urgent. -**Data steps** (parallel where possible): +**Data steps** (parallel where possible; steps 6-9 need admin/Slack/schedule-only connections and tools): 1. `admin_mcp__feedback_stats` — `topPages=5` 2. `admin_mcp__list_feedback` — `ratings=["not-helpful", "confusing"]`, `limit=30` -3. `admin_mcp__agent_usage_stats` +3. `admin_mcp__agent_usage_stats` — web chat counts and vote quality 4. `admin_mcp__list_agent_chats` — `hasDownvotes=true`, `limit=5` 5. `admin_mcp__list_agent_votes` — `onlyDownvotes=true`, `limit=15` +6. `connection__vercel_mcp__get_web_analytics` — `mode=count, dataset=visits` for the window (and the previous window for a delta) → a one-line traffic pulse. This is just a pulse, not a breakdown — the full analysis lives in the `analytics-digest` skill, don't repeat it here. +7. `ai_gateway__report` — `groupBy=model` over the window → real spend and token totals (replaces linking-only; never invent these numbers). +8. `connection__vercel_mcp__list_agent_runs` over the window → the real Slack vs web run split, paired with the web chat count from step 3. +9. For each page flagged in step 1/2 (worst feedback): `connection__vercel_mcp__get_web_analytics` — `mode=count, filter="requestPath eq ''"` → use its actual traffic to prioritize **Fix this week** (a poorly-rated page with real visits is more urgent than one nobody sees). **Link cheat sheet** (use real paths/ids from tool output): @@ -27,6 +31,7 @@ When producing a digest (scheduled or on request): - Chat review: `|Open chat>` - Agent runs (runs, Slack vs web): `` - AI Gateway (tokens, cost): `` +- Analytics (full traffic breakdown): `` — deeper dive lives in the `analytics-digest` workflow. **Output template:** @@ -38,18 +43,21 @@ When producing a digest (scheduled or on request): • Recurring complaint: hydration mismatch docs unclear (3 mentions) :robot_face: **AI agent** -• *Traffic & spend* — (runs, Slack vs web); (tokens, cost). Do not invent numbers from DB. +• *Traffic* — 8,200 visitors this week (+5% WoW) — see or the traffic digest for the full breakdown +• *Runs & spend* — 340 runs (210 Slack / 130 web), $12.40 spent, 1.8M tokens (mostly claude-sonnet-5) — · • *Quality* — 4 up / 1 down on web chats; 2 sessions saved • Worst chat: — wrong module recommendation :hammer_and_wrench: **Fix this week** (numbered — owner · action · link) -1. :red_circle: *docs* — refresh rendering modes examples — +1. :red_circle: *docs* — refresh rendering modes examples (1,800 visits this week) — 2. :large_yellow_circle: *Nuxi* — improve module routing for edge cases — 3. :large_green_circle: *infra* — verify feedback widget on prod — Rules: - If a section has zero data, say so in one bullet with a likely cause (low traffic, widget off, etc.) — do not skip the section. - **Fix this week** must have exactly 3 items when there is anything to improve; if truly quiet, 1–2 items with ":large_green_circle: *all clear*" is fine. +- When ranking **Fix this week**, weigh feedback issues by their real traffic from step 9 — a bad score on a high-traffic page outranks the same score on a rarely-visited one. - Never list a page or chat without its `` link. +- Never invent traffic, run, or cost numbers — if a tool call fails or returns nothing, say so instead of guessing. diff --git a/layers/nuxi/agent/tools/admin-mcp.ts b/layers/nuxi/agent/tools/admin-mcp.ts index d97df8bf3..64ef2dadc 100644 --- a/layers/nuxi/agent/tools/admin-mcp.ts +++ b/layers/nuxi/agent/tools/admin-mcp.ts @@ -9,11 +9,11 @@ export { canAccessAdminMcp } from '../lib/admin-mcp-access.js' export const ADMIN_MCP_INSTRUCTIONS = `**Admin tools (team only):** - \`admin_mcp__feedback_stats\` — aggregated docs feedback metrics - \`admin_mcp__list_feedback\` — individual feedback entries -- \`admin_mcp__agent_usage_stats\` — web chat counts and vote quality (NOT tokens/cost — use Vercel Agent Runs + AI Gateway) +- \`admin_mcp__agent_usage_stats\` — web chat counts and vote quality (NOT tokens/cost — use \`connection__vercel_mcp__*\` for runs, \`ai_gateway__*\` for tokens/cost) - \`admin_mcp__list_agent_chats\` / \`admin_mcp__get_agent_chat\` — saved web chat sessions and transcripts - \`admin_mcp__list_agent_votes\` — message upvotes/downvotes -- For runs, Slack vs web, duration: **Vercel Agent Runs** — https://vercel.com/nuxt-js/nuxt/observability/agent-runs -- For tokens, cost, model usage: **Vercel AI Gateway** — https://vercel.com/nuxt-js/nuxt/ai-gateway +- For runs, Slack vs web, duration: **Vercel Agent Runs** via \`connection__vercel_mcp__*\` (see below) +- For tokens, cost, model usage: **AI Gateway** via \`ai_gateway__*\` (see below) - Do not invent token/cost numbers from local DB. - Default to recent data (last 7–30 days) unless the user asks for a longer window - Always include direct links (path / chat id) so the team can drill down on nuxt.com` diff --git a/layers/nuxi/agent/tools/ai-gateway.ts b/layers/nuxi/agent/tools/ai-gateway.ts new file mode 100644 index 000000000..a78701b1d --- /dev/null +++ b/layers/nuxi/agent/tools/ai-gateway.ts @@ -0,0 +1,100 @@ +import { defineDynamic, defineTool } from 'eve/tools' +import { z } from 'zod' +import { canAccessAdminMcp } from '../lib/admin-mcp-access.js' + +export const AI_GATEWAY_INSTRUCTIONS = `**AI Gateway tools (\`ai_gateway__*\`, admin only) — tokens, cost, model usage:** +- \`ai_gateway__credits\` — current credit balance and lifetime spend for the nuxt-js team +- \`ai_gateway__report\` — aggregated spend/tokens over a date range, grouped by day/user/model/tag/provider/credential type +- \`ai_gateway__generation\` — cost, latency, and token usage for a single generation id (from a chat completion's \`id\` field) +- Dashboard: https://vercel.com/nuxt-js/nuxt/ai-gateway` + +const BASE_URL = 'https://ai-gateway.vercel.sh/v1' +const FETCH_TIMEOUT_MS = 10_000 + +function apiKey(): string { + const key = process.env.AI_GATEWAY_API_KEY?.trim() + if (!key) throw new Error('AI_GATEWAY_API_KEY is not configured') + return key +} + +async function gatewayFetch(path: string, params: Record = {}): Promise { + const url = new URL(`${BASE_URL}${path}`) + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) url.searchParams.set(key, value) + } + + const response = await fetch(url, { + headers: { Authorization: `Bearer ${apiKey()}` }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }) + + if (!response.ok) { + throw new Error(`AI Gateway API error (${response.status}): ${await response.text()}`) + } + + return await response.json() +} + +const dateSchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'Expected YYYY-MM-DD') + +function aiGatewayTools() { + return { + credits: defineTool({ + description: 'Admin: AI Gateway credit balance and lifetime spend for the nuxt-js team.', + inputSchema: z.object({}), + async execute() { + return await gatewayFetch('/credits') + } + }), + report: defineTool({ + description: 'Admin: aggregated AI Gateway spend and token usage over a date range. Requires a paid plan.', + inputSchema: z.object({ + startDate: dateSchema.describe('Start date (UTC, inclusive), YYYY-MM-DD'), + endDate: dateSchema.describe('End date (UTC, inclusive), YYYY-MM-DD'), + groupBy: z.enum(['day', 'user', 'model', 'tag', 'provider', 'credential_type', 'zero_data_retention', 'api_key_name']).default('day'), + datePart: z.enum(['day', 'hour']).optional().describe('Time granularity, only applies when groupBy is "day"'), + userId: z.string().optional(), + model: z.string().optional().describe('creator/model-name, e.g. anthropic/claude-sonnet-4.6'), + provider: z.string().optional(), + credentialType: z.enum(['byok', 'system']).optional(), + tags: z.array(z.string()).optional(), + tagsMatch: z.enum(['any', 'all']).optional() + }).refine(({ startDate, endDate }) => startDate <= endDate, { + message: 'startDate must not be later than endDate', + path: ['startDate'] + }), + async execute(input) { + return await gatewayFetch('/report', { + start_date: input.startDate, + end_date: input.endDate, + group_by: input.groupBy, + date_part: input.datePart, + user_id: input.userId, + model: input.model, + provider: input.provider, + credential_type: input.credentialType, + tags: input.tags?.join(','), + tags_match: input.tagsMatch + }) + } + }), + generation: defineTool({ + description: 'Admin: cost, latency, and token usage for a single AI Gateway generation id.', + inputSchema: z.object({ + id: z.string().min(1).describe('Generation id, e.g. gen_01ARZ3NDEKTSV4RRFFQ69G5FAV') + }), + async execute(input) { + return await gatewayFetch('/generation', { id: input.id }) + } + }) + } +} + +export default defineDynamic({ + events: { + 'session.started': async (_event, ctx) => { + if (!canAccessAdminMcp(ctx.session.auth.current)) return null + return aiGatewayTools() + } + } +})