diff --git a/src/composables/__tests__/useAiDiagnostics.spec.ts b/src/composables/__tests__/useAiDiagnostics.spec.ts new file mode 100644 index 00000000..79a17ed7 --- /dev/null +++ b/src/composables/__tests__/useAiDiagnostics.spec.ts @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ref } from 'vue'; +import { useDataApi } from '@/composables/axios'; +import { useAiDiagnostics } from '@/composables/useAiDiagnostics'; + +vi.mock('@/composables/axios', () => ({ + useDataApi: vi.fn(), +})); + +function mockDataApi(partial: { + execute: ReturnType; + isLoading?: ReturnType>; + error?: ReturnType>; +}) { + return { + execute: partial.execute, + isLoading: partial.isLoading ?? ref(false), + error: partial.error ?? ref(null), + } as unknown as ReturnType; +} + +describe('useAiDiagnostics', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('does not start a pagination request while one is already loading', async () => { + const summaryRequest = vi.fn(); + const runsRequest = vi.fn(async () => ({ + data: { + value: { + data: [ + { + id: 'run-1', + status: 'completed', + }, + ], + meta: { + nextCursor: 'cursor-2', + }, + }, + }, + })); + const runsPageRequest = vi.fn(); + const runDetailRequest = vi.fn(); + + vi.mocked(useDataApi) + .mockReturnValueOnce( + mockDataApi({ + execute: summaryRequest, + isLoading: ref(false), + error: ref(null), + }), + ) + .mockReturnValueOnce( + mockDataApi({ + execute: runsRequest, + isLoading: ref(false), + error: ref(null), + }), + ) + .mockReturnValueOnce( + mockDataApi({ + execute: runsPageRequest, + isLoading: ref(true), + error: ref(null), + }), + ) + .mockReturnValueOnce( + mockDataApi({ + execute: runDetailRequest, + isLoading: ref(false), + error: ref(null), + }), + ); + + const diagnostics = useAiDiagnostics(); + await diagnostics.refreshRuns(); + diagnostics.paginationError.value = 'previous pagination error'; + + await diagnostics.loadMoreRuns(); + + expect(runsPageRequest).not.toHaveBeenCalled(); + expect(diagnostics.paginationError.value).toBe('previous pagination error'); + expect(diagnostics.runs.value).toHaveLength(1); + expect(diagnostics.nextCursor.value).toBe('cursor-2'); + }); +}); diff --git a/src/composables/useAiDiagnostics.ts b/src/composables/useAiDiagnostics.ts new file mode 100644 index 00000000..76fc7279 --- /dev/null +++ b/src/composables/useAiDiagnostics.ts @@ -0,0 +1,204 @@ +import { computed, onUnmounted, ref, shallowRef } from 'vue'; +import { useDataApi } from '@/composables/axios'; +import { + buildAiDiagnosticsRunDetailEndpoint, + buildAiDiagnosticsRunsEndpoint, + buildAiDiagnosticsSummaryEndpoint, + type AiDiagnosticsRun, + type AiDiagnosticsRunDetail, + type AiDiagnosticsRunsFilters, + type AiDiagnosticsRunsResponse, + type AiDiagnosticsSummary, +} from '@/views/admin/partials/ai-diagnostics'; + +const labelKeyStopPaths = ['data.scope.labelSets', 'data.events.metadata']; + +function unavailableMessage(fallback: string, error: unknown): string { + if (error instanceof Error && error.message) { + return error.message; + } + + return fallback; +} + +export function useAiDiagnostics() { + const summary = shallowRef(null); + const runs = shallowRef([]); + const nextCursor = ref(null); + const runsError = ref(null); + const paginationError = ref(null); + const summaryError = ref(null); + const selectedRunDetail = shallowRef(null); + const runDetailError = ref(null); + const pollTimer = ref(); + + const { + execute: summaryRequest, + isLoading: summaryLoading, + error: summaryRequestError, + } = useDataApi(null, {}, { immediate: false }); + + const { + execute: runsRequest, + isLoading: runsLoading, + error: runsRequestError, + } = useDataApi(null, {}, { immediate: false }); + + const { execute: runsPageRequest, isLoading: paginationLoading } = useDataApi< + AiDiagnosticsRun[] + >(null, {}, { immediate: false }); + + const { execute: runDetailRequest, isLoading: runDetailLoading } = + useDataApi(null, {}, { immediate: false }); + + async function refreshSummary() { + summaryError.value = null; + + try { + const response = await summaryRequest( + buildAiDiagnosticsSummaryEndpoint(), + ); + summary.value = response?.data.value?.data ?? null; + } catch (error) { + summary.value = null; + summaryError.value = unavailableMessage( + 'AI diagnostics summary is unavailable.', + error, + ); + } + } + + async function refreshRuns(filters: AiDiagnosticsRunsFilters = {}) { + runsError.value = null; + paginationError.value = null; + nextCursor.value = null; + + try { + const response = await runsRequest( + buildAiDiagnosticsRunsEndpoint(filters), + { camelcaseStopPaths: labelKeyStopPaths }, + ); + const payload = response?.data.value as + | AiDiagnosticsRunsResponse + | undefined; + + runs.value = payload?.data ?? []; + nextCursor.value = payload?.meta?.nextCursor ?? null; + } catch (error) { + runs.value = []; + runsError.value = unavailableMessage( + 'AI diagnostics runs are unavailable.', + error, + ); + } + } + + async function loadMoreRuns(filters: AiDiagnosticsRunsFilters = {}) { + if (!nextCursor.value || paginationLoading.value) { + return; + } + + paginationError.value = null; + + try { + const response = await runsPageRequest( + buildAiDiagnosticsRunsEndpoint({ + ...filters, + cursor: nextCursor.value, + }), + { camelcaseStopPaths: labelKeyStopPaths }, + ); + const payload = response?.data.value as + | AiDiagnosticsRunsResponse + | undefined; + + runs.value = [...runs.value, ...(payload?.data ?? [])]; + nextCursor.value = payload?.meta?.nextCursor ?? null; + } catch (error) { + paginationError.value = unavailableMessage( + 'More AI diagnostics runs are unavailable.', + error, + ); + } + } + + async function fetchRunDetail(runId: string) { + runDetailError.value = null; + selectedRunDetail.value = null; + + try { + const response = await runDetailRequest( + buildAiDiagnosticsRunDetailEndpoint(runId), + { camelcaseStopPaths: labelKeyStopPaths }, + ); + selectedRunDetail.value = response?.data.value?.data ?? null; + } catch (error) { + runDetailError.value = unavailableMessage( + 'AI diagnostics run details are unavailable.', + error, + ); + } + + return selectedRunDetail.value; + } + + function stopPolling() { + if (!pollTimer.value) { + return; + } + + window.clearInterval(pollTimer.value); + pollTimer.value = undefined; + } + + function pollWhileActive(callback: () => Promise | void) { + if (pollTimer.value) { + return; + } + + pollTimer.value = window.setInterval(() => { + void callback(); + }, 30000); + } + + onUnmounted(stopPolling); + + return { + summary, + runs, + nextCursor, + selectedRunDetail, + summaryLoading, + runsLoading, + paginationLoading, + runDetailLoading, + summaryError: computed( + () => + summaryError.value ?? + (summaryRequestError.value + ? unavailableMessage( + 'AI diagnostics summary is unavailable.', + summaryRequestError.value, + ) + : null), + ), + runsError: computed( + () => + runsError.value ?? + (runsRequestError.value + ? unavailableMessage( + 'AI diagnostics runs are unavailable.', + runsRequestError.value, + ) + : null), + ), + paginationError, + runDetailError, + refreshSummary, + refreshRuns, + loadMoreRuns, + fetchRunDetail, + pollWhileActive, + stopPolling, + }; +} diff --git a/src/router/index.ts b/src/router/index.ts index 5c02a8e8..a3c1ff70 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -381,11 +381,38 @@ const authenticatedRoutes = [ { path: '/admin/notifications', name: 'admin-notifications', - component: () => import('../views/admin/NotificationsView.vue'), + redirect: { name: 'admin-diagnostics-notifications' }, meta: { requiresAuth: true, }, }, + { + path: '/admin/diagnostics', + name: 'admin-diagnostics', + component: () => import('../views/admin/DiagnosticsView.vue'), + redirect: { name: 'admin-diagnostics-notifications' }, + meta: { + requiresAuth: true, + }, + children: [ + { + path: 'notifications', + name: 'admin-diagnostics-notifications', + component: () => import('../views/admin/NotificationsView.vue'), + meta: { + requiresAuth: true, + }, + }, + { + path: 'ai-suggestions', + name: 'admin-diagnostics-ai-suggestions', + component: () => import('../views/admin/AiDiagnosticsView.vue'), + meta: { + requiresAuth: true, + }, + }, + ], + }, { path: '/admin/risks', name: 'admin-risks', diff --git a/src/views/LeftSideNav.vue b/src/views/LeftSideNav.vue index 0685b532..4532dc3a 100644 --- a/src/views/LeftSideNav.vue +++ b/src/views/LeftSideNav.vue @@ -141,9 +141,9 @@ const links = ref>([ title: 'Risk Templates', }, { - name: 'admin-notifications', - title: 'Notifications', - abbr: 'NTF', + name: 'admin-diagnostics', + title: 'Diagnostics', + abbr: 'DIAG', }, { name: 'admin-import', diff --git a/src/views/__tests__/AiDiagnosticsView.spec.ts b/src/views/__tests__/AiDiagnosticsView.spec.ts new file mode 100644 index 00000000..01f63bf9 --- /dev/null +++ b/src/views/__tests__/AiDiagnosticsView.spec.ts @@ -0,0 +1,430 @@ +import { flushPromises, mount } from '@vue/test-utils'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { nextTick } from 'vue'; +import { + buildAiDiagnosticsCacheHitChartData, + buildAiDiagnosticsTokenChartData, + type AiDiagnosticsRun, + type AiDiagnosticsRunDetail, + type AiDiagnosticsSummary, +} from '@/views/admin/partials/ai-diagnostics'; + +const state = vi.hoisted(() => { + function testRef(value: T) { + return { __v_isRef: true, value }; + } + + return { + aiEnabled: true, + aiFetched: true, + fetchConfig: vi.fn(), + summary: testRef(null), + runs: testRef([]), + detail: testRef(null), + nextCursor: testRef(null), + summaryLoading: testRef(false), + runsLoading: testRef(false), + paginationLoading: testRef(false), + detailLoading: testRef(false), + summaryError: testRef(null), + runsError: testRef(null), + paginationError: testRef(null), + detailError: testRef(null), + refreshSummary: vi.fn(), + refreshRuns: vi.fn(), + loadMoreRuns: vi.fn(), + fetchRunDetail: vi.fn(), + pollWhileActive: vi.fn(), + stopPolling: vi.fn(), + }; +}); + +vi.mock('@/stores/ai-config', () => ({ + useAiConfigStore: () => ({ + get dashboardSuggestionsEnabled() { + return state.aiEnabled; + }, + get dashboardSuggestionsConfigFetched() { + return state.aiFetched; + }, + fetchDashboardSuggestionsConfig: state.fetchConfig, + }), +})); + +vi.mock('@/composables/useAiDiagnostics', () => ({ + useAiDiagnostics: () => ({ + summary: state.summary, + runs: state.runs, + nextCursor: state.nextCursor, + selectedRunDetail: state.detail, + summaryLoading: state.summaryLoading, + runsLoading: state.runsLoading, + paginationLoading: state.paginationLoading, + runDetailLoading: state.detailLoading, + summaryError: state.summaryError, + runsError: state.runsError, + paginationError: state.paginationError, + runDetailError: state.detailError, + refreshSummary: state.refreshSummary, + refreshRuns: state.refreshRuns, + loadMoreRuns: state.loadMoreRuns, + fetchRunDetail: state.fetchRunDetail, + pollWhileActive: state.pollWhileActive, + stopPolling: state.stopPolling, + }), +})); + +vi.mock('@/components/charts/LineChart.vue', () => ({ + default: { + name: 'LineChart', + props: ['data', 'options'], + template: '
{{ data.datasets.length }}
', + }, +})); + +import AiDiagnosticsView from '../admin/AiDiagnosticsView.vue'; + +function mountView() { + return mount(AiDiagnosticsView, { + global: { + stubs: { + PageCard: { template: '
' }, + Message: { template: '
' }, + RouterLink: { + props: ['to'], + template: '', + }, + Drawer: { + props: ['visible', 'header'], + emits: ['update:visible'], + template: '', + }, + SecondaryButton: { + emits: ['click'], + props: ['disabled', 'type'], + template: + '', + }, + Tabs: { template: '
' }, + TabList: { template: '
' }, + Tab: { template: '' }, + TabPanels: { template: '
' }, + TabPanel: { template: '
' }, + }, + }, + }); +} + +describe('AiDiagnosticsView', () => { + beforeEach(() => { + vi.clearAllMocks(); + state.aiEnabled = true; + state.aiFetched = true; + state.fetchConfig.mockResolvedValue(true); + state.summary.value = { + enabled: true, + config: { + model: 'gpt-5.2', + promptVersion: 'v4', + maxControlsPerChunk: 5, + maxLabelSetsPerChunk: 4, + maxCallsPerRun: 25, + queueWorkers: 2, + }, + totals: { + runs: 3, + runsByStatus: { + pending: 0, + running: 1, + completed: 1, + failed: 1, + }, + cellsCompleted: 8, + cellsFailed: 2, + inputTokens: 12000, + outputTokens: 1800, + cacheReadInputTokens: 4000, + cacheCreationInputTokens: 3000, + cacheHitRatio: 0.42, + mappingsReturned: 18, + mappingsRejected: 3, + suggestionsAccepted: 6, + suggestionsRejected: 2, + suggestionsPending: 4, + rateLimitedTotal: 5, + }, + queue: { + name: 'suggestion', + available: 2, + running: 1, + retryable: 0, + scheduled: 3, + oldestAvailableAt: '2026-06-16T12:00:00Z', + }, + checks: [ + { + id: 'cache_engaging', + status: 'warn', + message: + "caching never wrote - likely under the model's 4,096-token minimum", + recommendedActions: ['Increase chunk sizes.'], + }, + { + id: 'rate_limit_pressure', + status: 'fail', + message: 'Rate limits are slowing runs.', + recommendedActions: ['Lower queue workers.'], + }, + { + id: 'queue_available', + status: 'pass', + message: 'Suggestion queue is healthy.', + }, + ], + }; + state.runs.value = [ + { + id: 'run-1', + sspId: 'ssp-1', + sspName: 'Payments SSP', + status: 'completed', + model: 'gpt-5.2', + promptVersion: 'v4', + plannedCalls: 4, + completedCells: 4, + failedCells: 0, + inputTokens: 1000, + outputTokens: 200, + cacheReadInputTokens: 500, + cacheCreationInputTokens: 100, + cacheHitRatio: 0.5, + mappingsReturned: 8, + mappingsRejected: 1, + rateLimitedTotal: 2, + startedAt: '2026-06-16T12:00:00Z', + completedAt: '2026-06-16T12:01:00Z', + durationMs: 60000, + triggeredBy: { id: 'u-1', name: 'Ada' }, + }, + ]; + state.detail.value = null; + state.nextCursor.value = 'next-1'; + state.summaryLoading.value = false; + state.runsLoading.value = false; + state.paginationLoading.value = false; + state.detailLoading.value = false; + state.summaryError.value = null; + state.runsError.value = null; + state.paginationError.value = null; + state.detailError.value = null; + state.fetchRunDetail.mockImplementation(async () => { + state.detail.value = { + ...state.runs.value[0], + scope: { + controlKeys: ['AC-1'], + labelSetHashes: ['hash-1'], + labelSets: [{ _policy: 'secret_scanning', repository: 'api' }], + }, + cells: [ + { + cellIndex: 0, + status: 'completed', + controlKeys: ['AC-1'], + labelSetHashes: ['hash-1'], + inputTokens: 600, + outputTokens: 100, + cacheReadInputTokens: 300, + cacheCreationInputTokens: 50, + rateLimitedCount: 0, + mappingsReturned: 4, + mappingsRejected: 0, + completedAt: '2026-06-16T12:00:30Z', + }, + { + cellIndex: 1, + status: 'failed', + controlKeys: ['AC-2'], + labelSetHashes: ['hash-2'], + inputTokens: 400, + outputTokens: 0, + cacheReadInputTokens: 200, + cacheCreationInputTokens: 50, + rateLimitedCount: 1, + mappingsReturned: 0, + mappingsRejected: 1, + error: 'model timeout', + }, + ], + events: [ + { + id: 'evt-1', + action: 'run_started', + actor: 'Ada', + message: 'Run started.', + createdAt: '2026-06-16T12:00:00Z', + }, + ], + }; + return state.detail.value; + }); + }); + + it('renders the disabled message without loading diagnostics when AI is off', async () => { + state.aiEnabled = false; + state.aiFetched = false; + state.fetchConfig.mockImplementation(async () => { + state.aiFetched = true; + return false; + }); + + const wrapper = mountView(); + await flushPromises(); + await wrapper.vm.$forceUpdate(); + await nextTick(); + + expect(wrapper.text()).toContain( + 'AI is not configured, so AI suggestions diagnostics are unavailable.', + ); + expect(state.refreshSummary).not.toHaveBeenCalled(); + expect(state.refreshRuns).not.toHaveBeenCalled(); + }); + + it('renders summary totals, checks, and the full cache warning', async () => { + const wrapper = mountView(); + await flushPromises(); + + expect(wrapper.text()).toContain('3'); + expect(wrapper.text()).toContain('42%'); + expect(wrapper.text()).toContain('12,000'); + expect(wrapper.text()).toContain( + "caching never wrote - likely under the model's 4,096-token minimum", + ); + expect(wrapper.text()).toContain('Increase chunk sizes.'); + expect(wrapper.text()).toContain('Rate limits are slowing runs.'); + }); + + it('passes filters and cursor pagination params through the composable', async () => { + const wrapper = mountView(); + await flushPromises(); + + const selects = wrapper.findAll('select'); + await selects[0].setValue('failed'); + await wrapper + .find('[data-testid="ai-diagnostics-ssp-filter"]') + .setValue('ssp-1'); + await selects[1].setValue('25'); + + expect(state.refreshRuns).toHaveBeenLastCalledWith({ + status: 'failed', + sspId: 'ssp-1', + limit: 25, + }); + + await wrapper + .findAll('button') + .find((button) => button.text() === 'Load More') + ?.trigger('click'); + + expect(state.loadMoreRuns).toHaveBeenCalledWith({ + status: 'failed', + sspId: 'ssp-1', + limit: 25, + }); + }); + + it('disables Load More while a pagination request is running', async () => { + state.paginationLoading.value = true; + + const wrapper = mountView(); + await flushPromises(); + + const loadMoreButton = wrapper + .findAll('button') + .find((button) => button.text() === 'Load More'); + + expect(loadMoreButton?.attributes('disabled')).toBeDefined(); + }); + + it('opens the run drawer with cells, failed errors, scope, and events', async () => { + const wrapper = mountView(); + await flushPromises(); + + await wrapper + .findAll('tbody tr') + .find((row) => row.text().includes('Payments SSP')) + ?.trigger('click'); + await flushPromises(); + + expect(state.fetchRunDetail).toHaveBeenCalledWith('run-1'); + expect(wrapper.text()).toContain('Cell 1: model timeout'); + expect(wrapper.text()).toContain('_policy=secret_scanning'); + expect(wrapper.text()).toContain('AC-1'); + expect(wrapper.text()).toContain('run_started'); + expect(wrapper.text()).toContain('Run started.'); + }); +}); + +describe('AI diagnostics chart builders', () => { + it('buckets token and cache-hit series by run start day', () => { + const runs: AiDiagnosticsRun[] = [ + { + id: 'run-1', + status: 'completed', + plannedCalls: 1, + completedCells: 1, + failedCells: 0, + inputTokens: 100, + outputTokens: 20, + cacheReadInputTokens: 40, + cacheCreationInputTokens: 10, + cacheHitRatio: 0.4, + mappingsReturned: 1, + mappingsRejected: 0, + rateLimitedTotal: 0, + startedAt: '2026-06-15T10:00:00Z', + }, + { + id: 'run-2', + status: 'completed', + plannedCalls: 1, + completedCells: 1, + failedCells: 0, + inputTokens: 200, + outputTokens: 30, + cacheReadInputTokens: 80, + cacheCreationInputTokens: 20, + cacheHitRatio: 0.6, + mappingsReturned: 1, + mappingsRejected: 0, + rateLimitedTotal: 0, + startedAt: '2026-06-15T11:00:00Z', + }, + { + id: 'run-3', + status: 'failed', + plannedCalls: 1, + completedCells: 0, + failedCells: 1, + inputTokens: 50, + outputTokens: 0, + cacheReadInputTokens: 10, + cacheCreationInputTokens: 5, + cacheHitRatio: 0.2, + mappingsReturned: 0, + mappingsRejected: 1, + rateLimitedTotal: 1, + startedAt: '2026-06-16T10:00:00Z', + }, + ]; + + const tokenData = buildAiDiagnosticsTokenChartData(runs); + expect(tokenData.labels).toEqual(['2026-06-15', '2026-06-16']); + expect(tokenData.datasets[0].data).toEqual([300, 50]); + expect(tokenData.datasets[1].data).toEqual([120, 10]); + expect(tokenData.datasets[2].data).toEqual([30, 5]); + + const cacheData = buildAiDiagnosticsCacheHitChartData(runs); + expect(cacheData.labels).toEqual(['2026-06-15', '2026-06-16']); + expect(cacheData.datasets[0].data).toEqual([50, 20]); + }); +}); diff --git a/src/views/__tests__/DiagnosticsRouting.spec.ts b/src/views/__tests__/DiagnosticsRouting.spec.ts new file mode 100644 index 00000000..11471506 --- /dev/null +++ b/src/views/__tests__/DiagnosticsRouting.spec.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import router from '@/router'; + +describe('diagnostics routes', () => { + it('configures /admin/notifications as a compatibility redirect', () => { + const route = router + .getRoutes() + .find( + (route) => + route.path === '/admin/notifications' || + route.name === 'admin-notifications', + ); + + expect(route?.redirect).toEqual({ + name: 'admin-diagnostics-notifications', + }); + }); +}); diff --git a/src/views/__tests__/DiagnosticsView.spec.ts b/src/views/__tests__/DiagnosticsView.spec.ts new file mode 100644 index 00000000..be811921 --- /dev/null +++ b/src/views/__tests__/DiagnosticsView.spec.ts @@ -0,0 +1,73 @@ +import { mount } from '@vue/test-utils'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ + aiEnabled: false, + aiFetched: true, + fetchConfig: vi.fn(), + routeName: 'admin-diagnostics-notifications', +})); + +vi.mock('vue-router', () => ({ + RouterLink: { + props: ['to'], + template: '', + }, + RouterView: { template: '
' }, + useRoute: () => ({ + get name() { + return state.routeName; + }, + }), +})); + +vi.mock('@/stores/ai-config', () => ({ + useAiConfigStore: () => ({ + get dashboardSuggestionsEnabled() { + return state.aiEnabled; + }, + get dashboardSuggestionsConfigFetched() { + return state.aiFetched; + }, + fetchDashboardSuggestionsConfig: state.fetchConfig, + }), +})); + +import DiagnosticsView from '../admin/DiagnosticsView.vue'; + +function mountView() { + return mount(DiagnosticsView, { + global: { + stubs: { + PageHeader: { template: '

' }, + PageSubHeader: { template: '

' }, + Message: { template: '
' }, + }, + }, + }); +} + +describe('DiagnosticsView', () => { + beforeEach(() => { + vi.clearAllMocks(); + state.aiEnabled = false; + state.aiFetched = true; + state.fetchConfig.mockResolvedValue(false); + }); + + it('hides the AI Suggestions tab when the feature flag is disabled', () => { + const wrapper = mountView(); + + expect(wrapper.text()).toContain('Diagnostics'); + expect(wrapper.text()).toContain('Notifications'); + expect(wrapper.text()).not.toContain('AI Suggestions'); + }); + + it('shows the AI Suggestions tab when the feature flag is enabled', () => { + state.aiEnabled = true; + + const wrapper = mountView(); + + expect(wrapper.text()).toContain('AI Suggestions'); + }); +}); diff --git a/src/views/__tests__/LeftSideNav.spec.ts b/src/views/__tests__/LeftSideNav.spec.ts index 038cabb1..63ebe845 100644 --- a/src/views/__tests__/LeftSideNav.spec.ts +++ b/src/views/__tests__/LeftSideNav.spec.ts @@ -45,7 +45,7 @@ describe('LeftSideNav', () => { const systemUsersIndex = linkTexts.indexOf('System Users'); const agentsIndex = linkTexts.indexOf('Agents'); - const notificationsIndex = linkTexts.indexOf('Notifications'); + const diagnosticsIndex = linkTexts.indexOf('Diagnostics'); const risksIndex = linkTexts.indexOf('Risks'); const subjectTemplatesIndex = linkTexts.indexOf('Subject Templates'); const riskTemplatesIndex = linkTexts.indexOf('Risk Templates'); @@ -54,7 +54,7 @@ describe('LeftSideNav', () => { for (const index of [ systemUsersIndex, agentsIndex, - notificationsIndex, + diagnosticsIndex, risksIndex, subjectTemplatesIndex, riskTemplatesIndex, @@ -67,8 +67,8 @@ describe('LeftSideNav', () => { expect(risksIndex).toBeGreaterThan(agentsIndex); expect(subjectTemplatesIndex).toBeGreaterThan(risksIndex); expect(riskTemplatesIndex).toBeGreaterThan(subjectTemplatesIndex); - expect(notificationsIndex).toBeGreaterThan(riskTemplatesIndex); - expect(importIndex).toBeGreaterThan(notificationsIndex); + expect(diagnosticsIndex).toBeGreaterThan(riskTemplatesIndex); + expect(importIndex).toBeGreaterThan(diagnosticsIndex); }); it('exposes import only from the admin navigation category', () => { diff --git a/src/views/admin/AiDiagnosticsView.vue b/src/views/admin/AiDiagnosticsView.vue new file mode 100644 index 00000000..41eacdac --- /dev/null +++ b/src/views/admin/AiDiagnosticsView.vue @@ -0,0 +1,968 @@ + + + diff --git a/src/views/admin/DiagnosticsView.vue b/src/views/admin/DiagnosticsView.vue new file mode 100644 index 00000000..1f033053 --- /dev/null +++ b/src/views/admin/DiagnosticsView.vue @@ -0,0 +1,70 @@ + + + diff --git a/src/views/admin/partials/ai-diagnostics.ts b/src/views/admin/partials/ai-diagnostics.ts new file mode 100644 index 00000000..a4022c4c --- /dev/null +++ b/src/views/admin/partials/ai-diagnostics.ts @@ -0,0 +1,279 @@ +import type { ChartData } from 'chart.js'; + +export type AiDiagnosticsStatus = + | 'pending' + | 'running' + | 'completed' + | 'failed'; + +export type AiDiagnosticsCheckStatus = 'pass' | 'warn' | 'fail'; + +export interface AiDiagnosticsConfig { + model?: string; + promptVersion?: string; + maxControlsPerChunk?: number; + maxLabelSetsPerChunk?: number; + maxCallsPerRun?: number; + queueWorkers?: number; +} + +export interface AiDiagnosticsTotals { + runs: number; + runsByStatus: Record; + cellsCompleted: number; + cellsFailed: number; + inputTokens: number; + outputTokens: number; + cacheReadInputTokens: number; + cacheCreationInputTokens: number; + cacheHitRatio: number; + mappingsReturned: number; + mappingsRejected: number; + suggestionsAccepted: number; + suggestionsRejected: number; + suggestionsPending: number; + rateLimitedTotal: number; +} + +export interface AiDiagnosticsQueue { + name: string; + maxWorkers?: number; + available?: number; + running?: number; + retryable?: number; + scheduled?: number; + completed24h?: number; + discarded24h?: number; + oldestAvailableAt?: string | null; +} + +export interface AiDiagnosticsCheck { + id: string; + status: AiDiagnosticsCheckStatus; + message: string; + recommendedActions?: string[]; +} + +export interface AiDiagnosticsSummary { + enabled: boolean; + config: AiDiagnosticsConfig; + totals: AiDiagnosticsTotals; + queue: AiDiagnosticsQueue | null; + checks: AiDiagnosticsCheck[]; +} + +export interface AiDiagnosticsActor { + id?: string; + name?: string; +} + +export interface AiDiagnosticsScope { + controlKeys?: string[]; + labelSetHashes?: string[]; + labelSets?: Array>; +} + +export interface AiDiagnosticsRun { + id: string; + sspId?: string; + sspName?: string; + status: AiDiagnosticsStatus; + model?: string; + promptVersion?: string; + plannedCalls: number; + completedCells: number; + failedCells: number; + inputTokens: number; + outputTokens: number; + cacheReadInputTokens: number; + cacheCreationInputTokens: number; + cacheHitRatio: number; + mappingsReturned: number; + mappingsRejected: number; + rateLimitedTotal: number; + startedAt?: string | null; + completedAt?: string | null; + durationMs?: number | null; + triggeredBy?: AiDiagnosticsActor | null; + scope?: AiDiagnosticsScope; +} + +export interface AiDiagnosticsCell { + cellIndex: number; + status: AiDiagnosticsStatus; + controlKeys?: string[]; + labelSetHashes?: string[]; + inputTokens?: number; + outputTokens?: number; + cacheReadInputTokens?: number; + cacheCreationInputTokens?: number; + rateLimitedCount?: number; + mappingsReturned?: number; + mappingsRejected?: number; + error?: string | null; + completedAt?: string | null; +} + +export interface AiDiagnosticsEvent { + id?: string; + uuid?: string; + action?: string; + actor?: string; + message?: string; + reasoning?: string; + createdAt?: string; + metadata?: Record; +} + +export interface AiDiagnosticsRunDetail extends AiDiagnosticsRun { + cells: AiDiagnosticsCell[]; + events: AiDiagnosticsEvent[]; +} + +export interface AiDiagnosticsRunsFilters { + status?: string; + sspId?: string; + limit?: number; + cursor?: string; +} + +export interface AiDiagnosticsRunsMeta { + nextCursor?: string | null; +} + +export interface AiDiagnosticsRunsResponse { + data: AiDiagnosticsRun[]; + meta?: AiDiagnosticsRunsMeta; +} + +export function buildAiDiagnosticsSummaryEndpoint(): string { + return '/api/admin/ai-diagnostics/summary'; +} + +export function buildAiDiagnosticsRunsEndpoint( + filters: AiDiagnosticsRunsFilters = {}, +): string { + const params = new URLSearchParams(); + + if (filters.status) params.set('status', filters.status); + if (filters.sspId) params.set('sspId', filters.sspId); + if (filters.limit) params.set('limit', String(filters.limit)); + if (filters.cursor) params.set('cursor', filters.cursor); + + const query = params.toString(); + return `/api/admin/ai-diagnostics/runs${query ? `?${query}` : ''}`; +} + +export function buildAiDiagnosticsRunDetailEndpoint(runId: string): string { + return `/api/admin/ai-diagnostics/runs/${encodeURIComponent(runId)}`; +} + +function runDay(run: AiDiagnosticsRun): string | null { + if (!run.startedAt) return null; + + const date = new Date(run.startedAt); + if (Number.isNaN(date.getTime())) return null; + + return date.toISOString().slice(0, 10); +} + +function sortedBucketEntries(runs: AiDiagnosticsRun[]) { + const buckets = new Map< + string, + { + inputTokens: number; + cacheReadInputTokens: number; + cacheCreationInputTokens: number; + cacheHitRatioTotal: number; + runCount: number; + } + >(); + + for (const run of runs) { + const day = runDay(run); + if (!day) continue; + + const bucket = buckets.get(day) ?? { + inputTokens: 0, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + cacheHitRatioTotal: 0, + runCount: 0, + }; + + bucket.inputTokens += run.inputTokens ?? 0; + bucket.cacheReadInputTokens += run.cacheReadInputTokens ?? 0; + bucket.cacheCreationInputTokens += run.cacheCreationInputTokens ?? 0; + bucket.cacheHitRatioTotal += run.cacheHitRatio ?? 0; + bucket.runCount += 1; + buckets.set(day, bucket); + } + + return Array.from(buckets.entries()).sort(([left], [right]) => + left.localeCompare(right), + ); +} + +export function buildAiDiagnosticsTokenChartData( + runs: AiDiagnosticsRun[], +): ChartData<'line'> { + const buckets = sortedBucketEntries(runs); + + return { + labels: buckets.map(([day]) => day), + datasets: [ + { + label: 'Input tokens', + data: buckets.map(([, bucket]) => bucket.inputTokens), + borderColor: '#2563eb', + backgroundColor: 'rgba(37, 99, 235, 0.12)', + pointRadius: 3, + tension: 0.3, + fill: false, + }, + { + label: 'Cache read tokens', + data: buckets.map(([, bucket]) => bucket.cacheReadInputTokens), + borderColor: '#059669', + backgroundColor: 'rgba(5, 150, 105, 0.12)', + pointRadius: 3, + tension: 0.3, + fill: false, + }, + { + label: 'Cache creation tokens', + data: buckets.map(([, bucket]) => bucket.cacheCreationInputTokens), + borderColor: '#d97706', + backgroundColor: 'rgba(217, 119, 6, 0.12)', + pointRadius: 3, + tension: 0.3, + fill: false, + }, + ], + }; +} + +export function buildAiDiagnosticsCacheHitChartData( + runs: AiDiagnosticsRun[], +): ChartData<'line'> { + const buckets = sortedBucketEntries(runs); + + return { + labels: buckets.map(([day]) => day), + datasets: [ + { + label: 'Cache hit %', + data: buckets.map(([, bucket]) => + bucket.runCount + ? Math.round((bucket.cacheHitRatioTotal / bucket.runCount) * 100) + : 0, + ), + borderColor: '#7c3aed', + backgroundColor: 'rgba(124, 58, 237, 0.12)', + pointRadius: 3, + tension: 0.3, + fill: false, + }, + ], + }; +}