Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions src/composables/__tests__/useDashboardSuggestions.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,43 @@ describe('useSuggestionRunPoller', () => {
scope.stop();
});

it('surfaces failure details when a completed run reports failed cells', async () => {
mocks.execute.mockResolvedValueOnce({
data: {
value: {
data: {
id: 'run-1',
status: 'completed',
plannedCalls: 2,
completedCells: 1,
failedCells: 1,
stats: {
failedCells: [{ cellIndex: 2, error: 'model timed out' }],
},
},
},
},
});

const scope = effectScope();
const poller = scope.run(() => useSuggestionRunPoller(ref('ssp-1')));

await poller?.pollLatest();

expect(mocks.toastAdd).toHaveBeenCalledWith(
expect.objectContaining({
severity: 'warn',
summary: '1 suggestion failed',
detail: 'Cell 2: model timed out',
}),
);
expect(mocks.toastAdd).not.toHaveBeenCalledWith(
expect.objectContaining({ summary: 'Suggestions ready' }),
);

scope.stop();
});

it('keeps polling after a transient poll error', async () => {
mocks.execute
.mockRejectedValueOnce(new Error('temporary outage'))
Expand Down
55 changes: 45 additions & 10 deletions src/composables/useDashboardSuggestions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
type GenerateDashboardSuggestionsPayload,
type SuggestionRun,
isRunActive,
runCellFailures,
} from '@/views/dashboard/partials/dashboard-suggestions';

export function useDashboardSuggestions(
Expand Down Expand Up @@ -72,7 +73,11 @@ export function useDashboardSuggestions(

return fetchPendingSuggestions(
buildDashboardSuggestionsEndpoint(sspId.value, 'pending'),
{ camelcaseStopPaths: ['data.labelSet'] },
// Preserve raw label keys (e.g. `_policy`) on both the originating label
// set and the proposed filter; otherwise camelcase conversion strips `_`.
{
camelcaseStopPaths: ['data.labelSet', 'data.proposedFilterLabelSet'],
},
);
}

Expand Down Expand Up @@ -132,7 +137,9 @@ export function useDashboardSuggestions(
for (const status of statuses) {
const response = await fetchHistoryRequest(
buildDashboardSuggestionsEndpoint(sspId.value, status),
{ camelcaseStopPaths: ['data.labelSet'] },
{
camelcaseStopPaths: ['data.labelSet', 'data.proposedFilterLabelSet'],
},
);
collected.push(...(response?.data.value?.data ?? []));
}
Expand Down Expand Up @@ -192,6 +199,28 @@ export function useDashboardSuggestions(
};
}

// Builds a human-readable detail string for a run that reported failures,
// preferring per-cell error messages (from `stats.failedCells`) over the
// generic top-level error.
function formatRunFailureDetail(run: SuggestionRun): string {
const failures = runCellFailures(run);
if (failures.length === 0) {
return run.error ?? 'Dashboard suggestion generation failed.';
}

const shown = failures.slice(0, 3).map((failure) => {
const label =
failure.cellIndex === undefined ? 'Cell' : `Cell ${failure.cellIndex}`;
return `${label}: ${failure.error ?? 'Failed'}`;
});

const remaining = failures.length - shown.length;
if (remaining > 0) {
shown.push(`and ${remaining} more`);
}
return shown.join('; ');
}

interface SuggestionRunPollerOptions {
stopOnPollError?: boolean;
}
Expand Down Expand Up @@ -257,20 +286,26 @@ export function useSuggestionRunPoller(
}
terminalToastShownFor.value = runKey;

if (latestRun.status === 'completed') {
const hasFailures =
latestRun.status === 'failed' || (latestRun.failedCells ?? 0) > 0;

if (hasFailures) {
toast.add({
severity: latestRun.status === 'failed' ? 'error' : 'warn',
summary:
latestRun.status === 'failed'
? 'Generation failed'
: `${latestRun.failedCells} suggestion${latestRun.failedCells === 1 ? '' : 's'} failed`,
detail: formatRunFailureDetail(latestRun),
life: 8000,
});
} else if (latestRun.status === 'completed') {
toast.add({
severity: 'success',
summary: 'Suggestions ready',
detail: `${latestRun.completedCells} cells completed`,
life: 3000,
});
} else if (latestRun.status === 'failed') {
toast.add({
severity: 'error',
summary: 'Generation failed',
detail: latestRun.error ?? 'Dashboard suggestion generation failed.',
life: 5000,
});
}
} catch {
if (options.stopOnPollError) {
Expand Down
119 changes: 81 additions & 38 deletions src/views/dashboard/SuggestionsView.vue
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,24 @@
</div>

<Message
v-if="run?.status === 'failed'"
severity="error"
v-if="run && (run.status === 'failed' || (run.failedCells ?? 0) > 0)"
:severity="run.status === 'failed' ? 'error' : 'warn'"
variant="outlined"
>
<p>{{ run.error ?? 'Suggestion generation failed.' }}</p>
<ul v-if="run.failures?.length" class="mt-2 list-disc pl-5">
<li v-for="failure in run.failures" :key="failureKey(failure)">
{{ failure.controlKey ?? 'Unknown control' }} /
{{ failure.labelSetHash ?? 'Unknown label set' }}:
{{ failure.message ?? 'Failed' }}
<p>
{{
run.status === 'failed'
? (run.error ?? 'Suggestion generation failed.')
: `${run.failedCells} of ${run.plannedCalls} cells failed to generate.`
}}
</p>
<ul v-if="runCellFailures(run).length" class="mt-2 list-disc pl-5">
<li
v-for="(failure, index) in runCellFailures(run)"
:key="failure.cellIndex ?? index"
>
Cell {{ failure.cellIndex ?? index }}:
{{ failure.error ?? 'Failed' }}
</li>
</ul>
</Message>
Expand Down Expand Up @@ -199,20 +207,8 @@
class="mt-3 rounded-md bg-zinc-50 p-3 text-sm dark:bg-slate-800"
>
<p>
<span class="font-semibold">Control fit:</span>
{{
suggestion.controlFitReasoning ??
suggestion.reasoning ??
'No control-fit reasoning provided.'
}}
</p>
<p class="mt-2">
<span class="font-semibold">System relevance:</span>
{{
suggestion.systemRelevanceReasoning ??
suggestion.reasoning ??
'No system-relevance reasoning provided.'
}}
<span class="font-semibold">Reasoning:</span>
{{ suggestion.reasoning ?? 'No reasoning provided.' }}
</p>
</div>
</div>
Expand Down Expand Up @@ -357,10 +353,10 @@ import Textarea from '@/volt/Textarea.vue';
import SuggestionScopeDialog from './partials/SuggestionScopeDialog.vue';
import {
formatLabelSet,
runCellFailures,
type DashboardSuggestion,
type DashboardSuggestionEvent,
type GenerateDashboardSuggestionsPayload,
type SuggestionRunFailure,
} from './partials/dashboard-suggestions';

const route = useRoute();
Expand Down Expand Up @@ -487,23 +483,74 @@ const pendingGroups = computed(() => {
>();

for (const suggestion of pendingSuggestions.value ?? []) {
const matched = labelSetByHash.value.get(suggestion.labelSetHash);
const group = groups.get(suggestion.labelSetHash) ?? {
hash: suggestion.labelSetHash,
labels: formatLabelSet(
matched?.labels ?? suggestion.labelSet ?? suggestion.labels ?? {},
),
evidenceCount: matched?.evidenceCount ?? 0,
sampleTitles: matched?.sampleTitles ?? [],
suggestions: [],
};
const proposedFilter = suggestion.proposedFilterLabelSet;
const hasProposedFilter =
!!proposedFilter && Object.keys(proposedFilter).length > 0;

// The proposed dashboard is defined by `proposedFilterLabelSet` (a subset of
// the originating evidence's labels), so group/label/link by that filter.
// Fall back to the full label set only when no proposed filter is present.
const key = hasProposedFilter
? `filter:${proposedFilterKey(proposedFilter)}`
: suggestion.labelSetHash;

let group = groups.get(key);
if (!group) {
const matched = labelSetByHash.value.get(suggestion.labelSetHash);
const evidence = hasProposedFilter
? evidenceMatchingFilter(proposedFilter)
: {
count: matched?.evidenceCount ?? 0,
sampleTitles: matched?.sampleTitles ?? [],
};

group = {
hash: key,
labels: formatLabelSet(
hasProposedFilter
? proposedFilter
: (matched?.labels ??
suggestion.labelSet ??
suggestion.labels ??
{}),
),
evidenceCount: evidence.count,
sampleTitles: evidence.sampleTitles,
suggestions: [],
};
groups.set(key, group);
}
group.suggestions.push(suggestion);
groups.set(suggestion.labelSetHash, group);
}

return Array.from(groups.values());
});

// Stable identity for a proposed filter, independent of key insertion order.
function proposedFilterKey(filter: Record<string, string>): string {
return Object.entries(filter)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, value]) => `${key}=${value}`)
.join('&');
}

// Evidence matching a proposed filter is every full label set that is a superset
// of the filter, since the dashboard filters on that subset of labels.
function evidenceMatchingFilter(filter: Record<string, string>) {
const matching = (labelSets.value ?? []).filter((labelSet) =>
Object.entries(filter).every(
([key, value]) => labelSet.labels?.[key] === value,
),
);
return {
count: matching.reduce(
(total, labelSet) => total + (labelSet.evidenceCount ?? 0),
0,
),
sampleTitles: matching.flatMap((labelSet) => labelSet.sampleTitles ?? []),
};
}

onMounted(async () => {
await aiConfig.fetchDashboardSuggestionsConfig();
if (!aiConfig.dashboardSuggestionsEnabled) {
Expand Down Expand Up @@ -769,10 +816,6 @@ function confidenceLabel(confidence: number | undefined) {
return `${Math.round(confidence * 100)}% confidence`;
}

function failureKey(failure: SuggestionRunFailure) {
return `${failure.controlKey ?? 'control'}-${failure.labelSetHash ?? 'label'}-${failure.message ?? 'failed'}`;
}

function toggleReasoning(id: string) {
const next = new Set(expandedReasoning.value);
if (next.has(id)) {
Expand Down
65 changes: 59 additions & 6 deletions src/views/dashboard/__tests__/SuggestionsView.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,7 @@ describe('SuggestionsView', () => {
controlTitle: 'Access control policy',
labelSetHash: 'hash-1',
confidence: 0.91,
controlFitReasoning: 'Fits AC-1',
systemRelevanceReasoning: 'Relevant to payments',
reasoning: 'Fits AC-1 and is relevant to payments',
action: 'create',
proposedFilterName: 'Production access',
},
Expand All @@ -146,8 +145,7 @@ describe('SuggestionsView', () => {
controlTitle: 'Account management',
labelSetHash: 'hash-1',
confidence: 0.8,
controlFitReasoning: 'Fits AC-2',
systemRelevanceReasoning: 'Relevant to accounts',
reasoning: 'Fits AC-2 and is relevant to accounts',
action: 'create',
proposedFilterName: 'Production access',
},
Expand Down Expand Up @@ -243,8 +241,63 @@ describe('SuggestionsView', () => {
await reasoningButton?.trigger('click');
await nextTick();

expect(wrapper.text()).toContain('Fits AC-1');
expect(wrapper.text()).toContain('Relevant to payments');
expect(wrapper.text()).toContain('Fits AC-1 and is relevant to payments');
});

it('labels, links and counts groups by the proposed filter label set', async () => {
const policy =
'compliance_framework.secret_scanning_push_protection_enabled';
state.pendingSuggestions.value = [
{
id: 'sug-1',
status: 'pending',
controlId: 'GD.Conf.C05',
controlTitle: 'Detect and Block Secret Leakage',
labelSetHash: 'full-hash-1',
labelSet: { _policy: policy, repository: 'todo-app', team: 'ccf' },
proposedFilterLabelSet: { _policy: policy },
confidence: 0.95,
action: 'create',
proposedFilterName: 'Secret scanning push protection enabled',
},
];
state.labelSets.value = [
{
hash: 'full-hash-1',
labels: { _policy: policy, repository: 'todo-app' },
evidenceCount: 3,
sampleTitles: ['todo-app evidence'],
},
{
hash: 'full-hash-2',
labels: { _policy: policy, repository: 'api' },
evidenceCount: 4,
sampleTitles: ['api evidence'],
},
{
hash: 'full-hash-3',
labels: { _policy: 'other.policy', repository: 'api' },
evidenceCount: 9,
sampleTitles: ['unrelated evidence'],
},
];

const wrapper = mountView();

expect(wrapper.findAll('[data-testid^="suggestion-group-"]')).toHaveLength(
1,
);
// Chips and link use the proposed filter, not the full originating label set.
expect(wrapper.text()).toContain(`_policy=${policy}`);
expect(wrapper.text()).not.toContain('repository=todo-app');

const evidenceLink = wrapper.find('a[data-to]');
// Sums evidence across every label set that is a superset of the filter.
expect(evidenceLink.text()).toContain('7 matched evidence');
expect(JSON.parse(evidenceLink.attributes('data-to') ?? '{}')).toEqual({
name: 'evidence:index',
query: { filter: `_policy=${policy}` },
});
});

it('wires group accept and reject reason payloads', async () => {
Expand Down
Loading
Loading