diff --git a/src/app/analysis/components/KeyPointCoverageTable.tsx b/src/app/analysis/components/KeyPointCoverageTable.tsx index 12f870b0..1616c4f9 100644 --- a/src/app/analysis/components/KeyPointCoverageTable.tsx +++ b/src/app/analysis/components/KeyPointCoverageTable.tsx @@ -91,14 +91,86 @@ const ModelCard: React.FC = ({ modelId, promptCoverageScores, pr return bundles; }, [relatedIds, promptCoverageScores, promptResponses, historiesForPrompt]); + // If no valid variants, diagnose why and surface a specific message + // instead of the generic "Error loading evaluation data for this model." + const emptyStateDiagnosis = useMemo(() => { + if (tempVariants.length > 0) return null; + + const covErrors = new Set(); + let hasAnyCoverage = false; + let hasAnyResponse = false; + let hasEmptyCoverage = false; + + relatedIds.forEach(mId => { + const cov = promptCoverageScores[mId]; + const resp = promptResponses[mId]; + if (cov) { + hasAnyCoverage = true; + if ('error' in cov && cov.error) { + covErrors.add(cov.error); + } else if (!('error' in cov) && !((cov as any).pointAssessments?.length)) { + hasEmptyCoverage = true; + } + } + if (resp !== undefined) hasAnyResponse = true; + }); + + // Prefer specific coverage errors when we have them. + if (covErrors.size > 0) { + const messages = Array.from(covErrors); + const isGenerationFailure = messages.some(m => /^Generation failed/i.test(m)); + return { + title: isGenerationFailure + ? 'The model failed to generate a response.' + : 'The response could not be evaluated.', + details: messages, + showResponses: hasAnyResponse && !isGenerationFailure, + }; + } + + if (!hasAnyResponse && !hasAnyCoverage) { + return { + title: 'No evaluation data for this model.', + details: ['The model did not produce a response and no coverage record was written for this scenario.'], + showResponses: false, + }; + } + + if (!hasAnyResponse) { + return { + title: 'The model failed to generate a response.', + details: ['No response text was recorded for this model on this scenario.'], + showResponses: false, + }; + } + + if (hasEmptyCoverage) { + return { + title: 'The response could not be evaluated.', + details: ['A coverage record exists but contains no criterion assessments.'], + showResponses: hasAnyResponse, + }; + } + + return { + title: 'Evaluation data is unavailable for this model.', + details: ['The results page could not build a valid view from the underlying data.'], + showResponses: false, + }; + }, [tempVariants.length, relatedIds, promptCoverageScores, promptResponses]); + if (tempVariants.length === 0) { + const diag = emptyStateDiagnosis!; return ( {getModelDisplayLabel(modelId)} - -

Error loading evaluation data for this model.

+ +

{diag.title}

+ {diag.details.map((d, i) => ( +

{d}

+ ))} ); diff --git a/src/app/analysis/components/SharedEvaluationComponents.tsx b/src/app/analysis/components/SharedEvaluationComponents.tsx index 5b7bdd46..1b588646 100644 --- a/src/app/analysis/components/SharedEvaluationComponents.tsx +++ b/src/app/analysis/components/SharedEvaluationComponents.tsx @@ -277,14 +277,20 @@ export const EvaluationView: React.FC<{ } }); + const isEvaluated = (a: PointAssessment) => + a.coverageExtent !== undefined && a.coverageExtent !== null && !isNaN(a.coverageExtent); + const categorize = (list: PointAssessment[]) => { const criticalFailures: PointAssessment[] = []; const majorGaps: PointAssessment[] = []; const passed: PointAssessment[] = []; + const notEvaluated: PointAssessment[] = []; list.forEach(a => { - if (a.isInverted && a.coverageExtent !== undefined && a.coverageExtent < 0.7) { + if (!isEvaluated(a)) { + notEvaluated.push(a); + } else if (a.isInverted && a.coverageExtent! < 0.7) { criticalFailures.push(a); - } else if (!a.isInverted && a.coverageExtent !== undefined && a.coverageExtent < 0.4) { + } else if (!a.isInverted && a.coverageExtent! < 0.4) { majorGaps.push(a); } else { passed.push(a); @@ -293,7 +299,7 @@ export const EvaluationView: React.FC<{ criticalFailures.sort((a,b) => (a.coverageExtent ?? 1) - (b.coverageExtent ?? 1)); majorGaps.sort((a,b) => (a.coverageExtent ?? 1) - (b.coverageExtent ?? 1)); passed.sort((a,b) => (b.coverageExtent ?? 0) - (a.coverageExtent ?? 0)); - return { criticalFailures, majorGaps, passed }; + return { criticalFailures, majorGaps, passed, notEvaluated }; }; return { @@ -323,7 +329,7 @@ export const EvaluationView: React.FC<{ }) ); - const renderCategorizedAssessments = (categorized: { criticalFailures: PointAssessment[], majorGaps: PointAssessment[], passed: PointAssessment[] }) => ( + const renderCategorizedAssessments = (categorized: { criticalFailures: PointAssessment[], majorGaps: PointAssessment[], passed: PointAssessment[], notEvaluated: PointAssessment[] }) => ( <> {categorized.criticalFailures.length > 0 && (
@@ -349,6 +355,14 @@ export const EvaluationView: React.FC<{
{renderAssessmentList(categorized.passed)}
)} + {categorized.notEvaluated.length > 0 && ( +
+

+ Not Evaluated ({categorized.notEvaluated.length}) +

+
{renderAssessmentList(categorized.notEvaluated)}
+
+ )} ); @@ -520,9 +534,9 @@ export const EvaluationView: React.FC<{ )}
- {renderCategorizedAssessments({ + {renderCategorizedAssessments({ criticalFailures: [ - ...requiredPoints.criticalFailures, + ...requiredPoints.criticalFailures, ...alternativePaths.flatMap(p => p.criticalFailures) ], majorGaps: [ @@ -532,6 +546,10 @@ export const EvaluationView: React.FC<{ passed: [ ...requiredPoints.passed, ...alternativePaths.flatMap(p => p.passed) + ], + notEvaluated: [ + ...requiredPoints.notEvaluated, + ...alternativePaths.flatMap(p => p.notEvaluated) ] })} {assessments.length === 0 && ( @@ -707,7 +725,7 @@ export const EvaluationView: React.FC<{
)}
- {(requiredPoints.criticalFailures.length > 0 || requiredPoints.majorGaps.length > 0 || requiredPoints.passed.length > 0) && ( + {(requiredPoints.criticalFailures.length > 0 || requiredPoints.majorGaps.length > 0 || requiredPoints.passed.length > 0 || requiredPoints.notEvaluated.length > 0) && (
{/*

Required Criteria diff --git a/src/cli/evaluators/__tests__/llm-coverage-evaluator.test.ts b/src/cli/evaluators/__tests__/llm-coverage-evaluator.test.ts index 06ebe122..912a522c 100644 --- a/src/cli/evaluators/__tests__/llm-coverage-evaluator.test.ts +++ b/src/cli/evaluators/__tests__/llm-coverage-evaluator.test.ts @@ -398,7 +398,7 @@ describe('LLMCoverageEvaluator', () => { } else { // No primary successes + backup fails → total failure expect(assessment.coverageExtent).toBeUndefined(); - expect(assessment.error).toBe('All judges failed in consensus mode.'); + expect(assessment.error).toMatch(/^All \d+ judge\(s\) failed in consensus mode\. Reasons: /); expect(assessment.individualJudgements).toBeUndefined(); expect(assessment.reflection).toBeUndefined(); } @@ -441,7 +441,8 @@ describe('LLMCoverageEvaluator', () => { // Should have been called times: primary judges + 1 backup expect(requestIndividualJudgeSpy).toHaveBeenCalledTimes(DEFAULT_JUDGES.length + 1); - expect(assessment.error).toBe('All judges failed in consensus mode.'); + expect(assessment.error).toMatch(/^All \d+ judge\(s\) failed in consensus mode\. Reasons: /); + expect(assessment.error).toContain('All judges failed'); // sample from the mocked reason expect(assessment.coverageExtent).toBeUndefined(); expect(assessment.judgeModelId).toBeUndefined(); }); @@ -532,8 +533,8 @@ describe('LLMCoverageEvaluator', () => { const result = await evaluator.evaluate([input]); const assessment = (result.llmCoverageScores?.['prompt-all-fail']?.['model1'] as any)?.pointAssessments[0]; - - expect(assessment.error).toBe('All judges failed in consensus mode.'); + + expect(assessment.error).toMatch(/^All \d+ judge\(s\) failed in consensus mode\. Reasons: /); expect(assessment.coverageExtent).toBeUndefined(); expect(assessment.judgeModelId).toBeUndefined(); }); diff --git a/src/cli/evaluators/llm-coverage-evaluator.ts b/src/cli/evaluators/llm-coverage-evaluator.ts index b3d156c4..a346f3f3 100644 --- a/src/cli/evaluators/llm-coverage-evaluator.ts +++ b/src/cli/evaluators/llm-coverage-evaluator.ts @@ -100,6 +100,46 @@ interface JudgeResult { const CALL_TIMEOUT_MS_POINTWISE = 45000; +/** + * Build a short human-readable summary of why every judge failed, so operators + * can distinguish rate-limit vs auth vs parse errors from the results UI. + * Groups by failure category and lists which judges hit each. + */ +function summarizeJudgeFailures( + failures: Array<{ judgeId: string; model: string; reason: string }> +): string { + if (failures.length === 0) return ''; + + const categorize = (reason: string): string => { + const r = reason.toLowerCase(); + if (r.includes('rate limit') || r.includes('429')) return 'rate-limited'; + if (r.includes('401') || r.includes('403') || r.includes('unauthorized') || r.includes('forbidden')) return 'auth error'; + if (r.includes('timeout') || r.includes('timed out')) return 'timeout'; + if (r.includes('failed to parse xml') || r.includes('invalid classification')) return 'unparseable response'; + if (r.includes('empty response')) return 'empty response'; + if (r.includes('client/network error')) return 'network error'; + if (r.includes('502') || r.includes('503') || r.includes('504')) return 'upstream gateway error'; + return 'other'; + }; + + const buckets = new Map>(); + for (const f of failures) { + const cat = categorize(f.reason); + if (!buckets.has(cat)) buckets.set(cat, []); + buckets.get(cat)!.push({ judgeId: f.judgeId, sample: f.reason }); + } + + const parts: string[] = []; + for (const [cat, entries] of buckets) { + const judgeList = entries.map(e => e.judgeId).join(', '); + // Include one sample reason to help distinguish edge cases inside a category. + const sample = entries[0]?.sample?.slice(0, 120); + parts.push(`${cat} (${judgeList}${sample && cat === 'other' ? `: ${sample}` : ''})`); + } + + return `Reasons: ${parts.join('; ')}.`; +} + export class LLMCoverageEvaluator implements Evaluator { private logger: Logger; private useCache: boolean; @@ -259,6 +299,8 @@ export class LLMCoverageEvaluator implements Evaluator { ): Promise { const judgeLog: string[] = []; const successfulJudgements: (PointwiseCoverageLLMResult & { judgeModelId: string })[] = []; + // Track per-judge failure reasons so we can surface them if all judges fail. + const judgeFailures: Array<{ judgeId: string; model: string; reason: string }> = []; // Determine which judges to use let judgesToUse: Judge[]; @@ -298,9 +340,11 @@ export class LLMCoverageEvaluator implements Evaluator { ); if ('error' in singleEvalResult) { - judgeLog.push(`[${judgeIdentifier}] FAILED: ${singleEvalResult.error}`); + const reason = singleEvalResult.error ?? 'unknown error'; + judgeLog.push(`[${judgeIdentifier}] FAILED: ${reason}`); + judgeFailures.push({ judgeId: judgeIdentifier, model: judge.model, reason }); // Check if it's a rate limit error - if (singleEvalResult.error?.includes('rate limit') || singleEvalResult.error?.includes('429')) { + if (reason.includes('rate limit') || reason.includes('429')) { adaptiveLimiter?.onRateLimit(); } else { adaptiveLimiter?.onError(); @@ -328,11 +372,15 @@ export class LLMCoverageEvaluator implements Evaluator { judges, judgeLog, classificationScale, - providerLimiters + providerLimiters, + judgeFailures, ); if (successfulJudgements.length === 0) { - const errorMsg = "All judges failed in consensus mode."; + const summary = summarizeJudgeFailures(judgeFailures); + const errorMsg = judgeFailures.length > 0 + ? `All ${judgeFailures.length} judge(s) failed in consensus mode. ${summary}` + : 'All judges failed in consensus mode.'; this.logger.warn(`[LLMCoverageEvaluator-Pointwise] --- ${errorMsg}`); judgeLog.push(`FINAL_ERROR: ${errorMsg}`); return { error: errorMsg, judgeLog }; @@ -381,6 +429,7 @@ export class LLMCoverageEvaluator implements Evaluator { judgeLog: string[] = [], classificationScale: ClassificationScaleItem[] = CLASSIFICATION_SCALE, providerLimiters?: Map }>, + judgeFailures?: Array<{ judgeId: string; model: string; reason: string }>, ): Promise { // Only use backup judge if we have fewer successful judgements than expected // and we're not using custom judges (to preserve user configurations) @@ -412,9 +461,11 @@ export class LLMCoverageEvaluator implements Evaluator { ); if ('error' in backupEvalResult) { - judgeLog.push(`[${backupJudgeIdentifier}] BACKUP FAILED: ${backupEvalResult.error}`); + const reason = backupEvalResult.error ?? 'unknown error'; + judgeLog.push(`[${backupJudgeIdentifier}] BACKUP FAILED: ${reason}`); + judgeFailures?.push({ judgeId: backupJudgeIdentifier, model: DEFAULT_BACKUP_JUDGE.model, reason }); // Check if it's a rate limit error - if (backupEvalResult.error?.includes('rate limit') || backupEvalResult.error?.includes('429')) { + if (reason.includes('rate limit') || reason.includes('429')) { backupAdaptiveLimiter?.onRateLimit(); } else { backupAdaptiveLimiter?.onError(); @@ -1042,6 +1093,9 @@ Output: The text mentions empathy, which means the criterion is MET error: judgeResult.error, individualJudgements: judgeResult.individualJudgements, judgeModelId: judgeResult.judgeModelId, + // Surface the per-point judge log so operators can diagnose + // consensus failures (rate limits, parse errors, etc.) from the UI. + judgeLog: judgeResult.error ? judgeResult.judgeLog : undefined, multiplier: point.multiplier, citation: point.citation, isInverted: point.isInverted,