weval-org · nojibe · Jul 21, 2026
diff --git a/src/app/analysis/components/KeyPointCoverageTable.tsx b/src/app/analysis/components/KeyPointCoverageTable.tsx
@@ -91,14 +91,86 @@ const ModelCard: React.FC<ModelCardProps> = ({ modelId, promptCoverageScores, pr
         return bundles;
     }, [relatedIds, promptCoverageScores, promptResponses, historiesForPrompt]);
 
+    // If no valid variants, diagnose why and surface a specific message
+    // instead of the generic "Error loading evaluation data for this model."
+    const emptyStateDiagnosis = useMemo(() => {
+        if (tempVariants.length > 0) return null;
+
+        const covErrors = new Set<string>();
+        let hasAnyCoverage = false;
+        let hasAnyResponse = false;
+        let hasEmptyCoverage = false;
+
+        relatedIds.forEach(mId => {
+            const cov = promptCoverageScores[mId];
+            const resp = promptResponses[mId];
+            if (cov) {
+                hasAnyCoverage = true;
+                if ('error' in cov && cov.error) {
+                    covErrors.add(cov.error);
+                } else if (!('error' in cov) && !((cov as any).pointAssessments?.length)) {
+                    hasEmptyCoverage = true;
+                }
+            }
+            if (resp !== undefined) hasAnyResponse = true;
+        });
+
+        // Prefer specific coverage errors when we have them.
+        if (covErrors.size > 0) {
+            const messages = Array.from(covErrors);
+            const isGenerationFailure = messages.some(m => /^Generation failed/i.test(m));
+            return {
+                title: isGenerationFailure
+                    ? 'The model failed to generate a response.'
+                    : 'The response could not be evaluated.',
+                details: messages,
+                showResponses: hasAnyResponse && !isGenerationFailure,
+            };
+        }
+
+        if (!hasAnyResponse && !hasAnyCoverage) {
+            return {
+                title: 'No evaluation data for this model.',
+                details: ['The model did not produce a response and no coverage record was written for this scenario.'],
+                showResponses: false,
+            };
+        }
+
+        if (!hasAnyResponse) {
+            return {
+                title: 'The model failed to generate a response.',
+                details: ['No response text was recorded for this model on this scenario.'],
+                showResponses: false,
+            };
+        }
+
+        if (hasEmptyCoverage) {
+            return {
+                title: 'The response could not be evaluated.',
+                details: ['A coverage record exists but contains no criterion assessments.'],
+                showResponses: hasAnyResponse,
+            };
+        }
+
+        return {
+            title: 'Evaluation data is unavailable for this model.',
+            details: ['The results page could not build a valid view from the underlying data.'],
+            showResponses: false,
+        };
+    }, [tempVariants.length, relatedIds, promptCoverageScores, promptResponses]);
+
     if (tempVariants.length === 0) {
+        const diag = emptyStateDiagnosis!;
         return (
             <Card className="h-full w-full border-dashed border-destructive/50">
                 <CardHeader>
                     <CardTitle>{getModelDisplayLabel(modelId)}</CardTitle>
                 </CardHeader>
-                <CardContent>
-                    <p className="text-destructive">Error loading evaluation data for this model.</p>
+                <CardContent className="space-y-2">
+                    <p className="text-destructive font-medium">{diag.title}</p>
+                    {diag.details.map((d, i) => (
+                        <p key={i} className="text-sm text-muted-foreground whitespace-pre-wrap break-words">{d}</p>
+                    ))}
                 </CardContent>
             </Card>
         );

diff --git a/src/app/analysis/components/SharedEvaluationComponents.tsx b/src/app/analysis/components/SharedEvaluationComponents.tsx
@@ -277,14 +277,20 @@ export const EvaluationView: React.FC<{
             }
         });
 
+        const isEvaluated = (a: PointAssessment) =>
+            a.coverageExtent !== undefined && a.coverageExtent !== null && !isNaN(a.coverageExtent);
+
         const categorize = (list: PointAssessment[]) => {
             const criticalFailures: PointAssessment[] = [];
             const majorGaps: PointAssessment[] = [];
             const passed: PointAssessment[] = [];
+            const notEvaluated: PointAssessment[] = [];
             list.forEach(a => {
-                if (a.isInverted && a.coverageExtent !== undefined && a.coverageExtent < 0.7) {
+                if (!isEvaluated(a)) {
+                    notEvaluated.push(a);
+                } else if (a.isInverted && a.coverageExtent! < 0.7) {
                     criticalFailures.push(a);
-                } else if (!a.isInverted && a.coverageExtent !== undefined && a.coverageExtent < 0.4) {
+                } else if (!a.isInverted && a.coverageExtent! < 0.4) {
                     majorGaps.push(a);
                 } else {
                     passed.push(a);
@@ -293,7 +299,7 @@ export const EvaluationView: React.FC<{
             criticalFailures.sort((a,b) => (a.coverageExtent ?? 1) - (b.coverageExtent ?? 1));
             majorGaps.sort((a,b) => (a.coverageExtent ?? 1) - (b.coverageExtent ?? 1));
             passed.sort((a,b) => (b.coverageExtent ?? 0) - (a.coverageExtent ?? 0));
-            return { criticalFailures, majorGaps, passed };
+            return { criticalFailures, majorGaps, passed, notEvaluated };
         };
 
         return {
@@ -323,7 +329,7 @@ export const EvaluationView: React.FC<{
         })
     );
 
-    const renderCategorizedAssessments = (categorized: { criticalFailures: PointAssessment[], majorGaps: PointAssessment[], passed: PointAssessment[] }) => (
+    const renderCategorizedAssessments = (categorized: { criticalFailures: PointAssessment[], majorGaps: PointAssessment[], passed: PointAssessment[], notEvaluated: PointAssessment[] }) => (
         <>
             {categorized.criticalFailures.length > 0 && (
                 <div>
@@ -349,6 +355,14 @@ export const EvaluationView: React.FC<{
                     <div className="space-y-3 my-4">{renderAssessmentList(categorized.passed)}</div>
                 </div>
             )}
+            {categorized.notEvaluated.length > 0 && (
+                <div>
+                    <h4 className="font-bold text-base text-muted-foreground flex items-center" title="These criteria could not be scored — the judge(s) failed or produced no valid classification. They are neither passed nor failed.">
+                        <Icon name="help-circle" className="h-5 w-5 mr-2" /> Not Evaluated ({categorized.notEvaluated.length})
+                    </h4>
+                    <div className="space-y-3 my-4">{renderAssessmentList(categorized.notEvaluated)}</div>
+                </div>
+            )}
         </>
     );
 
@@ -520,9 +534,9 @@ export const EvaluationView: React.FC<{
                         )}
                     </div>
                     <div className="space-y-4">
-                        {renderCategorizedAssessments({ 
+                        {renderCategorizedAssessments({
                             criticalFailures: [
-                                ...requiredPoints.criticalFailures, 
+                                ...requiredPoints.criticalFailures,
                                 ...alternativePaths.flatMap(p => p.criticalFailures)
                             ],
                             majorGaps: [
@@ -532,6 +546,10 @@ export const EvaluationView: React.FC<{
                             passed: [
                                 ...requiredPoints.passed,
                                 ...alternativePaths.flatMap(p => p.passed)
+                            ],
+                            notEvaluated: [
+                                ...requiredPoints.notEvaluated,
+                                ...alternativePaths.flatMap(p => p.notEvaluated)
                             ]
                         })}
                         {assessments.length === 0 && (
@@ -707,7 +725,7 @@ export const EvaluationView: React.FC<{
                     </div>
                 )}
                 <div className="custom-scrollbar min-h-0 flex-grow space-y-3 overflow-y-auto pr-2 pt-2">
-                    {(requiredPoints.criticalFailures.length > 0 || requiredPoints.majorGaps.length > 0 || requiredPoints.passed.length > 0) && (
+                    {(requiredPoints.criticalFailures.length > 0 || requiredPoints.majorGaps.length > 0 || requiredPoints.passed.length > 0 || requiredPoints.notEvaluated.length > 0) && (
                         <div className="mb-4">
                             {/* <h4 className="font-bold text-base text-primary flex items-center mb-2">
                                 Required Criteria

diff --git a/src/cli/evaluators/__tests__/llm-coverage-evaluator.test.ts b/src/cli/evaluators/__tests__/llm-coverage-evaluator.test.ts
@@ -398,7 +398,7 @@ describe('LLMCoverageEvaluator', () => {
             } else {
                 // No primary successes + backup fails → total failure
                 expect(assessment.coverageExtent).toBeUndefined();
-                expect(assessment.error).toBe('All judges failed in consensus mode.');
+                expect(assessment.error).toMatch(/^All \d+ judge\(s\) failed in consensus mode\. Reasons: /);
                 expect(assessment.individualJudgements).toBeUndefined();
                 expect(assessment.reflection).toBeUndefined();
             }
@@ -441,7 +441,8 @@ describe('LLMCoverageEvaluator', () => {
 
             // Should have been called times: primary judges + 1 backup
             expect(requestIndividualJudgeSpy).toHaveBeenCalledTimes(DEFAULT_JUDGES.length + 1);
-            expect(assessment.error).toBe('All judges failed in consensus mode.');
+            expect(assessment.error).toMatch(/^All \d+ judge\(s\) failed in consensus mode\. Reasons: /);
+            expect(assessment.error).toContain('All judges failed'); // sample from the mocked reason
             expect(assessment.coverageExtent).toBeUndefined();
             expect(assessment.judgeModelId).toBeUndefined();
         });
@@ -532,8 +533,8 @@ describe('LLMCoverageEvaluator', () => {
 
             const result = await evaluator.evaluate([input]);
             const assessment = (result.llmCoverageScores?.['prompt-all-fail']?.['model1'] as any)?.pointAssessments[0];
-            
-            expect(assessment.error).toBe('All judges failed in consensus mode.');
+
+            expect(assessment.error).toMatch(/^All \d+ judge\(s\) failed in consensus mode\. Reasons: /);
             expect(assessment.coverageExtent).toBeUndefined();
             expect(assessment.judgeModelId).toBeUndefined();
         });

diff --git a/src/cli/evaluators/llm-coverage-evaluator.ts b/src/cli/evaluators/llm-coverage-evaluator.ts
@@ -100,6 +100,46 @@ interface JudgeResult {
 
 const CALL_TIMEOUT_MS_POINTWISE = 45000;
 
+/**
+ * Build a short human-readable summary of why every judge failed, so operators
+ * can distinguish rate-limit vs auth vs parse errors from the results UI.
+ * Groups by failure category and lists which judges hit each.
+ */
+function summarizeJudgeFailures(
+    failures: Array<{ judgeId: string; model: string; reason: string }>
+): string {
+    if (failures.length === 0) return '';
+
+    const categorize = (reason: string): string => {
+        const r = reason.toLowerCase();
+        if (r.includes('rate limit') || r.includes('429')) return 'rate-limited';
+        if (r.includes('401') || r.includes('403') || r.includes('unauthorized') || r.includes('forbidden')) return 'auth error';
+        if (r.includes('timeout') || r.includes('timed out')) return 'timeout';
+        if (r.includes('failed to parse xml') || r.includes('invalid classification')) return 'unparseable response';
+        if (r.includes('empty response')) return 'empty response';
+        if (r.includes('client/network error')) return 'network error';
+        if (r.includes('502') || r.includes('503') || r.includes('504')) return 'upstream gateway error';
+        return 'other';
+    };
+
+    const buckets = new Map<string, Array<{ judgeId: string; sample: string }>>();
+    for (const f of failures) {
+        const cat = categorize(f.reason);
+        if (!buckets.has(cat)) buckets.set(cat, []);
+        buckets.get(cat)!.push({ judgeId: f.judgeId, sample: f.reason });
+    }
+
+    const parts: string[] = [];
+    for (const [cat, entries] of buckets) {
+        const judgeList = entries.map(e => e.judgeId).join(', ');
+        // Include one sample reason to help distinguish edge cases inside a category.
+        const sample = entries[0]?.sample?.slice(0, 120);
+        parts.push(`${cat} (${judgeList}${sample && cat === 'other' ? `: ${sample}` : ''})`);
+    }
+
+    return `Reasons: ${parts.join('; ')}.`;
+}
+
 export class LLMCoverageEvaluator implements Evaluator {
     private logger: Logger;
     private useCache: boolean;
@@ -259,6 +299,8 @@ export class LLMCoverageEvaluator implements Evaluator {
     ): Promise<JudgeResult> {
         const judgeLog: string[] = [];
         const successfulJudgements: (PointwiseCoverageLLMResult & { judgeModelId: string })[] = [];
+        // Track per-judge failure reasons so we can surface them if all judges fail.
+        const judgeFailures: Array<{ judgeId: string; model: string; reason: string }> = [];
 
         // Determine which judges to use
         let judgesToUse: Judge[];
@@ -298,9 +340,11 @@ export class LLMCoverageEvaluator implements Evaluator {
                 );
 
                 if ('error' in singleEvalResult) {
-                    judgeLog.push(`[${judgeIdentifier}] FAILED: ${singleEvalResult.error}`);
+                    const reason = singleEvalResult.error ?? 'unknown error';
+                    judgeLog.push(`[${judgeIdentifier}] FAILED: ${reason}`);
+                    judgeFailures.push({ judgeId: judgeIdentifier, model: judge.model, reason });
                     // Check if it's a rate limit error
-                    if (singleEvalResult.error?.includes('rate limit') || singleEvalResult.error?.includes('429')) {
+                    if (reason.includes('rate limit') || reason.includes('429')) {
                         adaptiveLimiter?.onRateLimit();
                     } else {
                         adaptiveLimiter?.onError();
@@ -328,11 +372,15 @@ export class LLMCoverageEvaluator implements Evaluator {
             judges,
             judgeLog,
             classificationScale,
-            providerLimiters
+            providerLimiters,
+            judgeFailures,
         );
 
         if (successfulJudgements.length === 0) {
-            const errorMsg = "All judges failed in consensus mode.";
+            const summary = summarizeJudgeFailures(judgeFailures);
+            const errorMsg = judgeFailures.length > 0
+                ? `All ${judgeFailures.length} judge(s) failed in consensus mode. ${summary}`
+                : 'All judges failed in consensus mode.';
             this.logger.warn(`[LLMCoverageEvaluator-Pointwise] --- ${errorMsg}`);
             judgeLog.push(`FINAL_ERROR: ${errorMsg}`);
             return { error: errorMsg, judgeLog };
@@ -381,6 +429,7 @@ export class LLMCoverageEvaluator implements Evaluator {
         judgeLog: string[] = [],
         classificationScale: ClassificationScaleItem[] = CLASSIFICATION_SCALE,
         providerLimiters?: Map<string, { adaptive: AdaptiveRateLimiter; limit: ReturnType<typeof pLimit> }>,
+        judgeFailures?: Array<{ judgeId: string; model: string; reason: string }>,
     ): Promise<boolean> {
         // Only use backup judge if we have fewer successful judgements than expected 
         // and we're not using custom judges (to preserve user configurations)
@@ -412,9 +461,11 @@ export class LLMCoverageEvaluator implements Evaluator {
         );
 
         if ('error' in backupEvalResult) {
-            judgeLog.push(`[${backupJudgeIdentifier}] BACKUP FAILED: ${backupEvalResult.error}`);
+            const reason = backupEvalResult.error ?? 'unknown error';
+            judgeLog.push(`[${backupJudgeIdentifier}] BACKUP FAILED: ${reason}`);
+            judgeFailures?.push({ judgeId: backupJudgeIdentifier, model: DEFAULT_BACKUP_JUDGE.model, reason });
             // Check if it's a rate limit error
-            if (backupEvalResult.error?.includes('rate limit') || backupEvalResult.error?.includes('429')) {
+            if (reason.includes('rate limit') || reason.includes('429')) {
                 backupAdaptiveLimiter?.onRateLimit();
             } else {
                 backupAdaptiveLimiter?.onError();
@@ -1042,6 +1093,9 @@ Output: <reflection>The text mentions empathy, which means the criterion is MET
                                     error: judgeResult.error,
                                     individualJudgements: judgeResult.individualJudgements,
                                     judgeModelId: judgeResult.judgeModelId,
+                                    // Surface the per-point judge log so operators can diagnose
+                                    // consensus failures (rate limits, parse errors, etc.) from the UI.
+                                    judgeLog: judgeResult.error ? judgeResult.judgeLog : undefined,
                                     multiplier: point.multiplier,
                                     citation: point.citation,
                                     isInverted: point.isInverted,