From b9e8321a049301416641b437654b6cd8c8e4a2ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:55:29 +0000 Subject: [PATCH 1/3] Initial plan From ce122326129b45f31247d990b2b8fe47ba70f5ce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:00:31 +0000 Subject: [PATCH 2/3] Dashboard: show only highest-level status badge per model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously a model with both an L3 skip and an L5 xfail rendered two inline badges (e.g. youtu showed `Skip` next to `XFail`). The badges were redundant — the highest-level failure conveys the most useful information. Collapse the four per-level badges into a single badge whose source is chosen by precedence: L5 xfail > L4 xfail > L3 xfail > L3 skip > L2 xfail / xfail_graph_only. --- scripts/templates/dashboard.html.j2 | 53 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/scripts/templates/dashboard.html.j2 b/scripts/templates/dashboard.html.j2 index 2106d120..ab58a1f2 100644 --- a/scripts/templates/dashboard.html.j2 +++ b/scripts/templates/dashboard.html.j2 @@ -581,33 +581,32 @@ function renderModelRow(m) { return ``; }).join(''); - // Per-level inline badges next to the model_type. L3 surfaces all three - // pass/xfail/skip outcomes; L2/L4/L5 surface only xfail because their - // dot already conveys pass/skip unambiguously and xfail is the only - // state that is otherwise visually identical to a fail. - let l2Badge = ''; - if (m.l2_status === 'xfail') { - l2Badge = ` XFail`; - } else if (m.l2_status === 'xfail_graph_only') { - l2Badge = ` XFail`; - } - // Positive states (L3 parity pass, L5 e2e verified) are intentionally NOT - // badged here: they would duplicate the Confidence column and the L1-L5 dot - // strip. Only exception states (xfail / skip) get inline badges, since those - // are the signals not otherwise distinguishable at a glance. - let l3Badge = ''; - if (m.l3_status === 'xfail') { - l3Badge = ` XFail`; - } else if (m.l3_status === 'skip') { - l3Badge = ` Skip`; - } - let l4Badge = ''; - if (m.l4_status === 'xfail') { - l4Badge = ` XFail`; - } - let l5Badge = ''; + // Per-level inline badge next to the model_type. To avoid duplicated + // badges (e.g. an L3 Skip alongside an L5 XFail for the same model), we + // emit at most ONE badge per row, picking the highest-level non-pass + // status. Higher-level failures supersede lower-level ones because the + // confidence column already reflects the highest attained level — the + // badge calls out the most consequential outstanding issue. + // + // Precedence (highest → lowest): L5 xfail > L4 xfail > L3 xfail/skip > + // L2 xfail / xfail_graph_only. Positive states (L3 parity pass, L5 e2e + // verified) are intentionally NOT badged: they would duplicate the + // Confidence column and the L1-L5 dot strip. Only exception states + // (xfail / skip) get inline badges, since those are the signals not + // otherwise distinguishable at a glance. + let statusBadge = ''; if (m.l5_status === 'xfail') { - l5Badge = ` XFail`; + statusBadge = ` XFail`; + } else if (m.l4_status === 'xfail') { + statusBadge = ` XFail`; + } else if (m.l3_status === 'xfail') { + statusBadge = ` XFail`; + } else if (m.l3_status === 'skip') { + statusBadge = ` Skip`; + } else if (m.l2_status === 'xfail') { + statusBadge = ` XFail`; + } else if (m.l2_status === 'xfail_graph_only') { + statusBadge = ` XFail`; } // min_token_match_ratio badge @@ -626,7 +625,7 @@ function renderModelRow(m) { // Use single-quoted HTML attributes so JSON.stringify's double quotes don't // break attribute parsing when this string is set via innerHTML. let row = ` - ${esc(m.model_type)}${l2Badge}${l3Badge}${l4Badge}${l5Badge}${ratioBadge} + ${esc(m.model_type)}${statusBadge}${ratioBadge} ${esc(m.category)} ${esc(m.module_class)} ${LEVEL_LABELS[m.confidence_level]} From 59b474025aa69cd5f0f0c18ac01e47f572d94edc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:09:24 +0000 Subject: [PATCH 3/3] Dashboard: include L4/L5 skip in single-badge precedence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous single-badge logic only considered skip at L3 and xfail at L2-L5, which meant L4/L5 skip statuses (the dominant skip population — many models have YAML cases marked skip_reason) showed no badge at all. Replace the if/else cascade with a loop that walks L5 → L2 and emits the first xfail OR skip encountered. Both exception kinds are now reportable at every level where the data model permits them. --- scripts/templates/dashboard.html.j2 | 50 ++++++++++++++++++----------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/scripts/templates/dashboard.html.j2 b/scripts/templates/dashboard.html.j2 index ab58a1f2..3a560fc5 100644 --- a/scripts/templates/dashboard.html.j2 +++ b/scripts/templates/dashboard.html.j2 @@ -584,29 +584,41 @@ function renderModelRow(m) { // Per-level inline badge next to the model_type. To avoid duplicated // badges (e.g. an L3 Skip alongside an L5 XFail for the same model), we // emit at most ONE badge per row, picking the highest-level non-pass - // status. Higher-level failures supersede lower-level ones because the + // status. Higher-level outcomes supersede lower-level ones because the // confidence column already reflects the highest attained level — the // badge calls out the most consequential outstanding issue. // - // Precedence (highest → lowest): L5 xfail > L4 xfail > L3 xfail/skip > - // L2 xfail / xfail_graph_only. Positive states (L3 parity pass, L5 e2e - // verified) are intentionally NOT badged: they would duplicate the - // Confidence column and the L1-L5 dot strip. Only exception states - // (xfail / skip) get inline badges, since those are the signals not - // otherwise distinguishable at a glance. + // Precedence: walk L5 → L2 and emit the first xfail/skip encountered. + // Both xfail and skip are reportable at every level (each level can be + // skipped or marked xfail independently). Within a level, xfail and + // skip are mutually exclusive in the data model, so we just check each. + // L2 also admits "xfail_graph_only" (parse passes, full-graph build + // xfails). Positive states (L3 parity pass, L5 e2e verified) are + // intentionally NOT badged: they would duplicate the Confidence column + // and the L1-L5 dot strip. Only exception states (xfail / skip) get + // inline badges, since those are the signals not otherwise + // distinguishable at a glance. + const xfailBadge = (level, reason) => + ` XFail`; + const skipBadge = (level, reason) => + ` Skip`; let statusBadge = ''; - if (m.l5_status === 'xfail') { - statusBadge = ` XFail`; - } else if (m.l4_status === 'xfail') { - statusBadge = ` XFail`; - } else if (m.l3_status === 'xfail') { - statusBadge = ` XFail`; - } else if (m.l3_status === 'skip') { - statusBadge = ` Skip`; - } else if (m.l2_status === 'xfail') { - statusBadge = ` XFail`; - } else if (m.l2_status === 'xfail_graph_only') { - statusBadge = ` XFail`; + for (const [level, status, reason] of [ + [5, m.l5_status, m.l5_reason], + [4, m.l4_status, m.l4_reason], + [3, m.l3_status, m.l3_reason], + [2, m.l2_status, m.l2_reason], + ]) { + if (status === 'xfail' || status === 'xfail_graph_only') { + statusBadge = (level === 2 && status === 'xfail_graph_only') + ? ` XFail` + : xfailBadge(level, reason); + break; + } + if (status === 'skip') { + statusBadge = skipBadge(level, reason); + break; + } } // min_token_match_ratio badge