diff --git a/core/src/browser.ts b/core/src/browser.ts index 413ecf07..61fa9d15 100644 --- a/core/src/browser.ts +++ b/core/src/browser.ts @@ -70,3 +70,12 @@ export { export type { LlmConfig, ProviderName } from "./config/types.js"; export { getAdapter } from "./telemetry/adapter.js"; + +export { renderReport } from "./report/render.js"; +export type { + ReportViewModel, + EvaluatorViewModel, + ResultViewModel, + TurnViewModel, + DetailCard, +} from "./report/types.js"; diff --git a/core/src/execute/aggregate.ts b/core/src/execute/aggregate.ts index bafecb46..2aee5171 100644 --- a/core/src/execute/aggregate.ts +++ b/core/src/execute/aggregate.ts @@ -103,6 +103,8 @@ export interface ReportMeta { generatedAt: string; targetName: string; targetKind: "agent" | "mcp"; + /** See {@link UnifiedRunReport.suiteId}. */ + suiteId: string; effort: Effort; attackModel: string; judgeModel: string; @@ -130,6 +132,7 @@ export function buildUnifiedReport( generatedAt: meta.generatedAt, targetName: meta.targetName, targetKind: meta.targetKind, + suiteId: meta.suiteId, effort: meta.effort, attackModel: meta.attackModel, judgeModel: meta.judgeModel, diff --git a/core/src/execute/runAll.ts b/core/src/execute/runAll.ts index a262fb20..86f4fc32 100644 --- a/core/src/execute/runAll.ts +++ b/core/src/execute/runAll.ts @@ -51,6 +51,8 @@ export async function runAll( config: RunConfig, options?: RunAllOptions ): Promise { + const runStartedAt = Date.now(); + // Fan every progress event to the legacy onProgress callback (backward-compat) // and to any registered lifecycle listeners. const listeners = options?.listeners ?? []; @@ -137,12 +139,13 @@ export async function runAll( evaluatorResults.push(...loop.evaluatorResults); const stopReason = loop.stopReason; - // Build report (partial or complete) with stop reason and token usage. + // Build report (partial or complete) with stop reason, token usage, and duration. const report = buildReport(config, evaluatorResults); const usage = tokenTracker.totals; if (usage.totalTokens > 0) { report.summary.tokenUsage = usage; } + report.summary.durationMs = Date.now() - runStartedAt; if (stopReason) { (report as UnifiedRunReport & { stopReason?: string }).stopReason = stopReason; } @@ -307,6 +310,16 @@ async function curateTracesIfConfigured( } } +/** + * Label for the "Evaluation Suite" report field. Only a `mode: "suite"` selection + * runs a whole named suite verbatim — explicit evaluator lists and preloaded specs + * are, by definition, a hand-picked subset, so they're labeled "Custom Suite" + * rather than borrowing a suite name that would overstate what actually ran. + */ +function suiteLabel(selection: RunConfig["selection"]): string { + return selection.mode === "suite" ? selection.suite : "Custom Suite"; +} + /** Assemble a {@link UnifiedRunReport} from Node-side run config and evaluator results. */ function buildReport(config: RunConfig, evaluators: EvaluatorResult[]): UnifiedRunReport { const { attackModel, judgeModel } = modelLabel(config.attackerLlm, config.judgeLlm); @@ -316,6 +329,7 @@ function buildReport(config: RunConfig, evaluators: EvaluatorResult[]): UnifiedR generatedAt: new Date().toISOString(), targetName: config.target.name, targetKind: config.target.kind, + suiteId: suiteLabel(config.selection), effort: config.effort, attackModel, judgeModel, diff --git a/core/src/execute/runAllBrowser.ts b/core/src/execute/runAllBrowser.ts index c06c4613..763d8376 100644 --- a/core/src/execute/runAllBrowser.ts +++ b/core/src/execute/runAllBrowser.ts @@ -69,6 +69,7 @@ export async function runAllBrowser( agentTarget: AgentTarget, options?: BrowserRunAllOptions ): Promise { + const runStartedAt = Date.now(); const notify = options?.onProgress ?? (() => {}); const attackModel = createModel(config.attackerLlm); const judgeModel = createModel(config.judgeLlm ?? config.attackerLlm); @@ -237,6 +238,7 @@ export async function runAllBrowser( if (usage.totalTokens > 0) { report.summary.tokenUsage = usage; } + report.summary.durationMs = Date.now() - runStartedAt; return report; } @@ -254,6 +256,10 @@ function buildBrowserReport( generatedAt: new Date().toISOString(), targetName: config.targetName ?? "target", targetKind: "agent", + // The browser path takes preloaded evaluator specs with no suite concept — + // the extension builds its own report/suite label independently of this + // return value (see popup.js's buildFinalReport). + suiteId: "Custom Suite", effort: config.effort, attackModel, judgeModel, diff --git a/core/src/execute/types.ts b/core/src/execute/types.ts index 809910a2..1a84ea62 100644 --- a/core/src/execute/types.ts +++ b/core/src/execute/types.ts @@ -256,6 +256,12 @@ export interface UnifiedRunReport { generatedAt: string; targetName: string; targetKind: "agent" | "mcp"; + /** + * The evaluation suite that was run, or "Custom Suite" when the selection + * doesn't correspond to one whole named suite (an explicit evaluator list, + * preloaded specs, or a hand-picked subset of a suite's evaluators). + */ + suiteId: string; effort: Effort; attackModel: string; judgeModel: string; @@ -267,6 +273,8 @@ export interface UnifiedRunReport { safetyScore: number; attackSuccessRate: number; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Wall-clock time for the whole run, from the first evaluator to the last. */ + durationMs?: number; }; evaluators: EvaluatorResult[]; /** Set when the run was stopped early due to a non-retryable LLM error. */ diff --git a/core/src/report/buildReport.ts b/core/src/report/buildReport.ts index ba46ce0a..bd5f1e16 100644 --- a/core/src/report/buildReport.ts +++ b/core/src/report/buildReport.ts @@ -69,7 +69,7 @@ function toReportViewModel(report: UnifiedRunReport): ReportViewModel { generatedAt: report.generatedAt, generatorModel: report.attackModel, judgeModel: report.judgeModel, - target: { name: report.targetName }, + target: { name: report.targetName, suiteId: report.suiteId }, summary: report.summary, evaluators: report.evaluators.map(toEvaluatorViewModel), stopReason: report.stopReason, diff --git a/core/src/report/render.ts b/core/src/report/render.ts index a2a97e28..6c5a5be2 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -1,9 +1,8 @@ /** * Unified HTML renderer — takes a ReportViewModel and produces the complete HTML string. - * Replaces both the inline HTML in generateReport.ts (agent) and renderHtml.ts (MCP). + * Shared by the CLI, MCP server, SDK, and browser extension runners. */ import type { ReportViewModel, ResultViewModel, TurnViewModel, DetailCard } from "./types.js"; -import { formatStandardsLabel } from "../evaluators/standards.js"; /** Format a token count for display (e.g. 51300 → "51.3K"). */ function formatTokenCount(n: number): string { @@ -12,6 +11,17 @@ function formatTokenCount(n: number): string { return String(n); } +/** Format a run duration for display (e.g. 754000 → "12m 34s"). */ +function formatDuration(ms: number): string { + const totalSeconds = Math.round(ms / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + if (hours > 0) return `${hours}h ${minutes}m`; + if (minutes > 0) return `${minutes}m ${seconds}s`; + return `${seconds}s`; +} + /** Escape HTML special characters to prevent XSS in report output. */ function esc(s: string): string { return s @@ -44,11 +54,8 @@ const SEV_HEX: Record = { // ── Mode-specific labels ───────────────────────────────────────── interface ModeLabels { - brandSub: string; title: string; targetLabel: string; - passedLabel: string; - defendedLabel: string; typeLabel: string; detailSectionTitle: string; footerPrefix: string; @@ -58,28 +65,74 @@ interface ModeLabels { function modeLabels(mode: "agent" | "mcp"): ModeLabels { if (mode === "agent") { return { - brandSub: "Agent Red-team", - title: "LLM Agent Security Assessment", + title: "Agent Security Assessment Report", targetLabel: "Target System", - passedLabel: "Tests Passed", - defendedLabel: "Agent defended correctly", typeLabel: "LLM Agent", - detailSectionTitle: "Detailed Test Results", - footerPrefix: "Generated by Opfor v0.2", + detailSectionTitle: "Evaluation Results", + footerPrefix: "Generated by Opfor", }; } return { - brandSub: "MCP Red-team", - title: "MCP Server Security Assessment", + title: "MCP Server Security Assessment Report", targetLabel: "Target Server", - passedLabel: "Attacks Passed", - defendedLabel: "Server defended correctly", typeLabel: "MCP Server", - detailSectionTitle: "Detailed Attack Results", - footerPrefix: "Generated by Opfor MCP", + detailSectionTitle: "Evaluation Results", + footerPrefix: "Generated by Opfor", }; } +// ── Opfor wordmark (white text, red X) — self-contained, no external asset ── +const OPFOR_LOGO_SVG = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +`; + +const LOCK_ICON = ``; +const COPY_ICON = ``; +const ATTACKER_ICON = ``; +const AGENT_ICON = ``; +const SEVERITY_ICON = ``; + +/** Semi-circular SVG gauge for a 0-100 percentage score. */ +function gaugeSvg(pct: number, color: string): string { + const r = 52; + const circumference = Math.PI * r; + const offset = circumference * (1 - Math.max(0, Math.min(100, pct)) / 100); + return ` + + + `; +} + // ── Public API ─────────────────────────────────────────────────── /** Render a complete HTML report from a {@link ReportViewModel}. */ @@ -100,11 +153,6 @@ export function renderReport(model: ReportViewModel): string { const scoreDenominator = summary.passed + summary.failed; const noScoreableTests = scoreDenominator === 0; - // Verdict logic: - // - ERROR: all attacks were errors OR no real results at all - // - PASS: no failures and at least one real result (may be partial if stopReason) - // - FAIL: at least one failure - // Note: stopReason with real results shows PASS/FAIL with a warning, not ERROR const hasRealResults = summary.passed > 0 || summary.failed > 0; const overallVerdict = summary.errors > 0 && !hasRealResults @@ -123,76 +171,12 @@ export function renderReport(model: ReportViewModel): string { ? { label: "High Risk", color: "#DC2626" } : { label: "Critical Risk", color: "#991B1B" }; - const evalsFailed = evaluators.filter((e) => e.failed > 0).length; - - // ── Findings ────────────────────────────────────────────────── - type Finding = { - rank: number; - evaluator: string; - resultId: string; - score: number; - description: string; - }; - const criticalFindings: Finding[] = []; - const highFindings: Finding[] = []; - for (const e of evaluators) { - for (const r of e.results) { - if (r.judge.verdict !== "FAIL") continue; - const f: Finding = { - rank: 0, - evaluator: e.evaluatorName || e.evaluatorId, - resultId: r.id, - score: r.judge.score, - description: r.judge.reasoning, - }; - if (e.severity === "critical") criticalFindings.push(f); - else if (e.severity === "high") highFindings.push(f); - } - } - criticalFindings.sort((a, b) => b.score - a.score); - highFindings.sort((a, b) => b.score - a.score); - criticalFindings.forEach((f, i) => { - f.rank = i + 1; - }); - highFindings.forEach((f, i) => { - f.rank = i + 1; - }); - - // ── Findings HTML ───────────────────────────────────────────── - const findingBlock = (label: string, list: Finding[], color: string) => - list.length === 0 - ? "" - : `
-
- ${esc(label)} - ${list.length} -
-
    - ${list - .map( - (f) => `
  1. - ${esc(f.evaluator)} - Score ${f.score}/10 - ${esc(f.resultId)} -
    ${esc(f.description)}
    -
    2. ` - ) - .join("")} -
-
`; - - // ── Results table rows ──────────────────────────────────────── const anyErrors = evaluators.some((e) => e.errors > 0); - const tableRows = evaluators + + // ── Evaluator detail cards ───────────────────────────────────── + const appendix = evaluators .map((e, idx) => { const sevColor = SEV_HEX[e.severity] || "#64748B"; - const passDenom = e.passed + e.failed; - const passRate = passDenom > 0 ? Math.round((e.passed / passDenom) * 100) : 0; - const scoreable = e.results.filter((r) => r.judge.verdict !== "ERROR"); - const avgScore = - scoreable.length > 0 - ? (scoreable.reduce((s, r) => s + r.judge.score, 0) / scoreable.length).toFixed(1) - : "—"; const evalVerdict = e.errors > 0 && e.passed === 0 && e.failed === 0 ? "ERROR" @@ -205,105 +189,80 @@ export function renderReport(model: ReportViewModel): string { : evalVerdict === "ERROR" ? "verdict-error" : "verdict-fail"; + const scoreable = e.results.filter((r) => r.judge.verdict !== "ERROR"); + const avgScore = + scoreable.length > 0 + ? Math.round((scoreable.reduce((s, r) => s + r.judge.score, 0) / scoreable.length) * 10) / + 10 + : null; + const showTestHeading = e.results.length > 1; + const cards = e.results + .map((r, i) => resultDetailCard(r, i, model.mode, showTestHeading, idx)) + .join(""); return ` - - ${String(idx + 1).padStart(2, "0")} - ${esc(e.evaluatorName || e.evaluatorId)}${e.standards && Object.keys(e.standards).length ? `
${esc(formatStandardsLabel(e.standards))}` : ""} - ${esc(e.severity)} - ${evalVerdict} - ${e.total} - ${e.passed} - ${e.failed} - ${anyErrors ? `${e.errors > 0 ? e.errors : "—"}` : ""} - ${passRate}% - ${avgScore !== "—" ? `${avgScore}/10` : "—"} - `; - }) - .join(""); - - // ── Evaluator detail appendix ───────────────────────────────── - const appendix = evaluators - .map((e, idx) => { - const sevColor = SEV_HEX[e.severity] || "#64748B"; - const evalVerdict2 = - e.errors > 0 && e.passed === 0 && e.failed === 0 - ? "ERROR" - : e.failed === 0 && e.passed > 0 - ? "PASS" - : "FAIL"; - const verdictClass2 = - evalVerdict2 === "PASS" - ? "verdict-pass" - : evalVerdict2 === "ERROR" - ? "verdict-error" - : "verdict-fail"; - const cards = e.results.map((r, i) => resultCard(r, i, model.mode)).join(""); - return ` -
- +
+
${String(idx + 1).padStart(2, "0")}
${esc(e.evaluatorName || e.evaluatorId)} - ${esc(e.severity)} - ${e.standards && Object.keys(e.standards).length ? `${esc(formatStandardsLabel(e.standards))}` : ""} + | + ${SEVERITY_ICON}${esc(e.severity.toUpperCase())}
- ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens` : ""} - ${e.passed}/${e.total - e.errors} passed - ${evalVerdict2} - + ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens|` : ""} + ${evalVerdict === "PASS" ? "Pass" : evalVerdict === "ERROR" ? "Error" : "Fail"} + Safety score: ${avgScore ?? "—"}/10
-
+
${cards}
-
`; + `; }) .join(""); - // ── Cover meta items ────────────────────────────────────────── - let coverMeta: string; - if (model.mode === "agent") { - coverMeta = ` -
Target System
${esc(truncate(target.name, 60))}
-
Endpoint
${esc(truncate(target.endpoint ?? "", 60))}
-
Assessment Date
${esc(dateStr)}, ${esc(timeStr)}
-
Generator Model
${esc(model.generatorModel)}
-
Judge Model
${esc(model.judgeModel)}
-
Report ID
${esc(model.reportId)}
`; - } else { - coverMeta = ` -
Target Server
${esc(truncate(target.name, 60))}
-
Evaluation Suite
${esc(target.suiteId ?? "")}
-
Assessment Date
${esc(dateStr)}, ${esc(timeStr)}
-
Generator Model
${esc(model.generatorModel)}
-
Judge Model
${esc(model.judgeModel)}
-
Report ID
${esc(model.reportId)}
`; - } + // ── Cover meta (3-column row) ───────────────────────────────── + const coverMeta = ` +
${esc(labels.targetLabel)}
${esc(truncate(target.name, 60))}
+
Evaluation Suite
${esc(target.suiteId ?? "—")}
+
Attacker Model
${esc(model.generatorModel)}
`; // ── Assessment scope section ────────────────────────────────── - let scopeHtml: string; - if (model.mode === "agent") { - scopeHtml = ` -
Target
System${esc(target.name)}
Endpoint${esc(truncate(target.endpoint ?? "", 60))}
Type${labels.typeLabel}
-
Evaluation Parameters
Evaluators${evaluators.length}
Total Tests${summary.total}
Generator${esc(model.generatorModel)}
Judge${esc(model.judgeModel)}
`; - } else { - scopeHtml = ` -
Target
Server${esc(truncate(target.name, 80))}
Transport${esc(target.transport ?? "")}
Total Attacks${summary.total}
-
Evaluation Parameters
Suite${esc(target.suiteId ?? "")}
Generator${esc(model.generatorModel)}
Judge${esc(model.judgeModel)}
Evaluators${evaluators.length}
`; - } + const targetRows = + model.mode === "agent" + ? `
System${esc(target.name)}
+ ${target.type ? `
Type${esc(target.type)}
` : `
Type${esc(labels.typeLabel)}
`} + ${target.accessMethod ? `
Access Method${esc(target.accessMethod)}
` : ""} + ${target.endpoint ? `
Endpoint${esc(truncate(target.endpoint, 60))}
` : ""}` + : `
Server${esc(truncate(target.name, 80))}
+
Transport${esc(target.transport ?? "—")}
`; + + const paramRows = ` +
Suite${esc(target.suiteId ?? "—")}
+
Attacker Model${esc(model.generatorModel)}
+
Judge Model${esc(model.judgeModel)}
+ ${target.maxTurns != null ? `
Max Turns / Evaluator${target.maxTurns}
` : ""} + ${target.waitBetweenTurnsSec != null ? `
Wait Between Turns${target.waitBetweenTurnsSec}s
` : ""} + ${target.messageLengthLimit != null ? `
Message Length Limit${target.messageLengthLimit} chars
` : ""}`; + + const scopeHtml = ` +
Target
${targetRows}
+
Evaluation Parameters
${paramRows}
+ ${ + model.businessContext + ? `
Business Context
${esc(model.businessContext)}
` + : "" + }`; // ── Narrative text ──────────────────────────────────────────── let narrative: string; const targetLabel = model.mode === "agent" ? "target agent" : "MCP server"; - // For partial runs (stopReason OR errors with some real results), show simple warning if (model.stopReason && hasRealResults) { narrative = `⚠ Partial assessment. The run was stopped before all evaluators could complete.`; } else if (summary.errors > 0 && hasRealResults) { - // Mixed results: some passed/failed, some errors (e.g., target failed mid-run) narrative = `⚠ Partial assessment. Some attacks failed due to target errors. Results below are from attacks that completed.`; } else if (overallVerdict === "ERROR") { if (model.stopReason && summary.total === 0) { @@ -312,17 +271,18 @@ export function renderReport(model: ReportViewModel): string { narrative = `⚠ Assessment incomplete. All attacks failed due to errors. Please verify the target is accessible and retry.`; } } else if (overallVerdict === "PASS") { - narrative = `The ${targetLabel} ${esc(target.name)} passed all tests. No exploitable vulnerabilities were found.`; + narrative = `The ${targetLabel} ${esc(target.name)} passed all ${summary.total} evaluator${summary.total === 1 ? "" : "s"}${target.suiteId ? ` in the ${esc(target.suiteId)} suite` : ""}. No exploitable vulnerabilities were found.`; } else { - narrative = `The ${targetLabel} ${esc(target.name)} failed ${summary.failed} test${summary.failed === 1 ? "" : "s"}.${criticalFindings.length > 0 ? ` ${criticalFindings.length} critical.` : ""}`; + narrative = `The ${targetLabel} ${esc(target.name)} failed ${summary.failed} of ${summary.total} evaluator${summary.total === 1 ? "" : "s"}${target.suiteId ? ` in the ${esc(target.suiteId)} suite` : ""} (${summary.attackSuccessRate}% attack success rate).`; } - // ── Page title for tag ──────────────────────────────── const pageTitle = model.mode === "agent" ? `Opfor Agent Report — ${esc(target.name)}` : `Opfor MCP Report — ${esc(target.suiteId ?? target.name)}`; + const gaugeColor = noScoreableTests ? "#94A3B8" : safetyColor(summary.safetyScore); + // ── Full HTML ───────────────────────────────────────────────── return `<!doctype html> <html lang="en"> @@ -337,7 +297,7 @@ export function renderReport(model: ReportViewModel): string { --line:#E2E8F0;--line-2:#CBD5E1; --pass:#059669;--pass-bg:#D1FAE5;--pass-border:#6EE7B7; --fail:#DC2626;--fail-bg:#FEE2E2;--fail-border:#FCA5A5; - --accent:#f5ad5c; + --accent:#FF4D4F; } *{box-sizing:border-box;margin:0;padding:0} html{background:var(--bg)} @@ -348,20 +308,21 @@ export function renderReport(model: ReportViewModel): string { .page{max-width:960px;margin:0 auto;padding:0 24px} /* ── Cover band ── */ - .cover{background:#0F172A;color:#fff;padding:0;margin-bottom:32px} + .cover{background:#000;color:#fff;padding:0;margin-bottom:32px} .cover-inner{max-width:960px;margin:0 auto;padding:36px 24px 32px} - .cover-top{display:flex;align-items:flex-start;justify-content:space-between;gap:24px;margin-bottom:28px} - .cover-brand{display:flex;align-items:center;gap:10px} - .cover-brand-icon{width:36px;height:36px;background:linear-gradient(135deg,#f5ad5c,#c47a2a);border-radius:8px;display:flex;align-items:center;justify-content:center;flex-shrink:0} - .cover-brand-name{font-size:15px;font-weight:700;letter-spacing:0.04em;color:#fff} - .cover-brand-sub{font-size:11px;color:#94A3B8;letter-spacing:0.08em;text-transform:uppercase;margin-top:1px} - .cover-classification{padding:4px 12px;border:1px solid rgba(255,255,255,0.15);border-radius:4px;font-size:11px;font-weight:600;letter-spacing:0.1em;text-transform:uppercase;color:#CBD5E1} - .cover-title{font-size:26px;font-weight:700;color:#fff;letter-spacing:-0.01em;margin-bottom:6px} - .cover-subtitle{font-size:14px;color:#94A3B8;margin-bottom:24px} + .cover-top{display:flex;align-items:center;justify-content:space-between;gap:24px;margin-bottom:20px} + .cover-title{font-size:26px;font-weight:700;color:#fff;letter-spacing:-0.01em} + .cover-badges-row{display:flex;align-items:center;gap:12px;margin-bottom:24px;flex-wrap:wrap} + .badge-confidential{display:inline-flex;align-items:center;gap:6px;background:var(--accent);color:#fff;padding:5px 12px;border-radius:6px;font-size:11px;font-weight:700;letter-spacing:0.08em;text-transform:uppercase} + .reportid-chip{display:inline-flex;align-items:center;gap:8px;background:#262626;border:1px solid #3a3a3a;color:#D4D4D4;padding:5px 10px;border-radius:6px;font-size:11px} + .reportid-chip .mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:#fff} + .copy-btn{background:none;border:none;color:#9CA3AF;cursor:pointer;padding:2px;display:flex;align-items:center;border-radius:3px} + .copy-btn:hover{color:#fff;background:rgba(255,255,255,0.1)} + .cover-date{font-size:12px;color:#9CA3AF} .cover-meta{display:grid;grid-template-columns:repeat(3,1fr);gap:0;border:1px solid rgba(255,255,255,0.08);border-radius:10px;overflow:hidden} .cover-meta-item{padding:14px 18px;border-right:1px solid rgba(255,255,255,0.08)} .cover-meta-item:last-child{border-right:none} - .cover-meta-k{font-size:11px;color:#64748B;text-transform:uppercase;letter-spacing:0.08em;margin-bottom:4px} + .cover-meta-k{font-size:11px;color:#9CA3AF;text-transform:uppercase;letter-spacing:0.08em;margin-bottom:4px} .cover-meta-v{font-size:13px;color:#E2E8F0;font-weight:500;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} /* ── Section header ── */ @@ -371,33 +332,25 @@ export function renderReport(model: ReportViewModel): string { .section-title{font-size:15px;font-weight:600;color:var(--text);letter-spacing:-0.01em} .section-subtitle{font-size:12px;color:var(--muted);margin-left:auto} - /* ── Executive summary ── */ - .exec-banner{display:flex;align-items:center;justify-content:space-between;gap:16px;padding:18px 20px;border-radius:12px;border:1px solid var(--line-2);background:var(--surface);margin-bottom:12px} - .exec-banner.pass{border-color:var(--pass-border);background:var(--pass-bg)} - .exec-banner.fail{border-color:var(--fail-border);background:var(--fail-bg)} - .exec-banner.error{border-color:#FDE68A;background:#FFFBEB} - .exec-banner-left{display:flex;align-items:center;gap:14px} - .exec-verdict-icon{width:44px;height:44px;border-radius:10px;border:1px solid;display:flex;align-items:center;justify-content:center;flex-shrink:0} - .exec-banner.pass .exec-verdict-icon{border-color:var(--pass-border);color:var(--pass);background:var(--pass-bg)} - .exec-banner.fail .exec-verdict-icon{border-color:var(--fail-border);color:var(--fail);background:var(--fail-bg)} - .exec-banner.error .exec-verdict-icon{border-color:#FDE68A;color:#D97706;background:#FEF3C7} - .exec-verdict-label{font-size:11px;font-weight:600;letter-spacing:0.08em;text-transform:uppercase;color:var(--muted);margin-bottom:3px} - .exec-verdict-text{font-size:26px;font-weight:800;letter-spacing:0.04em;line-height:1} - .exec-banner.pass .exec-verdict-text{color:var(--pass)} - .exec-banner.fail .exec-verdict-text{color:var(--fail)} - .exec-banner.error .exec-verdict-text{color:#D97706} - .exec-risk{font-size:12px;font-weight:600;padding:4px 12px;border-radius:999px;border:1px solid;white-space:nowrap} - .exec-banner.pass .exec-risk{background:var(--pass-bg);color:var(--pass);border-color:var(--pass-border)} - .exec-banner.fail .exec-risk{background:var(--fail-bg);color:var(--fail);border-color:var(--fail-border)} - .exec-banner.error .exec-risk{background:#FEF3C7;color:#D97706;border-color:#FDE68A} - .summary-stats{display:grid;grid-template-columns:repeat(4,1fr);gap:10px} - .stat-card{background:var(--surface);border:1px solid var(--line);border-radius:10px;padding:14px 16px} - .stat-card .sc-label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:0.06em;margin-bottom:6px} - .stat-card .sc-value{font-size:22px;font-weight:700;line-height:1;color:var(--text)} - .stat-card .sc-bar{height:4px;background:var(--line);border-radius:2px;margin-top:8px;overflow:hidden} - .stat-card .sc-bar-fill{height:100%;border-radius:2px} - .stat-card .sc-sub{font-size:11px;color:var(--muted);margin-top:4px} - .summary-narrative{background:var(--surface);border:1px solid var(--line);border-radius:10px;padding:16px;margin-top:12px;font-size:13px;color:var(--text-2);line-height:1.7} + /* ── Executive summary strip ── */ + .exec-strip{display:flex;align-items:stretch;border:1px solid var(--line);border-radius:12px;background:var(--surface);overflow:hidden;margin-bottom:12px} + .exec-strip-item{flex:1;padding:18px 22px;border-right:1px solid var(--line);display:flex;flex-direction:column;justify-content:center;min-width:0} + .exec-strip-item:last-child{border-right:none} + .exec-strip-label{font-size:11px;font-weight:600;letter-spacing:0.08em;text-transform:uppercase;color:var(--muted);margin-bottom:8px} + .exec-verdict-row{display:flex;align-items:center;gap:10px;flex-wrap:wrap} + .exec-verdict-text{font-size:28px;font-weight:800;letter-spacing:0.02em;line-height:1} + .exec-verdict-text.pass{color:var(--pass)} + .exec-verdict-text.fail{color:var(--fail)} + .exec-verdict-text.error{color:#D97706} + .exec-risk{font-size:11px;font-weight:600;padding:4px 11px;border-radius:999px;border:1px solid;white-space:nowrap} + .gauge-value{font-size:22px;font-weight:800;color:var(--text);text-align:center;margin-top:-30px} + .gauge-sub{font-size:12px;color:var(--muted);text-align:center;margin-top:2px} + .sc-value{font-size:26px;font-weight:800;line-height:1;color:var(--text)} + .sc-dots{display:flex;flex-direction:column;gap:3px;margin-top:8px} + .sc-dot-row{display:flex;align-items:center;gap:6px;font-size:12px;color:var(--muted)} + .sc-dot{width:7px;height:7px;border-radius:50%;flex-shrink:0} + .sc-sub{font-size:12px;color:var(--muted);margin-top:4px} + .summary-narrative{font-size:13px;color:var(--text-2);line-height:1.7;padding:2px 2px} .summary-narrative strong{color:var(--text)} /* ── Assessment scope ── */ @@ -409,93 +362,58 @@ export function renderReport(model: ReportViewModel): string { .scope-k{font-size:12px;color:var(--muted);flex-shrink:0} .scope-v{font-size:12px;color:var(--text);font-weight:500;text-align:right;word-break:break-word;max-width:60%} .scope-v.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:11px} - - /* ── Findings ── */ - .findings-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px} - .finding-block{background:var(--surface);border:1px solid;border-left-width:3px;border-radius:10px;padding:16px;overflow:hidden} - .finding-block-head{display:flex;align-items:center;gap:8px;margin-bottom:12px} - .finding-label{font-size:12px;font-weight:700;text-transform:uppercase;letter-spacing:0.07em} - .finding-count{font-size:11px;font-weight:700;padding:2px 8px;border:1px solid;border-radius:999px} - .finding-list{margin:0;padding-left:18px;display:flex;flex-direction:column;gap:10px} - .finding-list li{font-size:13px;color:var(--text-2);line-height:1.5} - .finding-list strong{color:var(--text)} - .finding-score{margin-left:6px;font-size:11px;color:var(--muted);font-weight:600;background:var(--surface-2);padding:1px 6px;border-radius:4px;border:1px solid var(--line)} - .finding-desc{margin-top:3px;font-size:12px;color:var(--muted)} - .no-findings{background:var(--pass-bg);border:1px solid var(--pass-border);border-radius:10px;padding:16px;text-align:center;color:var(--pass);font-weight:600;font-size:13px} - - /* ── Results table ── */ - .results-table-wrap{background:var(--surface);border:1px solid var(--line);border-radius:10px;overflow:hidden} - table.results{width:100%;border-collapse:collapse} - table.results th{background:var(--surface-2);padding:10px 14px;text-align:left;font-size:11px;font-weight:600;color:var(--muted);text-transform:uppercase;letter-spacing:0.06em;border-bottom:1px solid var(--line)} - table.results td{padding:11px 14px;font-size:13px;border-bottom:1px solid var(--line);vertical-align:middle} - table.results tr:last-child td{border-bottom:none} - table.results tr:hover td{background:var(--surface-2)} - .td-num{color:var(--muted-2);font-size:11px;font-family:ui-monospace,monospace;width:36px} - .td-score{font-size:13px;font-weight:600;color:var(--text)} - .eval-link{color:var(--text);font-weight:500} - .eval-link:hover{color:var(--accent)} + .scope-full{grid-column:1/-1} /* ── Badges ── */ - .sev-tag{display:inline-block;padding:2px 8px;border:1px solid;border-radius:4px;font-size:11px;font-weight:600;letter-spacing:0.03em;white-space:nowrap} - .verdict-tag{display:inline-block;padding:2px 9px;border-radius:4px;font-size:11px;font-weight:700;letter-spacing:0.04em} - .verdict-pass{background:var(--pass-bg);color:var(--pass);border:1px solid var(--pass-border)} - .verdict-fail{background:var(--fail-bg);color:var(--fail);border:1px solid var(--fail-border)} - .verdict-error{background:#FEF3C7;color:#D97706;border:1px solid #FDE68A} + .eval-sep{color:var(--line-2);font-weight:400} + .sev-tag{display:inline-flex;align-items:center;gap:4px;font-size:11px;font-weight:700;letter-spacing:0.04em;white-space:nowrap;font-family:ui-monospace,SFMono-Regular,Menlo,monospace} + .verdict-tag{font-size:18px;font-weight:800;letter-spacing:0.01em} + .verdict-pass{color:var(--pass)} + .verdict-fail{color:var(--fail)} + .verdict-error{color:#D97706} + .pass-text{color:var(--pass);font-weight:600} + .fail-text{color:var(--fail);font-weight:600} /* ── Evaluator detail blocks ── */ .eval-detail{background:var(--surface);border:1px solid var(--line);border-radius:10px;overflow:hidden;margin-bottom:8px} - .eval-detail > summary{display:flex;align-items:center;justify-content:space-between;padding:13px 16px;cursor:pointer;list-style:none;gap:12px} - .eval-detail > summary::-webkit-details-marker{display:none} - .eval-detail > summary:hover{background:var(--surface-2)} - .eval-detail[open] > summary{background:var(--surface-2);border-bottom:1px solid var(--line)} + .eval-summary{display:flex;align-items:center;justify-content:space-between;padding:13px 16px;gap:12px} .eval-summary-left{display:flex;align-items:center;gap:10px;flex:1;min-width:0} .eval-num{font-size:11px;font-family:ui-monospace,monospace;color:var(--muted-2);flex-shrink:0;width:22px} .eval-summary-info{display:flex;align-items:center;gap:8px;flex-wrap:wrap} - .eval-summary-name{font-size:13px;font-weight:600;color:var(--text)} + .eval-summary-name{font-size:18px;font-weight:700;color:var(--text)} .eval-summary-right{display:flex;align-items:center;gap:10px;flex-shrink:0} - .chevron{color:var(--muted-2);transition:transform 0.2s;flex-shrink:0} - .eval-detail[open] .chevron{transform:rotate(180deg)} .eval-body{padding:16px} - - /* ── Result cards ── */ - .result-card{border:1px solid var(--line);border-left:3px solid var(--line-2);border-radius:0 8px 8px 0;padding:12px 14px;margin-bottom:10px} - .result-card.pass{border-left-color:var(--pass)} - .result-card.fail{border-left-color:var(--fail)} - .result-card.error{border-left-color:#F59E0B;background:#FFFDF5} - .result-header{display:flex;align-items:center;gap:8px;margin-bottom:8px} - .result-id{font-size:12px;font-weight:600;color:var(--text)} - .result-tool{font-size:11px;color:var(--muted);font-family:ui-monospace,monospace} - .result-section{margin-bottom:8px} - .result-section-label{font-size:10px;font-weight:600;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);margin-bottom:4px} - .result-code{font-size:12px;background:var(--surface-2);border:1px solid var(--line);padding:8px 10px;border-radius:6px;white-space:pre-wrap;word-break:break-word;font-family:ui-monospace,SFMono-Regular,Menlo,monospace;max-height:400px;overflow:auto} - .result-judge{margin-top:8px;font-size:12px;padding:8px 12px;border-radius:6px;border:1px solid var(--pass-border);background:var(--pass-bg);line-height:1.6} - .result-judge.fail{background:var(--fail-bg);border-color:var(--fail-border)} - - /* ── Expandable code blocks ── */ - .code-wrap{position:relative;margin-top:4px} - .code-wrap .result-code{max-height:180px;overflow:hidden;margin-top:0;border-radius:6px 6px 0 0;border-bottom:none} - .code-wrap .code-fade{position:absolute;bottom:28px;left:0;right:0;height:48px;pointer-events:none} - .code-wrap .code-more{display:block;width:100%;font-size:11px;font-weight:600;color:var(--muted);background:var(--surface);border:1px solid var(--line);border-radius:0 0 6px 6px;padding:4px 10px;cursor:pointer;text-align:center;letter-spacing:0.03em} - .code-wrap .code-more:hover{background:var(--surface-2);color:var(--text)} - .code-wrap.expanded .result-code{max-height:none;border-radius:6px 6px 0 0} - .code-wrap.expanded .code-fade{display:none} - .code-wrap.expanded .code-more{color:var(--muted-2)} - - /* ── Turn cards (multi-turn breakdown) ── */ - .turn-card{margin-bottom:8px;border:1px solid var(--line);border-radius:8px;overflow:hidden} - .turn-card-header{display:flex;align-items:center;gap:8px;padding:7px 12px;background:var(--surface-2);cursor:pointer;list-style:none;font-size:11px;font-weight:600;color:var(--text);border-bottom:1px solid var(--line)} - .turn-card-header::-webkit-details-marker{display:none} - .turn-attacker{background:var(--surface-2);padding:10px 12px;border-bottom:1px solid var(--line);border-left:3px solid var(--accent)} - .turn-attacker .turn-label{font-size:10px;font-weight:700;text-transform:uppercase;letter-spacing:0.07em;color:var(--accent);margin-bottom:5px} - .turn-attacker .code-wrap .result-code{background:var(--surface);border-color:var(--line)} - .turn-attacker .code-more{background:var(--surface-2);border-color:var(--line)} - .turn-attacker .code-more:hover{background:var(--surface)} - .turn-agent{background:var(--surface);padding:10px 12px;border-left:3px solid var(--line-2)} - .turn-agent .turn-label{font-size:10px;font-weight:700;text-transform:uppercase;letter-spacing:0.07em;color:var(--muted);margin-bottom:5px} - .turn-agent .code-wrap .result-code{background:var(--surface-2);border-color:var(--line)} - .turn-agent .code-more{background:var(--surface);border-color:var(--line)} - .turn-agent .code-more:hover{background:var(--surface-2)} - .turn-reasoning{padding:7px 12px;background:var(--surface-2);border-top:1px solid var(--line);font-size:11px;color:var(--muted);font-style:italic;line-height:1.5} + .test-heading{font-size:12px;font-weight:700;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);margin:16px 0 8px} + .test-heading:first-child{margin-top:0} + .meta-k{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:0.05em;margin-bottom:4px} + .meta-v{font-size:13px;font-weight:600;color:var(--text)} + .detail-section{margin-bottom:18px} + .detail-section-label{font-size:12px;font-weight:700;text-transform:uppercase;letter-spacing:0.06em;color:var(--text-2);margin-bottom:8px} + .detail-section-body{font-size:13px;color:var(--text-2);line-height:1.6;white-space:pre-wrap;word-break:break-word} + .eval-meta-row{display:flex;flex-wrap:wrap;gap:36px;margin-bottom:14px} + .eval-meta-col{display:flex;flex-direction:column;gap:6px} + .eval-meta-col .meta-v-lg{font-size:20px;font-weight:700;color:var(--text);line-height:1} + + /* ── Transcript ── */ + .transcript-toggle{display:inline-flex;align-items:center;gap:6px;font-size:13px;font-weight:600;color:var(--muted);background:none;border:none;cursor:pointer;padding:12px 0 0;margin-top:2px;border-top:1px solid var(--line);width:100%} + .transcript-toggle:hover{color:var(--text)} + .transcript-toggle svg{transition:transform 0.2s} + .transcript-wrap{display:none;margin-bottom:8px} + .transcript-wrap.open{display:block} + .transcript{border:1px solid var(--line);border-radius:8px;overflow:hidden} + .transcript-header{padding:8px 12px;background:var(--surface-2);font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);border-bottom:1px solid var(--line);display:flex;align-items:center;gap:8px} + .tc-count{font-weight:400;color:var(--muted-2)} + .turn{padding:12px 0} + .turn-row{padding:2px 16px} + .turn-row.attacker-row{padding-left:64px} + .turn-row.agent-row{margin-top:12px} + .turn-role{display:flex;align-items:center;gap:6px;font-size:12px;font-weight:700;color:var(--text);margin-bottom:6px} + .turn-role .turn-icon{display:inline-flex;align-items:center} + .turn-role .turn-sep{color:var(--muted-2);font-weight:400} + .turn-role .turn-num{color:var(--muted);font-weight:500} + .turn-row pre{margin:0;white-space:pre-wrap;word-break:break-word;font:13px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:var(--text-2)} + .agent-bubble{display:inline-block;max-width:78%;background:var(--surface-2);border-radius:10px;padding:10px 14px} + .agent-bubble.turn-highlight{background:var(--fail-bg);border:1px dashed var(--fail-border)} /* ── Footer ── */ .report-footer{max-width:960px;margin:40px auto 0;padding:16px 24px;border-top:1px solid var(--line);display:flex;justify-content:space-between;align-items:center} @@ -506,13 +424,15 @@ export function renderReport(model: ReportViewModel): string { body{background:#fff;padding:0} .cover{-webkit-print-color-adjust:exact;print-color-adjust:exact} .eval-detail{border:1px solid var(--line)} - .stat-card,.scope-card,.finding-block,.results-table-wrap,.eval-detail{break-inside:avoid;box-shadow:none} + .scope-card,.eval-detail,.exec-strip{break-inside:avoid;box-shadow:none} } @media(max-width:640px){ - .cover-meta{grid-template-columns:1fr 1fr} - .exec-banner{flex-direction:column;align-items:flex-start} - .summary-stats{grid-template-columns:1fr 1fr} - .scope-grid,.findings-grid{grid-template-columns:1fr} + .cover-meta{grid-template-columns:1fr} + .exec-strip{flex-direction:column} + .exec-strip-item{border-right:none;border-bottom:1px solid var(--line)} + .exec-strip-item:last-child{border-bottom:none} + .scope-grid{grid-template-columns:1fr} + .eval-meta-row{gap:20px} } </style> </head> @@ -521,19 +441,16 @@ export function renderReport(model: ReportViewModel): string { <div class="cover"> <div class="cover-inner"> <div class="cover-top"> - <div class="cover-brand"> - <div class="cover-brand-icon"> - <svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 2l8 4v6c0 5-3.5 9-8 10-4.5-1-8-5-8-10V6l8-4z"/></svg> - </div> - <div> - <div class="cover-brand-name">Opfor</div> - <div class="cover-brand-sub">${labels.brandSub}</div> - </div> - </div> - <div class="cover-classification">Confidential</div> + <div class="cover-title">${labels.title}</div> + ${OPFOR_LOGO_SVG} + </div> + <div class="cover-badges-row"> + <span class="badge-confidential">${LOCK_ICON} Confidential</span> + <span class="reportid-chip">Report ID: <span class="mono">${esc(model.reportId)}</span> + <button class="copy-btn" data-copy="${esc(model.reportId)}" title="Copy report ID">${COPY_ICON}</button> + </span> + <span class="cover-date">${esc(dateStr)}, ${esc(timeStr)}</span> </div> - <div class="cover-title">${labels.title}</div> - <div class="cover-subtitle">Automated adversarial evaluation · opfor v0.2 · ${esc(dateStr)}</div> <div class="cover-meta"> ${coverMeta} </div> @@ -548,56 +465,45 @@ export function renderReport(model: ReportViewModel): string { <div class="section-num">1</div> <div class="section-title">Executive Summary</div> </div> - <div class="exec-banner ${overallVerdict === "PASS" ? "pass" : overallVerdict === "ERROR" ? "error" : "fail"}"> - <div class="exec-banner-left"> - <div class="exec-verdict-icon"> - ${ - overallVerdict === "PASS" - ? `<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M20 6L9 17l-5-5"/></svg>` - : overallVerdict === "ERROR" - ? `<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="12"/><line x1="12" y1="16" x2="12.01" y2="16"/></svg>` - : `<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linejoin="round"><path d="M12 2l8 4v6c0 5-3.5 9-8 10-4.5-1-8-5-8-10V6l8-4z"/></svg>` - } + <div class="exec-strip"> + <div class="exec-strip-item"> + <div class="exec-strip-label">Overall Verdict</div> + <div class="exec-verdict-row"> + <div class="exec-verdict-text ${overallVerdict === "PASS" ? "pass" : overallVerdict === "ERROR" ? "error" : "fail"}">${overallVerdict === "PASS" ? "Pass" : overallVerdict === "ERROR" ? "Error" : "Fail"}</div> + <div class="exec-risk" style="color:${riskLevel.color};border-color:${riskLevel.color}66;background:${riskLevel.color}14">${overallVerdict === "ERROR" ? "Inconclusive" : riskLevel.label.toUpperCase()}</div> </div> - <div> - <div class="exec-verdict-label">Overall Verdict</div> - <div class="exec-verdict-text">${overallVerdict}</div> - </div> - </div> - <div class="exec-risk">${overallVerdict === "ERROR" ? "Inconclusive" : riskLevel.label}</div> - </div> - <div class="summary-stats"> - <div class="stat-card"> - <div class="sc-label">Safety Score</div> - <div class="sc-value" style="color:${safetyColor(noScoreableTests ? 0 : summary.safetyScore)}">${noScoreableTests ? "N/A" : `${summary.safetyScore}%`}</div> - ${noScoreableTests ? "" : `<div class="sc-bar"><div class="sc-bar-fill" style="width:${summary.safetyScore}%;background:${safetyColor(summary.safetyScore)}"></div></div>`} - <div class="sc-sub">${noScoreableTests ? "No scoreable tests" : `${summary.passed} of ${scoreDenominator} attacks defended`}</div> - </div> - <div class="stat-card"> - <div class="sc-label">Attack Success Rate</div> - <div class="sc-value" style="color:${summary.attackSuccessRate > 0 ? "#DC2626" : "#059669"}">${noScoreableTests ? "N/A" : `${summary.attackSuccessRate}%`}</div> - ${noScoreableTests ? "" : `<div class="sc-bar"><div class="sc-bar-fill" style="width:${summary.attackSuccessRate}%;background:${summary.attackSuccessRate > 0 ? "#DC2626" : "#059669"}"></div></div>`} - <div class="sc-sub">${summary.failed} attack${summary.failed !== 1 ? "s" : ""} breached defenses</div> </div> - <div class="stat-card"> - <div class="sc-label">${labels.passedLabel}</div> - <div class="sc-value" style="color:#059669">${summary.passed}</div> - <div class="sc-sub">${labels.defendedLabel}</div> + <div class="exec-strip-item" style="align-items:center"> + <div class="exec-strip-label" style="text-align:center">Safety Score</div> + ${gaugeSvg(noScoreableTests ? 0 : summary.safetyScore, gaugeColor)} + <div class="gauge-value">${noScoreableTests ? "N/A" : `${summary.safetyScore}%`}</div> </div> - <div class="stat-card"> - <div class="sc-label">Evaluators Failed</div> - <div class="sc-value" style="color:${evalsFailed > 0 ? "#DC2626" : "#059669"}">${evalsFailed}</div> - <div class="sc-sub">${criticalFindings.length} critical · ${highFindings.length} high</div> + <div class="exec-strip-item"> + <div class="exec-strip-label">Evaluators Run</div> + <div class="sc-value">${evaluators.length}</div> + <div class="sc-dots"> + <div class="sc-dot-row"><span class="sc-dot" style="background:var(--pass)"></span>${summary.passed} passed</div> + <div class="sc-dot-row"><span class="sc-dot" style="background:var(--fail)"></span>${summary.failed} failed</div> + ${anyErrors ? `<div class="sc-dot-row"><span class="sc-dot" style="background:#D97706"></span>${summary.errors} errored</div>` : ""} + </div> </div> ${ summary.tokenUsage - ? `<div class="stat-card"> - <div class="sc-label">Token Usage</div> - <div class="sc-value" style="color:#6B7280">${formatTokenCount(summary.tokenUsage.totalTokens)}</div> + ? `<div class="exec-strip-item"> + <div class="exec-strip-label">Token Usage</div> + <div class="sc-value">${formatTokenCount(summary.tokenUsage.totalTokens)}</div> <div class="sc-sub">${summary.tokenUsage.inputTokens.toLocaleString()} in · ${summary.tokenUsage.outputTokens.toLocaleString()} out</div> </div>` : "" } + ${ + summary.durationMs !== undefined + ? `<div class="exec-strip-item"> + <div class="exec-strip-label">Duration</div> + <div class="sc-value">${formatDuration(summary.durationMs)}</div> + </div>` + : "" + } </div> <div class="summary-narrative"> ${narrative} @@ -615,51 +521,12 @@ export function renderReport(model: ReportViewModel): string { </div> </div> - <!-- 3. Findings --> - ${ - criticalFindings.length + highFindings.length > 0 - ? `<div class="section"> - <div class="section-header"> - <div class="section-num">3</div> - <div class="section-title">Key Findings</div> - <div class="section-subtitle">${criticalFindings.length + highFindings.length} finding${criticalFindings.length + highFindings.length === 1 ? "" : "s"} requiring attention</div> - </div> - <div class="findings-grid"> - ${findingBlock("Critical", criticalFindings, "#DC2626")} - ${findingBlock("High", highFindings, "#D97706")} - </div> - </div>` - : `<div class="section"> - <div class="section-header"> - <div class="section-num">3</div> - <div class="section-title">Key Findings</div> - </div> - <div class="no-findings">No critical or high severity findings — ${model.mode === "agent" ? "agent" : "server"} passed all evaluated attack patterns.</div> - </div>` - } - - <!-- 4. Evaluation Results --> - <div class="section"> - <div class="section-header"> - <div class="section-num">4</div> - <div class="section-title">Evaluation Results</div> - <div class="section-subtitle">${evaluators.length} evaluator${evaluators.length === 1 ? "" : "s"} · ${summary.total} ${model.mode === "agent" ? "tests" : "attacks"}</div> - </div> - <div class="results-table-wrap" style="margin-bottom:16px"> - <table class="results"> - <thead><tr> - <th>#</th><th>Evaluator</th><th>Severity</th><th>Verdict</th><th>Tests</th><th>Passed</th><th>Failed</th>${anyErrors ? "<th>Errors</th>" : ""}<th>Pass Rate</th><th>Avg Score</th> - </tr></thead> - <tbody>${tableRows}</tbody> - </table> - </div> - </div> - - <!-- 5. Detailed Results --> + <!-- 3. Evaluation Results --> <div class="section"> <div class="section-header"> - <div class="section-num">5</div> + <div class="section-num">3</div> <div class="section-title">${labels.detailSectionTitle}</div> + <div class="section-subtitle">${evaluators.length} evaluation${evaluators.length === 1 ? "" : "s"}</div> </div> ${appendix} </div> @@ -673,12 +540,21 @@ export function renderReport(model: ReportViewModel): string { <script> (function(){ - document.querySelectorAll('.code-wrap').forEach(function(wrap){ - var pre=wrap.querySelector('.result-code'); - if(pre&&pre.scrollHeight<=pre.clientHeight+4){ - wrap.querySelector('.code-fade').style.display='none'; - wrap.querySelector('.code-more').style.display='none'; - } + document.querySelectorAll('.copy-btn').forEach(function(btn){ + btn.addEventListener('click', function(){ + navigator.clipboard.writeText(btn.dataset.copy || ''); + var svg = btn.querySelector('svg'); + if (svg) svg.style.color = '#4ADE80'; + }); + }); + document.querySelectorAll('.transcript-toggle').forEach(function(btn){ + btn.addEventListener('click', function(){ + var wrap = btn.closest('.detail-section, .eval-body').querySelector('.transcript-wrap[data-for="' + btn.dataset.target + '"]'); + if(!wrap) return; + var open = wrap.classList.toggle('open'); + btn.querySelector('.tt-label').textContent = open ? 'View less details' : 'View more details'; + btn.querySelector('svg').style.transform = open ? 'rotate(180deg)' : 'rotate(0deg)'; + }); }); })(); </script> @@ -686,158 +562,151 @@ export function renderReport(model: ReportViewModel): string { </html>`; } -// ── Result card helper ─────────────────────────────────────────── +// ── Result detail card ──────────────────────────────────────────── -/** Wrap content in a collapsible block with a fade-out gradient and toggle button. */ -function expandableBlock(content: string, fadeColor: string, extraStyle = ""): string { - return `<div class="code-wrap"> - <pre class="result-code"${extraStyle ? ` style="${extraStyle}"` : ""}>${content}</pre> - <div class="code-fade" style="background:linear-gradient(to bottom,${fadeColor}00,${fadeColor})"></div> - <button class="code-more" onclick="var w=this.closest('.code-wrap');w.classList.toggle('expanded');this.textContent=w.classList.contains('expanded')?'▲ Show less':'▼ Show more'">▼ Show more</button> - </div>`; +/** Role label row: icon + name, optionally " | Turn N". */ +function roleLabel(icon: string, name: string, turnLabel?: string): string { + const suffix = turnLabel + ? `<span class="turn-sep">|</span><span class="turn-num">${turnLabel}</span>` + : ""; + return `<div class="turn-role"><span class="turn-icon">${icon}</span>${name}${suffix}</div>`; } -/** Render the prompt/response or tool-call detail for a single attack result. */ -function renderDetailContent(detail: DetailCard, _mode: "agent" | "mcp"): string { - if (detail.kind === "prompt") { +/** Render a single conversation turn: the attacker prompt as plain text, the agent + * response as a left-anchored bubble (highlighted when the judge cited this turn + * in its `failingTurns`). There's no per-turn judge — the transcript is judged once, + * at the end, over the whole exchange — so the flagged-turn signal comes from the + * result-level `judge.failingTurns` list, not from `turn.judge`. */ +function renderTurn(turn: TurnViewModel, failingTurns: Set<number>): string { + const bubbleClass = failingTurns.has(turn.turnIndex) + ? "agent-bubble turn-highlight" + : "agent-bubble"; + const turnLabel = `Turn ${turn.turnIndex}`; + + if (turn.detail.kind === "prompt") { return ` - <details class="result-section"> - <summary style="font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:0.07em;color:var(--accent);cursor:pointer;padding:2px 0">▸ Attacker Prompt</summary> - ${expandableBlock(esc(truncate(detail.prompt, 8000)), "#F1F5F9")} - </details> - <details class="result-section"> - <summary style="font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:0.07em;color:var(--muted);cursor:pointer;padding:2px 0">▸ Agent Response</summary> - ${expandableBlock(esc(truncate(detail.response, 8000)), "#F1F5F9")} - </details>`; + <div class="turn"> + <div class="turn-row attacker-row"> + ${roleLabel(ATTACKER_ICON, "Attacker", turnLabel)} + <pre>${esc(truncate(turn.detail.prompt, 8000))}</pre> + </div> + <div class="turn-row agent-row"> + <div class="${bubbleClass}"> + ${roleLabel(AGENT_ICON, "Agent", turnLabel)} + <pre>${esc(truncate(turn.detail.response, 8000))}</pre> + </div> + </div> + </div>`; } - const argsFormatted = esc(JSON.stringify(detail.args, null, 2)); - const responseText = detail.error - ? `Error: ${esc(truncate(detail.error, 4000))}` - : esc(truncate(detail.response, 8000)); + const tArgs = esc(JSON.stringify(turn.detail.args, null, 2)); + const tResp = turn.detail.error + ? `Error: ${esc(truncate(turn.detail.error, 4000))}` + : esc(truncate(turn.detail.response, 8000)); + const toolLabel = turn.detail.toolName ? ` · ${esc(turn.detail.toolName)}` : ""; return ` - <details class="result-section"> - <summary style="font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);cursor:pointer">Arguments</summary> - ${expandableBlock(argsFormatted, "#F1F5F9")} - </details> - <details class="result-section"> - <summary style="font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:0.06em;color:var(--muted);cursor:pointer">Tool Response</summary> - ${expandableBlock(responseText, "#F1F5F9")} - </details>`; + <div class="turn"> + <div class="turn-row attacker-row"> + ${roleLabel(ATTACKER_ICON, `Arguments${toolLabel}`, turnLabel)} + <pre>${tArgs}</pre> + </div> + <div class="turn-row agent-row"> + <div class="${bubbleClass}"> + ${roleLabel(AGENT_ICON, "Tool Response", turnLabel)} + <pre>${tResp}</pre> + </div> + </div> + </div>`; } -/** Render a single conversation turn (prompt + response or tool-call). */ -function renderTurnContent(turn: TurnViewModel): string { - const tVerdict = turn.judge?.verdict; - const tColor = - tVerdict === "PASS" - ? "var(--pass)" - : tVerdict === "ERROR" - ? "#D97706" - : tVerdict === "FAIL" - ? "var(--fail)" - : "var(--muted)"; - const borderColor = tVerdict ? tColor : "var(--line-2)"; - - const verdictBadge = tVerdict - ? `<span style="margin-left:auto;font-size:10px;font-weight:700;padding:1px 8px;border-radius:3px;background:${tVerdict === "PASS" ? "var(--pass-bg)" : tVerdict === "FAIL" ? "var(--fail-bg)" : "#FEF3C7"};color:${tColor};border:1px solid ${tVerdict === "PASS" ? "var(--pass-border)" : tVerdict === "FAIL" ? "var(--fail-border)" : "#FDE68A"}">${tVerdict}${tVerdict !== "ERROR" ? ` · ${turn.judge?.score}/10` : ""}</span>` - : ""; - - let detailHtml: string; - if (turn.detail.kind === "prompt") { - detailHtml = ` - <div class="turn-attacker"> - <div class="turn-label">Attacker Prompt</div> - ${expandableBlock(esc(truncate(turn.detail.prompt, 8000)), "#F1F5F9")} +/** Build the single-shot (non-multi-turn) transcript from a top-level DetailCard. */ +function singleTurnTranscript(detail: DetailCard): string { + if (detail.kind === "prompt") { + return `<div class="turn"> + <div class="turn-row attacker-row"> + ${roleLabel(ATTACKER_ICON, "Attacker")} + <pre>${esc(truncate(detail.prompt, 8000))}</pre> </div> - <div class="turn-agent"> - <div class="turn-label">Agent Response</div> - ${expandableBlock(esc(truncate(turn.detail.response, 8000)), "#FFFFFF")} - </div>`; - } else { - const tArgs = esc(JSON.stringify(turn.detail.args, null, 2)); - const tResp = turn.detail.error - ? `Error: ${esc(truncate(turn.detail.error, 4000))}` - : esc(truncate(turn.detail.response, 8000)); - const toolLabel = turn.detail.toolName - ? ` · <code style="background:var(--surface-2);padding:1px 5px;border-radius:3px;font-size:10px">${esc(turn.detail.toolName)}</code>` - : ""; - detailHtml = ` - <div class="turn-attacker"> - <div class="turn-label">Arguments${toolLabel}</div> - ${expandableBlock(tArgs, "#F1F5F9")} + <div class="turn-row agent-row"> + <div class="agent-bubble"> + ${roleLabel(AGENT_ICON, "Agent")} + <pre>${esc(truncate(detail.response, 8000))}</pre> + </div> </div> - <div class="turn-agent"> - <div class="turn-label">Tool Response</div> - ${expandableBlock(tResp, "#FFFFFF")} - </div>`; + </div>`; } - - return ` - <details class="turn-card" open> - <summary class="turn-card-header" style="border-left:3px solid ${borderColor}"> - <span style="font-family:ui-monospace,monospace;font-size:10px;color:var(--muted-2)">T${turn.turnIndex}</span> - <span>Turn ${turn.turnIndex}</span> - ${verdictBadge} - </summary> - ${detailHtml} - ${turn.judge?.reasoning ? `<div class="turn-reasoning">${esc(turn.judge.reasoning)}</div>` : ""} - </details>`; + const args = esc(JSON.stringify(detail.args, null, 2)); + const resp = detail.error + ? `Error: ${esc(truncate(detail.error, 4000))}` + : esc(truncate(detail.response, 8000)); + const toolLabel = detail.toolName ? ` · ${esc(detail.toolName)}` : ""; + return `<div class="turn"> + <div class="turn-row attacker-row"> + ${roleLabel(ATTACKER_ICON, `Arguments${toolLabel}`)} + <pre>${args}</pre> + </div> + <div class="turn-row agent-row"> + <div class="agent-bubble"> + ${roleLabel(AGENT_ICON, "Tool Response")} + <pre>${resp}</pre> + </div> + </div> + </div>`; } -/** Render a collapsible result card for one attack pattern. */ -function resultCard(r: ResultViewModel, index: number, mode: "agent" | "mcp"): string { +/** Render one attack result: reasoning/evidence, confidence + standards, and a collapsible transcript. */ +function resultDetailCard( + r: ResultViewModel, + index: number, + _mode: "agent" | "mcp", + showTestHeading: boolean, + evalIndex: number +): string { const verdict = r.judge.verdict; - const cardClass = verdict === "PASS" ? "pass" : verdict === "ERROR" ? "error" : "fail"; - const hasTurns = r.turns && r.turns.length > 0; + const failingTurns = new Set(r.judge.failingTurns ?? []); - // Header line - let headerLabel: string; - if (mode === "agent") { - headerLabel = `<span class="result-id">Test ${r.id}: ${esc(r.label)}</span>`; - if (r.traceId) { - headerLabel += ` <span style="font-size:11px;color:var(--muted);font-family:ui-monospace,monospace">trace: ${esc(r.traceId.slice(0, 12))}…</span>`; - } - } else { - headerLabel = `<span class="result-id">#${index + 1} ${esc(r.id)}</span><span class="result-tool">${esc(r.label)}</span>`; - } - - // For multi-turn agent cards, skip the top-level prompt/response (it's shown in turns) - const showTopDetail = !(mode === "agent" && hasTurns); - - // Turns HTML - let turnsHtml = ""; - if (hasTurns) { - turnsHtml = r.turns!.map((turn) => renderTurnContent(turn)).join(""); - turnsHtml = ` - <div class="result-section"> - <div class="result-section-label">Multi-turn breakdown (${r.turns!.length} turns)</div> - ${turnsHtml} - </div>`; - } - - // Judge block - const judgeBlock = + const hasTurns = r.turns && r.turns.length > 0; + const turnCount = hasTurns ? r.turns!.length : 1; + const turnsHtml = hasTurns + ? r.turns!.map((t) => renderTurn(t, failingTurns)).join("") + : singleTurnTranscript(r.detail); + + // Deterministic per document (not a module-level counter) — repeated renders + // of the same ReportViewModel (long-lived MCP server, snapshot tests) then + // produce identical data-for/data-target values. + const tId = `t${evalIndex}-${index}`; + + const evidenceHtml = + r.judge.evidence && r.judge.evidence !== "N/A" + ? `<div class="detail-section"><div class="detail-section-label">Evidence</div><div class="detail-section-body">${esc(r.judge.evidence)}</div></div>` + : ""; + const reasoningHtml = verdict === "ERROR" - ? `<div class="result-judge fail" style="background:#FFFBEB;border-color:#FDE68A;color:#92400E"><strong>ERROR:</strong> ${esc(r.judge.errorMessage ?? (r.detail.kind === "tool" ? (r.detail.error ?? "") : r.detail.response))}</div>` - : `<div class="result-judge ${verdict === "FAIL" ? "fail" : ""}"> - <strong style="color:${verdict === "PASS" ? "var(--pass)" : "var(--fail)"}">${verdict}</strong> - · Score ${r.judge.score}/10 - · Confidence ${r.judge.confidence}% - ${r.judge.evidence && r.judge.evidence !== "N/A" ? `· Evidence: <em>${esc(truncate(r.judge.evidence, 200))}</em>` : ""} - ${verdict === "FAIL" && hasTurns && r.judge.failingTurns && r.judge.failingTurns.length > 0 ? `· Failing turn${r.judge.failingTurns.length === 1 ? "" : "s"}: ${r.judge.failingTurns.join(", ")}` : ""} - <br><span style="color:var(--text-2)">${esc(r.judge.reasoning)}</span> - </div>`; + ? `<div class="detail-section"><div class="detail-section-label">Error</div><div class="detail-section-body">${esc(r.judge.errorMessage ?? "")}</div></div>` + : r.judge.reasoning + ? `<div class="detail-section"><div class="detail-section-label">Reasoning</div><div class="detail-section-body">${esc(r.judge.reasoning)}</div></div>` + : ""; + + const confidenceCol = ` + <div class="eval-meta-col"> + <div class="detail-section-label">Confidence</div> + <div class="meta-v-lg">${verdict === "ERROR" ? "—" : `${r.judge.confidence}%`}</div> + </div>`; - const resultVerdictClass = - verdict === "PASS" ? "verdict-pass" : verdict === "ERROR" ? "verdict-error" : "verdict-fail"; return ` - <div class="result-card ${cardClass}"> - <div class="result-header"> - ${headerLabel} - <span class="verdict-tag ${resultVerdictClass}" style="margin-left:auto">${verdict}</span> + ${showTestHeading ? `<div class="test-heading">Test ${index + 1} — ${esc(r.label)}</div>` : ""} + ${reasoningHtml} + ${evidenceHtml} + <div class="eval-meta-row"> + ${confidenceCol} + </div> + <div class="transcript-wrap" data-for="${tId}"> + <div class="transcript"> + <div class="transcript-header">Conversation Transcript <span class="tc-count">${turnCount} turn${turnCount === 1 ? "" : "s"}</span></div> + ${turnsHtml} </div> - ${showTopDetail ? renderDetailContent(r.detail, mode) : ""} - ${turnsHtml} - ${judgeBlock} - </div>`; + </div> + <button class="transcript-toggle" data-target="${tId}"> + <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M6 9l6 6 6-6"/></svg> + <span class="tt-label">View more details</span> + </button>`; } diff --git a/core/src/report/types.ts b/core/src/report/types.ts index 4faab7cb..468b9a63 100644 --- a/core/src/report/types.ts +++ b/core/src/report/types.ts @@ -58,6 +58,13 @@ export interface ReportViewModel { endpoint?: string; transport?: string; suiteId?: string; + /** Human-readable target type, e.g. "LLM Chatbot Interface". Extension-only today. */ + type?: string; + /** e.g. "Browser automation (live tab)". Extension-only today. */ + accessMethod?: string; + maxTurns?: number; + waitBetweenTurnsSec?: number; + messageLengthLimit?: number; }; summary: { total: number; @@ -67,8 +74,12 @@ export interface ReportViewModel { safetyScore: number; attackSuccessRate: number; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Wall-clock time for the whole run, from the first evaluator to the last. */ + durationMs?: number; }; evaluators: EvaluatorViewModel[]; /** Set when the run was stopped early due to a non-retryable LLM error. */ stopReason?: string; + /** Free-text business/application context. Extension-only today. */ + businessContext?: string; } diff --git a/core/tests/orchestrator.equivalence.test.ts b/core/tests/orchestrator.equivalence.test.ts index 1bbe7583..fb14fc65 100644 --- a/core/tests/orchestrator.equivalence.test.ts +++ b/core/tests/orchestrator.equivalence.test.ts @@ -172,6 +172,7 @@ test("buildUnifiedReport derives safetyScore + attackSuccessRate identically for generatedAt: "t", targetName: "x", targetKind: "agent" as const, + suiteId: "custom-suite-test", effort: "adaptive" as const, attackModel: "p/m", judgeModel: "p/m", @@ -238,11 +239,13 @@ test("runAll and runAllBrowser produce matching reports for the same input", asy ); // The two paths must agree on the verdict tallies and derived summary. - assert.deepEqual( - browserReport.summary, - nodeReport.summary, - "summary must match across orchestrators" - ); + // durationMs is wall-clock and legitimately differs between the two separately + // timed runs, so it's checked for shape only, not cross-orchestrator equality. + const { durationMs: nodeDurationMs, ...nodeSummary } = nodeReport.summary; + const { durationMs: browserDurationMs, ...browserSummary } = browserReport.summary; + assert.deepEqual(browserSummary, nodeSummary, "summary must match across orchestrators"); + assert.ok(typeof nodeDurationMs === "number" && nodeDurationMs >= 0); + assert.ok(typeof browserDurationMs === "number" && browserDurationMs >= 0); assert.equal(browserReport.evaluators.length, nodeReport.evaluators.length); for (let i = 0; i < nodeReport.evaluators.length; i++) { const n = nodeReport.evaluators[i]; diff --git a/runners/cli/tests/jsonlEventListener.test.ts b/runners/cli/tests/jsonlEventListener.test.ts index 3443d75b..61e1f5cc 100644 --- a/runners/cli/tests/jsonlEventListener.test.ts +++ b/runners/cli/tests/jsonlEventListener.test.ts @@ -34,6 +34,7 @@ test("writes one NDJSON line per lifecycle event with type + payload", async () reportId: "r1", generatedAt: "2026-01-01T00:00:00.000Z", targetName: "target", + suiteId: "quick-smoke", targetKind: "agent", effort: "adaptive", attackModel: "attacker-model", diff --git a/runners/cli/tests/knowledge.test.ts b/runners/cli/tests/knowledge.test.ts index bc3031c6..35a1a73a 100644 --- a/runners/cli/tests/knowledge.test.ts +++ b/runners/cli/tests/knowledge.test.ts @@ -8,8 +8,10 @@ test("loadKnowledge loads the bundled seed libraries", async () => { assert.ok(kb.personas.length >= 3, "expected several persona seeds"); assert.ok(kb.strategies.length >= 3, "expected several strategy seeds"); - const injection = kb.vulnClasses.find((v) => v.id === "prompt-injection"); - assert.ok(injection, "prompt-injection vuln-class present"); + // Vuln-class ids are the evaluator *category* ids (evaluators/agent/<id>/README.md), + // not individual evaluator ids — "injection", not "prompt-injection". + const injection = kb.vulnClasses.find((v) => v.id === "injection"); + assert.ok(injection, "injection vuln-class present"); assert.ok(injection!.failRubric.length > 0, "fail rubric parsed"); assert.ok(injection!.passRubric.length > 0, "pass rubric parsed"); assert.equal(injection!.severity, "critical"); diff --git a/runners/extension/popup.js b/runners/extension/popup.js index ea1ec222..7d62296c 100644 --- a/runners/extension/popup.js +++ b/runners/extension/popup.js @@ -6,7 +6,12 @@ // OPFOR_UI_RUN / RESUME / STOP / DISCARD_PAUSED message contracts. // ───────────────────────────────────────────────────────────────── -import { PROVIDERS, PROVIDER_CAPABILITIES, PROVIDER_DISPLAY_NAMES } from "./dist/core.bundle.js"; +import { + PROVIDERS, + PROVIDER_CAPABILITIES, + PROVIDER_DISPLAY_NAMES, + renderReport, +} from "./dist/core.bundle.js"; const PROVIDER_OPTIONS = Object.values(PROVIDERS).map((value) => ({ value, @@ -112,6 +117,9 @@ const state = { /** @type {{id:string;name:string;sev:string;verdict:string;summary:string;raw:any}[]} */ ([]), lastReport: /** @type {any | null} */ (null), running: false, + // Wall-clock start of the current/last run, persisted alongside the queue so + // it survives a popup close/reopen mid-run — used to compute report duration. + runStartedAt: /** @type {number | null} */ (null), // True only while THIS popup instance's startRun() loop is actively // driving a run — as opposed to `running`, which also gets set to true // when the popup reopens mid-run and merely restores the running screen @@ -732,6 +740,7 @@ async function checkPausedRun() { state.queue = opforPopupRun.queue; state.results = Array.isArray(opforPopupRun.results) ? opforPopupRun.results : []; state.maxTurns = opforPopupRun.maxTurns || state.maxTurns; + state.runStartedAt = opforPopupRun.runStartedAt || state.runStartedAt; // Point at the paused evaluator when it's still in the queue. const idx = state.queue.findIndex((q) => q.id === evId); state.evIdx = idx >= 0 ? idx : Math.min(opforPopupRun.evIdx || 0, state.queue.length - 1); @@ -1142,6 +1151,27 @@ function judgedCount(summary) { return Number(summary?.judged ?? summary?.totalEvaluators ?? summary?.totalTests ?? 0) || 0; } +/** + * Label for the report's "Evaluation Suite" field. A suite name is only + * accurate when every one of its evaluators was actually selected — picking + * a subset (or the catch-all "Custom Evaluators" bucket) is a hand-picked + * list, not that suite, so it's labeled "Custom Suite" instead of borrowing + * a name that would overstate what actually ran. + */ +function resolveSuiteLabel() { + const suite = state.catalog?.suites?.find((s) => s.id === state.suiteId); + if (!suite) return "Custom Suite"; + // Based on what actually completed, not what was selected — a run stopped + // partway through a full-suite selection is not that suite either. + const completedIds = new Set( + state.results.filter((r) => r.verdict !== "CANCELLED").map((r) => r.id) + ); + const fullSuite = + suite.evaluatorIds.length === completedIds.size && + suite.evaluatorIds.every((id) => completedIds.has(id)); + return fullSuite ? suite.id : "Custom Suite"; +} + function buildReport() { const total = state.results.length; const passed = state.results.filter((r) => r.verdict === "PASS").length; @@ -1243,16 +1273,18 @@ function buildReport() { const stamp = now.toISOString().replace(/[-:]/g, "").replace(/\..+/, "").replace("T", "-"); const reportId = `opfor-${state.suiteId || "run"}-${stamp}`; + const durationMs = state.runStartedAt ? Math.max(0, Date.now() - state.runStartedAt) : undefined; + const targetUrl = state.results[0]?.raw?.siteUrl || state.results[0]?.raw?.frameUrl || "active tab"; return { metadata: { reportId, - configId: state.suiteId || "run", + configId: resolveSuiteLabel(), framework: "opfor v0.2", generated: now.toISOString(), - duration: "—", + duration: durationMs != null ? formatDurationMs(durationMs) : "—", llmJudge: state.model, }, target: { @@ -1284,6 +1316,7 @@ function buildReport() { criticalFindings: criticalFindings.length, highFindings: highFindings.length, tokenUsage, + durationMs, }, cancelled: state.runCancelled, evaluatorResults, @@ -1292,613 +1325,137 @@ function buildReport() { }; } -function escapeHtml(s) { - return String(s ?? "") - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, """) - .replace(/'/g, "'"); +function formatTokenCount(n) { + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`; + return String(n); } -function truncate(s, n) { - const str = String(s ?? ""); - return str.length > n ? str.slice(0, n) + "…" : str; +/** Format a run duration for display (e.g. 754000 → "12m 34s"). */ +function formatDurationMs(ms) { + const totalSeconds = Math.round(ms / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + if (hours > 0) return `${hours}h ${minutes}m`; + if (minutes > 0) return `${minutes}m ${seconds}s`; + return `${seconds}s`; +} + +// --------------------------------------------------------------------------- +// Adapter: extension's buildReport() shape → core's shared ReportViewModel +// +// Renders through the same `renderReport` the CLI/MCP/SDK runners use, so the +// extension's downloaded HTML report looks identical to theirs instead of +// maintaining its own template. +// --------------------------------------------------------------------------- + +/** Extract {user, assistant} turn pairs from a raw evaluator record's transcript/turns. */ +function turnsForReport(raw) { + const tr = Array.isArray(raw?.transcript) ? raw.transcript : []; + if (tr.length) { + const pairs = []; + for (let i = 0; i < tr.length; i += 2) { + const u = tr[i]?.role === "user" ? String(tr[i].content || "") : ""; + const next = tr[i + 1]; + const a = next && next.role === "assistant" ? String(next.content || "") : ""; + if (u || a) pairs.push({ user: u, assistant: a }); + } + if (pairs.length) return pairs; + } + const tl = Array.isArray(raw?.turns) ? raw.turns : []; + return tl.map((t) => ({ + user: String( + t.userMessage || t.user || t.attacker || (t.role === "user" ? t.content : "") || "" + ), + assistant: String( + t.assistantPreview || + t.bot || + t.agent || + t.assistant || + (t.role === "assistant" ? t.content : "") || + "" + ), + })); } -function safetyColor(score) { - if (score >= 70) return "#059669"; - if (score >= 50) return "#D97706"; - return "#DC2626"; +function toTurnViewModels(raw) { + return turnsForReport(raw).map((t, i) => ({ + turnIndex: i + 1, + detail: { kind: "prompt", prompt: t.user, response: t.assistant }, + })); } -function formatTokenCount(n) { - if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; - if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`; - return String(n); +function toResultViewModel(tr, evaluator) { + const cancelled = tr.verdict === "CANCELLED"; + return { + id: evaluator.id + (tr.testNumber > 1 ? `-${tr.testNumber}` : ""), + label: tr.pattern, + judge: { + verdict: cancelled ? "ERROR" : tr.verdict, + score: tr.score ?? 0, + confidence: tr.confidence ?? 0, + evidence: tr.evidence ?? "", + reasoning: cancelled + ? "Run cancelled by user before this evaluator completed." + : (tr.reasoning ?? ""), + errorMessage: cancelled ? "Cancelled by user" : undefined, + }, + detail: { kind: "prompt", prompt: "", response: "" }, + turns: toTurnViewModels(evaluator.raw), + }; } -function sevDot(sev) { - return { critical: "🔴", high: "🟠", medium: "🟡", low: "🟢" }[sev] || "⚪"; +function toEvaluatorViewModel(e) { + return { + evaluatorId: e.id, + evaluatorName: e.name, + standards: e.standards, + severity: e.severity, + total: e.totalTests, + passed: e.passed, + failed: e.failed, + errors: e.testResults[0]?.verdict === "CANCELLED" ? 1 : 0, + passRate: e.passRate, + results: e.testResults.map((tr) => toResultViewModel(tr, e)), + tokenUsage: e.raw?.tokenUsage, + }; } -function generateHtmlReport(report) { - const { metadata, target, summary, evaluatorResults, criticalFindings, highFindings } = report; - // Absent when nothing was judged — grading an unassessed run as "Critical - // Risk" would be a fabricated conclusion, not a conservative default. - const scored = summary.safetyScore != null; - const judged = judgedCount(summary); - const overallVerdict = - report.cancelled || summary.cancelled > 0 || judged === 0 - ? "CANCELLED" - : summary.failed === 0 && summary.totalTests > 0 - ? "PASS" - : "FAIL"; - const riskLevel = !scored - ? { label: "Not Assessed", color: "#64748B", bg: "#F1F5F9", border: "#CBD5E1" } - : summary.safetyScore >= 80 - ? { label: "Low Risk", color: "#059669", bg: "#D1FAE5", border: "#6EE7B7" } - : summary.safetyScore >= 60 - ? { label: "Medium Risk", color: "#D97706", bg: "#FEF3C7", border: "#FCD34D" } - : summary.safetyScore >= 40 - ? { label: "High Risk", color: "#DC2626", bg: "#FEE2E2", border: "#FCA5A5" } - : { label: "Critical Risk", color: "#991B1B", bg: "#FEE2E2", border: "#EF4444" }; - const genDate = new Date(metadata.generated); - const dateStr = genDate.toLocaleDateString("en-US", { - year: "numeric", - month: "long", - day: "numeric", - }); - const timeStr = genDate.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit" }); - - // ── Turns helper ─────────────────────────────────────────────── - const turnsForReport = (raw) => { - const tr = Array.isArray(raw?.transcript) ? raw.transcript : []; - if (tr.length) { - const pairs = []; - for (let i = 0; i < tr.length; i += 2) { - const u = tr[i]?.role === "user" ? String(tr[i].content || "") : ""; - const next = tr[i + 1]; - const a = next && next.role === "assistant" ? String(next.content || "") : ""; - if (u || a) pairs.push({ user: u, assistant: a }); - } - if (pairs.length) return pairs; - } - const tl = Array.isArray(raw?.turns) ? raw.turns : []; - return tl.map((t) => ({ - user: String( - t.userMessage || t.user || t.attacker || (t.role === "user" ? t.content : "") || "" - ), - assistant: String( - t.assistantPreview || - t.bot || - t.agent || - t.assistant || - (t.role === "assistant" ? t.content : "") || - "" - ), - })); +function toReportViewModel(report) { + const { metadata, target, summary } = report; + return { + mode: "agent", + reportId: metadata.reportId, + generatedAt: metadata.generated, + generatorModel: target.model, + judgeModel: metadata.llmJudge || target.model, + target: { + name: target.name, + suiteId: metadata.configId, + type: "LLM Chatbot Interface", + accessMethod: "Browser automation (live tab)", + maxTurns: state.maxTurns, + waitBetweenTurnsSec: state.waitSec, + messageLengthLimit: state.messageCharLimit, + }, + summary: { + total: summary.totalEvaluators, + passed: summary.passed, + failed: summary.failed, + errors: summary.cancelled || 0, + safetyScore: summary.safetyScore ?? 0, + attackSuccessRate: summary.attackSuccessRate ?? 0, + tokenUsage: summary.tokenUsage, + durationMs: summary.durationMs, + }, + evaluators: report.evaluatorResults.map(toEvaluatorViewModel), + stopReason: report.cancelled ? "user-interrupted" : undefined, + businessContext: + report.applicationContext?.purpose && report.applicationContext.purpose !== "—" + ? report.applicationContext.purpose + : undefined, }; - - // ── Evaluator details ────────────────────────────────────────── - const appendix = evaluatorResults - .map((e, idx) => { - const turns = turnsForReport(e.raw); - const tr = e.testResults[0] || {}; - const sevColor = SEV_HEX[e.severity] || "#64748B"; - const verdictPass = tr.verdict === "PASS"; - const verdictCancelled = tr.verdict === "CANCELLED"; - const verdictClass = verdictPass - ? "verdict-pass" - : verdictCancelled - ? "verdict-cancelled" - : "verdict-fail"; - const standardsLabel = formatStandardsLabel(e.standards); - const transcript = turns.length - ? `<div class="transcript"> - <div class="transcript-header">Conversation Transcript <span class="tc-count">${turns.length} turn${turns.length === 1 ? "" : "s"}</span></div> - ${turns - .map( - (t, i) => ` - <div class="turn"> - <div class="turn-row"> - <div class="turn-role attacker-role">Attacker · Turn ${i + 1}</div> - <pre>${escapeHtml(truncate(t.user, 4000))}</pre> - </div> - <div class="turn-row"> - <div class="turn-role agent-role">Agent · Turn ${i + 1}</div> - <pre>${escapeHtml(truncate(t.assistant, 4000))}</pre> - </div> - </div>` - ) - .join("")} - </div>` - : ""; - return ` - <details class="eval-detail" id="eval-${idx}"> - <summary> - <div class="eval-summary-left"> - <span class="eval-num">${String(idx + 1).padStart(2, "0")}</span> - <div class="eval-summary-info"> - <span class="eval-summary-name">${escapeHtml(e.name)}</span> - <span class="sev-tag" style="background:${sevColor}18;color:${sevColor};border-color:${sevColor}44">${escapeHtml(e.severity)}</span> - ${standardsLabel ? `<span class="standards-tag">${escapeHtml(standardsLabel)}</span>` : ""} - </div> - </div> - <div class="eval-summary-right"> - ${e.raw?.tokenUsage?.totalTokens ? `<span style="font-size:11px;color:var(--muted)">${formatTokenCount(e.raw.tokenUsage.totalTokens)} tokens</span>` : ""} - <span class="score-badge">${tr.score ?? "—"}<span class="score-denom">/10</span></span> - <span class="verdict-tag ${verdictClass}">${tr.verdict || "—"}</span> - <svg class="chevron" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M6 9l6 6 6-6"/></svg> - </div> - </summary> - <div class="eval-body"> - <div class="eval-meta-grid"> - <div class="meta-item"><div class="meta-k">Verdict</div><div class="meta-v ${verdictPass ? "pass-text" : verdictCancelled ? "cancelled-text" : "fail-text"}">${tr.verdict || "—"}</div></div> - <div class="meta-item"><div class="meta-k">Safety Score</div><div class="meta-v">${tr.score ?? "—"} / 10</div></div> - <div class="meta-item"><div class="meta-k">Confidence</div><div class="meta-v">${tr.confidence != null ? tr.confidence + "%" : "—"}</div></div> - <div class="meta-item"><div class="meta-k">Severity</div><div class="meta-v"><span class="sev-tag" style="background:${sevColor}18;color:${sevColor};border-color:${sevColor}44">${escapeHtml(e.severity)}</span></div></div> - ${ - standardsLabel - ? `<div class="meta-item meta-item-wide"><div class="meta-k">Standards</div><div class="meta-v standards-meta">${escapeHtml(standardsLabel)}</div></div>` - : "" - } - </div> - ${ - tr.evidence && tr.evidence !== "N/A" - ? `<div class="detail-section"><div class="detail-section-label">Evidence</div><div class="detail-section-body">${escapeHtml(tr.evidence)}</div></div>` - : "" - } - ${ - tr.reasoning - ? `<div class="detail-section"><div class="detail-section-label">Reasoning</div><div class="detail-section-body">${escapeHtml(tr.reasoning)}</div></div>` - : "" - } - ${transcript} - </div> - </details>`; - }) - .join(""); - - // ── Findings list ────────────────────────────────────────────── - const findingBlock = (label, list, color) => - list.length === 0 - ? "" - : `<div class="finding-block" style="--fc:${color}"> - <div class="finding-block-head"> - <span class="finding-label" style="color:${color}">${escapeHtml(label)}</span> - <span class="finding-count" style="background:${color}18;color:${color};border-color:${color}44">${list.length}</span> - </div> - <ol class="finding-list"> - ${list - .map( - (f) => `<li> - <strong>${escapeHtml(f.evaluator)}</strong> - <span class="finding-score">Score ${f.score}/10</span> - <div class="finding-desc">${escapeHtml(truncate(f.description, 240))}</div> - </li>` - ) - .join("")} - </ol> - </div>`; - - // ── Results table rows ───────────────────────────────────────── - const tableRows = evaluatorResults - .map((e, idx) => { - const sevColor = SEV_HEX[e.severity] || "#64748B"; - const rowVerdict = - e.testResults[0]?.verdict || (e.passed > 0 && e.failed === 0 ? "PASS" : "FAIL"); - const verdictClass = - rowVerdict === "PASS" - ? "verdict-pass" - : rowVerdict === "CANCELLED" - ? "verdict-cancelled" - : "verdict-fail"; - const standardsLabel = formatStandardsLabel(e.standards); - return ` - <tr> - <td class="td-num">${String(idx + 1).padStart(2, "0")}</td> - <td><a href="#eval-${idx}" class="eval-link">${escapeHtml(e.name)}</a>${standardsLabel ? `<br><span class="standards-tag">${escapeHtml(standardsLabel)}</span>` : ""}</td> - <td><span class="sev-tag" style="background:${sevColor}18;color:${sevColor};border-color:${sevColor}44">${escapeHtml(e.severity)}</span></td> - <td><span class="verdict-tag ${verdictClass}">${escapeHtml(rowVerdict)}</span></td> - <td class="td-score">${e.avgScore != null ? `${e.avgScore.toFixed(1)}<span style="color:#94A3B8">/10</span>` : "—"}</td> - </tr>`; - }) - .join(""); - - return `<!doctype html> -<html lang="en"> -<head> -<meta charset="utf-8"/> -<meta name="viewport" content="width=device-width,initial-scale=1"/> -<title>Opfor Security Report — ${escapeHtml(target.name)} - - - - -
-
-
-
-
- -
-
-
Opfor
-
Red-team Platform
-
-
-
Confidential
-
-
LLM Security Assessment Report
-
Automated adversarial evaluation · ${escapeHtml(metadata.framework)} · ${dateStr}
-
-
-
Target System
-
${escapeHtml(target.name)}
-
-
-
Evaluation Suite
-
${escapeHtml(metadata.configId)}
-
-
-
Assessment Date
-
${dateStr}, ${timeStr}
-
-
-
Evaluators Run
-
${summary.totalEvaluators}
-
-
-
Attacker Model
-
${escapeHtml(target.model)}
-
-
-
Report ID
-
${escapeHtml(metadata.reportId)}
-
-
-
-
- -
- - -
-
-
1
-
Executive Summary
-
-
-
-
- ${ - overallVerdict === "PASS" - ? `` - : overallVerdict === "CANCELLED" - ? `` - : `` - } -
-
-
Overall Verdict
-
${overallVerdict}
-
-
-
${overallVerdict === "CANCELLED" ? "Run Cancelled" : riskLevel.label}
-
-
-
-
Safety Score
-
${scored ? `${summary.safetyScore}%` : "—"}
-
-
${scored ? `Based on ${judged} evaluator${judged === 1 ? "" : "s"}` : "No evaluator completed"}${summary.cancelled ? ` · ${summary.cancelled} cancelled` : ""}
-
-
-
Attack Success Rate
-
0 ? "#DC2626" : "#059669"}">${scored ? `${summary.attackSuccessRate}%` : "—"}
-
0 ? "#DC2626" : "#059669"}">
-
${scored ? `${summary.failed} of ${judged} evaluators breached` : "Nothing was evaluated"}
-
-
-
Evaluators Passed
-
${summary.passed}
-
No vulnerability surfaced
-
-
-
Evaluators Failed
-
${summary.failed}
-
${criticalFindings.length} critical · ${highFindings.length} high severity
-
- ${ - summary.tokenUsage - ? `
-
Token Usage
-
${formatTokenCount(summary.tokenUsage.totalTokens)}
-
${summary.tokenUsage.inputTokens.toLocaleString()} in · ${summary.tokenUsage.outputTokens.toLocaleString()} out
-
` - : "" - } -
-
- ${ - overallVerdict === "CANCELLED" - ? `⚠ Run cancelled by user. The assessment of ${escapeHtml(target.name)} in the ${escapeHtml(metadata.configId)} suite was stopped before all evaluators could complete. Of ${summary.totalTests} evaluator${summary.totalTests === 1 ? "" : "s"} recorded, ${summary.passed} passed and ${summary.failed} failed${summary.cancelled ? `, and ${summary.cancelled} ${summary.cancelled === 1 ? "was" : "were"} interrupted mid-evaluation` : ""}. This report reflects only the evaluators completed up to the point of cancellation.` - : overallVerdict === "PASS" - ? `The target system ${escapeHtml(target.name)} passed all ${summary.totalTests} evaluator${summary.totalTests === 1 ? "" : "s"} in the ${escapeHtml(metadata.configId)} suite. No exploitable vulnerabilities were surfaced under sustained adversarial pressure with the configured turn budget. The system demonstrates adequate resistance to the evaluated attack patterns at the time of assessment.` - : `The target system ${escapeHtml(target.name)} failed ${summary.failed} of ${summary.totalTests} evaluator${summary.totalTests === 1 ? "" : "s"} (${summary.attackSuccessRate}% attack success rate) in the ${escapeHtml(metadata.configId)} suite. ${summary.failed === 1 ? "One vulnerability was" : "Multiple vulnerabilities were"} surfaced under adversarial pressure.${criticalFindings.length > 0 ? ` ${criticalFindings.length} critical finding${criticalFindings.length === 1 ? "" : "s"} require immediate remediation.` : ""} Refer to the Findings section for a prioritised remediation plan.` - } -
-
- - -
-
-
2
-
Assessment Scope
-
-
-
-
Target
-
System${escapeHtml(target.name)}
-
TypeLLM Chatbot Interface
-
Access methodBrowser automation (live tab)
-
-
-
Evaluation Parameters
-
Suite${escapeHtml(metadata.configId)}
-
Attacker model${escapeHtml(target.model)}
-
Max turns / evaluator${state.maxTurns}
-
Wait between turns${state.waitSec}s
-
Message length limit${state.messageCharLimit} chars
-
- ${ - state.businessUseCase - ? `
-
Business Context
-
${escapeHtml(state.businessUseCase)}
-
` - : "" - } -
-
- - - ${ - criticalFindings.length + highFindings.length > 0 - ? `
-
-
3
-
Key Findings
-
${criticalFindings.length + highFindings.length} finding${criticalFindings.length + highFindings.length === 1 ? "" : "s"} requiring attention
-
-
- ${findingBlock("Critical", criticalFindings, "#DC2626")} - ${findingBlock("High", highFindings, "#D97706")} -
-
` - : `
-
-
3
-
Key Findings
-
-
${ - judged === 0 - ? "No critical or high severity findings — no evaluator was assessed before the run was cancelled." - : "No critical or high severity findings — system passed all evaluated attack patterns." - }
-
` - } - - -
-
-
4
-
Evaluation Results
-
${summary.totalEvaluators} evaluator${summary.totalEvaluators === 1 ? "" : "s"}
-
-
- - - - - ${tableRows || ``} -
#EvaluatorSeverityVerdictSafety Score
No evaluators executed.
-
-
-
-
5
-
Detailed Results
-
- ${appendix} -
- -
- -
-
Generated by ${escapeHtml(metadata.framework)} · ${dateStr}
-
${escapeHtml(metadata.reportId)}
-
- - -`; } const REPORT_HISTORY_KEY = "opforReportHistory"; @@ -2038,7 +1595,7 @@ async function finalizeAndPersistCurrentReport() { } function downloadReportHtml(report) { - const html = generateHtmlReport(report); + const html = renderReport(toReportViewModel(report)); const blob = new Blob([html], { type: "text/html" }); const url = URL.createObjectURL(blob); const a = document.createElement("a"); @@ -2700,6 +2257,7 @@ async function startRun({ resume = false } = {}) { // Build queue from current selection (or use existing queue if resuming) if (!resume) { + state.runStartedAt = Date.now(); await saveSettings(); const suite = state.catalog?.suites.find((s) => s.id === state.suiteId); if (!suite) { @@ -3480,6 +3038,7 @@ async function persistPopupRunQueue({ running = true, paused = false } = {}) { evIdx: state.evIdx, results: state.results, maxTurns: state.maxTurns, + runStartedAt: state.runStartedAt, updatedAt: Date.now(), }, }); @@ -3549,6 +3108,12 @@ async function checkActiveRun() { } } + // Restored independently of `popupQueueActive`: a run can still be genuinely + // active per `opforRunStatus` well past the 5-minute popup-queue freshness + // window (long evaluators, slow targets), and losing the original start time + // there would silently drop the report's duration. + state.runStartedAt = opforPopupRun?.runStartedAt || state.runStartedAt; + if (popupQueueActive) { state.suiteId = opforPopupRun.suiteId || state.suiteId; state.maxTurns = opforPopupRun.maxTurns || state.maxTurns; diff --git a/runners/sdk/src/report.ts b/runners/sdk/src/report.ts index f19a5617..b41fe27b 100644 --- a/runners/sdk/src/report.ts +++ b/runners/sdk/src/report.ts @@ -1,7 +1,13 @@ import { writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import type { RunResults } from "./types.js"; +import { renderReport } from "@keyvaluesystems/agent-opfor-core/report/render.js"; +import type { + ReportViewModel, + EvaluatorViewModel, + ResultViewModel, +} from "@keyvaluesystems/agent-opfor-core/report/types.js"; +import type { RunResults, AttackResult, EvaluatorResult } from "./types.js"; export interface ReportBuilder { /** @@ -38,7 +44,7 @@ export function report(results: RunResults): ReportBuilder { const resolvedPath = path.resolve(toFsPath(outputPath)); await mkdir(path.dirname(resolvedPath), { recursive: true }); - const html = renderHtmlReport(results); + const html = renderReport(toReportViewModel(results)); await writeFile(resolvedPath, html, "utf8"); return resolvedPath; }, @@ -49,253 +55,59 @@ function toFsPath(p: string | URL): string { return p instanceof URL ? fileURLToPath(p) : p; } -function renderHtmlReport(results: RunResults): string { - const findingRows = results.findings - .map( - (f) => ` - - ${f.severity.toUpperCase()} - ${escapeHtml(f.title)} - ${escapeHtml(f.evaluatorId)} - ${escapeHtml(f.evidence ?? "")} - ` - ) - .join("\n"); +// --------------------------------------------------------------------------- +// Adapter: SDK RunResults → core's shared ReportViewModel +// +// Renders through the same `renderReport` the CLI/MCP runners and the +// browser extension use, so the SDK's HTML report looks identical to theirs +// instead of maintaining its own template. +// --------------------------------------------------------------------------- - const evaluatorRows = results.evaluators - .map( - (e) => ` - - ${escapeHtml(e.evaluatorName)} - ${e.severity} - ${e.total} - ${e.passed} - ${e.failed} - ${e.errors} - ${(e.passRate * 100).toFixed(0)}% - ` - ) - .join("\n"); +function toReportViewModel(results: RunResults): ReportViewModel { + return { + mode: results.targetKind, + reportId: results.id, + generatedAt: results.timestamp, + generatorModel: results.attackerModel, + judgeModel: results.judgeModel, + target: { name: results.targetName, suiteId: results.suiteId }, + summary: results.summary, + evaluators: results.evaluators.map(toEvaluatorViewModel), + }; +} - return ` - - - - - Opfor Security Report - ${escapeHtml(results.targetName)} - - - -
-
-
-

Opfor Security Report

-

Target: ${escapeHtml(results.targetName)} (${results.targetKind})

-

Generated: ${new Date(results.timestamp).toLocaleString()}

-

Models: ${escapeHtml(results.attackerModel)} / ${escapeHtml(results.judgeModel)}

-
-
-
${results.score}%
-
Safety Score
-
-
- -
-
-
${results.summary.total}
-
Total Attacks
-
-
-
${results.summary.passed}
-
Passed
-
-
-
${results.summary.failed}
-
Failed
-
-
-
${results.summary.errors}
-
Errors
-
-
-
${results.findings.length}
-
Findings
-
-
- -

Findings

- ${ - results.findings.length === 0 - ? '
✓ No vulnerabilities detected
' - : ` - - - - - - - - - - - ${findingRows} - -
SeverityFindingEvaluatorEvidence
` - } - -

Evaluators

- - - - - - - - - - - - - - ${evaluatorRows} - -
EvaluatorSeverityTotalPassedFailedErrorsPass Rate
- -

- Report ID: ${results.id} | Effort: ${results.effort} -

-
- -`; +function toEvaluatorViewModel(e: EvaluatorResult): EvaluatorViewModel { + return { + evaluatorId: e.evaluatorId, + evaluatorName: e.evaluatorName, + standards: e.standards, + severity: e.severity, + total: e.total, + passed: e.passed, + failed: e.failed, + errors: e.errors, + passRate: e.passRate, + results: e.attacks.map(toResultViewModel), + }; } -function escapeHtml(str: string): string { - return str - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """) - .replace(/'/g, "'"); +function toResultViewModel(a: AttackResult): ResultViewModel { + return { + id: a.attackId, + label: a.patternName, + judge: { + verdict: a.verdict, + score: a.score ?? 0, + confidence: a.confidence ?? 0, + evidence: a.evidence ?? "", + reasoning: a.reasoning ?? "", + failingTurns: a.failingTurns, + errorMessage: a.errorMessage, + }, + detail: { kind: "prompt", prompt: a.prompt, response: a.response }, + turns: a.turns?.map((t) => ({ + turnIndex: t.turnIndex, + detail: { kind: "prompt", prompt: t.prompt, response: t.response }, + })), + }; } diff --git a/runners/sdk/src/run.ts b/runners/sdk/src/run.ts index 2d41b1f4..beea411d 100644 --- a/runners/sdk/src/run.ts +++ b/runners/sdk/src/run.ts @@ -81,6 +81,7 @@ function transformReport(coreReport: UnifiedRunReport): RunResults { id: coreReport.reportId, timestamp: coreReport.generatedAt, targetName: coreReport.targetName, + suiteId: coreReport.suiteId, targetKind: coreReport.targetKind, effort: coreReport.effort, attackerModel: coreReport.attackModel, @@ -107,6 +108,8 @@ function extractFindings(report: UnifiedRunReport): Finding[] { description: attack.judge.reasoning ?? "", evidence: attack.judge.evidence, standards: evaluator.standards, + score: attack.judge.score, + confidence: attack.judge.confidence, }); } } @@ -145,6 +148,11 @@ function transformAttackResult( response, verdict: core.judge.verdict, evidence: core.judge.evidence, + score: core.judge.score, + confidence: core.judge.confidence, + reasoning: core.judge.reasoning, + errorMessage: core.judge.errorMessage, + failingTurns: core.judge.failingTurns, turns: core.turns?.map((t) => { if (t.kind === "agent") { return { diff --git a/runners/sdk/src/types.ts b/runners/sdk/src/types.ts index 4f9d20eb..9b926aec 100644 --- a/runners/sdk/src/types.ts +++ b/runners/sdk/src/types.ts @@ -150,6 +150,10 @@ export interface Finding { description: string; evidence?: string; standards?: Record; + /** Judge safety score (0-10). */ + score?: number; + /** Judge confidence (0-100). */ + confidence?: number; } export interface AttackResult { @@ -160,6 +164,16 @@ export interface AttackResult { response: string; verdict: "PASS" | "FAIL" | "ERROR"; evidence?: string; + /** Judge safety score (0-10). */ + score?: number; + /** Judge confidence (0-100). */ + confidence?: number; + /** Judge's reasoning for the verdict. */ + reasoning?: string; + /** Set when verdict is "ERROR" — why the judge/attack failed to complete. */ + errorMessage?: string; + /** For multi-turn attacks, the 1-indexed turns that caused a FAIL verdict. */ + failingTurns?: number[]; turns?: Array<{ turnIndex: number; prompt: string; @@ -184,6 +198,8 @@ export interface RunResults { id: string; timestamp: string; targetName: string; + /** The evaluation suite that was run, or "Custom Suite" for a hand-picked selection. */ + suiteId: string; targetKind: "agent" | "mcp"; effort: Effort; attackerModel: string; @@ -196,6 +212,8 @@ export interface RunResults { errors: number; safetyScore: number; attackSuccessRate: number; + /** Wall-clock time for the whole run, from the first evaluator to the last. */ + durationMs?: number; }; findings: Finding[]; evaluators: EvaluatorResult[]; diff --git a/runners/sdk/tests/opfor-class.test.ts b/runners/sdk/tests/opfor-class.test.ts index 63ad05a6..4f06d4e9 100644 --- a/runners/sdk/tests/opfor-class.test.ts +++ b/runners/sdk/tests/opfor-class.test.ts @@ -35,6 +35,7 @@ describe("Opfor class", () => { id: "test-id", timestamp: new Date().toISOString(), targetName: "Test", + suiteId: "quick-smoke", targetKind: "agent" as const, effort: "adaptive" as const, attackerModel: "test", diff --git a/runners/sdk/tests/report.test.ts b/runners/sdk/tests/report.test.ts index 2fd1c6d4..c4ebc434 100644 --- a/runners/sdk/tests/report.test.ts +++ b/runners/sdk/tests/report.test.ts @@ -22,6 +22,7 @@ function createMockResults(overrides: Partial = {}): RunResults { id: "test-report-123", timestamp: new Date().toISOString(), targetName: "Test Target", + suiteId: "quick-smoke", targetKind: "agent", effort: "adaptive", attackerModel: "claude-sonnet-4", diff --git a/runners/sdk/tests/types.test.ts b/runners/sdk/tests/types.test.ts index 7b8c79f0..3bbc3a1b 100644 --- a/runners/sdk/tests/types.test.ts +++ b/runners/sdk/tests/types.test.ts @@ -185,6 +185,7 @@ describe("SDK types", () => { id: "report-123", timestamp: new Date().toISOString(), targetName: "Test", + suiteId: "quick-smoke", targetKind: "agent", effort: "adaptive", attackerModel: "claude-sonnet-4", @@ -236,6 +237,7 @@ describe("SDK types", () => { id: "r1", timestamp: "2026-01-01T00:00:00Z", targetName: "T", + suiteId: "quick-smoke", targetKind: "agent", effort: "adaptive", attackerModel: "m",