Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ jobs:
/internal-explorer/bundles/0x1 /api/internal-explorer/blocks \
/tips /tips/block/0x1 /tips/bundles/0x1 /api/tips/blocks \
/benchmark /benchmark/run/latest /benchmark/run-comparison/1 \
/benchmark/load-tests/sepolia /scaling /scaling/da /scaling/guardrails /scaling/runs; do
/benchmark/load-tests/sepolia /scaling /scaling/da /scaling/guardrails /scaling/runs /scaling/runs/bench/1 \
/scaling/runs/load/sepolia/1; do
code=$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:3000${route}")
if [ "${code}" != "404" ]; then
echo "FAIL: ${route} returned ${code}, expected 404"
Expand Down
2 changes: 1 addition & 1 deletion app/benchmark/components/RunList.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import StatusSummary from "./StatusSummary";
import ConfigurationTags from "./ConfigurationTags";
import Tooltip from "./Tooltip";
import MachineInfo from "./MachineInfo";
import { downloadRunInfo, downloadRunInfoCSV } from "../utils/downloadUtils";
import { downloadRunInfo, downloadRunInfoCSV } from "../../scaling/runs/lib/downloadUtils";
import FloatingDropdown from "./FloatingDropdown";
import clsx from "classnames";

Expand Down
2 changes: 1 addition & 1 deletion app/benchmark/components/RunListFilter.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { useState } from "react";
import { camelToTitleCase, formatLabel } from "../../scaling/runs/lib/formatters";
import { FilterValue } from "../filter";
import { BenchmarkRunWithStatus } from "../../scaling/runs/lib/types";
import { downloadAllRuns, downloadAllRunsCSV } from "../utils/downloadUtils";
import { downloadAllRuns, downloadAllRunsCSV } from "../../scaling/runs/lib/downloadUtils";
import Select from "./Select";
import FloatingDropdown from "./FloatingDropdown";
import clsx from "classnames";
Expand Down
71 changes: 71 additions & 0 deletions app/scaling/runs/bench/[id]/BenchRun.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import { renderToStaticMarkup } from 'react-dom/server';
import { expect, it, vi } from 'vitest';

import fixture from '../../fixtures/synthetic.json';
import { CHART_GROUPS } from '../../lib/chartGroups';
import { SORTED_CHART_CONFIG } from '../../lib/metricDefinitions';
import type { BenchmarkRuns } from '../../lib/types';
import { BenchRun } from './BenchRun';

const swr = vi.hoisted(() => ({ meta: {} as object, metrics: {} as object }));
vi.mock('swr', () => ({ default: (key: unknown) => (Array.isArray(key) ? swr.metrics : swr.meta) }));

const [run, other] = (fixture.metadata as BenchmarkRuns).runs;
const secondConfig = {
...run,
outputDir: 'synthetic-1b',
testConfig: { ...run.testConfig, GasLimit: 200000000 },
thresholds: { error: { 'validator/latency/new_payload': 25000000 } },
};
swr.meta = { data: { runs: [run, secondConfig, other] } };
const block = (n: number, metrics: Record<string, number>) => ({ BlockNumber: n, ExecutionMetrics: metrics });
swr.metrics = {
data: [
[block(1, { 'latency/get_payload': 0.01, 'gas/per_block': 1 }), block(2, { 'latency/get_payload': 0.02, 'gas/per_block': 2 })],
[block(1, { 'gas/per_block': 1, reth_sync_execution_execution_duration: 0.005 }), block(2, { 'gas/per_block': 2, reth_sync_execution_execution_duration: 0.006 })],
],
};

it('shows every configuration of the run with thresholds, role-prefixed ones included, and raw config', () => {
const html = renderToStaticMarkup(<BenchRun id="test-synthetic-1" />);
expect(html).toContain('href="/scaling/runs"');
expect(html).toContain('2 configurations');
expect(html).toContain('100M gas · 1000 ms blocks');
expect(html).toContain('200M gas · 1000 ms blocks');
expect(html).toContain('420.0');
expect(html).not.toContain('test-synthetic-2');
expect(html).toMatch(/latency\/get_payload.*?20\.0 ms.*?&gt; 50\.0 ms.*?success/);
expect(html).toMatch(/validator\/latency\/new_payload.*?30\.0 ms.*?&gt; 25\.0 ms.*?error/);
expect(html).toMatch(/200M gas · 1000 ms blocks<\/td>.*?<\/tr>/);
expect(html.match(/200M gas · 1000 ms blocks<\/td>.*?<\/tr>/)![0]).toContain('error');
expect(html).toContain('&quot;TransactionPayload&quot;: &quot;synthetic-payload&quot;');
});

it('charts sequencer and validator by stage with only the first stage open', () => {
const html = renderToStaticMarkup(<BenchRun id="test-synthetic-1" />);
const groups = html.split('<details').slice(1);
expect(groups.map((g) => g.match(/<summary[^>]*>([^<]+)/)![1].trim())).toEqual(CHART_GROUPS.map(([g]) => g));
expect(groups.map((g) => g.startsWith(' open=""'))).toEqual([true, false, false, false, false]);
expect(groups[0]).toContain('Get Payload');
expect(groups[0]).toMatch(/Gas Per Block.*?sequencer.*?validator/);
expect(groups[4]).toContain('Reth Sync Execution Duration');
expect(groups[3]).toContain('No data for this stage in this run.');
});

it('reports a failed metrics fetch instead of drawing partial charts', () => {
const saved = swr.metrics;
swr.metrics = { error: new Error('Failed to fetch metrics: 500') };
const html = renderToStaticMarkup(<BenchRun id="test-synthetic-1" />);
swr.metrics = saved;
expect(html).toContain('Metrics could not be loaded: Failed to fetch metrics: 500');
expect(html).not.toContain('<details');
});

it('assigns every metric to exactly one stage', () => {
const assigned = CHART_GROUPS.flatMap(([, metrics]) => metrics);
expect(assigned.toSorted()).toEqual(SORTED_CHART_CONFIG.map(([key]) => key).toSorted());
});

it('says when no run has the id', () => {
expect(renderToStaticMarkup(<BenchRun id="missing" />)).toContain('No base-bench run has id missing.');
});
160 changes: 160 additions & 0 deletions app/scaling/runs/bench/[id]/BenchRun.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
'use client';

import { useState } from 'react';
import useSWR from 'swr';

import { Button } from '../../../../components/ui/Button';
import { Card } from '../../../../components/ui/Card';
import { cn } from '../../../../components/ui/cn';
import { EmptyState } from '../../../../components/ui/EmptyState';
import { Text } from '../../../../components/ui/Text';
import { Info } from '../../../components/Info';
import { LineChart } from '../../components/LineChart';
import { Crumbs, Section, Status, Table, TD } from '../../components/parts';
import { CHART_GROUPS } from '../../lib/chartGroups';
import { getDataService } from '../../lib/dataService';
import { downloadRunInfo, downloadRunInfoCSV } from '../../lib/downloadUtils';
import { formatValue } from '../../lib/formatters';
import { SORTED_CHART_CONFIG } from '../../lib/metricDefinitions';
import { getTestRunsWithStatus, thresholdSummary, type BenchmarkRunWithStatus } from '../../lib/types';
import { toUnifiedRun } from '../../lib/unified';

const configLabel = ({ testConfig: c, outputDir }: BenchmarkRunWithStatus) =>
[c.GasLimit && `${Number(c.GasLimit) / 1e6}M gas`, c.BlockTimeMilliseconds && `${c.BlockTimeMilliseconds} ms blocks`].filter(Boolean).join(' · ') || outputDir;

const mgas = (gps?: number) => (gps === undefined ? '—' : (gps / 1e6).toFixed(1));

// Thresholds are in nanoseconds; summary latencies are in seconds.
function thresholdRows(run: BenchmarkRunWithStatus) {
const { warning = {}, error = {} } = run.thresholds ?? {};
return [...new Set([...Object.keys(warning), ...Object.keys(error)])].map((metric) => {
const [field, role, scale] = thresholdSummary(metric) ?? [];
const value = field ? (run.result?.[`${role}Metrics`]?.[field as never] as number | undefined) : undefined;
const ns = value === undefined ? undefined : value * scale!;
const status = ns === undefined ? 'unknown' : ns > (error[metric] ?? Infinity) ? 'error' : ns > (warning[metric] ?? Infinity) ? 'warning' : 'success';
return { metric, value, warning: warning[metric], error: error[metric], status };
});
}

export function BenchRun({ id }: { id: string }) {
const { data, error } = useSWR('scaling-bench-metadata', () => getDataService().getMetadata());
const runs = data ? getTestRunsWithStatus({ runs: data.runs.filter((r) => `${r.testConfig.BenchmarkRun}` === id) }) : [];
const first = runs[0];
const thresholds = runs.flatMap((run) => thresholdRows(run).map((t) => ({ run, ...t })));

return (
<div className="flex flex-col gap-6 pb-8">
<header className="flex flex-col gap-3">
<Crumbs here={id} />
<Text as="h1" variant="title2">
<span className="inline-flex items-center gap-2">
{first?.testName ?? 'Bench run'}
{first?.testDescription ? <Info label="About this benchmark"><span className="whitespace-pre-line">{first.testDescription}</span></Info> : null}
</span>
</Text>
{first ? <Text variant="label.regular" tone="muted">{first.createdAt.slice(0, 16).replace('T', ' ')} UTC · {first.sourceFile} · {runs.length} configuration{runs.length === 1 ? '' : 's'}</Text> : null}
</header>
{error ? (
<EmptyState title="Run could not be loaded" description={error instanceof Error ? error.message : String(error)} />
) : !data ? (
<Text tone="muted">Loading run…</Text>
) : !first ? (
<EmptyState title="Run not found" description={`No base-bench run has id ${id}.`} />
) : (
<>
<Section title="Summary" action={
<div className="flex gap-2">
<Button size="sm" variant="outline" onClick={() => downloadRunInfo(runs, id)}>Download JSON</Button>
<Button size="sm" variant="outline" onClick={() => downloadRunInfoCSV(runs, id)}>Download CSV</Button>
</div>
}>
<Table head={['Configuration', 'Workload', 'Sequencer Mgas/s', 'Validator Mgas/s', 'Latency', 'Status', 'Machine', 'Client']}>
{runs.map((run) => {
const u = toUnifiedRun(run);
return (
<tr key={run.outputDir} className="border-b border-bds-gray-10 last:border-0 dark:border-white/10">
<td className={cn(TD, 'whitespace-nowrap')}>{configLabel(run)}</td>
<td className={TD}>{u.workload}</td>
<td className={cn(TD, 'font-mono')}>{mgas(run.result?.sequencerMetrics?.gasPerSecond)}</td>
<td className={cn(TD, 'font-mono')}>{mgas(run.result?.validatorMetrics?.gasPerSecond)}</td>
<td className={cn(TD, 'whitespace-nowrap font-mono')}>{u.latency.length ? u.latency.map((l) => <div key={l.label}>{l.label} {formatValue(l.seconds, 's')}</div>) : '—'}</td>
<td className={TD}><Status status={run.status} /></td>
<td className={TD}>{u.machine ?? '—'}</td>
<td className={TD}>{u.commit ?? '—'}</td>
</tr>
);
})}
</Table>
</Section>
<Section title="Thresholds">
{thresholds.length ? (
<Table head={['Metric', 'Value', 'Warning', 'Error', 'Status']}>
{thresholds.map(({ run, ...t }) => (
<tr key={`${run.outputDir}-${t.metric}`} className="border-b border-bds-gray-10 last:border-0 dark:border-white/10">
<td className={TD}>{t.metric}{runs.length > 1 ? <p className="text-bds-gray-60">{configLabel(run)}</p> : null}</td>
<td className={cn(TD, 'whitespace-nowrap font-mono')}>{t.value === undefined ? '—' : formatValue(t.value, 's')}</td>
<td className={cn(TD, 'whitespace-nowrap font-mono')}>{t.warning === undefined ? '—' : `> ${formatValue(t.warning, 'ns')}`}</td>
<td className={cn(TD, 'whitespace-nowrap font-mono')}>{t.error === undefined ? '—' : `> ${formatValue(t.error, 'ns')}`}</td>
<td className={TD}><Status status={t.status} /></td>
</tr>
))}
</Table>
) : <Text tone="muted">No thresholds recorded.</Text>}
</Section>
<Charts runs={runs} />
<Section title="Raw config">
{runs.map((run) => (
<Card key={run.outputDir} className="overflow-x-auto bg-background p-4 dark:bg-white/5">
{runs.length > 1 ? <Text variant="label.regular" tone="muted">{configLabel(run)}</Text> : null}
<pre className="font-mono text-[12px]">{JSON.stringify(run.testConfig, null, 2)}</pre>
</Card>
))}
</Section>
</>
)}
</div>
);
}

const CONFIG = new Map(SORTED_CHART_CONFIG);
const ROLES = [
{ name: 'sequencer', stroke: 'stroke-base-blue', dot: 'bg-base-blue' },
{ name: 'validator', stroke: 'stroke-bds-orange-50', dot: 'bg-bds-orange-50' },
];

function Charts({ runs }: { runs: BenchmarkRunWithStatus[] }) {
const [selected, setSelected] = useState(0);
const run = runs[selected];
const { data, error } = useSWR(['scaling-bench-metrics', run.outputDir], () => Promise.all(ROLES.map((r) => getDataService().getMetrics(run.outputDir, r.name))));
const groups = CHART_GROUPS.map(([group, metrics]) => [group, metrics.map((key) => {
const config = CONFIG.get(key)!;
const keys = [key, ...(config.aliases ?? [])];
const series = ROLES.map((role, i) => ({ ...role, points: (data?.[i] ?? []).flatMap((m): [number, number][] => {
const k = keys.find((k) => m.ExecutionMetrics[k] !== undefined);
return k ? [[m.BlockNumber, m.ExecutionMetrics[k]]] : [];
}) }));
return { key, config, series };
}).filter((c) => c.series.some((s) => s.points.length))] as const);

return (
<Section title="Per-block metrics">
{runs.length > 1 ? (
<div className="flex flex-wrap gap-2">
{runs.map((r, i) => (
<button key={r.outputDir} type="button" aria-pressed={i === selected} onClick={() => setSelected(i)} className={cn('rounded-full border px-3 py-1 text-[12px]', i === selected ? 'border-base-blue text-base-blue' : 'border-bds-gray-20 text-bds-gray-60')}>{configLabel(r)}</button>
))}
</div>
) : null}
{error ? <Text tone="muted">Metrics could not be loaded: {error instanceof Error ? error.message : String(error)}</Text> : !data ? <Text tone="muted">Loading metrics…</Text> : groups.map(([group, charts], i) => (
<details key={group} open={i === 0} className="rounded-lg border border-bds-gray-10 px-4 py-3 dark:border-white/10">
<summary className="cursor-pointer text-[14px] font-[500]">{group} <span className="font-[400] text-bds-gray-60">· {charts.length} chart{charts.length === 1 ? '' : 's'}</span></summary>
{charts.length ? (
<div className="mt-3 grid grid-cols-1 gap-4 md:grid-cols-2">
{charts.map((c) => <LineChart key={c.key} title={c.config.title} unit={c.config.unit} series={c.series} xLabel={(x) => `block ${x}`} />)}
</div>
) : <Text tone="muted">No data for this stage in this run.</Text>}
</details>
))}
</Section>
);
}
6 changes: 6 additions & 0 deletions app/scaling/runs/bench/[id]/page.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { BenchRun } from './BenchRun';

export default async function BenchRunPage({ params }: { params: Promise<{ id: string }> }) {
const { id } = await params;
return <BenchRun id={id} />;
}
34 changes: 34 additions & 0 deletions app/scaling/runs/components/LineChart.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { cn } from '../../../components/ui/cn';
import { formatValue } from '../lib/formatters';
import type { ChartConfig } from '../lib/types';

export type Series = { name: string; stroke: string; dot: string; points: [x: number, y: number][] };

const W = 600;
const H = 140;

export function LineChart({ title, unit, series, xLabel }: { title: string; unit?: ChartConfig['unit']; series: Series[]; xLabel: (x: number) => string }) {
const drawn = series.filter((s) => s.points.length > 0);
const all = drawn.flatMap((s) => s.points);
if (all.length === 0) return null;
const [x0, x1, y0, y1] = all.reduce(([a, b, c, d], [x, y]) => [Math.min(a, x), Math.max(b, x), Math.min(c, y), Math.max(d, y)], [Infinity, -Infinity, 0, 0]);
const px = (x: number) => (x1 === x0 ? W / 2 : ((x - x0) / (x1 - x0)) * W);
const py = (y: number) => (y1 === y0 ? H : H - ((y - y0) / (y1 - y0)) * H);
return (
<figure className="flex min-w-0 flex-col gap-1 text-[12px]">
<figcaption className="flex flex-wrap items-center justify-between gap-x-3">
<span>{title}</span>
<span className="flex items-center gap-3 text-bds-gray-60">
{drawn.length > 1 ? drawn.map((s) => <span key={s.name} className="flex items-center gap-1"><span aria-hidden="true" className={cn('h-2 w-2 rounded-full', s.dot)} />{s.name}</span>) : null}
<span className="font-mono">max {formatValue(y1, unit)}</span>
</span>
</figcaption>
<svg viewBox={`0 0 ${W} ${H}`} preserveAspectRatio="none" role="img" aria-label={title} className="h-[140px] w-full rounded-md border border-bds-gray-10 dark:border-white/10">
{drawn.map((s) => (
<polyline key={s.name} fill="none" strokeWidth={1.5} vectorEffect="non-scaling-stroke" className={s.stroke} points={s.points.map(([x, y]) => `${px(x)},${py(y)}`).join(' ')} />
))}
</svg>
<span className="flex justify-between font-mono text-[11px] text-bds-gray-60"><span>{xLabel(x0)}</span><span>{xLabel(x1)}</span></span>
</figure>
);
}
44 changes: 44 additions & 0 deletions app/scaling/runs/components/parts.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import Link from 'next/link';
import type { ReactNode } from 'react';

import { Card } from '../../../components/ui/Card';
import { cn } from '../../../components/ui/cn';
import { Text } from '../../../components/ui/Text';

export const TD = 'px-3 py-2 align-top';
const TH = 'px-3 py-2 text-left text-[12px] font-[500] text-bds-gray-60';
const DOT: Record<string, string> = { success: 'bg-bds-green-50', warning: 'bg-bds-yellow-50', error: 'bg-bds-red-50', fatal: 'bg-bds-red-50' };

export function Crumbs({ here }: { here: string }) {
return (
<Text variant="label.regular" tone="muted">
<Link className="underline" href="/scaling">Scaling</Link> / <Link className="underline" href="/scaling/runs">Runs</Link> / {here}
</Text>
);
}

export function Section({ title, action, children }: { title: ReactNode; action?: ReactNode; children: ReactNode }) {
return (
<section className="flex flex-col gap-3">
<div className="flex flex-wrap items-center justify-between gap-2"><Text as="h2" variant="headline">{title}</Text>{action}</div>
{children}
</section>
);
}

export function Table({ head, children }: { head: string[]; children: ReactNode }) {
return (
<Card className="overflow-x-auto bg-background dark:bg-white/5">
<table className="w-full border-collapse text-[13px]">
<thead><tr className="border-b border-bds-gray-10 dark:border-white/10">{head.map((h) => <th key={h} className={TH}>{h}</th>)}</tr></thead>
<tbody>{children}</tbody>
</table>
</Card>
);
}

export function Status({ status }: { status: string }) {
return (
<span className="flex items-center gap-2 whitespace-nowrap"><span aria-hidden="true" className={cn('h-2 w-2 rounded-full', DOT[status] ?? 'bg-bds-gray-30')} />{status}</span>
);
}
Loading
Loading