diff --git a/app/scaling/components/GlamsterdamComparison.test.tsx b/app/scaling/components/GlamsterdamComparison.test.tsx new file mode 100644 index 0000000..cf55de8 --- /dev/null +++ b/app/scaling/components/GlamsterdamComparison.test.tsx @@ -0,0 +1,70 @@ +import { renderToStaticMarkup } from 'react-dom/server'; +import { expect, it } from 'vitest'; + +import { GlamsterdamComparisonSection } from './GlamsterdamComparison'; +import type { Run } from '../proven/capacity'; + +const run = (id: string, mgas: number, tps: number, gps: number, workload: Run['workload'] = 'mix'): Run => ({ + id, + outputDir: id, + workload, + mgas, + gps, + tps, + reverted: 0, + passed: true, + commit: 'abc123', + createdAt: 1, + steps: [], + replay: null, + proofNode: null, + da: null, + storage: null, + blockLatency: null, +}); + +it('shows the benchmark source setup message when jobs are unavailable', () => { + const html = renderToStaticMarkup(); + expect(html).toContain('Benchmark source is not configured.'); +}); + +it('shows Glamsterdam and Main jobs side by side without pairing or TPS-gap calculations', () => { + const glamsterdamRuns = [run('glam-mix-200', 200, 2238.2, 204.5e6), run('glam-pay-300', 300, 1520, 302e6, 'payment')]; + const mainRuns = [run('main-mix-200', 200, 2970.6, 203e6), run('main-mix-400', 400, 5942.5, 406.5e6)]; + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain('Glamsterdam jobs'); + expect(html).toContain('All Main targets'); + expect(html).toContain('All Glamsterdam targets'); + expect(html).toContain('Latest Glamsterdam mix run'); + expect(html).toContain('Latest Main mix run'); + expect(html).toContain('glam-mix-200'); + expect(html).toContain('glam-pay-300'); + expect(html).toContain('main-mix-200'); + expect(html).toContain('main-mix-400'); + expect(html).toContain('2,238.2'); + expect(html).toContain('2,970.6'); + expect(html).toContain('5,942.5'); + expect(html).toContain('Actual Mgas/s'); + expect(html).toContain('Successful TPS'); + expect(html).not.toContain('closest TPS'); + expect(html).not.toContain('TPS gap'); +}); + +/** Verifies highlights select by recency, exclude non-mix runs, and retain the inventory. */ +it('highlights the latest mix run rather than the largest target or newest other workload', () => { + const older = { ...run('older-high-target', 400, 5000, 406e6), createdAt: 1 }; + const newest = { ...run('newest-mix', 100, 1200, 102e6), createdAt: 3, reverted: 0.25 }; + const payment = { ...run('newest-payment', 300, 2000, 302e6, 'payment'), createdAt: 4 }; + const html = renderToStaticMarkup(); + const cards = html.slice(0, html.indexOf('All Glamsterdam targets')); + expect(cards, 'latest mix card uses the newest mix job link').toContain('href="/scaling/runs/bench/newest-mix"'); + expect(cards, 'largest target is not assumed newest').not.toContain('older-high-target'); + expect(cards, 'newer payment does not replace latest mix').not.toContain('newest-payment'); + expect(cards, 'successful TPS excludes 25% reverted').toContain('900 successful TPS'); + expect(cards, 'empty Main has an explicit empty state').toContain('No eligible mix jobs in the last 14 days.'); + expect(html, 'older mix remains in the full inventory').toContain('older-high-target'); + expect(html, 'payment remains in the full inventory').toContain('newest-payment'); +}); diff --git a/app/scaling/components/GlamsterdamComparison.tsx b/app/scaling/components/GlamsterdamComparison.tsx new file mode 100644 index 0000000..538c0f1 --- /dev/null +++ b/app/scaling/components/GlamsterdamComparison.tsx @@ -0,0 +1,135 @@ +import { Text } from '../../components/ui/Text'; +import { DA_WORKLOADS } from '../da/workloads'; +import { runHref, successfulTps, type Run } from '../proven/capacity'; + +type Props = { + configured: boolean; + glamsterdamRuns?: Run[]; + mainRuns?: Run[]; + glamsterdamErrors?: number; + error?: unknown; +}; + +const TH = 'px-3 py-2 text-left font-[600] text-bds-gray-60'; +const TD = 'border-t border-bds-gray-10 px-3 py-2 align-top dark:border-white/10'; +const tps = (run: Run) => successfulTps(run).toLocaleString('en-US', { maximumFractionDigits: 1 }); +const rate = (run: Run) => (run.gps / 1e6).toLocaleString('en-US', { maximumFractionDigits: 1 }); + +/** Highlights the newest eligible mix run without matching it to the other branch. */ +function LatestRun({ title, runs, accent = false }: { title: string; runs: Run[]; accent?: boolean }) { + const latest = runs.filter((run) => run.workload === 'mix').reduce( + (newest, run) => !newest || run.createdAt > newest.createdAt ? run : newest, undefined, + ); + return ( +
+

{title}

+ {latest ? ( + <> +
+ {rate(latest)} + Mgas/s actual +
+

+ Target {latest.mgas.toLocaleString('en-US')} Mgas/s · {tps(latest)} successful TPS ·{' '} + View job ↗ +

+ + ) : No eligible mix jobs in the last 14 days.} +
+ ); +} + +/** Lists independent measured runs across all workloads and targets. */ +function RunTable({ title, runs }: { title: string; runs: Run[] }) { + return ( +
+ + {title} + + {runs.length === 0 ? ( + No eligible jobs in the last 14 days. + ) : ( +
+ + + + + + + + + + + + {runs.map((run) => { + const workload = DA_WORKLOADS.find((item) => item.payload === run.workload)?.label ?? run.workload; + return ( + + + + + + + + ); + })} + +
WorkloadTargetJobActual Mgas/sSuccessful TPS
{workload}{run.mgas.toLocaleString('en-US')} Mgas/s + + {run.id} + + {rate(run)}{tps(run)}
+
+ )} +
+ ); +} + +/** Shows latest mix results above separate Glamsterdam and Main run inventories. */ +export function GlamsterdamComparisonSection({ + configured, + glamsterdamRuns, + mainRuns, + glamsterdamErrors = 0, + error, +}: Props) { + const message = !configured + ? 'Benchmark source is not configured.' + : error + ? 'Could not load benchmark data.' + : !glamsterdamRuns || !mainRuns + ? 'Loading benchmark runs…' + : null; + + return ( +
+ + Glamsterdam + + + Latest run per workload and target in the last 14 days. TPS excludes reverted transactions; gas throughput is measured. + + {message ? ( + + {message} + + ) : ( + <> + {glamsterdamErrors > 0 ? ( + + {glamsterdamErrors} latest Glamsterdam job{glamsterdamErrors === 1 ? '' : 's'} could not be loaded. + + ) : null} +
+ + +
+
+ + +
+ + )} +
+ ); +} diff --git a/app/scaling/components/Scorecard.tsx b/app/scaling/components/Scorecard.tsx index eb01a8f..3d1bf18 100644 --- a/app/scaling/components/Scorecard.tsx +++ b/app/scaling/components/Scorecard.tsx @@ -9,6 +9,7 @@ import { Ladder, type ProvenMark } from './Ladder'; import { day, useProvenCapacity } from './ProvenCapacity'; import { Work } from './Work'; import { subsystemCards, WorkloadSubsystems } from './WorkloadSubsystems'; +import { GlamsterdamComparisonSection } from './GlamsterdamComparison'; function benchMark({ mgas, atTop, run }: Proven): ProvenMark { const top = atTop ? ` ≥ because ${mgas} Mgas/s is the highest rate the bench runs.` : ''; @@ -51,6 +52,7 @@ export function Scorecard({ snapshot, benchUrl }: { snapshot: Snapshot; benchUrl + ); diff --git a/app/scaling/components/scorecard.test.tsx b/app/scaling/components/scorecard.test.tsx index e8c6edc..260247d 100644 --- a/app/scaling/components/scorecard.test.tsx +++ b/app/scaling/components/scorecard.test.tsx @@ -54,6 +54,9 @@ const storage = (currentBytes: number | null = null): NonNullable = {}): ProvenCapacity => ({ current: { ...none, mix: { mgas, atTop, run: run(mgas) } }, da: none, + glamsterdamRuns: [], + mainRuns: [], + glamsterdamErrors: 0, history: [ { at: T0, mgas: 40, commit: 'aaaaaaaaaaaaaaaa', previousCommit: null, compareUrl: null }, { at: T0 + 5 * DAY, mgas, commit: 'bbbbbbbbbbbbbbbb', previousCommit: 'aaaaaaaaaaaaaaaa', compareUrl: 'https://github.com/base/base/compare/aaaa...bbbb' }, @@ -66,8 +69,8 @@ const fixture = (): Snapshot => JSON.parse(readFileSync(new URL('../fixtures/syn // In page order: the freshness line, then headline, ladder, ranked limits, workloads, work. const render = (props: { benchUrl?: string } = { benchUrl: BENCH }, snapshot = fixture()) => { const html = renderToStaticMarkup(); - const [fresh, head, ladder, ranked, subsystems, work] = html.split(' { @@ -124,6 +127,40 @@ describe('freshness and headline', () => { }); +describe('Glamsterdam jobs', () => { + it('shows Glamsterdam and Main jobs side by side with measured rates and run links', () => { + const main = { ...run(400), id: 'main-mix-400', outputDir: 'main-mix-400', tps: 2200.24 }; + const glamsterdam = { ...run(200), id: 'glam-mix-200', outputDir: 'glam-mix-200', gps: 197e6, tps: 2100.14 }; + swr.result = { data: bench(400, true, { mainRuns: [main], glamsterdamRuns: [glamsterdam] }) }; + const section = render().glamsterdam; + + expect(text(section)).toContain('All Glamsterdam targets'); + expect(text(section)).toContain('All Main targets'); + expect(text(section)).toContain('glam-mix-200'); + expect(text(section)).toContain('main-mix-400'); + expect(text(section)).toContain('2,100.1'); + expect(text(section)).toContain('2,200.2'); + expect(section).toContain('href="/scaling/runs/bench/glam-mix-200"'); + expect(section).toContain('href="/scaling/runs/bench/main-mix-400"'); + expect(section).not.toContain('closest TPS'); + expect(section).not.toContain('TPS gap'); + }); + + it('distinguishes unconfigured, loading, error and unavailable jobs', () => { + expect(text(render({}).glamsterdam)).toContain('Benchmark source is not configured.'); + expect(text(render().glamsterdam)).toContain('Loading benchmark runs…'); + + swr.result = { error: new Error('offline') }; + expect(text(render().glamsterdam)).toContain('Could not load benchmark data.'); + + swr.result = { data: bench(400, true) }; + expect(text(render().glamsterdam)).toContain('No eligible jobs in the last 14 days.'); + swr.result = { data: bench(400, true, { glamsterdamErrors: 1 }) }; + expect(text(render().glamsterdam)).toContain('1 latest Glamsterdam job could not be loaded.'); + }); +}); + + describe('workload tiles', () => { it('takes the lowest of the proven, DA and gas-limit ceilings, preferring DA then the gas limit on ties', () => { expect(sustainable(400, 82, 200)).toEqual({ mgas: 82, cap: 'DA' }); diff --git a/app/scaling/page.test.tsx b/app/scaling/page.test.tsx index 159770f..771fefa 100644 --- a/app/scaling/page.test.tsx +++ b/app/scaling/page.test.tsx @@ -22,6 +22,7 @@ const area = async (id: string) => renderToStaticMarkup(await ScalingAreaPage({ it('ends the first screen with the DA link, and no guardrails or benchmark links', async () => { const html = renderToStaticMarkup(await ScalingPage()); expect(html).toContain('DA by workload'); + expect(html).toContain('Block Gas Limit: 40 Mgas'); // The guardrails page stays, unlinked, while the first screen leaves guardrails out. expect(html).not.toMatch(/href="\/scaling\/guardrails"|Guardrails<\/h2>|SLOs/); expect(html).not.toMatch(/href="\/benchmark|href="\/scaling\/networking"/); @@ -33,7 +34,7 @@ it('ends the first screen with the DA link, and no guardrails or benchmark links it('orders the first screen: freshness, headline, ladder, ranked limits, subsystems by workload, work, links', async () => { const html = renderToStaticMarkup(await ScalingPage()); - const order = ['Usage to 2026-01-02', 'Proven capacity', 'How close are we, and to what?', 'What breaks first', 'Subsystems by workload', 'Work log', 'DA by workload'].map((h) => html.indexOf(h)); + const order = ['Block Gas Limit: 40 Mgas', 'Usage to 2026-01-02', 'Proven capacity', 'How close are we, and to what?', 'What breaks first', 'Subsystems by workload', 'Glamsterdam', 'Work log', 'DA by workload'].map((h) => html.indexOf(h)); expect(order.every((i, k) => i > (order[k - 1] ?? -1))).toBe(true); expect(html).not.toMatch(/Subsystem detail|>Progress<|Ceilings by bottleneck|Usage data as of/); }); diff --git a/app/scaling/page.tsx b/app/scaling/page.tsx index 0fec28a..4bfa4ea 100644 --- a/app/scaling/page.tsx +++ b/app/scaling/page.tsx @@ -9,6 +9,9 @@ import { Header, loadSnapshot, Unavailable } from './loadSnapshot'; // The snapshot is a runtime file, so render per request rather than at build. export const dynamic = 'force-dynamic'; +// Convert the configured Mgas/s rate to gas per 200ms block. +const BASE_BLOCK_TIME_SECONDS = 0.2; + export default async function ScalingPage() { const loaded = await loadSnapshot(); if (loaded.kind !== 'ok') return ; @@ -17,7 +20,12 @@ export default async function ScalingPage() { const dashboardUrl = process.env.SCALING_DATADOG_DASHBOARD_URL; return (
-
+
+
+ + Block Gas Limit: {snapshot.ladder.configuredLimit.value === null ? '—' : `${(snapshot.ladder.configuredLimit.value * BASE_BLOCK_TIME_SECONDS).toLocaleString('en-US', { maximumFractionDigits: 1 })} Mgas`} + +
diff --git a/app/scaling/proven/capacity.test.ts b/app/scaling/proven/capacity.test.ts index c4c0a6a..81a83cf 100644 --- a/app/scaling/proven/capacity.test.ts +++ b/app/scaling/proven/capacity.test.ts @@ -1,6 +1,6 @@ import { expect, it } from 'vitest'; -import { loadProvenCapacity, PAYLOADS, type Workload } from './capacity'; +import { loadProvenCapacity, PAYLOADS, successfulTps, type Workload } from './capacity'; const NOW = Date.parse('2026-02-01T00:00:00Z'); const DAY = 86_400_000; @@ -10,9 +10,12 @@ type Spec = { dir: string; mgas: number; daysAgo: number; + comparable?: boolean; workload?: Workload; ref?: string | null; - comparable?: boolean; + confirmed?: number; + reverted?: number; + tps?: number; delivered?: number; scenario?: string; benchmarkRun?: string; @@ -73,7 +76,7 @@ function load(specs: Spec[]) { switch (file) { case 'validation.json': return Response.json({ comparable: s.comparable ?? true, reasons: [] }); case 'load-test-result.json': return Response.json({ - throughput: { gps: delivered * 1e6, tps: delivered * 10, total_submitted: 1000, total_reverted: 155 }, + throughput: { gps: delivered * 1e6, tps: s.tps ?? delivered * 10, total_submitted: 1000, total_confirmed: s.confirmed ?? 1000, total_reverted: s.reverted ?? 155 }, ...(s.noLatency ? {} : { block_latency: { p50: { secs: 0, nanos: 205_382_593 }, p99: { secs: 1, nanos: 6_265_374 } } }), }); case 'pipeline-result.json': return Response.json({ suite: 'synthetic', comparable: s.comparable ?? true, steps: STEPS(s) }); @@ -171,6 +174,89 @@ it('ignores alias copies and non-sustained scenarios', async () => { expect(unknown).toEqual([]); }); +it('lists the latest eligible jobs per workload and target independently', async () => { + const { glamsterdamRuns, mainRuns, glamsterdamErrors } = await load([ + { dir: 'list-main-mix-150', mgas: 150, daysAgo: 2, tps: 2000, reverted: 0 }, + { dir: 'list-main-mix-200', mgas: 200, daysAgo: 1, tps: 2970, reverted: 0 }, + { dir: 'list-main-payment-300', mgas: 300, daysAgo: 1, workload: 'payment', tps: 1500, reverted: 0 }, + { dir: 'list-glam-mix-200', mgas: 200, daysAgo: 0, scenario: 'glamsterdam-sustained-200mgas', ref: 'glamsterdam', tps: 2238.2, reverted: 0 }, + { dir: 'list-glam-payment-250', mgas: 250, daysAgo: 0, workload: 'payment', scenario: 'glamsterdam-sustained-250mgas', ref: 'glamsterdam', tps: 1520, reverted: 0 }, + ]); + + expect(mainRuns.map(({ workload, mgas, outputDir }) => [workload, mgas, outputDir])).toEqual([ + ['mix', 150, 'list-main-mix-150'], + ['mix', 200, 'list-main-mix-200'], + ['payment', 300, 'list-main-payment-300'], + ]); + expect(glamsterdamRuns.map(({ workload, mgas, outputDir }) => [workload, mgas, outputDir])).toEqual([ + ['mix', 200, 'list-glam-mix-200'], + ['payment', 250, 'list-glam-payment-250'], + ]); + expect(glamsterdamErrors).toBe(0); +}); + +it('keeps the two job lists separate from main proven capacity and selects the latest rate run', async () => { + const { glamsterdamRuns, mainRuns, glamsterdamErrors, current, history, latestAt } = await load([ + { dir: 'main-mix-200-old', mgas: 200, daysAgo: 3 }, + { dir: 'main-mix-200-new', mgas: 200, daysAgo: 1, tps: 2200, reverted: 100 }, + { dir: 'main-payment-200', mgas: 200, daysAgo: 2, workload: 'payment' }, + { dir: 'glam-mix-200-old', mgas: 200, daysAgo: 2, scenario: 'glamsterdam-sustained-200mgas', ref: 'glamsterdam' }, + { dir: 'glam-mix-200-latest', mgas: 200, daysAgo: 0, scenario: 'glamsterdam-sustained-200mgas', ref: 'feature', comparable: false, tps: 2100, reverted: 100 }, + { dir: 'glam-payment-200', mgas: 200, daysAgo: 1, workload: 'payment', scenario: 'glamsterdam-sustained-200mgas', ref: 'glamsterdam' }, + { dir: 'glam-mix-300', mgas: 300, daysAgo: 1, scenario: 'glamsterdam-sustained-300mgas', ref: 'glamsterdam' }, + { dir: 'main-swap-200', mgas: 200, daysAgo: 1, workload: 'swap' }, + ]); + + expect(mainRuns.map(({ workload, outputDir }) => [workload, outputDir])).toEqual([ + ['mix', 'main-mix-200-new'], + ['payment', 'main-payment-200'], + ['swap', 'main-swap-200'], + ]); + expect(glamsterdamRuns.map(({ workload, outputDir }) => [workload, outputDir])).toEqual([ + ['mix', 'glam-mix-200-latest'], + ['mix', 'glam-mix-300'], + ['payment', 'glam-payment-200'], + ]); + expect(successfulTps(mainRuns[0])).toBe(1980); + expect(successfulTps(glamsterdamRuns[0])).toBe(1890); + expect(glamsterdamRuns[0].passed).toBe(false); + expect(current.mix).toMatchObject({ mgas: 200, run: { outputDir: 'main-mix-200-new' } }); + expect(history.map(({ mgas }) => mgas)).toEqual([200]); + expect(latestAt).toBe(NOW - DAY); + expect(glamsterdamErrors).toBe(0); +}); + +it('omits a stale Glamsterdam job when the latest run cannot be loaded', async () => { + const { glamsterdamRuns, mainRuns, glamsterdamErrors } = await load([ + { dir: 'failed-latest-main-200', mgas: 200, daysAgo: 1 }, + { dir: 'failed-latest-glam-old-200', mgas: 200, daysAgo: 3, scenario: 'glamsterdam-sustained-200mgas' }, + { dir: 'failed-latest-glam-new-200', mgas: 200, daysAgo: 0, scenario: 'glamsterdam-sustained-200mgas', fail: 'load-test-result.json' }, + ]); + + expect(glamsterdamRuns).toEqual([]); + expect(mainRuns.map(({ outputDir }) => outputDir)).toEqual(['failed-latest-main-200']); + expect(glamsterdamErrors).toBe(1); +}); + +it('keeps a loaded latest Glamsterdam job when only an older run fails', async () => { + const { glamsterdamRuns, glamsterdamErrors } = await load([ + { dir: 'older-failure-main-200', mgas: 200, daysAgo: 1 }, + { dir: 'older-failure-glam-old-200', mgas: 200, daysAgo: 3, scenario: 'glamsterdam-sustained-200mgas', fail: 'load-test-result.json' }, + { dir: 'older-failure-glam-new-200', mgas: 200, daysAgo: 0, scenario: 'glamsterdam-sustained-200mgas' }, + ]); + + expect(glamsterdamRuns.map(({ outputDir }) => outputDir)).toEqual(['older-failure-glam-new-200']); + expect(glamsterdamErrors).toBe(0); +}); + +it('computes successful TPS by excluding reverts from confirmed transactions, not all submissions', async () => { + const { current } = await load([{ dir: 'success-tps-denominator', mgas: 100, daysAgo: 1, confirmed: 800, reverted: 100 }]); + + expect(current.mix?.run.reverted).toBe(0.125, 'reverts should be a share of confirmed transactions'); + expect(successfulTps(current.mix!.run)).toBe(875, 'successful TPS should exclude confirmed reverts from measured TPS'); +}); + + it('fails runs that are not comparable or deliver under 95% of target', async () => { const { current } = await load([ { dir: 'fail-100', mgas: 100, daysAgo: 3 }, diff --git a/app/scaling/proven/capacity.ts b/app/scaling/proven/capacity.ts index 30faab2..b41fcc3 100644 --- a/app/scaling/proven/capacity.ts +++ b/app/scaling/proven/capacity.ts @@ -61,7 +61,7 @@ export type Run = { mgas: number; gps: number; tps: number; - // Share of submitted transactions that reverted; they still use gas. + // Share of confirmed transactions that reverted; they still use gas. reverted: number; passed: boolean; commit: string; @@ -75,7 +75,7 @@ export type Run = { blockLatency: { p50Ms: number; p99Ms: number } | null; }; -// Reverted transactions count toward gas/s but not toward transactions per second. +// The load result's tps includes confirmed transactions that reverted, so exclude them for successful TPS. export const successfulTps = (run: Run) => run.tps * (1 - run.reverted); export type Proven = { mgas: number; atTop: boolean; run: Run }; @@ -91,19 +91,25 @@ export type Step = { // `history` is the mainnet mix's; `da` is the latest counting run per workload in the window whose DA // step passed; `latestAt` is when the newest counting run was created. Replay, proof-node and DA results are loaded only for the runs in `current` and `da`: // replay files list every block, and every other run would download them for nothing. + export type ProvenCapacity = { current: Record; da: Record; + glamsterdamRuns: Run[]; + mainRuns: Run[]; + glamsterdamErrors: number; history: Step[]; latestAt: number | null; unknown: string[]; }; + const WINDOW_MS = 14 * 24 * 60 * 60 * 1000; const PASS_RATIO = 0.95; const CONCURRENCY = 6; -const SCENARIO = /^sustained-(\d+)mgas$/; +const SCENARIO = /^(glamsterdam-)?sustained-(\d+)mgas$/; const WORKLOADS = Object.keys(PAYLOADS) as Workload[]; +type Candidate = { meta: BenchmarkRun; kind: 'main' | 'glamsterdam'; mgas: number; workload: Workload }; // Run outputs never change once written. `null` marks a run that does not count: not built from // main, or written before the step pipeline. @@ -122,14 +128,24 @@ export async function loadProvenCapacity( const res = await fetchImpl(`${root}metadata.json`, { cache: 'no-cache' }); if (!res.ok) throw new Error(`Failed to fetch metadata: ${res.status} ${res.statusText}`); const runs: Run[] = []; + const glamRuns: Run[] = []; const unknown: string[] = []; - await forEachLimit(candidates(await res.json()), CONCURRENCY, async (meta) => { + const failedGlamRuns = new Map(); + await forEachLimit(candidates(await res.json()), CONCURRENCY, async (candidate) => { + const { meta, kind } = candidate; try { - if (!runCache.has(meta.outputDir)) runCache.set(meta.outputDir, await loadRun(root, meta, fetchImpl)); + if (!runCache.has(meta.outputDir)) runCache.set(meta.outputDir, await loadRun(root, candidate, fetchImpl)); const run = runCache.get(meta.outputDir); - if (run) runs.push(run); + if (run) (kind === 'glamsterdam' ? glamRuns : runs).push(run); } catch { - unknown.push(meta.outputDir); + if (kind === 'main') { + unknown.push(meta.outputDir); + } else { + const key = `${candidate.workload}:${candidate.mgas}`; + const failedAt = Date.parse(meta.createdAt); + const previous = failedGlamRuns.get(key); + if (previous === undefined || failedAt > previous) failedGlamRuns.set(key, failedAt); + } } }); const inWindow = (r: Run) => r.createdAt > now - WINDOW_MS && r.createdAt <= now; @@ -149,27 +165,37 @@ export async function loadProvenCapacity( const run = latestDa(of(full, w)); return run?.da ? run : null; }; + const jobs = benchmarkRuns(full, glamRuns, now, failedGlamRuns); return { current: Object.fromEntries(WORKLOADS.map((w) => [w, provenAt(of(full, w), now)])) as ProvenCapacity['current'], da: Object.fromEntries(WORKLOADS.map((w) => [w, daRun(w)])) as ProvenCapacity['da'], + glamsterdamRuns: jobs.glamsterdamRuns, + mainRuns: jobs.mainRuns, + glamsterdamErrors: jobs.errors, history: history(of(full, 'mix'), now), latestAt: runs.length ? Math.max(...runs.map((r) => r.createdAt)) : null, unknown, }; } -function candidates({ runs }: BenchmarkRuns): BenchmarkRun[] { +function candidates({ runs }: BenchmarkRuns): Candidate[] { const payloads: string[] = Object.values(PAYLOADS); - const byDir = new Map(); + const byDir = new Map(); for (const run of runs) { const c = run.testConfig; + const scenario = SCENARIO.exec(String(c.Scenario)); if ( c.ChainId === 8453 && payloads.includes(String(c.TransactionPayload)) && - SCENARIO.test(String(c.Scenario)) && + scenario && !String(c.BenchmarkRun).endsWith('--latest') ) { - byDir.set(run.outputDir, run); + byDir.set(run.outputDir, { + meta: run, + kind: scenario[1] ? 'glamsterdam' : 'main', + workload: WORKLOADS.find((w) => PAYLOADS[w] === c.TransactionPayload)!, + mgas: Number(scenario[2]), + }); } } return [...byDir.values()]; @@ -182,25 +208,26 @@ async function getJson(fetchImpl: typeof fetch, url: string) { return res.json().catch(() => null); } -async function loadRun(root: string, meta: BenchmarkRun, fetchImpl: typeof fetch): Promise { +async function loadRun(root: string, candidate: Candidate, fetchImpl: typeof fetch): Promise { + const { meta, kind, mgas } = candidate; const get = (file: string) => getJson(fetchImpl, `${root}${meta.outputDir}/${file}`); const [pipeline, provenance] = await Promise.all([get('pipeline-result.json'), get('provenance.json')]); - if (!pipeline || provenance?.source?.ref !== 'main') return null; + const source = provenance?.source; + if (!pipeline || !source || (kind === 'main' && source.ref !== 'main')) return null; const [validation, load] = await Promise.all([get('validation.json'), get('load-test-result.json')]); if (!validation || !load) throw new Error(`Missing results for ${meta.outputDir}`); - const mgas = Number(SCENARIO.exec(String(meta.testConfig.Scenario))![1]); const gps: number = load.throughput.gps; return { id: String(meta.testConfig.BenchmarkRun), outputDir: meta.outputDir, - workload: WORKLOADS.find((w) => PAYLOADS[w] === meta.testConfig.TransactionPayload)!, + workload: candidate.workload, mgas, gps, tps: load.throughput.tps, - reverted: load.throughput.total_submitted ? (load.throughput.total_reverted ?? 0) / load.throughput.total_submitted : 0, + reverted: load.throughput.total_confirmed ? (load.throughput.total_reverted ?? 0) / load.throughput.total_confirmed : 0, blockLatency: load.block_latency?.p50 && load.block_latency?.p99 ? { p50Ms: durationMs(load.block_latency.p50), p99Ms: durationMs(load.block_latency.p99) } : null, passed: validation.comparable === true && gps >= PASS_RATIO * mgas * 1e6, - commit: provenance.source.revision, + commit: source.revision, createdAt: Date.parse(meta.createdAt), steps: pipeline.steps, replay: null, @@ -228,6 +255,44 @@ async function loadDetails(root: string, run: Run, fetchImpl: typeof fetch): Pro }; } +/** Keeps the latest in-window run for each workload and target gas rate. */ +function latestByWorkloadAndRate(runs: Run[], now: number): Map { + const latest = new Map(); + for (const run of runs) { + if (run.createdAt <= now - WINDOW_MS || run.createdAt > now) continue; + const key = `${run.workload}:${run.mgas}`; + const previous = latest.get(key); + if (!previous || run.createdAt > previous.createdAt) latest.set(key, run); + } + return latest; +} + +/** Lists the latest in-window run for each workload and target, without pairing jobs. */ +function benchmarkRuns( + mainRuns: Run[], + glamRuns: Run[], + now: number, + failedGlamRuns: Map, +): { glamsterdamRuns: Run[]; mainRuns: Run[]; errors: number } { + const latestMain = latestByWorkloadAndRate(mainRuns, now); + const latestGlam = latestByWorkloadAndRate(glamRuns, now); + const blocked = new Set( + [...failedGlamRuns].flatMap(([key, failedAt]) => { + const loaded = latestGlam.get(key); + return failedAt > now - WINDOW_MS && failedAt <= now && (!loaded || failedAt >= loaded.createdAt) ? [key] : []; + }), + ); + const sortRuns = (a: Run, b: Run) => WORKLOADS.indexOf(a.workload) - WORKLOADS.indexOf(b.workload) || a.mgas - b.mgas; + return { + mainRuns: [...latestMain.values()].sort(sortRuns), + glamsterdamRuns: [...latestGlam.entries()] + .filter(([key]) => !blocked.has(key)) + .map(([, run]) => run) + .sort(sortRuns), + errors: blocked.size, + }; +} + // Rust Durations serialize as { secs, nanos }. const durationMs = (d: { secs: number; nanos: number }) => d.secs * 1e3 + d.nanos / 1e6;