From 1221f2029efd544a2a36a27d9c9a581cc3385895 Mon Sep 17 00:00:00 2001 From: Mariana Martinho Date: Wed, 11 Nov 2026 19:01:57 +0100 Subject: [PATCH 1/9] feat(benchmarks): group runs by workflow and hardware, one comparison per group Same change as #2005, now based on main instead of the closed #1981. Co-Authored-By: Claude Opus 5.5 --- locales/en.json | 63 +- locales/zh.json | 63 +- .../src/components/BrandedPageHeader.vue | 19 +- src/renderer/src/panel/PanelApp.test.ts | 4 +- src/renderer/src/views/BenchmarksView.test.ts | 915 ++++---- src/renderer/src/views/BenchmarksView.vue | 1875 +++++++++-------- 6 files changed, 1678 insertions(+), 1261 deletions(-) diff --git a/locales/en.json b/locales/en.json index 896dbac76..6a82a93df 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1613,16 +1613,10 @@ }, "benchmarks": { "title": "Benchmarks", - "description": "Compare results from your performance tests.", - "runsLibrary": "Runs library", "openFolder": "Select folder", "refresh": "Refresh sessions", "searchPlaceholder": "Search runs...", - "allWorkspaces": "All workspaces", - "allInstances": "All builds", - "allHardware": "All hardware", - "allWorkflows": "All workflows", - "columns": "Properties", + "columns": "Columns", "columnsToDisplay": "Columns to display", "unmanagedWorkspace": "Unmanaged", "unknownHardware": "Unknown hardware", @@ -1638,8 +1632,7 @@ "median": "Median", "slowest": "Slowest", "measuredRuns": "Measured runs", - "selectVisible": "Select all visible runs", - "selectRun": "Select {workflow}", + "selectRun": "Select session {session}", "actions": "Actions", "deleteRecord": "Delete {workflow} benchmark", "deleteConfirmTitle": "Delete {workflow}?", @@ -1652,12 +1645,7 @@ "renameErrorTitle": "Could not rename session", "renameErrorMessage": "The session folder could not be renamed.", "sessionNameRequired": "Enter a session name.", - "comparison": "Comparison", - "sortComparison": "Sort comparison", - "manualSort": "Manual sort", - "switchSortAscending": "Switch to ascending sort", - "switchSortDescending": "Switch to descending sort", - "reorderComparisonColumn": "Reorder {workflow} comparison column", + "manualSort": "Manual", "reorderComparisonHint": "Drag to reorder. You can also press Alt+Left or Alt+Right.", "comparisonImageTitle": "Benchmark Comparison", "exportResultsImage": "Export results", @@ -1665,22 +1653,51 @@ "exportImageFailed": "Could not export the comparison image.", "metric": "Metric", "baseline": "Baseline", - "selectBaseline": "Use {workflow}, session {session}, as the baseline", - "visualization": "Comparison visualization", - "speed": "Speed", "relativePerformance": "Relative speed", "relativePerformanceHint": "Choose a baseline. Relative speed factor = baseline duration / run duration.", "faster": "Faster", "slower": "Slower", "same": "Same speed", "missing": "Unavailable", - "duration": "Duration", "durationRange": "Duration range (min → max)", "durationRangeHint": "Each line shows the fastest and slowest run, with the average marked.", - "selectPrompt": "Select runs from the library to compare them.", - "empty": "No performance test results yet. Complete a Performance Test to add a run.", - "noMatches": "No runs match these filters.", - "loadError": "Could not load performance test results." + "noMatches": "No runs match these filters", + "loadError": "Could not load performance test results", + "loadErrorHint": "Check the folder and try again.", + "refreshButton": "Refresh", + "emptyTitle": "No performance test results yet", + "emptyHint": "Complete a Performance Test to add a run.", + "openPerformanceTest": "Open Performance Test", + "noMatchesHint": "Try another search or filter.", + "filterAll": "All", + "filterWorkspace": "Workspace", + "filterBuild": "Build", + "filterHardware": "Hardware", + "filterWorkflow": "Workflow", + "compareBy": "Compare by", + "groupMeta": "{hardware} · {sessions} · {runs}", + "sessionCount": "{count} session | {count} sessions", + "runCount": "{count} run | {count} runs", + "groupActions": "More actions for {workflow}", + "selectGroup": "Select all runs", + "exportGroup": "Export this group", + "singleRun": "Only one build ran this workflow here. Run it on another build to compare.", + "metricAndRange": "{metric} and range", + "vsFastest": "Vs fastest", + "vsBaseline": "Vs baseline", + "selection": "Selected runs", + "reorderRun": "Reorder session {session}", + "noMeasuredRuns": "No measured runs", + "timesFaster": "{factor}× faster", + "timesSlower": "{factor}× slower", + "moreActions": "More actions for session {session}", + "setBaseline": "Set as baseline", + "renameSession": "Rename session", + "speedFormula": "Speed = baseline time ÷ run time", + "resetBaseline": "Reset baseline", + "selectedCount": "{count} selected", + "exportLimit": "Up to {max} runs", + "clearSelection": "Clear selection" }, "downloadsPopup": { "title": "Downloads", diff --git a/locales/zh.json b/locales/zh.json index ac9e96a8c..fdc3187d9 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -1613,16 +1613,10 @@ }, "benchmarks": { "title": "性能比较", - "description": "比较性能测试结果。", - "runsLibrary": "运行记录", "openFolder": "选择文件夹", "refresh": "刷新会话", "searchPlaceholder": "搜索工作流、构建或硬件...", - "allWorkspaces": "所有工作区", - "allInstances": "所有构建", - "allHardware": "所有硬件", - "allWorkflows": "所有工作流", - "columns": "属性", + "columns": "列", "columnsToDisplay": "要显示的列", "unmanagedWorkspace": "未管理", "unknownHardware": "未知硬件", @@ -1638,8 +1632,7 @@ "median": "中位数", "slowest": "最慢", "measuredRuns": "测量运行", - "selectVisible": "选择所有可见运行", - "selectRun": "选择 {workflow}", + "selectRun": "选择会话 {session}", "actions": "操作", "deleteRecord": "删除 {workflow} 基准测试", "deleteConfirmTitle": "删除 {workflow}?", @@ -1652,12 +1645,7 @@ "renameErrorTitle": "无法重命名会话", "renameErrorMessage": "无法重命名会话文件夹。", "sessionNameRequired": "请输入会话名称。", - "comparison": "比较", - "sortComparison": "排序比较", - "manualSort": "手动排序", - "switchSortAscending": "切换为升序", - "switchSortDescending": "切换为降序", - "reorderComparisonColumn": "重新排列 {workflow} 比较列", + "manualSort": "手动", "reorderComparisonHint": "拖动以重新排序。也可以按 Alt+左箭头或 Alt+右箭头。", "comparisonImageTitle": "基准比较", "exportResultsImage": "导出结果", @@ -1665,22 +1653,51 @@ "exportImageFailed": "无法导出比较图像。", "metric": "指标", "baseline": "基准", - "selectBaseline": "将 {workflow}(会话 {session})设为基准", - "visualization": "比较可视化", - "speed": "速度", "relativePerformance": "相对速度", "relativePerformanceHint": "选择基准。相对速度系数 = 基准耗时 / 运行耗时。", "faster": "更快", "slower": "更慢", "same": "速度相同", "missing": "不可用", - "duration": "耗时", "durationRange": "耗时范围(最短 → 最长)", "durationRangeHint": "每条线显示最快和最慢运行,并标记平均值。", - "selectPrompt": "从运行记录中选择要比较的运行。", - "empty": "暂无性能测试结果。请完成一次性能测试以添加运行记录。", - "noMatches": "没有符合筛选条件的运行。", - "loadError": "无法加载性能测试结果。" + "noMatches": "没有符合筛选条件的运行", + "loadError": "无法加载性能测试结果", + "loadErrorHint": "请检查文件夹后重试。", + "refreshButton": "刷新", + "emptyTitle": "暂无性能测试结果", + "emptyHint": "完成一次性能测试以添加运行记录。", + "openPerformanceTest": "打开性能测试", + "noMatchesHint": "请尝试其他搜索或筛选条件。", + "filterAll": "全部", + "filterWorkspace": "工作区", + "filterBuild": "构建", + "filterHardware": "硬件", + "filterWorkflow": "工作流", + "compareBy": "比较依据", + "groupMeta": "{hardware} · {sessions} · {runs}", + "sessionCount": "{count} 个会话 | {count} 个会话", + "runCount": "{count} 次运行 | {count} 次运行", + "groupActions": "{workflow} 的更多操作", + "selectGroup": "选择所有运行", + "exportGroup": "导出此组", + "singleRun": "此处只有一个构建运行过此工作流。在另一个构建上运行它即可比较。", + "metricAndRange": "{metric}及范围", + "vsFastest": "对比最快", + "vsBaseline": "对比基准", + "selection": "已选运行", + "reorderRun": "重新排列会话 {session}", + "noMeasuredRuns": "没有测量的运行", + "timesFaster": "快 {factor}×", + "timesSlower": "慢 {factor}×", + "moreActions": "会话 {session} 的更多操作", + "setBaseline": "设为基准", + "renameSession": "重命名会话", + "speedFormula": "速度 = 基准耗时 ÷ 运行耗时", + "resetBaseline": "重置基准", + "selectedCount": "已选择 {count} 项", + "exportLimit": "最多 {max} 次运行", + "clearSelection": "清除选择" }, "downloadsPopup": { "title": "下载", diff --git a/src/renderer/src/components/BrandedPageHeader.vue b/src/renderer/src/components/BrandedPageHeader.vue index 2a3fc7045..fa182cede 100644 --- a/src/renderer/src/components/BrandedPageHeader.vue +++ b/src/renderer/src/components/BrandedPageHeader.vue @@ -1,19 +1,28 @@ diff --git a/src/renderer/src/panel/PanelApp.test.ts b/src/renderer/src/panel/PanelApp.test.ts index 42ce7c79f..4bc122a28 100644 --- a/src/renderer/src/panel/PanelApp.test.ts +++ b/src/renderer/src/panel/PanelApp.test.ts @@ -612,9 +612,7 @@ describe('PanelApp', () => { expect(wrapper.find('[data-testid="benchmarks"]').exists()).toBe(true) expect(wrapper.get('.branded-page-header h1').text()).toBe('Benchmarks') - expect(wrapper.get('.branded-page-header__description').text()).toBe( - 'Compare results from your performance tests.' - ) + expect(wrapper.find('.branded-page-header__description').exists()).toBe(false) }) it('preserves the dashboard workspace when opening Performance Test', async () => { diff --git a/src/renderer/src/views/BenchmarksView.test.ts b/src/renderer/src/views/BenchmarksView.test.ts index 1a5761ad0..3b451ed97 100644 --- a/src/renderer/src/views/BenchmarksView.test.ts +++ b/src/renderer/src/views/BenchmarksView.test.ts @@ -1,26 +1,39 @@ import { defineComponent, h, KeepAlive } from 'vue' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { flushPromises, mount } from '@vue/test-utils' +import { flushPromises, mount, type VueWrapper } from '@vue/test-utils' import type { PerformanceTestBenchmark } from '../types/ipc' import type * as PerformanceTestResultsSvg from '../lib/performanceTestResultsSvg' -import BaseSelect from '../components/ui/BaseSelect.vue' import { useDialogs } from '../composables/useDialogs' import BenchmarksView from './BenchmarksView.vue' import { setChecked } from './testHelpers' const createResultsPngMock = vi.hoisted(() => - vi.fn(async () => new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer) + vi.fn(async (_svg: string) => new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer) ) const exportResultsImageMock = vi.hoisted(() => - vi.fn(async () => ({ ok: true, filePath: 'C:\\Exports\\benchmark-comparison.png' })) + vi.fn( + async (): Promise<{ + ok: boolean + canceled?: boolean + message?: string + filePath?: string + }> => ({ ok: true, filePath: 'C:\\Exports\\benchmark-comparison.png' }) + ) ) const deletePerformanceTestBenchmarkMock = vi.hoisted(() => vi.fn(async () => ({ ok: true }))) const renamePerformanceTestBenchmarkMock = vi.hoisted(() => - vi.fn(async (_folderPath: string, _sessionId: string, newSessionId: string) => ({ - ok: true, - sessionId: newSessionId - })) + vi.fn( + async ( + _folderPath: string, + _sessionId: string, + newSessionId: string + ): Promise<{ ok: boolean; sessionId?: string; message?: string }> => ({ + ok: true, + sessionId: newSessionId + }) + ) ) +const navigatePanelMock = vi.hoisted(() => vi.fn()) vi.mock('../lib/performanceTestResultsSvg', async (importOriginal) => ({ ...(await importOriginal()), @@ -35,10 +48,11 @@ function benchmark( id: string, workflowName: string, average: number, - hardwareName = 'NVIDIA RTX 4090' + hardwareName: string | null = 'NVIDIA RTX 4090', + build = 'Nightly 0.3.62' ): PerformanceTestBenchmark { const createdAt = `2026-09-${id.padStart(2, '0')}T10:00:00.000Z` - const instance = { id: `instance-${id}`, name: `Instance ${id}` } + const instance = { id: `instance-${build}`, name: build } const workspace = { id: 'workspace-1', name: 'Comfy' } return { id, @@ -69,15 +83,111 @@ function benchmark( } } +function emptyRun(id: string): PerformanceTestBenchmark { + return { + ...benchmark(id, 'portrait.json', 1, 'NVIDIA RTX 4090', 'Broken build'), + fastestJobDurationSeconds: null, + slowestJobDurationSeconds: null, + averageJobDurationSeconds: null, + medianJobDurationSeconds: null, + measuredJobCount: 0 + } +} + const sampleBenchmarks = [ benchmark('13', 'portrait.json', 2), - benchmark('12', 'product.json', 3.4), + benchmark('12', 'portrait.json', 2.2, 'NVIDIA RTX 4090', 'Stable 0.3.60'), benchmark('11', 'portrait.json', 1.2, 'NVIDIA RTX 5090'), - benchmark('10', 'landscape.json', 8.7, 'Apple M3 Max') + benchmark('10', 'landscape.json', 8.7, 'Apple M3 Max', 'Stable 0.3.60'), + { + ...benchmark('9', 'portrait.json', 3, 'NVIDIA RTX 4090', 'Stable 0.3.55'), + fastestJobDurationSeconds: 1 + } ] +const PORTRAIT_4090 = 'benchmark-group-portrait.json-NVIDIA RTX 4090' + +function listMock(): ReturnType { + return window.api.listPerformanceTestBenchmarks as unknown as ReturnType +} + +function useBenchmarks(list: PerformanceTestBenchmark[]): void { + listMock().mockResolvedValue({ folderPath: 'C:\\results\\performance-tests', benchmarks: list }) +} function mountView() { - return mount(BenchmarksView) + return mount(BenchmarksView, { attachTo: document.body }) +} + +function group(wrapper: VueWrapper, testId = PORTRAIT_4090) { + return wrapper.get(`[data-testid="${testId}"]`) +} + +function rowIds(wrapper: VueWrapper, testId = PORTRAIT_4090): Array { + return group(wrapper, testId) + .findAll('[data-testid^="benchmark-row-"]') + .map((row) => row.attributes('data-testid')?.replace('benchmark-row-', '')) +} + +function vsLabels(wrapper: VueWrapper, testId = PORTRAIT_4090): string[] { + return group(wrapper, testId) + .findAll('.benchmarks__vs') + .map((cell) => cell.text()) +} + +async function openMenu(wrapper: VueWrapper, selector: string): Promise { + await wrapper.get(selector).trigger('keydown', { key: 'ArrowDown' }) + await flushPromises() +} + +async function filterOptions(wrapper: VueWrapper, key: string): Promise { + await openMenu(wrapper, `[data-testid="benchmarks-filter-${key}"]`) + const labels = Array.from( + document.body.querySelectorAll('[role="menu"] [role="menuitemradio"]'), + (item) => item.textContent!.trim() + ) + await wrapper + .get(`[data-testid="benchmarks-filter-${key}"]`) + .trigger('keydown', { key: 'Escape' }) + document.body + .querySelector('[role="menu"]') + ?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })) + await vi.waitFor(() => expect(document.body.querySelector('[role="menu"]')).toBeNull()) + return labels +} + +async function chooseFilter(wrapper: VueWrapper, key: string, value: string): Promise { + await openMenu(wrapper, `[data-testid="benchmarks-filter-${key}"]`) + await chooseMenuItem(`[data-value="${value}"]`) +} + +async function compareBy(wrapper: VueWrapper, value: string): Promise { + await chooseFilter(wrapper, 'compare-by', value) +} + +function compareByCurrent(wrapper: VueWrapper): string { + return wrapper.get('[data-testid="benchmarks-filter-compare-by"]').text() +} + +function menuItem(selector: string): HTMLElement | null { + return document.body.querySelector(`[role="menu"] ${selector}`) +} + +async function chooseMenuItem(selector: string): Promise { + menuItem(selector)!.click() + await vi.waitFor(() => expect(document.body.querySelector('[role="menu"]')).toBeNull()) + await flushPromises() +} + +async function openRowMenu(wrapper: VueWrapper, id: string): Promise { + await openMenu(wrapper, `[data-testid="benchmark-row-${id}"] .benchmarks__row-menu`) +} + +async function selectRow(wrapper: VueWrapper, id: string, checked = true): Promise { + await setChecked(wrapper.get(`[data-testid="benchmark-row-${id}"] [role="checkbox"]`), checked) +} + +function lastExportedSvg(): string { + return createResultsPngMock.mock.calls.at(-1)![0] } function mountKeptAlive() { @@ -94,13 +204,20 @@ function mountKeptAlive() { describe('BenchmarksView', () => { beforeEach(() => { useDialogs().cancel() + document.body.innerHTML = '' createResultsPngMock.mockClear() exportResultsImageMock.mockClear() + exportResultsImageMock.mockResolvedValue({ + ok: true, + filePath: 'C:\\Exports\\benchmark-comparison.png' + }) deletePerformanceTestBenchmarkMock.mockClear() deletePerformanceTestBenchmarkMock.mockResolvedValue({ ok: true }) renamePerformanceTestBenchmarkMock.mockClear() + navigatePanelMock.mockClear() ;(window as unknown as { api: object }).api = { browseFolder: vi.fn(), + navigatePanel: navigatePanelMock, exportResultsImage: exportResultsImageMock, deletePerformanceTestBenchmark: deletePerformanceTestBenchmarkMock, renamePerformanceTestBenchmark: renamePerformanceTestBenchmarkMock, @@ -112,8 +229,6 @@ describe('BenchmarksView', () => { }) describe('returning to the page', () => { - const list = (): ReturnType => - window.api.listPerformanceTestBenchmarks as unknown as ReturnType const browseFolder = (): ReturnType => window.api.browseFolder as unknown as ReturnType @@ -127,29 +242,25 @@ describe('BenchmarksView', () => { mountKeptAlive() await flushPromises() - expect(list()).toHaveBeenCalledOnce() + expect(listMock()).toHaveBeenCalledOnce() }) it('keeps the folder, search and picked runs', async () => { browseFolder().mockResolvedValue('D:\\other') - list().mockImplementation(async (folderPath?: string) => ({ + listMock().mockImplementation(async (folderPath?: string) => ({ folderPath: folderPath ?? 'C:\\results\\performance-tests', benchmarks: sampleBenchmarks })) const wrapper = mountKeptAlive() await flushPromises() - const selectFolder = wrapper - .findAll('button') - .find((button) => button.text() === 'Select folder') - await selectFolder?.trigger('click') + await wrapper.get('.benchmarks__open-folder').trigger('click') await flushPromises() - await setChecked(wrapper.get('[data-testid="benchmark-row-13"] [role="checkbox"]'), false) - await setChecked(wrapper.get('[data-testid="benchmark-row-10"] [role="checkbox"]'), true) + await selectRow(wrapper, '10') await wrapper.get('input[type="text"]').setValue('json') await leaveAndReturn(wrapper) - expect(list()).toHaveBeenLastCalledWith('D:\\other') + expect(listMock()).toHaveBeenLastCalledWith('D:\\other') expect(wrapper.get('input[type="text"]').element.value).toBe('json') const checked = (id: string): string | undefined => wrapper @@ -162,10 +273,9 @@ describe('BenchmarksView', () => { it('shows runs that finished meanwhile and drops picks that are gone', async () => { const wrapper = mountKeptAlive() await flushPromises() - list().mockResolvedValue({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: [benchmark('14', 'new.json', 1), ...sampleBenchmarks.slice(1)] - }) + await selectRow(wrapper, '13') + await selectRow(wrapper, '12') + useBenchmarks([benchmark('14', 'new.json', 1), ...sampleBenchmarks.slice(1)]) await leaveAndReturn(wrapper) @@ -174,398 +284,429 @@ describe('BenchmarksView', () => { expect( wrapper.get('[data-testid="benchmark-row-14"] [role="checkbox"]').attributes('aria-checked') ).toBe('false') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('1 selected') }) it('keeps the list on screen when the refresh fails', async () => { const wrapper = mountKeptAlive() await flushPromises() - list().mockRejectedValue(new Error('offline')) + listMock().mockRejectedValue(new Error('offline')) await leaveAndReturn(wrapper) - expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(4) + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(5) }) }) - it('selects the three newest runs and highlights only the best duration in each metric', async () => { + it('groups runs by workflow and hardware, newest group first', async () => { const wrapper = mountView() await flushPromises() - expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(4) - expect(wrapper.findAll('.benchmarks__table thead th').map((header) => header.text())).toEqual([ - '', - 'Workflow', - 'Session', - 'GPU / Hardware', - 'Runs', - 'Date / Time ↓', - '' + const groups = wrapper.findAll('.benchmarks__group') + expect(groups.map((card) => card.get('.benchmarks__group-toggle').text())).toEqual([ + 'portrait.json', + 'portrait.json', + 'landscape.json' ]) - expect(wrapper.get('[data-testid="benchmark-row-13"]').text()).toContain('13') - expect(wrapper.find('.benchmarks__selection-tray').exists()).toBe(false) - expect(wrapper.get('[data-testid="benchmark-row-13"]').attributes('style')).toContain( - '--series-color: #55e0d1' - ) - expect(wrapper.findAll('.benchmarks__best').map((cell) => cell.text())).toEqual([ - '0.8 s', - '1.2 s', - '1.1 s', - '1.9 s' + expect(groups.map((card) => card.get('.benchmarks__group-meta').text())).toEqual([ + 'NVIDIA RTX 4090 · 3 sessions · 15 runs', + 'NVIDIA RTX 5090 · 1 session · 5 runs', + 'Apple M3 Max · 1 session · 5 runs' ]) + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) expect( - wrapper.findAll('[data-testid^="benchmark-comparison-13-"]').map((field) => field.text()) - ).toEqual(['portrait.json', 'Session: 13', 'GPU / Hardware: NVIDIA RTX 4090']) - expect( - wrapper.findAll('[data-testid^="benchmark-chart-13-"]').map((field) => field.text()) - ).toEqual([ - 'portrait.json', - 'Session: 13', - 'GPU / Hardware: NVIDIA RTX 4090', - '1.6 s', - '2.7 s', - '2 s' - ]) + group(wrapper) + .findAll('thead th') + .map((header) => header.text()) + ).toEqual(['', 'Session', 'Build', 'Median and range', 'Median ↑', 'Vs fastest', '']) + expect(group(wrapper).get('[data-testid="benchmark-row-13"]').text()).toContain( + 'Nightly 0.3.62' + ) expect( - wrapper - .get('[data-testid="benchmark-chart-13-slowest-label"]') - .classes('benchmarks__chart-point-label--slowest') - ).toBe(true) - - const exportButton = wrapper.get('.benchmarks__export-results') - expect(exportButton.text()).toBe('Export results') - await exportButton.trigger('click') - await flushPromises() - expect(exportResultsImageMock).toHaveBeenCalledTimes(1) - const [png, imageType, defaultPath] = exportResultsImageMock.mock.calls[0]! - expect(png).toBeInstanceOf(ArrayBuffer) - expect(imageType).toBe('benchmark-comparison') - expect(defaultPath).toBe('C:\\results\\performance-tests') - const svg = createResultsPngMock.mock.calls[0]![0] - expect(svg).toContain('Benchmark Comparison') - expect(svg).toContain('portrait.json') - expect(svg).toContain('Duration range') - expect(svg).toContain('Each line shows the fastest and slowest run, with the average marked.') - expect(svg).toContain('role="img" aria-label="Comfy"') - expect(svg).toContain('class="footer-date"') - expect(svg.match(/class="table-cell best-cell"/g)).toHaveLength(4) - expect(svg).toContain('.best-cell { fill: #f2ff59; fill-opacity: 0.08; }') - expect(svg.match(/class="chart-point-label"/g)).toHaveLength(9) + group(wrapper).get('[data-testid="benchmark-row-13"] .benchmarks__session-meta').text() + ).toContain('5 runs') }) - it('filters the run library and updates the comparison selection directly', async () => { + it('names the fastest build and compares it with the runner-up', async () => { const wrapper = mountView() await flushPromises() - expect(wrapper.find('[data-testid="benchmark-column-systemInfo.cpu_model"]').exists()).toBe( - true - ) - expect(wrapper.find('[data-testid="benchmark-comparison-13-customScore"]').exists()).toBe(false) - await setChecked(wrapper.get('[data-testid="benchmark-column-customScore"]'), true) - expect(wrapper.get('.benchmarks__table thead').text()).toContain('Custom Score') - expect(wrapper.get('[data-testid="benchmark-row-13"]').text()).toContain('13') - expect(wrapper.find('[data-testid="benchmark-comparison-13-customScore"]').exists()).toBe(false) - await setChecked(wrapper.get('[data-testid="benchmark-comparison-column-customScore"]'), true) - expect(wrapper.get('[data-testid="benchmark-comparison-13-customScore"]').text()).toBe( - 'Custom Score: 13' - ) - expect(wrapper.get('[data-testid="benchmark-comparison-12-customScore"]').text()).toBe( - 'Custom Score: 12' - ) - expect(wrapper.get('[data-testid="benchmark-chart-13-customScore"]').text()).toBe( - 'Custom Score: 13' - ) + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.11× slower', '1.53× slower']) expect( - wrapper.get('[data-testid="benchmark-column-workflowName"]').attributes('disabled') - ).toBe('') - expect(wrapper.get('.benchmarks__table thead').text()).toContain('Workflow') - expect(wrapper.get('[data-testid="benchmark-comparison-13-workflowName"]').text()).toContain( - 'portrait.json' - ) + group(wrapper) + .findAll('.benchmarks__value') + .map((cell) => cell.text()) + ).toEqual(['1.9 s', '2.1 s', '2.9 s']) expect( - wrapper.get('[data-testid="benchmark-comparison-column-workflowName"]').attributes('disabled') - ).toBe('') - expect(wrapper.get('[data-testid="benchmark-comparison-13-workflowName"]').text()).toContain( - 'portrait.json' - ) - expect(wrapper.get('[data-testid="benchmark-chart-13-workflowName"]').text()).toContain( - 'portrait.json' - ) - - await wrapper.get('input[type="text"]').setValue('landscape') - expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) - expect(wrapper.get('[data-testid="benchmark-row-10"]').text()).toContain('landscape.json') - - await setChecked(wrapper.get('[data-testid="benchmark-row-10"] [role="checkbox"]'), true) - expect(wrapper.get('[data-testid="benchmark-row-10"]').attributes('style')).toContain( - '--series-color: #ff8a65' - ) - expect(wrapper.get('.benchmarks__comparison').text()).toContain('landscape.json') + group(wrapper) + .findAll('.benchmarks__bar-fill--best') + .map((fill) => fill.element.closest('tr')?.getAttribute('data-testid')) + ).toEqual(['benchmark-row-13']) + expect(group(wrapper).find('.benchmarks__axis').exists()).toBe(false) + expect( + Number.parseFloat( + ( + group(wrapper).get('[data-testid="benchmark-bar-13"] .benchmarks__bar-fill') + .element as HTMLElement + ).style.width + ) + ).toBeCloseTo((1.9 / 3.7) * 100, 3) + }) - const comparisonToggle = wrapper.get('#comparison-title') - expect(comparisonToggle.attributes('aria-expanded')).toBe('true') - await comparisonToggle.trigger('click') - expect(comparisonToggle.attributes('aria-expanded')).toBe('false') - expect(wrapper.get('.benchmarks__comparison').attributes('style')).toContain('display: none') - await comparisonToggle.trigger('click') + it('describes a single-run group without a comparison', async () => { + const wrapper = mountView() + await flushPromises() + const single = group(wrapper, 'benchmark-group-landscape.json-Apple M3 Max') - await setChecked(wrapper.get('[data-testid="benchmark-row-10"] [role="checkbox"]'), false) - expect(wrapper.get('[data-testid="benchmark-row-10"]').attributes('style')).toBeUndefined() - await wrapper.get('input[type="text"]').setValue('') - await setChecked(wrapper.get('[data-testid="benchmark-row-13"] [role="checkbox"]'), false) - await setChecked(wrapper.get('[data-testid="benchmark-row-12"] [role="checkbox"]'), false) - await setChecked(wrapper.get('[data-testid="benchmark-row-11"] [role="checkbox"]'), false) - expect(wrapper.get('.benchmarks__comparison').text()).toContain( - 'Select runs from the library to compare them.' + expect(single.get('.benchmarks__single-note').text()).toBe( + 'Only one build ran this workflow here. Run it on another build to compare.' ) + expect(single.get('.benchmarks__vs').text()).toBe('') }) - it('compares the selected metric against a selected baseline', async () => { + it('compares against a chosen baseline and resets it', async () => { const wrapper = mountView() await flushPromises() - const chartElement = wrapper.get('.benchmarks__chart').element - const workflowLabelElements = wrapper - .findAll('[data-testid$="-workflowName"]') - .map((label) => label.element) - const visualizationTabs = wrapper.findAll('.benchmarks__visualization-tabs button') - expect(visualizationTabs.map((tab) => tab.text())).toEqual(['Duration', 'Speed']) - expect(visualizationTabs[0]!.attributes('aria-pressed')).toBe('true') - await visualizationTabs[1]!.trigger('click') - expect(wrapper.get('.benchmarks__chart').element).toBe(chartElement) - expect(wrapper.findAll('[data-testid$="-workflowName"]').map((label) => label.element)).toEqual( - workflowLabelElements - ) - const metricSelect = wrapper - .findAllComponents(BaseSelect) - .find((select) => select.props('ariaLabel') === 'Metric') - expect(metricSelect?.props('modelValue')).toBe('medianJobDurationSeconds') + await openRowMenu(wrapper, '12') + await chooseMenuItem('.benchmarks__set-baseline') - const relativeRows = () => wrapper.findAll('.benchmarks__relative-row') - expect(relativeRows().map((row) => row.get('.benchmarks__relative-result').text())).toEqual([ - '1.00×Baseline', - '0.58×Slower', - '1.73×Faster' - ]) - - expect(wrapper.find('[aria-label="Baseline"]').exists()).toBe(false) - const selectedHeader = wrapper.get('[data-testid="benchmark-comparison-column-title-12"]') - await selectedHeader.get('.benchmarks__matrix-title').trigger('click') - expect(selectedHeader.classes()).toContain('benchmarks__matrix-column--baseline') - expect(selectedHeader.get('.benchmarks__matrix-title').attributes('aria-pressed')).toBe('true') - expect(selectedHeader.get('.benchmarks__baseline-badge').element.parentElement).toBe( - selectedHeader.element + expect(vsLabels(wrapper)).toEqual(['1.11× faster', 'Baseline', '1.38× slower']) + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') + expect(group(wrapper).get('.benchmarks__baseline-foot').text()).toContain( + 'Speed = baseline time ÷ run time' ) - expect(relativeRows().map((row) => row.get('.benchmarks__relative-result').text())).toEqual([ - '1.74×Faster', - '1.00×Baseline', - '3.00×Faster' - ]) + expect( + group(wrapper, 'benchmark-group-portrait.json-NVIDIA RTX 5090') + .find('.benchmarks__baseline-foot') + .exists() + ).toBe(false) - metricSelect?.vm.$emit('update:modelValue', 'fastestJobDurationSeconds') - await wrapper.vm.$nextTick() - expect(relativeRows()[0]!.get('.benchmarks__relative-duration').text()).toBe('1.6 s') - expect(relativeRows()[0]!.get('.benchmarks__relative-result').text()).toBe('1.88×Faster') + await compareBy(wrapper, 'fastestJobDurationSeconds') + expect(vsLabels(wrapper)).toEqual(['1.80× faster', '1.13× faster', 'Baseline']) - await wrapper.get('.benchmarks__export-results').trigger('click') - await flushPromises() - const svg = createResultsPngMock.mock.calls[0]![0] - expect(svg).toContain('Relative speed') - expect(svg).toContain('Relative speed factor = baseline duration / run duration.') - expect(svg).toContain('1.88× Faster') - expect(svg).toContain('1.00× Baseline') - expect(svg).toContain('class="relative-bar relative-faster"') - expect(svg).not.toContain('>Duration range (min → max)') + await group(wrapper).get('.benchmarks__reset-baseline').trigger('click') + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.60× slower', '1.80× slower']) + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs fastest') }) it('does not label another equally fast run as the baseline', async () => { - vi.mocked(window.api.listPerformanceTestBenchmarks).mockResolvedValueOnce({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: [benchmark('21', 'a.json', 2), benchmark('22', 'b.json', 2)] - }) + useBenchmarks([ + benchmark('21', 'a.json', 2, 'GPU', 'Build A'), + benchmark('22', 'a.json', 2, 'GPU', 'Build B') + ]) const wrapper = mountView() await flushPromises() - await wrapper.findAll('.benchmarks__visualization-tabs button')[1]!.trigger('click') - expect(wrapper.findAll('.benchmarks__relative-result').map((result) => result.text())).toEqual([ - '1.00×Baseline', - '1.00×Same speed' - ]) + expect(vsLabels(wrapper, 'benchmark-group-a.json-GPU')).toEqual(['Fastest', 'Same speed']) }) - it('closes property menus when clicking outside', async () => { + it('shows runs without measurements as unavailable', async () => { + useBenchmarks([...sampleBenchmarks, emptyRun('8')]) const wrapper = mountView() await flushPromises() - const pickers = wrapper.findAll('.benchmarks__columns-picker') - for (const picker of pickers) { - const details = picker.element as HTMLDetailsElement - details.open = true - await picker.get('[role="switch"]').trigger('pointerdown') - expect(details.open).toBe(true) - - document.body.dispatchEvent(new Event('pointerdown', { bubbles: true })) - expect(details.open).toBe(false) - } + const row = wrapper.get('[data-testid="benchmark-row-8"]') + expect(rowIds(wrapper)).toEqual(['13', '12', '9', '8']) + expect(row.get('.benchmarks__no-data').text()).toBe('No measured runs') + expect(row.find('.benchmarks__bar').exists()).toBe(false) + expect(row.get('.benchmarks__value').text()).toBe('—') + expect(row.get('.benchmarks__vs').text()).toBe('Unavailable') + await openRowMenu(wrapper, '8') + expect(menuItem('.benchmarks__set-baseline')?.hasAttribute('data-disabled')).toBe(true) }) - it('reorders comparison columns by dragging a column title', async () => { + it('shows every time of a run when its bar is hovered', async () => { const wrapper = mountView() await flushPromises() - const columnIds = () => - wrapper - .findAll('[data-testid^="benchmark-comparison-column-title-"]') - .map((header) => header.attributes('data-testid')?.split('-').at(-1)) + await group(wrapper).get('[data-testid="benchmark-row-12"] .tooltip-wrap').trigger('focusin') + await flushPromises() - expect(columnIds()).toEqual(['13', '12', '11']) + const tip = document.body.querySelector('.benchmarks__bar-tip')! + expect(tip.textContent).toContain('12') + expect(tip.textContent).toContain('Stable 0.3.60') + expect(tip.textContent).toContain('5 runs') + expect(tip.textContent).toContain('Fastest1.8 s') + expect(tip.textContent).toContain('Average2.2 s') + expect(tip.textContent).toContain('Median2.1 s') + expect(tip.textContent).toContain('Slowest2.9 s') + }) + it('compares by another metric and sorts by any header', async () => { + const wrapper = mountView() + await flushPromises() + + expect(await filterOptions(wrapper, 'compare-by')).toEqual([ + 'Fastest', + 'Average', + 'Median', + 'Slowest' + ]) + await compareBy(wrapper, 'fastestJobDurationSeconds') + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + expect(group(wrapper).get('.benchmarks__bar-head').text()).toBe('Fastest and range') + + await group(wrapper).get('.benchmarks__metric-sort').trigger('click') + expect(rowIds(wrapper)).toEqual(['12', '13', '9']) + expect(group(wrapper).get('.benchmarks__metric-sort').text()).toBe('Fastest ↓') + + const sessionHeader = group(wrapper) + .findAll('.benchmarks__sort') + .find((header) => header.text().startsWith('Session'))! + await sessionHeader.trigger('click') + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + await sessionHeader.trigger('click') + expect(rowIds(wrapper)).toEqual(['12', '13', '9']) + }) + + it('reorders rows by dragging and with Alt+arrow keys', async () => { + const wrapper = mountView() + await flushPromises() const dataTransfer = new DataTransfer() + await wrapper - .get('[data-testid="benchmark-comparison-column-title-13"] .benchmarks__column-grip') + .get('[data-testid="benchmark-row-9"] .benchmarks__grip') .trigger('dragstart', { dataTransfer }) - await wrapper.get('[data-testid="benchmark-comparison-column-title-11"]').trigger('dragover', { - dataTransfer - }) + await wrapper.get('[data-testid="benchmark-row-13"]').trigger('dragover', { dataTransfer }) expect( - wrapper - .get('[data-testid="benchmark-comparison-column-title-11"]') - .classes('benchmarks__matrix-column--drop-target') + wrapper.get('[data-testid="benchmark-row-13"]').classes('benchmarks__row--drop-target') ).toBe(true) - await wrapper.get('[data-testid="benchmark-comparison-column-title-11"]').trigger('drop', { - dataTransfer - }) + await wrapper.get('[data-testid="benchmark-row-13"]').trigger('drop', { dataTransfer }) - expect(columnIds()).toEqual(['12', '13', '11']) + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + expect(compareByCurrent(wrapper)).toBe('Compare byManual') + expect((await filterOptions(wrapper, 'compare-by'))[0]).toBe('Manual') + expect(group(wrapper).get('.benchmarks__metric-sort').text()).toBe('Median') + expect(vsLabels(wrapper)).toEqual(['1.53× slower', 'Fastest', '1.11× slower']) + + await wrapper + .get('[data-testid="benchmark-row-12"] .benchmarks__grip') + .trigger('keydown', { key: 'ArrowLeft', altKey: true }) + expect(rowIds(wrapper)).toEqual(['9', '12', '13']) + await wrapper + .get('[data-testid="benchmark-row-9"] .benchmarks__grip') + .trigger('keydown', { key: 'ArrowRight', altKey: true }) + expect(rowIds(wrapper)).toEqual(['12', '9', '13']) await wrapper - .get('[data-testid="benchmark-comparison-column-title-11"] .benchmarks__column-grip') + .get('[data-testid="benchmark-row-11"] .benchmarks__grip') .trigger('dragstart', { dataTransfer }) - await wrapper.get('[data-testid="benchmark-comparison-column-title-12"]').trigger('drop', { - dataTransfer - }) + await wrapper.get('[data-testid="benchmark-row-12"]').trigger('drop', { dataTransfer }) + expect(rowIds(wrapper)).toEqual(['12', '9', '13']) - expect(columnIds()).toEqual(['11', '12', '13']) - expect(wrapper.findAll('.benchmarks__chart-row strong').map((title) => title.text())).toEqual([ - 'portrait.json', - 'product.json', - 'portrait.json' - ]) + await compareBy(wrapper, 'medianJobDurationSeconds') + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + expect(await filterOptions(wrapper, 'compare-by')).toHaveLength(4) }) - it('sorts comparison columns by duration metrics in either direction', async () => { + it('collapses a group', async () => { const wrapper = mountView() await flushPromises() + const toggle = group(wrapper).get('.benchmarks__group-toggle') + + expect(toggle.attributes('aria-expanded')).toBe('true') + await toggle.trigger('click') + expect(toggle.attributes('aria-expanded')).toBe('false') + expect(group(wrapper).find('table').exists()).toBe(false) + await toggle.trigger('click') + expect(group(wrapper).find('table').exists()).toBe(true) + }) - const columnIds = () => - wrapper - .findAll('[data-testid^="benchmark-comparison-column-title-"]') - .map((header) => header.attributes('data-testid')?.split('-').at(-1)) - const sortSelect = wrapper - .findAllComponents(BaseSelect) - .find((select) => select.props('ariaLabel') === 'Sort comparison') - - expect(sortSelect?.props('options').map((option) => option.label)).toEqual([ - 'Manual sort', - 'Fastest', + it('adds picked columns to the table and the export, best value in green', async () => { + const wrapper = mountView() + await flushPromises() + + expect(wrapper.find('[data-testid="benchmark-column-systemInfo.cpu_model"]').exists()).toBe( + true + ) + expect(wrapper.find('[data-testid="benchmark-column-workflowName"]').exists()).toBe(false) + await setChecked( + wrapper.get('[data-testid="benchmark-column-averageJobDurationSeconds"]'), + true + ) + await setChecked(wrapper.get('[data-testid="benchmark-column-customScore"]'), true) + await setChecked(wrapper.get('[data-testid="benchmark-column-instance.name"]'), false) + + expect( + group(wrapper) + .findAll('thead th') + .map((header) => header.text()) + ).toEqual([ + '', + 'Session', + 'Median and range', 'Average', - 'Median', - 'Slowest' + 'Custom Score', + 'Median ↑', + 'Vs fastest', + '' ]) + expect( + group(wrapper) + .findAll('.benchmarks__best') + .map((cell) => cell.text()) + ).toEqual(['2 s']) + expect(group(wrapper).get('[data-testid="benchmark-cell-12-customScore"]').text()).toBe('12') + + await openMenu(wrapper, `[data-testid="${PORTRAIT_4090}"] .benchmarks__group-menu`) + await chooseMenuItem('.benchmarks__export-group') + const svg = lastExportedSvg() + expect(svg).toContain('Custom Score') + expect(svg).toContain('GPU / Hardware') + expect(svg).toContain('Duration range') + }) + + it('closes the columns menu when clicking outside', async () => { + const wrapper = mountView() + await flushPromises() + const picker = wrapper.get('.benchmarks__columns-picker') + const details = picker.element as HTMLDetailsElement - sortSelect?.vm.$emit('update:modelValue', 'fastestJobDurationSeconds') - await wrapper.vm.$nextTick() - expect(columnIds()).toEqual(['11', '13', '12']) + details.open = true + await picker.get('[role="checkbox"]').trigger('pointerdown') + expect(details.open).toBe(true) - await wrapper.get('.benchmarks__sort-direction').trigger('click') - expect(columnIds()).toEqual(['12', '13', '11']) - expect(wrapper.get('.benchmarks__sort-direction').attributes('aria-label')).toBe( - 'Switch to ascending sort' - ) + document.body.dispatchEvent(new Event('pointerdown', { bubbles: true })) + expect(details.open).toBe(false) }) - it('defaults the baseline to the first displayed column until one is picked', async () => { + it('filters by search and the toolbar selects', async () => { const wrapper = mountView() await flushPromises() - await wrapper.findAll('.benchmarks__visualization-tabs button')[1]!.trigger('click') - const baselineColumnId = () => - wrapper - .get('.benchmarks__matrix-column--baseline') - .attributes('data-testid') - ?.split('-') - .at(-1) - const sortSelect = wrapper - .findAllComponents(BaseSelect) - .find((select) => select.props('ariaLabel') === 'Sort comparison') - - expect(baselineColumnId()).toBe('13') - sortSelect?.vm.$emit('update:modelValue', 'fastestJobDurationSeconds') - await wrapper.vm.$nextTick() - expect(baselineColumnId()).toBe('11') - await wrapper - .get('[data-testid="benchmark-comparison-column-title-12"] .benchmarks__matrix-title') - .trigger('click') - await wrapper.get('.benchmarks__sort-direction').trigger('click') - expect(baselineColumnId()).toBe('12') + await wrapper.get('input[type="text"]').setValue('stable 0.3.55') + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) + expect(wrapper.find('[data-testid="benchmark-row-9"]').exists()).toBe(true) + + await wrapper.get('input[type="text"]').setValue('nothing here') + expect(wrapper.get('.benchmarks__no-results').text()).toContain('No runs match these filters') + + await wrapper.get('input[type="text"]').setValue('') + await chooseFilter(wrapper, 'hardware', 'Apple M3 Max') + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) + expect(wrapper.find('[data-testid="benchmark-row-10"]').exists()).toBe(true) }) - it('places endpoint labels immediately outside their data point markers', async () => { - const edgeBenchmark = { - ...benchmark('14', 'qwen_image_2.1_int8_bf16.json', 45.83), - fastestJobDurationSeconds: 44.7, - averageJobDurationSeconds: 45.83, - medianJobDurationSeconds: 45.61, - slowestJobDurationSeconds: 47.24 - } - vi.mocked(window.api.listPerformanceTestBenchmarks).mockResolvedValueOnce({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: [edgeBenchmark] - }) + it('labels unmanaged workspaces and unknown hardware', async () => { + useBenchmarks([ + { + ...benchmark('31', 'mystery.json', 1, null), + workspace: { id: null, name: null } + } + ]) const wrapper = mountView() await flushPromises() - const fastestLabel = wrapper.get('[data-testid="benchmark-chart-14-fastest-label"]') - expect(fastestLabel.classes()).toContain('benchmarks__chart-point-label--fastest') - expect(wrapper.get('[data-testid="benchmark-chart-14-slowest-label"]').classes()).toContain( - 'benchmarks__chart-point-label--slowest' + expect(wrapper.get('.benchmarks__group-meta').text()).toBe( + 'Unknown hardware · 1 session · 5 runs' ) - expect(wrapper.get('[data-testid="benchmark-chart-14-average-label"]').classes()).toContain( - 'benchmarks__chart-point-label--center' + expect(await filterOptions(wrapper, 'workspace')).toEqual(['All', 'Unmanaged']) + await chooseFilter(wrapper, 'workspace', '__unmanaged__') + expect(wrapper.find('[data-testid="benchmark-row-31"]').exists()).toBe(true) + }) + + it('exports the selected runs from the selection bar', async () => { + const wrapper = mountView() + await flushPromises() + + await selectRow(wrapper, '12') + await selectRow(wrapper, '13') + expect(wrapper.get('[data-testid="benchmark-row-12"]').classes()).toContain( + 'benchmarks__row--selected' ) + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('2 selected') + const exportButton = wrapper.get('.benchmarks__export-results') + expect(exportButton.text()).toBe('Export results') + await exportButton.trigger('click') + await flushPromises() + expect(exportResultsImageMock).toHaveBeenCalledTimes(1) + const [png, imageType, defaultPath] = exportResultsImageMock.mock.calls[0] as unknown as [ + ArrayBuffer, + string, + string + ] + expect(png).toBeInstanceOf(ArrayBuffer) + expect(imageType).toBe('benchmark-comparison') + expect(defaultPath).toBe('C:\\results\\performance-tests') + const svg = lastExportedSvg() + expect(svg).toContain('Benchmark Comparison') + expect(svg).toContain('portrait.json') + expect(svg).toContain('Duration range') + expect(svg).toContain('Each line shows the fastest and slowest run, with the average marked.') + expect(svg.match(/class="table-cell best-cell"/g)).toHaveLength(4) + + await wrapper.get('.benchmarks__clear-selection').trigger('click') + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) + }) + + it('exports relative speed once a baseline is chosen', async () => { + const wrapper = mountView() + await flushPromises() + + await openRowMenu(wrapper, '12') + await chooseMenuItem('.benchmarks__set-baseline') + await openMenu(wrapper, `[data-testid="${PORTRAIT_4090}"] .benchmarks__group-menu`) + await chooseMenuItem('.benchmarks__select-group') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('3 selected') await wrapper.get('.benchmarks__export-results').trigger('click') await flushPromises() - const svg = createResultsPngMock.mock.calls[0]![0] - const markerMatches = [...svg.matchAll(/]*\/>/g)] - const fastestLabelMatch = svg.match( - /44\.7 s<\/text>/ - ) - const slowestLabelMatch = svg.match( - /47\.24 s<\/text>/ + + const svg = lastExportedSvg() + expect(svg).toContain('Relative speed') + expect(svg).toContain('Relative speed factor = baseline duration / run duration.') + expect(svg).toContain('1.11× Faster') + expect(svg).toContain('1.00× Baseline') + expect(svg).not.toContain('>Duration range (min → max)') + }) + + it('shows export errors in the selection bar', async () => { + exportResultsImageMock.mockResolvedValueOnce({ ok: false, message: 'Disk full' }) + const wrapper = mountView() + await flushPromises() + + await selectRow(wrapper, '13') + await wrapper.get('.benchmarks__export-results').trigger('click') + await flushPromises() + + expect(wrapper.get('.benchmarks__export-error').text()).toBe('Disk full') + await wrapper.get('.benchmarks__clear-selection').trigger('click') + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) + }) + + it('limits exports to twenty runs', async () => { + useBenchmarks( + Array.from({ length: 21 }, (_, index) => + benchmark(String(index + 1), 'bulk.json', 2 + index / 10, 'GPU', `Build ${index}`) + ) ) - const averageMarkerX = svg.match( - /]+r="5"[^>]+class="chart-average"/ - )?.[1] - expect(svg).toContain('r="5" fill="#55e0d1" class="chart-average"') - const averageLabelX = svg.match( - /]+class="chart-point-label" text-anchor="middle">45\.83 s<\/text>/ - )?.[1] - expect(Number(fastestLabelMatch?.[1])).toBe(Number(markerMatches[0]?.[1]) - 4) - expect(Number(slowestLabelMatch?.[1])).toBe(Number(markerMatches[1]?.[1]) + 4) - expect(Number(fastestLabelMatch?.[2])).toBe(Number(markerMatches[0]?.[2]) + 4) - expect(Number(slowestLabelMatch?.[2])).toBe(Number(markerMatches[1]?.[2]) + 4) - expect(averageLabelX).toBe(averageMarkerX) + const wrapper = mountView() + await flushPromises() + + await openMenu(wrapper, '.benchmarks__group-menu') + expect(menuItem('.benchmarks__export-group')?.hasAttribute('data-disabled')).toBe(true) + await chooseMenuItem('.benchmarks__select-group') + + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('21 selected') + expect(wrapper.get('.benchmarks__export-results').attributes()).toHaveProperty('disabled') + await wrapper.get('.benchmarks__selection-bar .tooltip-wrap').trigger('focusin') + await flushPromises() + expect(document.body.textContent).toContain('Up to 20 runs') + + await selectRow(wrapper, '21', false) + expect(wrapper.get('.benchmarks__export-results').attributes()).not.toHaveProperty('disabled') }) it('confirms before deleting all files for a benchmark record', async () => { const wrapper = mountView() await flushPromises() const dialogs = useDialogs() + await selectRow(wrapper, '13') - expect(wrapper.findAll('.benchmarks__delete-record')).toHaveLength(4) - await wrapper - .get('[data-testid="benchmark-row-13"] .benchmarks__delete-record') - .trigger('click') + await openRowMenu(wrapper, '13') + await chooseMenuItem('.benchmarks__delete-record') expect(dialogs.state.open).toBe(true) expect(dialogs.state.kind).toBe('confirm') expect(dialogs.state.confirm.title).toBe('Delete portrait.json?') @@ -580,9 +721,7 @@ describe('BenchmarksView', () => { '13' ) expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(false) - expect(wrapper.find('[data-testid="benchmark-comparison-column-title-13"]').exists()).toBe( - false - ) + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) }) it('shows an error when the benchmark delete bridge rejects', async () => { @@ -591,9 +730,8 @@ describe('BenchmarksView', () => { await flushPromises() const dialogs = useDialogs() - await wrapper - .get('[data-testid="benchmark-row-13"] .benchmarks__delete-record') - .trigger('click') + await openRowMenu(wrapper, '13') + await chooseMenuItem('.benchmarks__delete-record') dialogs.confirmPrimary() await flushPromises() @@ -603,26 +741,19 @@ describe('BenchmarksView', () => { expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(true) }) - it('renames a session inline and preserves its comparison selection', async () => { + it('renames a session inline and keeps its selection, order and baseline', async () => { const wrapper = mountView() await flushPromises() - await wrapper - .findAll('.benchmarks__visualization-tabs button') - .find((tab) => tab.text() === 'Speed')! - .trigger('click') - await wrapper - .get('[data-testid="benchmark-comparison-column-title-13"] .benchmarks__matrix-title') - .trigger('click') + await openRowMenu(wrapper, '12') + await chooseMenuItem('.benchmarks__set-baseline') + await selectRow(wrapper, '12') - const sessionButton = wrapper.get('[data-testid="benchmark-row-13"] .benchmarks__session-name') - expect(sessionButton.text()).toBe('13') - expect(sessionButton.attributes('aria-label')).toBe('Edit session 13') - - await sessionButton.trigger('click') + await openRowMenu(wrapper, '12') + await chooseMenuItem('.benchmarks__rename-session') const input = wrapper.get('.benchmarks__session-name-input') - expect(input.element.value).toBe('13') + expect(input.element.value).toBe('12') expect( - wrapper.get('[data-testid="benchmark-row-13"] .benchmarks__delete-record').attributes() + wrapper.get('[data-testid="benchmark-row-12"] .benchmarks__row-menu').attributes() ).toHaveProperty('disabled') await input.setValue('gpu-baseline') await input.trigger('keydown', { key: 'Enter' }) @@ -630,21 +761,49 @@ describe('BenchmarksView', () => { expect(renamePerformanceTestBenchmarkMock).toHaveBeenCalledWith( 'C:\\results\\performance-tests', - '13', + '12', 'gpu-baseline' ) - expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(false) - expect(wrapper.get('[data-testid="benchmark-row-gpu-baseline"]').text()).toContain( - 'gpu-baseline' - ) - expect( - wrapper.find('[data-testid="benchmark-comparison-column-title-gpu-baseline"]').exists() - ).toBe(true) + expect(wrapper.find('[data-testid="benchmark-row-12"]').exists()).toBe(false) + expect(rowIds(wrapper)).toEqual(['13', 'gpu-baseline', '9']) + expect(vsLabels(wrapper)).toEqual(['1.11× faster', 'Baseline', '1.38× slower']) expect( wrapper - .get('[data-testid="benchmark-comparison-column-title-gpu-baseline"]') - .classes('benchmarks__matrix-column--baseline') - ).toBe(true) + .get('[data-testid="benchmark-row-gpu-baseline"] [role="checkbox"]') + .attributes('aria-checked') + ).toBe('true') + }) + + it('edits a session name from the name itself and reports rename errors', async () => { + renamePerformanceTestBenchmarkMock.mockResolvedValueOnce({ ok: false, message: 'Name taken' }) + const wrapper = mountView() + await flushPromises() + const dialogs = useDialogs() + + const sessionButton = wrapper.get('[data-testid="benchmark-row-13"] .benchmarks__session-name') + expect(sessionButton.attributes('aria-label')).toBe('Edit session 13') + await sessionButton.trigger('click') + const input = wrapper.get('.benchmarks__session-name-input') + await input.setValue('12') + await input.trigger('keydown', { key: 'Enter' }) + await flushPromises() + + expect(dialogs.state.kind).toBe('alert') + expect(dialogs.state.alert.title).toBe('Could not rename session') + expect(dialogs.state.alert.message).toBe('Name taken') + dialogs.cancel() + await flushPromises() + + await input.setValue(' ') + await input.trigger('keydown', { key: 'Enter' }) + await flushPromises() + expect(dialogs.state.alert.message).toBe('Enter a session name.') + dialogs.cancel() + await flushPromises() + + await input.trigger('keydown', { key: 'Escape' }) + expect(wrapper.find('.benchmarks__session-name-input').exists()).toBe(false) + expect(renamePerformanceTestBenchmarkMock).toHaveBeenCalledTimes(1) }) it('opens the current results folder in the picker and loads a selected folder', async () => { @@ -664,13 +823,16 @@ describe('BenchmarksView', () => { const wrapper = mountView() await flushPromises() + expect(wrapper.get('.benchmarks__open-folder').attributes('title')).toBe( + 'C:\\results\\performance-tests' + ) await wrapper.get('.benchmarks__open-folder').trigger('click') await flushPromises() expect(api.browseFolder).toHaveBeenCalledWith('C:\\results\\performance-tests') expect(api.listPerformanceTestBenchmarks).toHaveBeenLastCalledWith(customFolder) expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) - expect(wrapper.get('[data-testid="benchmark-row-10"]').text()).toContain('landscape.json') + expect(wrapper.get('[data-testid="benchmark-row-10"]').text()).toContain('Stable 0.3.60') }) it('refreshes the currently selected sessions folder', async () => { @@ -680,8 +842,45 @@ describe('BenchmarksView', () => { await wrapper.get('[data-testid="benchmarks-refresh"]').trigger('click') await flushPromises() - expect(window.api.listPerformanceTestBenchmarks).toHaveBeenLastCalledWith( - 'C:\\results\\performance-tests' + expect(listMock()).toHaveBeenLastCalledWith('C:\\results\\performance-tests') + }) + + it('shows a skeleton while loading', async () => { + listMock().mockReturnValue(new Promise(() => {})) + const wrapper = mountView() + await flushPromises() + + expect(wrapper.find('.benchmarks__state--loading').exists()).toBe(true) + expect( + wrapper + .get('[data-testid="benchmarks-refresh"] svg') + .classes('benchmarks__refresh-icon--busy') + ).toBe(true) + }) + + it('offers a refresh when loading fails', async () => { + listMock().mockRejectedValueOnce(new Error('broken')) + const wrapper = mountView() + await flushPromises() + + expect(wrapper.get('.benchmarks__state--error').text()).toContain( + 'Could not load performance test results' + ) + useBenchmarks(sampleBenchmarks) + await wrapper.get('.benchmarks__retry').trigger('click') + await flushPromises() + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(5) + }) + + it('points to the Performance Test when there are no results', async () => { + useBenchmarks([]) + const wrapper = mountView() + await flushPromises() + + expect(wrapper.get('.benchmarks__state--empty').text()).toContain( + 'No performance test results yet' ) + await wrapper.get('.benchmarks__open-performance-test').trigger('click') + expect(navigatePanelMock).toHaveBeenCalledExactlyOnceWith('performance-test') }) }) diff --git a/src/renderer/src/views/BenchmarksView.vue b/src/renderer/src/views/BenchmarksView.vue index 85cb6358f..03a04b196 100644 --- a/src/renderer/src/views/BenchmarksView.vue +++ b/src/renderer/src/views/BenchmarksView.vue @@ -1,13 +1,19 @@ + + diff --git a/src/renderer/src/views/benchmarks/BenchmarkSessionName.vue b/src/renderer/src/views/benchmarks/BenchmarkSessionName.vue new file mode 100644 index 000000000..36ad9cdf6 --- /dev/null +++ b/src/renderer/src/views/benchmarks/BenchmarkSessionName.vue @@ -0,0 +1,43 @@ + + + diff --git a/src/types/ipc.ts b/src/types/ipc.ts index cd7bdbcea..9bbab6f45 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -748,10 +748,17 @@ export interface PerformanceTestBenchmark { averageJobDurationSeconds: number | null medianJobDurationSeconds: number | null measuredJobCount: number - hardwareName: string | null + machine: PerformanceTestMachine result: Record } +export interface PerformanceTestMachine { + gpu: string | null + cpu: string | null + memoryGb: number | null + platform: string | null +} + export interface SnapshotDiffEntry { createdAt: string trigger: string From 43e25ea12ddce1a361b8673ff1490745cae139a0 Mon Sep 17 00:00:00 2001 From: Mariana Martinho Date: Thu, 12 Nov 2026 01:36:03 +0100 Subject: [PATCH 4/9] feat(benchmarks): group by choice, responsive table, column reorder and baseline flag Co-Authored-By: Claude Opus 5.5 --- locales/en.json | 6 +- locales/zh.json | 6 +- src/renderer/src/lib/benchmarkGroups.ts | 17 +- src/renderer/src/views/BenchmarksView.test.ts | 222 +++++++++++----- src/renderer/src/views/BenchmarksView.vue | 45 +++- .../views/benchmarks/BenchmarkGroupCard.vue | 249 ++++++++++++++---- 6 files changed, 422 insertions(+), 123 deletions(-) diff --git a/locales/en.json b/locales/en.json index 877333619..3f749db1a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1677,7 +1677,10 @@ "filterHardware": "Hardware", "filterWorkflow": "Workflow", "compareBy": "Compare by", - "groupMeta": "{hardware} · {sessions} · {runs}", + "groupMeta": "{sessions} · {runs}", + "groupBy": "Group by", + "groupByNone": "None", + "allRuns": "All runs", "sessionCount": "{count} session | {count} sessions", "runCount": "{count} run | {count} runs", "groupActions": "More actions for {workflow}", @@ -1685,7 +1688,6 @@ "exportGroup": "Export this group", "singleRun": "Only one build ran this workflow here. Run it on another build to compare.", "metricAndRange": "{metric} and range", - "vsFastest": "Vs fastest", "vsBaseline": "Vs baseline", "selection": "Selected runs", "reorderRun": "Reorder session {session}", diff --git a/locales/zh.json b/locales/zh.json index c7cd05d28..2757ddbf4 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -1677,7 +1677,10 @@ "filterHardware": "硬件", "filterWorkflow": "工作流", "compareBy": "比较依据", - "groupMeta": "{hardware} · {sessions} · {runs}", + "groupMeta": "{sessions} · {runs}", + "groupBy": "分组依据", + "groupByNone": "无", + "allRuns": "所有运行", "sessionCount": "{count} 个会话 | {count} 个会话", "runCount": "{count} 次运行 | {count} 次运行", "groupActions": "{workflow} 的更多操作", @@ -1685,7 +1688,6 @@ "exportGroup": "导出此组", "singleRun": "此处只有一个构建运行过此工作流。在另一个构建上运行它即可比较。", "metricAndRange": "{metric}及范围", - "vsFastest": "对比最快", "vsBaseline": "对比基准", "selection": "已选运行", "reorderRun": "重新排列会话 {session}", diff --git a/src/renderer/src/lib/benchmarkGroups.ts b/src/renderer/src/lib/benchmarkGroups.ts index 018bf9135..04e477ff6 100644 --- a/src/renderer/src/lib/benchmarkGroups.ts +++ b/src/renderer/src/lib/benchmarkGroups.ts @@ -18,6 +18,10 @@ export const DURATION_KEYS = [ export type DurationKey = (typeof DURATION_KEYS)[number] +export const GROUP_BY_OPTIONS = ['none', 'workflow', 'hardware'] as const + +export type GroupBy = (typeof GROUP_BY_OPTIONS)[number] + export const DURATION_LABEL_KEYS: Record = { fastestJobDurationSeconds: 'benchmarks.fastest', averageJobDurationSeconds: 'benchmarks.average', @@ -53,6 +57,7 @@ export interface BenchmarkGroup { export interface BenchmarkGroupsInput { benchmarks: readonly PerformanceTestBenchmark[] + groupBy: GroupBy isVisible: (benchmark: PerformanceTestBenchmark) => boolean metric: DurationKey sort: ColumnSort @@ -84,8 +89,14 @@ export function machineKey(benchmark: PerformanceTestBenchmark): string { return JSON.stringify([gpu, cpu, memoryGb, platform]) } -export function benchmarkGroupKey(benchmark: PerformanceTestBenchmark): string { - return JSON.stringify([benchmark.workflowName, machineKey(benchmark)]) +export function isGroupBy(value: string): value is GroupBy { + return (GROUP_BY_OPTIONS as readonly string[]).includes(value) +} + +export function benchmarkGroupKey(benchmark: PerformanceTestBenchmark, groupBy: GroupBy): string { + if (groupBy === 'workflow') return JSON.stringify(['workflow', benchmark.workflowName]) + if (groupBy === 'hardware') return JSON.stringify(['hardware', machineKey(benchmark)]) + return 'all' } export function machineLabels( @@ -227,7 +238,7 @@ function buildGroup( export function buildBenchmarkGroups(input: BenchmarkGroupsInput): BenchmarkGroup[] { const membersByKey = new Map() for (const benchmark of input.benchmarks) { - const key = benchmarkGroupKey(benchmark) + const key = benchmarkGroupKey(benchmark, input.groupBy) const members = membersByKey.get(key) if (members) members.push(benchmark) else membersByKey.set(key, [benchmark]) diff --git a/src/renderer/src/views/BenchmarksView.test.ts b/src/renderer/src/views/BenchmarksView.test.ts index 3ece1d082..31c2fbf76 100644 --- a/src/renderer/src/views/BenchmarksView.test.ts +++ b/src/renderer/src/views/BenchmarksView.test.ts @@ -109,9 +109,8 @@ const sampleBenchmarks = [ run('10', 'landscape.json', 8.7, { gpu: 'Apple M3 Max', build: 'Stable 0.3.60' }), run('9', 'portrait.json', 3, { build: 'Stable 0.3.55', fastest: 1 }) ] -const PORTRAIT_4090 = 'benchmark-group-portrait.json-NVIDIA RTX 4090' -const PORTRAIT_5090 = 'benchmark-group-portrait.json-NVIDIA RTX 5090' -const LANDSCAPE_M3 = 'benchmark-group-landscape.json-Apple M3 Max' +const PORTRAIT_4090 = 'benchmark-group-NVIDIA RTX 4090' +const PORTRAIT_5090 = 'benchmark-group-NVIDIA RTX 5090' function listMock(): ReturnType { return window.api.listPerformanceTestBenchmarks as unknown as ReturnType @@ -125,6 +124,13 @@ function mountView() { return mount(BenchmarksView, { attachTo: document.body }) } +async function mountGrouped(groupBy: string) { + const wrapper = mountView() + await flushPromises() + await chooseOption(wrapper, 'group-by', groupBy) + return wrapper +} + function group(wrapper: VueWrapper, testId = PORTRAIT_4090) { return wrapper.get(`[data-testid="${testId}"]`) } @@ -341,28 +347,28 @@ describe('BenchmarksView', () => { }) }) - it('compares the builds that ran each workflow on each machine', async () => { - const wrapper = mountView() - await flushPromises() + it('compares the builds that ran on each machine when grouped by hardware', async () => { + const wrapper = await mountGrouped('Hardware') const cards = wrapper.findAll('.benchmarks__group') expect(cards.map((card) => card.get('.benchmarks__group-toggle').text())).toEqual([ - 'portrait.json', - 'portrait.json', - 'landscape.json' + 'NVIDIA RTX 4090', + 'NVIDIA RTX 5090', + 'Apple M3 Max' ]) expect(cards.map((card) => card.get('.benchmarks__group-meta').text())).toEqual([ - 'NVIDIA RTX 4090 · 3 sessions · 15 runs', - 'NVIDIA RTX 5090 · 1 session · 5 runs', - 'Apple M3 Max · 1 session · 5 runs' + '3 sessions · 15 runs', + '1 session · 5 runs', + '1 session · 5 runs' ]) expect(texts(wrapper, 'thead th')).toEqual([ '', 'Session', + 'Workflow', 'Build', 'Median and range', 'Median ↑', - 'Vs fastest', + 'Vs baseline', '' ]) expect(rowIds(wrapper)).toEqual(['13', '12', '9']) @@ -383,10 +389,7 @@ describe('BenchmarksView', () => { (row(wrapper, '13').get('.benchmarks__bar-fill').element as HTMLElement).style.width ) ).toBeCloseTo((1.9 / 3.7) * 100, 3) - expect(group(wrapper).find('.benchmarks__single-note').exists()).toBe(false) - expect(group(wrapper, PORTRAIT_5090).get('.benchmarks__single-note').text()).toBe( - 'Only one build ran this workflow here. Run it on another build to compare.' - ) + expect(wrapper.find('.benchmarks__single-note').exists()).toBe(false) expect(vsLabels(wrapper, PORTRAIT_5090)).toEqual(['']) expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) }) @@ -397,12 +400,15 @@ describe('BenchmarksView', () => { run('22', 'a.json', 3, { cpu: 'Intel Core i9-14900K' }), run('23', 'a.json', 2.5, { cpu: 'AMD Ryzen 9 7950X', build: 'Stable' }) ]) - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') + expect(texts(wrapper, '.benchmarks__group-toggle', 'benchmarks')).toEqual([ + 'NVIDIA RTX 4090 · AMD Ryzen 9 7950X', + 'NVIDIA RTX 4090 · Intel Core i9-14900K' + ]) expect(texts(wrapper, '.benchmarks__group-meta', 'benchmarks')).toEqual([ - 'NVIDIA RTX 4090 · AMD Ryzen 9 7950X · 2 sessions · 10 runs', - 'NVIDIA RTX 4090 · Intel Core i9-14900K · 1 session · 5 runs' + '2 sessions · 10 runs', + '1 session · 5 runs' ]) expect(await menuOptions(wrapper, 'hardware')).toEqual([ 'All', @@ -415,29 +421,33 @@ describe('BenchmarksView', () => { }) it('compares filtered runs with the fastest run of the whole group', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await search(wrapper, 'stable 0.3.55') expect(rowIds(wrapper)).toEqual(['9']) expect(vsLabels(wrapper)).toEqual(['1.53× slower']) - expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs fastest') + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') expect(group(wrapper).get('.benchmarks__reference-hidden').text()).toBe( 'Compared with session 13, which the filters hide.' ) - expect(group(wrapper).find('.benchmarks__single-note').exists()).toBe(false) await search(wrapper, '') await chooseOption(wrapper, 'build', 'Stable 0.3.60') expect(rowIds(wrapper)).toEqual(['12']) expect(vsLabels(wrapper)).toEqual(['1.11× slower']) - expect(group(wrapper).find('.benchmarks__single-note').exists()).toBe(false) - expect(group(wrapper, LANDSCAPE_M3).find('.benchmarks__single-note').exists()).toBe(true) + + await chooseOption(wrapper, 'group-by', 'Workflow') + expect(rowIds(wrapper, 'benchmark-group-portrait.json')).toEqual(['12']) + expect( + group(wrapper, 'benchmark-group-portrait.json').find('.benchmarks__single-note').exists() + ).toBe(false) + expect( + group(wrapper, 'benchmark-group-landscape.json').get('.benchmarks__single-note').text() + ).toBe('Only one build ran this workflow here. Run it on another build to compare.') }) it('keeps a chosen baseline while filters hide it, and resets it', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await rowAction(wrapper, '12', '.benchmarks__set-baseline') expect(vsLabels(wrapper)).toEqual(['1.11× faster', 'Baseline', '1.38× slower']) @@ -462,7 +472,7 @@ describe('BenchmarksView', () => { await group(wrapper).get('.benchmarks__reset-baseline').trigger('click') expect(vsLabels(wrapper)).toEqual(['Fastest', '1.60× slower', '1.80× slower']) - expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs fastest') + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') }) it('calls equally fast runs the same speed', async () => { @@ -470,16 +480,14 @@ describe('BenchmarksView', () => { run('21', 'a.json', 2, { gpu: 'GPU', build: 'Build A' }), run('22', 'a.json', 2, { gpu: 'GPU', build: 'Build B' }) ]) - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') - expect(vsLabels(wrapper, 'benchmark-group-a.json-GPU')).toEqual(['Fastest', 'Same speed']) + expect(vsLabels(wrapper, 'benchmark-group-GPU')).toEqual(['Fastest', 'Same speed']) }) it('shows runs without measurements as unavailable', async () => { useBenchmarks([...sampleBenchmarks, unmeasuredRun('8')]) - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') expect(rowIds(wrapper)).toEqual(['13', '12', '9', '8']) expect(row(wrapper, '8').get('.benchmarks__no-data').text()).toBe('No measured runs') @@ -510,8 +518,7 @@ describe('BenchmarksView', () => { }) it('compares by another metric and sorts by any header', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') expect(await menuOptions(wrapper, 'compare-by')).toEqual([ 'Fastest', @@ -540,8 +547,7 @@ describe('BenchmarksView', () => { }) it('moves rows up and down by dragging and with Alt+arrow keys', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await drag(wrapper, '13', '12') expect(rowIds(wrapper)).toEqual(['12', '13', '9']) @@ -564,8 +570,7 @@ describe('BenchmarksView', () => { }) it('clears the drop highlight when a drag leaves the table', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') const dataTransfer = new DataTransfer() await row(wrapper, '9').get('.benchmarks__grip').trigger('dragstart', { dataTransfer }) @@ -586,8 +591,7 @@ describe('BenchmarksView', () => { }) it('keeps every row movable after filters change, runs arrive or another folder loads', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await search(wrapper, 'stable') await drag(wrapper, '9', '12') @@ -614,8 +618,7 @@ describe('BenchmarksView', () => { }) it('sorts again by any header after a manual order', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await drag(wrapper, '9', '13') expect(rowIds(wrapper)).toEqual(['9', '13', '12']) @@ -626,8 +629,7 @@ describe('BenchmarksView', () => { }) it('collapses a group', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') const toggle = group(wrapper).get('.benchmarks__group-toggle') expect(toggle.attributes('aria-expanded')).toBe('true') @@ -639,8 +641,7 @@ describe('BenchmarksView', () => { }) it('adds picked columns to the table and the export, best value in green', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await openMenu(wrapper, '[data-testid="benchmarks-columns"]') expect(menuItem('[data-testid="benchmark-column-systemInfo.cpu_model"]')).not.toBeNull() @@ -659,11 +660,12 @@ describe('BenchmarksView', () => { expect(texts(wrapper, 'thead th')).toEqual([ '', 'Session', + 'Workflow', 'Median and range', 'Average', 'Custom Score', 'Median ↑', - 'Vs fastest', + 'Vs baseline', '' ]) expect(texts(wrapper, '.benchmarks__best')).toEqual(['2 s']) @@ -678,6 +680,103 @@ describe('BenchmarksView', () => { expect(svg).toContain('Duration range') }) + it('compares every run in one table by default and groups on demand', async () => { + const wrapper = mountView() + await flushPromises() + + expect(wrapper.findAll('.benchmarks__group')).toHaveLength(1) + expect(group(wrapper, 'benchmark-group-all').get('.benchmarks__group-toggle').text()).toBe( + 'All runs' + ) + expect(rowIds(wrapper, 'benchmark-group-all')).toHaveLength(5) + expect(texts(wrapper, '.benchmarks__workflow', 'benchmark-group-all')).toContain( + 'landscape.json' + ) + expect(wrapper.find('.benchmarks__single-note').exists()).toBe(false) + expect(await menuOptions(wrapper, 'group-by')).toEqual(['None', 'Workflow', 'Hardware']) + + await chooseOption(wrapper, 'group-by', 'Workflow') + expect(wrapper.findAll('.benchmarks__group-toggle').map((toggle) => toggle.text())).toEqual([ + 'portrait.json', + 'landscape.json' + ]) + expect(wrapper.find('.benchmarks__workflow').exists()).toBe(false) + expect(texts(wrapper, 'thead th', 'benchmark-group-portrait.json')).toContain('Vs baseline') + + await chooseOption(wrapper, 'group-by', 'None') + expect(wrapper.findAll('.benchmarks__group')).toHaveLength(1) + }) + + it('truncates long workflow names and keeps the full name in a tooltip', async () => { + const longName = 'my_client_project_final_v3_portrait_upscale_with_face_detailer_2x.json' + useBenchmarks([run('5', longName, 2), run('6', longName, 3, { build: 'Stable 0.3.60' })]) + const wrapper = mountView() + await flushPromises() + + const cell = row(wrapper, '5').get('.benchmarks__workflow') + expect(cell.classes()).toContain('truncate') + expect(cell.attributes('title')).toBe(longName) + + await chooseOption(wrapper, 'group-by', 'Workflow') + const toggle = group(wrapper, `benchmark-group-${longName}`).get('.benchmarks__group-toggle') + expect(toggle.attributes('title')).toBe(longName) + expect(toggle.get('.benchmarks__group-label').classes()).toContain('truncate') + }) + + it('sets and resets the baseline from the flag on a row', async () => { + const wrapper = await mountGrouped('Hardware') + + expect(row(wrapper, '13').find('[data-testid="benchmark-baseline-13"]').exists()).toBe(false) + const flag = row(wrapper, '12').get('[data-testid="benchmark-baseline-12"]') + expect(flag.attributes('aria-pressed')).toBe('false') + expect(flag.attributes('aria-label')).toBe('Set as baseline') + + await flag.trigger('click') + const chosen = row(wrapper, '12').get('[data-testid="benchmark-baseline-12"]') + expect(chosen.attributes('aria-pressed')).toBe('true') + expect(chosen.attributes('aria-label')).toBe('Reset baseline') + expect(group(wrapper).find('.benchmarks__reset-baseline').exists()).toBe(true) + expect(row(wrapper, '13').find('[data-testid="benchmark-baseline-13"]').exists()).toBe(true) + + await chosen.trigger('click') + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.11× slower', '1.53× slower']) + expect(group(wrapper).find('.benchmarks__reset-baseline').exists()).toBe(false) + }) + + it('reorders picked columns by dragging headers and with Alt+arrow keys', async () => { + const wrapper = mountView() + await flushPromises() + await openMenu(wrapper, '[data-testid="benchmarks-columns"]') + menuItem('[data-testid="benchmark-column-averageJobDurationSeconds"]')!.click() + menuItem('[data-testid="benchmark-column-customScore"]')!.click() + await flushPromises() + await closeMenu() + const heads = () => + wrapper + .findAll('[data-testid^="benchmark-column-head-"]') + .map((head) => head.attributes('data-testid')!.replace('benchmark-column-head-', '')) + const head = (key: string) => wrapper.get(`[data-testid="benchmark-column-head-${key}"]`) + expect(heads()).toEqual(['averageJobDurationSeconds', 'customScore']) + + const dataTransfer = new DataTransfer() + await head('customScore').trigger('dragstart', { dataTransfer }) + await head('averageJobDurationSeconds').trigger('dragover', { dataTransfer }) + expect(head('averageJobDurationSeconds').classes()).toContain( + 'benchmarks__column-head--drop-target' + ) + await head('averageJobDurationSeconds').trigger('drop', { dataTransfer }) + await head('customScore').trigger('dragend', { dataTransfer }) + expect(heads()).toEqual(['customScore', 'averageJobDurationSeconds']) + expect(wrapper.find('.benchmarks__column-head--drop-target').exists()).toBe(false) + + await head('customScore').get('button').trigger('keydown', { key: 'ArrowRight', altKey: true }) + expect(heads()).toEqual(['averageJobDurationSeconds', 'customScore']) + await head('averageJobDurationSeconds') + .get('button') + .trigger('keydown', { key: 'ArrowLeft', altKey: true }) + expect(heads()).toEqual(['averageJobDurationSeconds', 'customScore']) + }) + it('filters by search and the toolbar menus', async () => { const wrapper = mountView() await flushPromises() @@ -702,12 +801,10 @@ describe('BenchmarksView', () => { useBenchmarks([ { ...run('31', 'mystery.json', 1, { gpu: null }), workspace: { id: null, name: null } } ]) - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') - expect(wrapper.get('.benchmarks__group-meta').text()).toBe( - 'Unknown hardware · 1 session · 5 runs' - ) + expect(wrapper.get('.benchmarks__group-toggle').text()).toBe('Unknown hardware') + expect(wrapper.get('.benchmarks__group-meta').text()).toBe('1 session · 5 runs') expect(await menuOptions(wrapper, 'workspace')).toEqual(['All', 'Unmanaged']) await chooseOption(wrapper, 'workspace', 'Unmanaged') expect(wrapper.find('[data-testid="benchmark-row-31"]').exists()).toBe(true) @@ -750,8 +847,7 @@ describe('BenchmarksView', () => { }) it('exports relative speed only for one group with a chosen baseline', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await rowAction(wrapper, '12', '.benchmarks__set-baseline') await groupAction(wrapper, '.benchmarks__select-group') @@ -769,8 +865,7 @@ describe('BenchmarksView', () => { it('shows export errors in the selection bar until the list reloads', async () => { exportResultsImageMock.mockResolvedValueOnce({ ok: false, message: 'Disk full' }) - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await groupAction(wrapper, '.benchmarks__export-group') expect(wrapper.get('.benchmarks__export-error').text()).toBe('Disk full') @@ -805,8 +900,7 @@ describe('BenchmarksView', () => { }) it('renames a session in place and keeps its selection, order and baseline', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') await rowAction(wrapper, '12', '.benchmarks__set-baseline') await selectRow(wrapper, '12') await drag(wrapper, '9', '13') @@ -829,8 +923,7 @@ describe('BenchmarksView', () => { it('puts the old name back when a rename is refused', async () => { renamePerformanceTestBenchmarkMock.mockResolvedValueOnce({ ok: false, message: 'Name taken' }) - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') const dialogs = useDialogs() expect(sessionField(wrapper, '13').getAttribute('aria-label')).toBe('Edit session 13') @@ -846,8 +939,7 @@ describe('BenchmarksView', () => { }) it('confirms before deleting all files for a session', async () => { - const wrapper = mountView() - await flushPromises() + const wrapper = await mountGrouped('Hardware') const dialogs = useDialogs() await selectRow(wrapper, '13') diff --git a/src/renderer/src/views/BenchmarksView.vue b/src/renderer/src/views/BenchmarksView.vue index 6735097d4..70d168eca 100644 --- a/src/renderer/src/views/BenchmarksView.vue +++ b/src/renderer/src/views/BenchmarksView.vue @@ -35,11 +35,14 @@ import { DURATION_KEYS, DURATION_LABEL_KEYS, formatSeconds, + GROUP_BY_OPTIONS, isDurationKey, + isGroupBy, machineKey, machineLabels, type ColumnSort, - type DurationKey + type DurationKey, + type GroupBy } from '../lib/benchmarkGroups' import { createResultsPng } from '../lib/performanceTestResultsSvg' import { useDialogs } from '../composables/useDialogs' @@ -78,6 +81,7 @@ const workspaceFilter = ref('') const buildFilter = ref('') const machineFilter = ref('') const workflowFilter = ref('') +const groupBy = ref('none') const metric = ref('medianJobDurationSeconds') const columnSort = ref({ key: 'medianJobDurationSeconds', ascending: true }) const manualOrders = ref>({}) @@ -85,6 +89,7 @@ const baselineIds = ref>({}) const collapsedGroupKeys = ref>(new Set()) const selectedIds = ref>(new Set()) const visibleColumnKeys = ref>(new Set(['instance.name'])) +const columnOrder = ref([]) const deletingIds = ref>(new Set()) const isExporting = ref(false) const exportError = ref(null) @@ -199,11 +204,16 @@ const availableColumnKeys = computed(() => { return [...ordered, ...[...keys].sort((a, b) => a.localeCompare(b))] }) const buildColumnVisible = computed(() => visibleColumnKeys.value.has('instance.name')) -const extraColumns = computed(() => - availableColumnKeys.value +const extraColumns = computed(() => { + const position = new Map(columnOrder.value.map((key, index) => [key, index])) + return availableColumnKeys.value .filter((key) => key !== 'instance.name' && visibleColumnKeys.value.has(key)) + .sort( + (a, b) => + (position.get(a) ?? Number.MAX_SAFE_INTEGER) - (position.get(b) ?? Number.MAX_SAFE_INTEGER) + ) .map((key) => ({ key, label: columnLabel(key) })) -) +}) const exportColumns = computed(() => [ ...IDENTITY_COLUMN_KEYS, @@ -278,6 +288,22 @@ const toolbarMenus = computed(() => [ benchmarks.value.map(({ workflowName }) => ({ value: workflowName, label: workflowName })) ) }, + { + key: 'group-by', + label: t('benchmarks.groupBy'), + value: groupBy.value, + set: setGroupBy, + options: GROUP_BY_OPTIONS.map((value) => ({ + value, + label: t( + { + none: 'benchmarks.groupByNone', + workflow: 'benchmarks.filterWorkflow', + hardware: 'benchmarks.filterHardware' + }[value] + ) + })) + }, { key: 'compare-by', label: t('benchmarks.compareBy'), @@ -315,6 +341,7 @@ function isVisible(benchmark: PerformanceTestBenchmark): boolean { const groups = computed(() => buildBenchmarkGroups({ benchmarks: benchmarks.value, + groupBy: groupBy.value, isVisible, metric: metric.value, sort: columnSort.value, @@ -330,6 +357,10 @@ const selectionOverLimit = computed( () => selectedRuns.value.length > MAX_BENCHMARK_COMPARISON_EXPORT_RUNS ) +function setGroupBy(value: string): void { + if (isGroupBy(value)) groupBy.value = value +} + function setMetric(value: string): void { if (!isDurationKey(value)) return if (columnSort.value.key === metric.value) columnSort.value = { ...columnSort.value, key: value } @@ -392,7 +423,7 @@ function isBestAmong( async function exportRuns(runs: PerformanceTestBenchmark[]): Promise { if (runs.length === 0 || runs.length > MAX_BENCHMARK_COMPARISON_EXPORT_RUNS) return - const groupKeys = new Set(runs.map(benchmarkGroupKey)) + const groupKeys = new Set(runs.map((run) => benchmarkGroupKey(run, groupBy.value))) const onlyGroup = groupKeys.size === 1 ? groups.value.find((group) => groupKeys.has(group.key)) : undefined const relativeGroup = onlyGroup?.customBaseline ? onlyGroup : undefined @@ -655,10 +686,10 @@ onActivated(() => { :key="option.value" :value="option.value" :data-value="option.value" + class="data-[state=checked]:bg-accent" @select="menu.set(option.value)" > {{ option.label }} -