diff --git a/locales/en.json b/locales/en.json index 570a0ac4b..f878e4e0b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1623,16 +1623,10 @@ }, "benchmarks": { "title": "Benchmarks", - "description": "Compare results from your performance tests.", - "runsLibrary": "Runs library", "openFolder": "Select folder", "refresh": "Refresh sessions", "searchPlaceholder": "Search runs...", - "allWorkspaces": "All workspaces", - "allInstances": "All builds", - "allHardware": "All hardware", - "allWorkflows": "All workflows", - "columns": "Properties", + "columns": "Columns", "columnsToDisplay": "Columns to display", "unmanagedWorkspace": "Unmanaged", "unknownHardware": "Unknown hardware", @@ -1648,8 +1642,7 @@ "median": "Median", "slowest": "Slowest", "measuredRuns": "Measured runs", - "selectVisible": "Select all visible runs", - "selectRun": "Select {workflow}", + "selectRun": "Select session {session}", "actions": "Actions", "deleteRecord": "Delete {workflow} benchmark", "deleteConfirmTitle": "Delete {workflow}?", @@ -1657,40 +1650,66 @@ "deleteFiles": "Delete files", "deleteErrorTitle": "Could not delete benchmark", "deleteErrorMessage": "The benchmark files could not be deleted.", - "sessionName": "Session name", "editSessionName": "Edit session {session}", "renameErrorTitle": "Could not rename session", "renameErrorMessage": "The session folder could not be renamed.", - "sessionNameRequired": "Enter a session name.", - "comparison": "Comparison", - "sortComparison": "Sort comparison", - "manualSort": "Manual sort", - "switchSortAscending": "Switch to ascending sort", - "switchSortDescending": "Switch to descending sort", - "reorderComparisonColumn": "Reorder {workflow} comparison column", - "reorderComparisonHint": "Drag to reorder. You can also press Alt+Left or Alt+Right.", + "reorderComparisonHint": "Drag to reorder. You can also press Alt+Up or Alt+Down.", + "reorderColumnHint": "Drag to reorder. You can also press Alt+Left or Alt+Right.", "comparisonImageTitle": "Benchmark Comparison", "exportResultsImage": "Export results", "exportingImage": "Exporting image...", "exportImageFailed": "Could not export the comparison image.", "metric": "Metric", "baseline": "Baseline", - "selectBaseline": "Use {workflow}, session {session}, as the baseline", - "visualization": "Comparison visualization", - "speed": "Speed", "relativePerformance": "Relative speed", "relativePerformanceHint": "Choose a baseline. Relative speed factor = baseline duration / run duration.", "faster": "Faster", "slower": "Slower", "same": "Same speed", "missing": "Unavailable", - "duration": "Duration", "durationRange": "Duration range (min → max)", "durationRangeHint": "Each line shows the fastest and slowest run, with the average marked.", - "selectPrompt": "Select runs from the library to compare them.", - "empty": "No performance test results yet. Complete a Performance Test to add a run.", - "noMatches": "No runs match these filters.", - "loadError": "Could not load performance test results." + "noMatches": "No runs match these filters", + "loadError": "Could not load performance test results", + "loadErrorHint": "Check the folder and try again.", + "refreshButton": "Refresh", + "emptyTitle": "No performance test results yet", + "emptyHint": "Complete a Performance Test to add a run.", + "openPerformanceTest": "Open Performance Test", + "noMatchesHint": "Try another search or filter.", + "filterAll": "All", + "filterWorkspace": "Workspace", + "filterBuild": "Build", + "filterHardware": "Hardware", + "filterWorkflow": "Workflow", + "compareBy": "Compare by", + "groupMeta": "{sessions} · {runs}", + "groupBy": "Group by", + "groupByNone": "None", + "allRuns": "All runs", + "sessionCount": "{count} session | {count} sessions", + "runCount": "{count} run | {count} runs", + "groupActions": "More actions for {workflow}", + "selectGroup": "Select all runs", + "exportGroup": "Export this group", + "singleRun": "Only one build ran this workflow here. Run it on another build to compare.", + "metricAndRange": "{metric} and range", + "vsBaseline": "Vs baseline", + "selection": "Selected runs", + "reorderRun": "Reorder session {session}", + "noMeasuredRuns": "No measured runs", + "timesFaster": "{factor}× faster", + "timesSlower": "{factor}× slower", + "moreActions": "More actions for session {session}", + "setBaseline": "Set as baseline", + "renameSession": "Rename session", + "speedFormula": "Speed = baseline time ÷ run time", + "resetBaseline": "Reset baseline", + "selectedCount": "{count} selected", + "exportLimit": "Up to {max} runs", + "clearSelection": "Clear selection", + "memory": "{gigabytes} GB RAM", + "referenceHidden": "Compared with session {session}, which the filters hide." }, "downloadsPopup": { "title": "Downloads", diff --git a/locales/zh.json b/locales/zh.json index 9d244cded..c01a429c8 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -1623,16 +1623,10 @@ }, "benchmarks": { "title": "性能比较", - "description": "比较性能测试结果。", - "runsLibrary": "运行记录", "openFolder": "选择文件夹", "refresh": "刷新会话", "searchPlaceholder": "搜索工作流、构建或硬件...", - "allWorkspaces": "所有工作区", - "allInstances": "所有构建", - "allHardware": "所有硬件", - "allWorkflows": "所有工作流", - "columns": "属性", + "columns": "列", "columnsToDisplay": "要显示的列", "unmanagedWorkspace": "未管理", "unknownHardware": "未知硬件", @@ -1648,8 +1642,7 @@ "median": "中位数", "slowest": "最慢", "measuredRuns": "测量运行", - "selectVisible": "选择所有可见运行", - "selectRun": "选择 {workflow}", + "selectRun": "选择会话 {session}", "actions": "操作", "deleteRecord": "删除 {workflow} 基准测试", "deleteConfirmTitle": "删除 {workflow}?", @@ -1657,40 +1650,66 @@ "deleteFiles": "删除文件", "deleteErrorTitle": "无法删除基准测试", "deleteErrorMessage": "无法删除基准测试文件。", - "sessionName": "会话名称", "editSessionName": "编辑会话 {session}", "renameErrorTitle": "无法重命名会话", "renameErrorMessage": "无法重命名会话文件夹。", - "sessionNameRequired": "请输入会话名称。", - "comparison": "比较", - "sortComparison": "排序比较", - "manualSort": "手动排序", - "switchSortAscending": "切换为升序", - "switchSortDescending": "切换为降序", - "reorderComparisonColumn": "重新排列 {workflow} 比较列", - "reorderComparisonHint": "拖动以重新排序。也可以按 Alt+左箭头或 Alt+右箭头。", + "reorderComparisonHint": "拖动以重新排序。也可以按 Alt+上箭头或 Alt+下箭头。", + "reorderColumnHint": "拖动以重新排序。也可以按 Alt+左箭头或 Alt+右箭头。", "comparisonImageTitle": "基准比较", "exportResultsImage": "导出结果", "exportingImage": "正在导出图像...", "exportImageFailed": "无法导出比较图像。", "metric": "指标", "baseline": "基准", - "selectBaseline": "将 {workflow}(会话 {session})设为基准", - "visualization": "比较可视化", - "speed": "速度", "relativePerformance": "相对速度", "relativePerformanceHint": "选择基准。相对速度系数 = 基准耗时 / 运行耗时。", "faster": "更快", "slower": "更慢", "same": "速度相同", "missing": "不可用", - "duration": "耗时", "durationRange": "耗时范围(最短 → 最长)", "durationRangeHint": "每条线显示最快和最慢运行,并标记平均值。", - "selectPrompt": "从运行记录中选择要比较的运行。", - "empty": "暂无性能测试结果。请完成一次性能测试以添加运行记录。", - "noMatches": "没有符合筛选条件的运行。", - "loadError": "无法加载性能测试结果。" + "noMatches": "没有符合筛选条件的运行", + "loadError": "无法加载性能测试结果", + "loadErrorHint": "请检查文件夹后重试。", + "refreshButton": "刷新", + "emptyTitle": "暂无性能测试结果", + "emptyHint": "完成一次性能测试以添加运行记录。", + "openPerformanceTest": "打开性能测试", + "noMatchesHint": "请尝试其他搜索或筛选条件。", + "filterAll": "全部", + "filterWorkspace": "工作区", + "filterBuild": "构建", + "filterHardware": "硬件", + "filterWorkflow": "工作流", + "compareBy": "比较依据", + "groupMeta": "{sessions} · {runs}", + "groupBy": "分组依据", + "groupByNone": "无", + "allRuns": "所有运行", + "sessionCount": "{count} 个会话 | {count} 个会话", + "runCount": "{count} 次运行 | {count} 次运行", + "groupActions": "{workflow} 的更多操作", + "selectGroup": "选择所有运行", + "exportGroup": "导出此组", + "singleRun": "此处只有一个构建运行过此工作流。在另一个构建上运行它即可比较。", + "metricAndRange": "{metric}及范围", + "vsBaseline": "对比基准", + "selection": "已选运行", + "reorderRun": "重新排列会话 {session}", + "noMeasuredRuns": "没有测量的运行", + "timesFaster": "快 {factor}×", + "timesSlower": "慢 {factor}×", + "moreActions": "会话 {session} 的更多操作", + "setBaseline": "设为基准", + "renameSession": "重命名会话", + "speedFormula": "速度 = 基准耗时 ÷ 运行耗时", + "resetBaseline": "重置基准", + "selectedCount": "已选择 {count} 项", + "exportLimit": "最多 {max} 次运行", + "clearSelection": "清除选择", + "memory": "{gigabytes} GB 内存", + "referenceHidden": "与会话 {session} 比较,该会话已被筛选条件隐藏。" }, "downloadsPopup": { "title": "下载", diff --git a/src/main/lib/performanceTestWorkflows.test.ts b/src/main/lib/performanceTestWorkflows.test.ts index b81763b33..df55ba010 100644 --- a/src/main/lib/performanceTestWorkflows.test.ts +++ b/src/main/lib/performanceTestWorkflows.test.ts @@ -147,6 +147,7 @@ describe('listPerformanceTestBenchmarks', () => { medianJobDurationSeconds: 2, measuredJobCount: 5, hardware: { deviceName: 'NVIDIA RTX 4090' }, + systemInfo: { cpu_model: 'AMD Ryzen 9 7950X', total_memory_gb: 64, platform: 'win32' }, customMetric: 42 } await Promise.all( @@ -191,7 +192,7 @@ describe('listPerformanceTestBenchmarks', () => { averageJobDurationSeconds: 2.125, medianJobDurationSeconds: 2, measuredJobCount: 5, - hardwareName: 'mps', + machine: { gpu: 'mps', cpu: 'AMD Ryzen 9 7950X', memoryGb: 64, platform: 'win32' }, result: { ...validSummary, createdAt: '2026-09-13T15:30:45.000Z', @@ -202,6 +203,7 @@ describe('listPerformanceTestBenchmarks', () => { } }) expect(benchmarks[1]!.createdAt).toBeNull() + expect(benchmarks[1]!.machine.gpu).toBe('NVIDIA RTX 4090') }) it('returns an empty list before any performance tests have been saved', async () => { diff --git a/src/main/lib/performanceTestWorkflows.ts b/src/main/lib/performanceTestWorkflows.ts index 410162a3a..36d26da26 100644 --- a/src/main/lib/performanceTestWorkflows.ts +++ b/src/main/lib/performanceTestWorkflows.ts @@ -3,6 +3,7 @@ import path from 'path' import type { AcceleratorSnapshot, PerformanceTestBenchmark, + PerformanceTestMachine, PerformanceTestResultValue, PerformanceTestResultsSummary, PerformanceTestStatistics, @@ -31,6 +32,24 @@ function isDuration(value: unknown): value is number | null { return value === null || (typeof value === 'number' && Number.isFinite(value) && value >= 0) } +function stringOrNull(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value : null +} + +function parsePerformanceTestMachine( + summary: Partial +): PerformanceTestMachine { + const hardware = summary.hardware + const systemInfo = summary.systemInfo as Partial | undefined + const memoryGb = systemInfo?.total_memory_gb + return { + gpu: stringOrNull(hardware?.deviceName) ?? stringOrNull(hardware?.deviceType), + cpu: stringOrNull(systemInfo?.cpu_model), + memoryGb: typeof memoryGb === 'number' && Number.isFinite(memoryGb) ? memoryGb : null, + platform: stringOrNull(systemInfo?.platform) + } +} + function parsePerformanceTestBenchmark( value: unknown, id: string @@ -58,13 +77,6 @@ function parsePerformanceTestBenchmark( ) { return null } - const hardware = summary.hardware - const hardwareName = - hardware && typeof hardware.deviceName === 'string' - ? hardware.deviceName - : hardware && typeof hardware.deviceType === 'string' - ? hardware.deviceType - : null return { id, createdAt: summary.createdAt ?? null, @@ -76,7 +88,7 @@ function parsePerformanceTestBenchmark( averageJobDurationSeconds: summary.averageJobDurationSeconds, medianJobDurationSeconds: summary.medianJobDurationSeconds, measuredJobCount: summary.measuredJobCount!, - hardwareName, + machine: parsePerformanceTestMachine(summary), result: value as Record } } diff --git a/src/renderer/src/components/BrandedPageHeader.vue b/src/renderer/src/components/BrandedPageHeader.vue index d3e3d6c27..7ec1bf3e0 100644 --- a/src/renderer/src/components/BrandedPageHeader.vue +++ b/src/renderer/src/components/BrandedPageHeader.vue @@ -2,12 +2,14 @@ defineProps<{ title: string description?: string + actionsInline?: boolean }>() diff --git a/src/renderer/src/lib/benchmarkComparisonSvg.ts b/src/renderer/src/lib/benchmarkComparisonSvg.ts index 53dad7e1c..c7321eece 100644 --- a/src/renderer/src/lib/benchmarkComparisonSvg.ts +++ b/src/renderer/src/lib/benchmarkComparisonSvg.ts @@ -1,4 +1,5 @@ import comfyWordmarkSource from '../components/icons/ComfyWordmark.vue?raw' +import { formatSeconds } from './benchmarkGroups' import { getRelativeSpeedOffset, getRelativeSpeedScale, @@ -99,10 +100,6 @@ function propertyLines( .join('') } -function formatDuration(value: number): string { - return `${value.toFixed(2).replace(/\.?0+$/, '')} s` -} - export const MAX_BENCHMARK_COMPARISON_EXPORT_RUNS = 20 export function createBenchmarkComparisonSvg(data: BenchmarkComparisonImageData): string { @@ -195,14 +192,14 @@ export function createBenchmarkComparisonSvg(data: BenchmarkComparisonImageData) ? ` - ${text(fastestX - 4, trackY + 4, formatDuration(fastest), 'chart-point-label', 'end')} - ${text(slowestX + 4, trackY + 4, formatDuration(slowest), 'chart-point-label', 'start')}` + ${text(fastestX - 4, trackY + 4, formatSeconds(fastest), 'chart-point-label', 'end')} + ${text(slowestX + 4, trackY + 4, formatSeconds(slowest), 'chart-point-label', 'start')}` : '' const averageMarker = average === null || averageX === null ? '' : ` - ${text(averageX, trackY + 22, formatDuration(average), 'chart-point-label', 'middle')}` + ${text(averageX, trackY + 22, formatSeconds(average), 'chart-point-label', 'middle')}` return [ ``, range, @@ -229,7 +226,7 @@ export function createBenchmarkComparisonSvg(data: BenchmarkComparisonImageData) kind === 'baseline' ? `` : '' - const duration = durationSeconds === null ? '—' : formatDuration(durationSeconds) + const duration = durationSeconds === null ? '—' : formatSeconds(durationSeconds) const factorLabel = factor === null ? '—' : `${factor.toFixed(2)}×` const result = relative?.status ? `${factorLabel} ${relative.status}` : factorLabel return [ diff --git a/src/renderer/src/lib/benchmarkGroups.ts b/src/renderer/src/lib/benchmarkGroups.ts new file mode 100644 index 000000000..04e477ff6 --- /dev/null +++ b/src/renderer/src/lib/benchmarkGroups.ts @@ -0,0 +1,250 @@ +import type { + PerformanceTestBenchmark, + PerformanceTestMachine, + PerformanceTestResultValue +} from '../types/ipc' +import { + calculateRelativeSpeedFactor, + getRelativeSpeedKind, + type RelativeSpeedKind +} from './relativeSpeed' + +export const DURATION_KEYS = [ + 'fastestJobDurationSeconds', + 'averageJobDurationSeconds', + 'medianJobDurationSeconds', + 'slowestJobDurationSeconds' +] as const + +export type DurationKey = (typeof DURATION_KEYS)[number] + +export const GROUP_BY_OPTIONS = ['none', 'workflow', 'hardware'] as const + +export type GroupBy = (typeof GROUP_BY_OPTIONS)[number] + +export const DURATION_LABEL_KEYS: Record = { + fastestJobDurationSeconds: 'benchmarks.fastest', + averageJobDurationSeconds: 'benchmarks.average', + medianJobDurationSeconds: 'benchmarks.median', + slowestJobDurationSeconds: 'benchmarks.slowest' +} + +export interface ColumnSort { + key: string + ascending: boolean +} + +export interface RunComparison { + factor: number | null + kind: RelativeSpeedKind +} + +export interface BenchmarkGroup { + key: string + workflowName: string + machineKey: string + members: PerformanceTestBenchmark[] + rows: PerformanceTestBenchmark[] + manual: boolean + buildCount: number + measuredCount: number + fastestId: string | null + referenceId: string | null + customBaseline: boolean + comparisons: ReadonlyMap + scaleMaximum: number +} + +export interface BenchmarkGroupsInput { + benchmarks: readonly PerformanceTestBenchmark[] + groupBy: GroupBy + isVisible: (benchmark: PerformanceTestBenchmark) => boolean + metric: DurationKey + sort: ColumnSort + valueOf: ( + benchmark: PerformanceTestBenchmark, + key: string + ) => PerformanceTestResultValue | undefined + manualOrders: Readonly> + baselineIds: Readonly> +} + +const PLATFORM_NAMES: Record = { + darwin: 'macOS', + win32: 'Windows', + linux: 'Linux' +} + +export function isDurationKey(key: string): key is DurationKey { + return (DURATION_KEYS as readonly string[]).includes(key) +} + +export function formatSeconds(value: number | null): string { + if (value === null) return '—' + return `${value.toFixed(2).replace(/\.?0+$/, '')} s` +} + +export function machineKey(benchmark: PerformanceTestBenchmark): string { + const { gpu, cpu, memoryGb, platform } = benchmark.machine + return JSON.stringify([gpu, cpu, memoryGb, platform]) +} + +export function isGroupBy(value: string): value is GroupBy { + return (GROUP_BY_OPTIONS as readonly string[]).includes(value) +} + +export function benchmarkGroupKey(benchmark: PerformanceTestBenchmark, groupBy: GroupBy): string { + if (groupBy === 'workflow') return JSON.stringify(['workflow', benchmark.workflowName]) + if (groupBy === 'hardware') return JSON.stringify(['hardware', machineKey(benchmark)]) + return 'all' +} + +export function machineLabels( + benchmarks: readonly PerformanceTestBenchmark[], + text: { unknownHardware: string; memory: (gigabytes: number) => string } +): Map { + const machines = new Map( + benchmarks.map((benchmark) => [machineKey(benchmark), benchmark.machine]) + ) + const details = [...machines].map(([key, machine]) => ({ + key, + parts: [ + machine.gpu ?? text.unknownHardware, + machine.cpu, + machine.memoryGb === null ? null : text.memory(machine.memoryGb), + machine.platform === null ? null : (PLATFORM_NAMES[machine.platform] ?? machine.platform) + ] + })) + const label = (parts: Array, depth: number): string => + parts + .slice(0, depth) + .filter((part): part is string => part !== null) + .join(' · ') + return new Map( + details.map(({ key, parts }) => { + let depth = 1 + while ( + depth < parts.length && + details.some( + (other) => other.key !== key && label(other.parts, depth) === label(parts, depth) + ) + ) { + depth += 1 + } + return [key, label(parts, depth)] + }) + ) +} + +export function compareFieldValues( + a: PerformanceTestResultValue | undefined, + b: PerformanceTestResultValue | undefined, + ascending: boolean +): number { + const aMissing = a === undefined || a === null + const bMissing = b === undefined || b === null + if (aMissing || bMissing) return Number(aMissing) - Number(bMissing) + const order = + typeof a === 'number' && typeof b === 'number' + ? a - b + : String(a).localeCompare(String(b), undefined, { numeric: true }) + return ascending ? order : -order +} + +export function moveRunInOrder(order: readonly string[], id: string, targetId: string): string[] { + const from = order.indexOf(id) + const to = order.indexOf(targetId) + if (from < 0 || to < 0 || from === to) return [...order] + const next = order.filter((candidate) => candidate !== id) + const targetIndex = next.indexOf(targetId) + next.splice(from < to ? targetIndex + 1 : targetIndex, 0, id) + return next +} + +function newestRunTime(members: readonly PerformanceTestBenchmark[]): number { + return Math.max( + 0, + ...members.map((benchmark) => + benchmark.createdAt === null ? 0 : Date.parse(benchmark.createdAt) || 0 + ) + ) +} + +function orderMembers( + members: readonly PerformanceTestBenchmark[], + input: BenchmarkGroupsInput, + manualIds: readonly string[] | undefined +): PerformanceTestBenchmark[] { + const sorted = [...members].sort((a, b) => + compareFieldValues( + input.valueOf(a, input.sort.key), + input.valueOf(b, input.sort.key), + input.sort.ascending + ) + ) + if (!manualIds) return sorted + const manualIndex = new Map(manualIds.map((id, index) => [id, index])) + const position = (benchmark: PerformanceTestBenchmark): number => + manualIndex.get(benchmark.id) ?? Number.MAX_SAFE_INTEGER + return sorted.sort((a, b) => position(a) - position(b)) +} + +function buildGroup( + key: string, + members: PerformanceTestBenchmark[], + input: BenchmarkGroupsInput +): BenchmarkGroup { + const { metric } = input + const manualIds = input.manualOrders[key] + const ordered = orderMembers(members, input, manualIds) + const measured = members + .filter((benchmark) => benchmark[metric] !== null) + .sort((a, b) => a[metric]! - b[metric]!) + const fastest = measured[0] ?? null + const chosen = measured.find((benchmark) => benchmark.id === input.baselineIds[key]) ?? null + const reference = chosen ?? fastest + const referenceDuration = reference?.[metric] ?? null + const comparisons = new Map( + members.map((benchmark) => { + const factor = calculateRelativeSpeedFactor(referenceDuration, benchmark[metric]) + return [ + benchmark.id, + { factor, kind: getRelativeSpeedKind(factor, benchmark.id === reference?.id) } + ] + }) + ) + const scaleValues = members.flatMap((benchmark) => + [benchmark.slowestJobDurationSeconds, benchmark[metric]].filter( + (value): value is number => value !== null && value > 0 + ) + ) + return { + key, + workflowName: members[0]!.workflowName, + machineKey: machineKey(members[0]!), + members: ordered, + rows: ordered.filter(input.isVisible), + manual: manualIds !== undefined, + buildCount: new Set(members.map((benchmark) => benchmark.instance.id)).size, + measuredCount: measured.length, + fastestId: fastest?.id ?? null, + referenceId: reference?.id ?? null, + customBaseline: chosen !== null && chosen.id !== fastest?.id, + comparisons, + scaleMaximum: scaleValues.length > 0 ? Math.max(...scaleValues) : 1 + } +} + +export function buildBenchmarkGroups(input: BenchmarkGroupsInput): BenchmarkGroup[] { + const membersByKey = new Map() + for (const benchmark of input.benchmarks) { + const key = benchmarkGroupKey(benchmark, input.groupBy) + const members = membersByKey.get(key) + if (members) members.push(benchmark) + else membersByKey.set(key, [benchmark]) + } + return [...membersByKey] + .map(([key, members]) => buildGroup(key, members, input)) + .filter((group) => group.rows.length > 0) + .sort((a, b) => newestRunTime(b.members) - newestRunTime(a.members)) +} diff --git a/src/renderer/src/panel/PanelApp.test.ts b/src/renderer/src/panel/PanelApp.test.ts index e3120adb7..a4d7823ec 100644 --- a/src/renderer/src/panel/PanelApp.test.ts +++ b/src/renderer/src/panel/PanelApp.test.ts @@ -656,9 +656,7 @@ describe('PanelApp', () => { expect(wrapper.find('[data-testid="benchmarks"]').exists()).toBe(true) expect(wrapper.get('.branded-page-header h1').text()).toBe('Benchmarks') - expect(wrapper.get('.branded-page-header__description').text()).toBe( - 'Compare results from your performance tests.' - ) + expect(wrapper.find('.branded-page-header__description').exists()).toBe(false) }) it('preserves the dashboard workspace when opening Performance Test', async () => { diff --git a/src/renderer/src/types/ipc.ts b/src/renderer/src/types/ipc.ts index 752c33ab5..4e0409a28 100644 --- a/src/renderer/src/types/ipc.ts +++ b/src/renderer/src/types/ipc.ts @@ -8,6 +8,7 @@ export type { Installation, RunningInstance, PerformanceTestBenchmark, + PerformanceTestMachine, ExampleWorkflowDownload, PerformanceTestResultValue, PerformanceTestResultsSummary, diff --git a/src/renderer/src/views/BenchmarksView.test.ts b/src/renderer/src/views/BenchmarksView.test.ts index 1a5761ad0..6ff874d47 100644 --- a/src/renderer/src/views/BenchmarksView.test.ts +++ b/src/renderer/src/views/BenchmarksView.test.ts @@ -1,26 +1,39 @@ import { defineComponent, h, KeepAlive } from 'vue' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { flushPromises, mount } from '@vue/test-utils' +import { flushPromises, mount, type VueWrapper } from '@vue/test-utils' import type { PerformanceTestBenchmark } from '../types/ipc' import type * as PerformanceTestResultsSvg from '../lib/performanceTestResultsSvg' -import BaseSelect from '../components/ui/BaseSelect.vue' import { useDialogs } from '../composables/useDialogs' import BenchmarksView from './BenchmarksView.vue' import { setChecked } from './testHelpers' const createResultsPngMock = vi.hoisted(() => - vi.fn(async () => new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer) + vi.fn(async (_svg: string) => new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer) ) const exportResultsImageMock = vi.hoisted(() => - vi.fn(async () => ({ ok: true, filePath: 'C:\\Exports\\benchmark-comparison.png' })) + vi.fn( + async (): Promise<{ + ok: boolean + canceled?: boolean + message?: string + filePath?: string + }> => ({ ok: true, filePath: 'C:\\Exports\\benchmark-comparison.png' }) + ) ) const deletePerformanceTestBenchmarkMock = vi.hoisted(() => vi.fn(async () => ({ ok: true }))) const renamePerformanceTestBenchmarkMock = vi.hoisted(() => - vi.fn(async (_folderPath: string, _sessionId: string, newSessionId: string) => ({ - ok: true, - sessionId: newSessionId - })) + vi.fn( + async ( + _folderPath: string, + _sessionId: string, + newSessionId: string + ): Promise<{ ok: boolean; sessionId?: string; message?: string }> => ({ + ok: true, + sessionId: newSessionId + }) + ) ) +const navigatePanelMock = vi.hoisted(() => vi.fn()) vi.mock('../lib/performanceTestResultsSvg', async (importOriginal) => ({ ...(await importOriginal()), @@ -31,53 +44,210 @@ vi.mock('./devplatform/DevPlatformAccountChip.vue', () => ({ default: { template: '
' } })) -function benchmark( +const DEFAULT_FOLDER = 'C:\\results\\performance-tests' + +interface RunOptions { + gpu?: string | null + cpu?: string + build?: string + fastest?: number +} + +function run( id: string, workflowName: string, average: number, - hardwareName = 'NVIDIA RTX 4090' + { gpu = 'NVIDIA RTX 4090', cpu = 'Test CPU', build = 'Nightly 0.3.62', fastest }: RunOptions = {} ): PerformanceTestBenchmark { const createdAt = `2026-09-${id.padStart(2, '0')}T10:00:00.000Z` - const instance = { id: `instance-${id}`, name: `Instance ${id}` } + const instance = { id: `instance-${build}`, name: build } const workspace = { id: 'workspace-1', name: 'Comfy' } + const durations = { + fastestJobDurationSeconds: fastest ?? average - 0.4, + slowestJobDurationSeconds: average + 0.7, + averageJobDurationSeconds: average, + medianJobDurationSeconds: average - 0.1 + } return { id, createdAt, instance, workspace, workflowName, - fastestJobDurationSeconds: average - 0.4, - slowestJobDurationSeconds: average + 0.7, - averageJobDurationSeconds: average, - medianJobDurationSeconds: average - 0.1, + ...durations, measuredJobCount: 5, - hardwareName, + machine: { gpu, cpu, memoryGb: 64, platform: 'win32' }, result: { createdAt, instance, workspace, workflowName, - fastestJobDurationSeconds: average - 0.4, - slowestJobDurationSeconds: average + 0.7, - averageJobDurationSeconds: average, - medianJobDurationSeconds: average - 0.1, + ...durations, measuredJobCount: 5, - hardware: { deviceName: hardwareName }, - systemInfo: { cpu_model: 'Test CPU' }, + hardware: { deviceName: gpu }, + systemInfo: { cpu_model: cpu }, customScore: Number(id) } } } +function unmeasuredRun(id: string): PerformanceTestBenchmark { + return { + ...run(id, 'portrait.json', 1, { build: 'Broken build' }), + fastestJobDurationSeconds: null, + slowestJobDurationSeconds: null, + averageJobDurationSeconds: null, + medianJobDurationSeconds: null, + measuredJobCount: 0 + } +} + const sampleBenchmarks = [ - benchmark('13', 'portrait.json', 2), - benchmark('12', 'product.json', 3.4), - benchmark('11', 'portrait.json', 1.2, 'NVIDIA RTX 5090'), - benchmark('10', 'landscape.json', 8.7, 'Apple M3 Max') + run('13', 'portrait.json', 2), + run('12', 'portrait.json', 2.2, { build: 'Stable 0.3.60' }), + run('11', 'portrait.json', 1.2, { gpu: 'NVIDIA RTX 5090' }), + run('10', 'landscape.json', 8.7, { gpu: 'Apple M3 Max', build: 'Stable 0.3.60' }), + run('9', 'portrait.json', 3, { build: 'Stable 0.3.55', fastest: 1 }) ] +const PORTRAIT_4090 = 'benchmark-group-NVIDIA RTX 4090' +const PORTRAIT_5090 = 'benchmark-group-NVIDIA RTX 5090' + +function listMock(): ReturnType { + return window.api.listPerformanceTestBenchmarks as unknown as ReturnType +} + +function useBenchmarks(list: PerformanceTestBenchmark[], folderPath = DEFAULT_FOLDER): void { + listMock().mockResolvedValue({ folderPath, benchmarks: list }) +} function mountView() { - return mount(BenchmarksView) + return mount(BenchmarksView, { attachTo: document.body }) +} + +async function mountGrouped(groupBy: string) { + const wrapper = mountView() + await flushPromises() + await chooseOption(wrapper, 'group-by', groupBy) + return wrapper +} + +function group(wrapper: VueWrapper, testId = PORTRAIT_4090) { + return wrapper.get(`[data-testid="${testId}"]`) +} + +function row(wrapper: VueWrapper, id: string) { + return wrapper.get(`[data-testid="benchmark-row-${id}"]`) +} + +function rowIds(wrapper: VueWrapper, testId = PORTRAIT_4090): Array { + return group(wrapper, testId) + .findAll('[data-testid^="benchmark-row-"]') + .map((element) => element.attributes('data-testid')?.replace('benchmark-row-', '')) +} + +function texts(wrapper: VueWrapper, selector: string, testId = PORTRAIT_4090): string[] { + return group(wrapper, testId) + .findAll(selector) + .map((element) => element.text()) +} + +function vsLabels(wrapper: VueWrapper, testId = PORTRAIT_4090): string[] { + return texts(wrapper, '.benchmarks__vs', testId) +} + +async function openMenu(wrapper: VueWrapper, selector: string): Promise { + await wrapper.get(selector).trigger('keydown', { key: 'ArrowDown' }) + await flushPromises() +} + +function openMenuElement(): HTMLElement | null { + return document.body.querySelector('[role="menu"]') +} + +async function closeMenu(): Promise { + openMenuElement()?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })) + await vi.waitFor(() => expect(openMenuElement()).toBeNull()) +} + +function menuItem(selector: string): HTMLElement | null { + return document.body.querySelector(`[role="menu"] ${selector}`) +} + +async function chooseMenuItem(selector: string): Promise { + menuItem(selector)!.click() + await vi.waitFor(() => expect(openMenuElement()).toBeNull()) + await flushPromises() +} + +async function menuOptions(wrapper: VueWrapper, key: string): Promise { + await openMenu(wrapper, `[data-testid="benchmarks-filter-${key}"]`) + const labels = Array.from( + document.body.querySelectorAll('[role="menu"] [role="menuitemradio"]'), + (item) => item.textContent!.trim() + ) + await closeMenu() + return labels +} + +async function chooseOption(wrapper: VueWrapper, key: string, label: string): Promise { + await openMenu(wrapper, `[data-testid="benchmarks-filter-${key}"]`) + const option = Array.from( + document.body.querySelectorAll('[role="menu"] [role="menuitemradio"]') + ).find((item) => item.textContent!.trim() === label) + expect(option, `option ${label}`).toBeDefined() + option!.click() + await vi.waitFor(() => expect(openMenuElement()).toBeNull()) + await flushPromises() +} + +async function rowAction(wrapper: VueWrapper, id: string, action: string): Promise { + await openMenu(wrapper, `[data-testid="benchmark-row-${id}"] .benchmarks__row-menu`) + await chooseMenuItem(action) +} + +async function groupAction(wrapper: VueWrapper, action: string, testId = PORTRAIT_4090) { + await openMenu(wrapper, `[data-testid="${testId}"] .benchmarks__group-menu`) + await chooseMenuItem(action) +} + +async function selectRow(wrapper: VueWrapper, id: string, checked = true): Promise { + await setChecked(row(wrapper, id).get('[role="checkbox"]'), checked) +} + +async function search(wrapper: VueWrapper, query: string): Promise { + await wrapper.get('.benchmarks__search input').setValue(query) +} + +async function drag(wrapper: VueWrapper, id: string, targetId: string): Promise { + const dataTransfer = new DataTransfer() + await row(wrapper, id).get('.benchmarks__grip').trigger('dragstart', { dataTransfer }) + await row(wrapper, targetId).trigger('dragover', { dataTransfer }) + await row(wrapper, targetId).trigger('drop', { dataTransfer }) + await row(wrapper, id).get('.benchmarks__grip').trigger('dragend', { dataTransfer }) +} + +async function pressOnGrip(wrapper: VueWrapper, id: string, key: string): Promise { + const grip = row(wrapper, id).get('.benchmarks__grip') + grip.element.focus() + await grip.trigger('keydown', { key, altKey: true }) + await flushPromises() +} + +async function exportSelection(wrapper: VueWrapper): Promise { + await wrapper.get('.benchmarks__export-results').trigger('click') + await flushPromises() + return createResultsPngMock.mock.calls.at(-1)![0] +} + +function sessionField(wrapper: VueWrapper, id: string): HTMLElement { + return row(wrapper, id).get('.benchmarks__session-name-text').element +} + +async function typeSessionName(field: HTMLElement, name: string): Promise { + field.focus() + field.textContent = name + field.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + await flushPromises() } function mountKeptAlive() { @@ -86,37 +256,37 @@ function mountKeptAlive() { setup: (props) => () => h(KeepAlive, null, { default: () => (props.visible ? h(BenchmarksView) : h('div')) }) }) - return mount(Page, { - props: { visible: true } - }) + return mount(Page, { props: { visible: true } }) } describe('BenchmarksView', () => { beforeEach(() => { useDialogs().cancel() + document.body.innerHTML = '' createResultsPngMock.mockClear() exportResultsImageMock.mockClear() + exportResultsImageMock.mockResolvedValue({ + ok: true, + filePath: 'C:\\Exports\\benchmark-comparison.png' + }) deletePerformanceTestBenchmarkMock.mockClear() deletePerformanceTestBenchmarkMock.mockResolvedValue({ ok: true }) renamePerformanceTestBenchmarkMock.mockClear() + navigatePanelMock.mockClear() ;(window as unknown as { api: object }).api = { browseFolder: vi.fn(), + navigatePanel: navigatePanelMock, exportResultsImage: exportResultsImageMock, deletePerformanceTestBenchmark: deletePerformanceTestBenchmarkMock, renamePerformanceTestBenchmark: renamePerformanceTestBenchmarkMock, listPerformanceTestBenchmarks: vi.fn(async () => ({ - folderPath: 'C:\\results\\performance-tests', + folderPath: DEFAULT_FOLDER, benchmarks: sampleBenchmarks })) } }) describe('returning to the page', () => { - const list = (): ReturnType => - window.api.listPerformanceTestBenchmarks as unknown as ReturnType - const browseFolder = (): ReturnType => - window.api.browseFolder as unknown as ReturnType - async function leaveAndReturn(wrapper: ReturnType): Promise { await wrapper.setProps({ visible: false }) await wrapper.setProps({ visible: true }) @@ -127,446 +297,727 @@ describe('BenchmarksView', () => { mountKeptAlive() await flushPromises() - expect(list()).toHaveBeenCalledOnce() + expect(listMock()).toHaveBeenCalledOnce() }) it('keeps the folder, search and picked runs', async () => { - browseFolder().mockResolvedValue('D:\\other') - list().mockImplementation(async (folderPath?: string) => ({ - folderPath: folderPath ?? 'C:\\results\\performance-tests', + vi.mocked(window.api.browseFolder).mockResolvedValue('D:\\other') + listMock().mockImplementation(async (folderPath?: string) => ({ + folderPath: folderPath ?? DEFAULT_FOLDER, benchmarks: sampleBenchmarks })) const wrapper = mountKeptAlive() await flushPromises() - const selectFolder = wrapper - .findAll('button') - .find((button) => button.text() === 'Select folder') - await selectFolder?.trigger('click') + await wrapper.get('.benchmarks__open-folder').trigger('click') await flushPromises() - await setChecked(wrapper.get('[data-testid="benchmark-row-13"] [role="checkbox"]'), false) - await setChecked(wrapper.get('[data-testid="benchmark-row-10"] [role="checkbox"]'), true) - await wrapper.get('input[type="text"]').setValue('json') + await selectRow(wrapper, '10') + await wrapper.get('.benchmarks__search input').setValue('json') await leaveAndReturn(wrapper) - expect(list()).toHaveBeenLastCalledWith('D:\\other') - expect(wrapper.get('input[type="text"]').element.value).toBe('json') - const checked = (id: string): string | undefined => - wrapper - .get(`[data-testid="benchmark-row-${id}"] [role="checkbox"]`) - .attributes('aria-checked') - expect(checked('13')).toBe('false') - expect(checked('10')).toBe('true') + expect(listMock()).toHaveBeenLastCalledWith('D:\\other') + expect(wrapper.get('.benchmarks__search input').element.value).toBe('json') + expect(row(wrapper, '13').get('[role="checkbox"]').attributes('aria-checked')).toBe('false') + expect(row(wrapper, '10').get('[role="checkbox"]').attributes('aria-checked')).toBe('true') }) it('shows runs that finished meanwhile and drops picks that are gone', async () => { const wrapper = mountKeptAlive() await flushPromises() - list().mockResolvedValue({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: [benchmark('14', 'new.json', 1), ...sampleBenchmarks.slice(1)] - }) + await selectRow(wrapper, '13') + await selectRow(wrapper, '12') + useBenchmarks([run('14', 'new.json', 1), ...sampleBenchmarks.slice(1)]) await leaveAndReturn(wrapper) expect(wrapper.find('[data-testid="benchmark-row-14"]').exists()).toBe(true) expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(false) - expect( - wrapper.get('[data-testid="benchmark-row-14"] [role="checkbox"]').attributes('aria-checked') - ).toBe('false') + expect(row(wrapper, '14').get('[role="checkbox"]').attributes('aria-checked')).toBe('false') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('1 selected') }) it('keeps the list on screen when the refresh fails', async () => { const wrapper = mountKeptAlive() await flushPromises() - list().mockRejectedValue(new Error('offline')) + listMock().mockRejectedValue(new Error('offline')) await leaveAndReturn(wrapper) - expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(4) + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(5) }) }) - it('selects the three newest runs and highlights only the best duration in each metric', async () => { + it('keeps the data filters apart from the view options', async () => { const wrapper = mountView() await flushPromises() + const keysIn = (selector: string) => + wrapper + .get(selector) + .findAll('[data-testid^="benchmarks-filter-"]') + .map((button) => button.attributes('data-testid')) + + expect(keysIn('.benchmarks__filters')).toEqual([ + 'benchmarks-filter-workspace', + 'benchmarks-filter-build', + 'benchmarks-filter-hardware', + 'benchmarks-filter-workflow' + ]) + expect(keysIn('.benchmarks__view-options')).toEqual([ + 'benchmarks-filter-group-by', + 'benchmarks-filter-compare-by' + ]) + }) - expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(4) - expect(wrapper.findAll('.benchmarks__table thead th').map((header) => header.text())).toEqual([ + it('compares the builds that ran on each machine when grouped by hardware', async () => { + const wrapper = await mountGrouped('Hardware') + + const cards = wrapper.findAll('.benchmarks__group') + expect(cards.map((card) => card.get('.benchmarks__group-toggle').text())).toEqual([ + 'NVIDIA RTX 4090', + 'NVIDIA RTX 5090', + 'Apple M3 Max' + ]) + expect(cards.map((card) => card.get('.benchmarks__group-meta').text())).toEqual([ + '3 sessions · 15 runs', + '1 session · 5 runs', + '1 session · 5 runs' + ]) + expect(texts(wrapper, 'thead th')).toEqual([ '', - 'Workflow', 'Session', - 'GPU / Hardware', - 'Runs', - 'Date / Time ↓', + 'Workflow', + 'Build', + 'Median and range', + 'Median ↑', + 'Vs baseline', '' ]) - expect(wrapper.get('[data-testid="benchmark-row-13"]').text()).toContain('13') - expect(wrapper.find('.benchmarks__selection-tray').exists()).toBe(false) - expect(wrapper.get('[data-testid="benchmark-row-13"]').attributes('style')).toContain( - '--series-color: #55e0d1' - ) - expect(wrapper.findAll('.benchmarks__best').map((cell) => cell.text())).toEqual([ - '0.8 s', - '1.2 s', - '1.1 s', - '1.9 s' + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + expect(texts(wrapper, '.benchmarks__build')).toEqual([ + 'Nightly 0.3.62', + 'Stable 0.3.60', + 'Stable 0.3.55' ]) + expect(texts(wrapper, '.benchmarks__value')).toEqual(['1.9 s', '2.1 s', '2.9 s']) + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.11× slower', '1.53× slower']) expect( - wrapper.findAll('[data-testid^="benchmark-comparison-13-"]').map((field) => field.text()) - ).toEqual(['portrait.json', 'Session: 13', 'GPU / Hardware: NVIDIA RTX 4090']) + group(wrapper) + .findAll('.benchmarks__bar-fill--best') + .map((fill) => fill.element.closest('tr')?.getAttribute('data-testid')) + ).toEqual(['benchmark-row-13']) expect( - wrapper.findAll('[data-testid^="benchmark-chart-13-"]').map((field) => field.text()) - ).toEqual([ - 'portrait.json', - 'Session: 13', - 'GPU / Hardware: NVIDIA RTX 4090', - '1.6 s', - '2.7 s', - '2 s' + Number.parseFloat( + (row(wrapper, '13').get('.benchmarks__bar-fill').element as HTMLElement).style.width + ) + ).toBeCloseTo((1.9 / 3.7) * 100, 3) + expect(wrapper.find('.benchmarks__single-note').exists()).toBe(false) + expect(vsLabels(wrapper, PORTRAIT_5090)).toEqual(['']) + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) + }) + + it('keeps different computers with the same GPU in separate comparisons', async () => { + useBenchmarks([ + run('21', 'a.json', 2, { cpu: 'AMD Ryzen 9 7950X' }), + run('22', 'a.json', 3, { cpu: 'Intel Core i9-14900K' }), + run('23', 'a.json', 2.5, { cpu: 'AMD Ryzen 9 7950X', build: 'Stable' }) ]) - expect( - wrapper - .get('[data-testid="benchmark-chart-13-slowest-label"]') - .classes('benchmarks__chart-point-label--slowest') - ).toBe(true) + const wrapper = await mountGrouped('Hardware') - const exportButton = wrapper.get('.benchmarks__export-results') - expect(exportButton.text()).toBe('Export results') - await exportButton.trigger('click') - await flushPromises() - expect(exportResultsImageMock).toHaveBeenCalledTimes(1) - const [png, imageType, defaultPath] = exportResultsImageMock.mock.calls[0]! - expect(png).toBeInstanceOf(ArrayBuffer) - expect(imageType).toBe('benchmark-comparison') - expect(defaultPath).toBe('C:\\results\\performance-tests') - const svg = createResultsPngMock.mock.calls[0]![0] - expect(svg).toContain('Benchmark Comparison') - expect(svg).toContain('portrait.json') - expect(svg).toContain('Duration range') - expect(svg).toContain('Each line shows the fastest and slowest run, with the average marked.') - expect(svg).toContain('role="img" aria-label="Comfy"') - expect(svg).toContain('class="footer-date"') - expect(svg.match(/class="table-cell best-cell"/g)).toHaveLength(4) - expect(svg).toContain('.best-cell { fill: #f2ff59; fill-opacity: 0.08; }') - expect(svg.match(/class="chart-point-label"/g)).toHaveLength(9) + expect(texts(wrapper, '.benchmarks__group-toggle', 'benchmarks')).toEqual([ + 'NVIDIA RTX 4090 · AMD Ryzen 9 7950X', + 'NVIDIA RTX 4090 · Intel Core i9-14900K' + ]) + expect(texts(wrapper, '.benchmarks__group-meta', 'benchmarks')).toEqual([ + '2 sessions · 10 runs', + '1 session · 5 runs' + ]) + expect(await menuOptions(wrapper, 'hardware')).toEqual([ + 'All', + 'NVIDIA RTX 4090 · AMD Ryzen 9 7950X', + 'NVIDIA RTX 4090 · Intel Core i9-14900K' + ]) + await chooseOption(wrapper, 'hardware', 'NVIDIA RTX 4090 · Intel Core i9-14900K') + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) + expect(wrapper.find('[data-testid="benchmark-row-22"]').exists()).toBe(true) }) - it('filters the run library and updates the comparison selection directly', async () => { - const wrapper = mountView() - await flushPromises() + it('compares filtered runs with the fastest run of the whole group', async () => { + const wrapper = await mountGrouped('Hardware') - expect(wrapper.find('[data-testid="benchmark-column-systemInfo.cpu_model"]').exists()).toBe( - true - ) - expect(wrapper.find('[data-testid="benchmark-comparison-13-customScore"]').exists()).toBe(false) - await setChecked(wrapper.get('[data-testid="benchmark-column-customScore"]'), true) - expect(wrapper.get('.benchmarks__table thead').text()).toContain('Custom Score') - expect(wrapper.get('[data-testid="benchmark-row-13"]').text()).toContain('13') - expect(wrapper.find('[data-testid="benchmark-comparison-13-customScore"]').exists()).toBe(false) - await setChecked(wrapper.get('[data-testid="benchmark-comparison-column-customScore"]'), true) - expect(wrapper.get('[data-testid="benchmark-comparison-13-customScore"]').text()).toBe( - 'Custom Score: 13' - ) - expect(wrapper.get('[data-testid="benchmark-comparison-12-customScore"]').text()).toBe( - 'Custom Score: 12' - ) - expect(wrapper.get('[data-testid="benchmark-chart-13-customScore"]').text()).toBe( - 'Custom Score: 13' + await search(wrapper, 'stable 0.3.55') + expect(rowIds(wrapper)).toEqual(['9']) + expect(vsLabels(wrapper)).toEqual(['1.53× slower']) + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') + expect(group(wrapper).get('.benchmarks__reference-hidden').text()).toBe( + 'Compared with session 13, which the filters hide.' ) + + await search(wrapper, '') + await chooseOption(wrapper, 'build', 'Stable 0.3.60') + expect(rowIds(wrapper)).toEqual(['12']) + expect(vsLabels(wrapper)).toEqual(['1.11× slower']) + + await chooseOption(wrapper, 'group-by', 'Workflow') + expect(rowIds(wrapper, 'benchmark-group-portrait.json')).toEqual(['12']) expect( - wrapper.get('[data-testid="benchmark-column-workflowName"]').attributes('disabled') - ).toBe('') - expect(wrapper.get('.benchmarks__table thead').text()).toContain('Workflow') - expect(wrapper.get('[data-testid="benchmark-comparison-13-workflowName"]').text()).toContain( - 'portrait.json' - ) + group(wrapper, 'benchmark-group-portrait.json').find('.benchmarks__single-note').exists() + ).toBe(false) expect( - wrapper.get('[data-testid="benchmark-comparison-column-workflowName"]').attributes('disabled') - ).toBe('') - expect(wrapper.get('[data-testid="benchmark-comparison-13-workflowName"]').text()).toContain( - 'portrait.json' - ) - expect(wrapper.get('[data-testid="benchmark-chart-13-workflowName"]').text()).toContain( - 'portrait.json' - ) + group(wrapper, 'benchmark-group-landscape.json').get('.benchmarks__single-note').text() + ).toBe('Only one build ran this workflow here. Run it on another build to compare.') + }) - await wrapper.get('input[type="text"]').setValue('landscape') - expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) - expect(wrapper.get('[data-testid="benchmark-row-10"]').text()).toContain('landscape.json') + it('keeps a chosen baseline while filters hide it, and resets it', async () => { + const wrapper = await mountGrouped('Hardware') - await setChecked(wrapper.get('[data-testid="benchmark-row-10"] [role="checkbox"]'), true) - expect(wrapper.get('[data-testid="benchmark-row-10"]').attributes('style')).toContain( - '--series-color: #ff8a65' + await rowAction(wrapper, '12', '.benchmarks__set-baseline') + expect(vsLabels(wrapper)).toEqual(['1.11× faster', 'Baseline', '1.38× slower']) + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') + expect(group(wrapper).get('.benchmarks__group-foot').text()).toContain( + 'Speed = baseline time ÷ run time' ) - expect(wrapper.get('.benchmarks__comparison').text()).toContain('landscape.json') - - const comparisonToggle = wrapper.get('#comparison-title') - expect(comparisonToggle.attributes('aria-expanded')).toBe('true') - await comparisonToggle.trigger('click') - expect(comparisonToggle.attributes('aria-expanded')).toBe('false') - expect(wrapper.get('.benchmarks__comparison').attributes('style')).toContain('display: none') - await comparisonToggle.trigger('click') - - await setChecked(wrapper.get('[data-testid="benchmark-row-10"] [role="checkbox"]'), false) - expect(wrapper.get('[data-testid="benchmark-row-10"]').attributes('style')).toBeUndefined() - await wrapper.get('input[type="text"]').setValue('') - await setChecked(wrapper.get('[data-testid="benchmark-row-13"] [role="checkbox"]'), false) - await setChecked(wrapper.get('[data-testid="benchmark-row-12"] [role="checkbox"]'), false) - await setChecked(wrapper.get('[data-testid="benchmark-row-11"] [role="checkbox"]'), false) - expect(wrapper.get('.benchmarks__comparison').text()).toContain( - 'Select runs from the library to compare them.' + expect(group(wrapper, PORTRAIT_5090).find('.benchmarks__group-foot').exists()).toBe(false) + + await search(wrapper, 'stable 0.3.55') + expect(vsLabels(wrapper)).toEqual(['1.38× slower']) + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') + expect(group(wrapper).get('.benchmarks__reference-hidden').text()).toBe( + 'Compared with session 12, which the filters hide.' ) + expect(group(wrapper).find('.benchmarks__reset-baseline').exists()).toBe(true) + + await search(wrapper, '') + await chooseOption(wrapper, 'compare-by', 'Fastest') + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + expect(vsLabels(wrapper)).toEqual(['1.80× faster', '1.13× faster', 'Baseline']) + + await group(wrapper).get('.benchmarks__reset-baseline').trigger('click') + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.60× slower', '1.80× slower']) + expect(group(wrapper).get('.benchmarks__vs-head').text()).toBe('Vs baseline') + }) + + it('calls equally fast runs the same speed', async () => { + useBenchmarks([ + run('21', 'a.json', 2, { gpu: 'GPU', build: 'Build A' }), + run('22', 'a.json', 2, { gpu: 'GPU', build: 'Build B' }) + ]) + const wrapper = await mountGrouped('Hardware') + + expect(vsLabels(wrapper, 'benchmark-group-GPU')).toEqual(['Fastest', 'Same speed']) }) - it('compares the selected metric against a selected baseline', async () => { + it('shows runs without measurements as unavailable', async () => { + useBenchmarks([...sampleBenchmarks, unmeasuredRun('8')]) + const wrapper = await mountGrouped('Hardware') + + expect(rowIds(wrapper)).toEqual(['13', '12', '9', '8']) + expect(row(wrapper, '8').get('.benchmarks__no-data').text()).toBe('No measured runs') + expect(row(wrapper, '8').find('.benchmarks__bar').exists()).toBe(false) + expect(row(wrapper, '8').get('.benchmarks__value').text()).toBe('—') + expect(row(wrapper, '8').get('.benchmarks__vs').text()).toBe('Unavailable') + await openMenu(wrapper, '[data-testid="benchmark-row-8"] .benchmarks__row-menu') + expect(menuItem('.benchmarks__set-baseline')?.hasAttribute('data-disabled')).toBe(true) + }) + + it('shows every time of a run from its bar, by pointer or keyboard', async () => { const wrapper = mountView() await flushPromises() - const chartElement = wrapper.get('.benchmarks__chart').element - const workflowLabelElements = wrapper - .findAll('[data-testid$="-workflowName"]') - .map((label) => label.element) - const visualizationTabs = wrapper.findAll('.benchmarks__visualization-tabs button') - expect(visualizationTabs.map((tab) => tab.text())).toEqual(['Duration', 'Speed']) - expect(visualizationTabs[0]!.attributes('aria-pressed')).toBe('true') - await visualizationTabs[1]!.trigger('click') - expect(wrapper.get('.benchmarks__chart').element).toBe(chartElement) - expect(wrapper.findAll('[data-testid$="-workflowName"]').map((label) => label.element)).toEqual( - workflowLabelElements + const bar = row(wrapper, '12').get('.benchmarks__bar') + + expect(bar.attributes('tabindex')).toBe('0') + expect(bar.attributes('aria-label')).toBe( + 'Fastest 1.8 s, Average 2.2 s, Median 2.1 s, Slowest 2.9 s' ) + await row(wrapper, '12').get('.tooltip-wrap').trigger('focusin') + await flushPromises() - const metricSelect = wrapper - .findAllComponents(BaseSelect) - .find((select) => select.props('ariaLabel') === 'Metric') - expect(metricSelect?.props('modelValue')).toBe('medianJobDurationSeconds') + const tip = document.body.querySelector('.benchmarks__bar-tip')! + expect(tip.textContent).toContain('Stable 0.3.60') + expect(tip.textContent).toContain('5 runs') + expect(tip.textContent).toContain('Fastest1.8 s') + expect(tip.textContent).toContain('Slowest2.9 s') + }) - const relativeRows = () => wrapper.findAll('.benchmarks__relative-row') - expect(relativeRows().map((row) => row.get('.benchmarks__relative-result').text())).toEqual([ - '1.00×Baseline', - '0.58×Slower', - '1.73×Faster' - ]) + it('compares by another metric and sorts by any header', async () => { + const wrapper = await mountGrouped('Hardware') - expect(wrapper.find('[aria-label="Baseline"]').exists()).toBe(false) - const selectedHeader = wrapper.get('[data-testid="benchmark-comparison-column-title-12"]') - await selectedHeader.get('.benchmarks__matrix-title').trigger('click') - expect(selectedHeader.classes()).toContain('benchmarks__matrix-column--baseline') - expect(selectedHeader.get('.benchmarks__matrix-title').attributes('aria-pressed')).toBe('true') - expect(selectedHeader.get('.benchmarks__baseline-badge').element.parentElement).toBe( - selectedHeader.element - ) - expect(relativeRows().map((row) => row.get('.benchmarks__relative-result').text())).toEqual([ - '1.74×Faster', - '1.00×Baseline', - '3.00×Faster' + expect(await menuOptions(wrapper, 'compare-by')).toEqual([ + 'Fastest', + 'Average', + 'Median', + 'Slowest' ]) + await chooseOption(wrapper, 'compare-by', 'Fastest') + expect(wrapper.get('[data-testid="benchmarks-filter-compare-by"]').text()).toBe( + 'Compare byFastest' + ) + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + expect(group(wrapper).get('.benchmarks__bar-head').text()).toBe('Fastest and range') + + await group(wrapper).get('.benchmarks__metric-sort').trigger('click') + expect(rowIds(wrapper)).toEqual(['12', '13', '9']) + expect(group(wrapper).get('.benchmarks__metric-sort').text()).toBe('Fastest ↓') + + const sessionHeader = group(wrapper) + .findAll('.benchmarks__sort') + .find((header) => header.text().startsWith('Session'))! + await sessionHeader.trigger('click') + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + await sessionHeader.trigger('click') + expect(rowIds(wrapper)).toEqual(['9', '12', '13']) + }) - metricSelect?.vm.$emit('update:modelValue', 'fastestJobDurationSeconds') - await wrapper.vm.$nextTick() - expect(relativeRows()[0]!.get('.benchmarks__relative-duration').text()).toBe('1.6 s') - expect(relativeRows()[0]!.get('.benchmarks__relative-result').text()).toBe('1.88×Faster') + it('moves rows up and down by dragging and with Alt+arrow keys', async () => { + const wrapper = await mountGrouped('Hardware') + + await drag(wrapper, '13', '12') + expect(rowIds(wrapper)).toEqual(['12', '13', '9']) + await drag(wrapper, '12', '9') + expect(rowIds(wrapper)).toEqual(['13', '9', '12']) + await drag(wrapper, '12', '13') + expect(rowIds(wrapper)).toEqual(['12', '13', '9']) + expect(group(wrapper).get('.benchmarks__metric-sort').text()).toBe('Median') + expect(vsLabels(wrapper)).toEqual(['1.11× slower', 'Fastest', '1.53× slower']) + + await pressOnGrip(wrapper, '9', 'ArrowUp') + expect(rowIds(wrapper)).toEqual(['12', '9', '13']) + expect(document.activeElement).toBe(row(wrapper, '9').get('.benchmarks__grip').element) + await pressOnGrip(wrapper, '12', 'ArrowDown') + expect(rowIds(wrapper)).toEqual(['9', '12', '13']) + + await drag(wrapper, '11', '12') + expect(rowIds(wrapper)).toEqual(['9', '12', '13']) + expect(rowIds(wrapper, PORTRAIT_5090)).toEqual(['11']) + }) - await wrapper.get('.benchmarks__export-results').trigger('click') + it('clears the drop highlight when a drag leaves the table', async () => { + const wrapper = await mountGrouped('Hardware') + const dataTransfer = new DataTransfer() + + await row(wrapper, '9').get('.benchmarks__grip').trigger('dragstart', { dataTransfer }) + await row(wrapper, '13').trigger('dragover', { dataTransfer }) + expect(row(wrapper, '13').classes()).toContain('benchmarks__row--drop-target') + + group(wrapper) + .get('tbody') + .element.dispatchEvent( + new DragEvent('dragleave', { + bubbles: true, + dataTransfer, + relatedTarget: row(wrapper, '11').element + }) + ) await flushPromises() - const svg = createResultsPngMock.mock.calls[0]![0] - expect(svg).toContain('Relative speed') - expect(svg).toContain('Relative speed factor = baseline duration / run duration.') - expect(svg).toContain('1.88× Faster') - expect(svg).toContain('1.00× Baseline') - expect(svg).toContain('class="relative-bar relative-faster"') - expect(svg).not.toContain('>Duration range (min → max)') + expect(row(wrapper, '13').classes()).not.toContain('benchmarks__row--drop-target') }) - it('does not label another equally fast run as the baseline', async () => { - vi.mocked(window.api.listPerformanceTestBenchmarks).mockResolvedValueOnce({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: [benchmark('21', 'a.json', 2), benchmark('22', 'b.json', 2)] - }) - const wrapper = mountView() + it('keeps every row movable after filters change, runs arrive or another folder loads', async () => { + const wrapper = await mountGrouped('Hardware') + + await search(wrapper, 'stable') + await drag(wrapper, '9', '12') + await search(wrapper, '') + expect(rowIds(wrapper)).toEqual(['13', '9', '12']) + await pressOnGrip(wrapper, '13', 'ArrowDown') + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + + useBenchmarks([...sampleBenchmarks, run('14', 'portrait.json', 5, { build: 'Old' })]) + await wrapper.get('[data-testid="benchmarks-refresh"]').trigger('click') await flushPromises() - await wrapper.findAll('.benchmarks__visualization-tabs button')[1]!.trigger('click') + expect(rowIds(wrapper)).toEqual(['9', '13', '12', '14']) + await pressOnGrip(wrapper, '14', 'ArrowUp') + expect(rowIds(wrapper)).toEqual(['9', '13', '14', '12']) - expect(wrapper.findAll('.benchmarks__relative-result').map((result) => result.text())).toEqual([ - '1.00×Baseline', - '1.00×Same speed' - ]) + vi.mocked(window.api.browseFolder).mockResolvedValue('D:\\shared') + useBenchmarks(sampleBenchmarks, 'D:\\shared') + await wrapper.get('.benchmarks__open-folder').trigger('click') + await flushPromises() + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + expect(group(wrapper).get('.benchmarks__metric-sort').text()).toBe('Median ↑') + await pressOnGrip(wrapper, '9', 'ArrowUp') + expect(rowIds(wrapper)).toEqual(['13', '9', '12']) }) - it('closes property menus when clicking outside', async () => { - const wrapper = mountView() - await flushPromises() - const pickers = wrapper.findAll('.benchmarks__columns-picker') + it('sorts again by any header after a manual order', async () => { + const wrapper = await mountGrouped('Hardware') - for (const picker of pickers) { - const details = picker.element as HTMLDetailsElement - details.open = true - await picker.get('[role="switch"]').trigger('pointerdown') - expect(details.open).toBe(true) + await drag(wrapper, '9', '13') + expect(rowIds(wrapper)).toEqual(['9', '13', '12']) + await group(wrapper).get('.benchmarks__metric-sort').trigger('click') - document.body.dispatchEvent(new Event('pointerdown', { bubbles: true })) - expect(details.open).toBe(false) - } + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + expect(group(wrapper).get('.benchmarks__metric-sort').text()).toBe('Median ↑') }) - it('reorders comparison columns by dragging a column title', async () => { - const wrapper = mountView() + it('collapses a group', async () => { + const wrapper = await mountGrouped('Hardware') + const toggle = group(wrapper).get('.benchmarks__group-toggle') + + expect(toggle.attributes('aria-expanded')).toBe('true') + await toggle.trigger('click') + expect(toggle.attributes('aria-expanded')).toBe('false') + expect(group(wrapper).find('table').exists()).toBe(false) + await toggle.trigger('click') + expect(group(wrapper).find('table').exists()).toBe(true) + }) + + it('checks every column the table shows and hides the ones you uncheck', async () => { + const wrapper = await mountGrouped('Hardware') + const checked = (key: string) => + menuItem(`[data-testid="benchmark-column-${key}"]`)!.getAttribute('aria-checked') + const locked = (key: string) => + menuItem(`[data-testid="benchmark-column-${key}"]`)!.hasAttribute('data-disabled') + + await openMenu(wrapper, '[data-testid="benchmarks-columns"]') + const title = document.querySelector('.benchmarks__columns-title')! + expect(title.textContent?.trim()).toBe('Columns to display') + expect(title.classList).toContain('uppercase') + for (const key of [ + 'id', + 'workflowName', + 'instance.name', + 'range', + 'medianJobDurationSeconds', + 'vsBaseline' + ]) { + expect(checked(key), key).toBe('true') + } + expect(checked('averageJobDurationSeconds')).toBe('false') + expect(['id', 'medianJobDurationSeconds', 'vsBaseline'].every(locked)).toBe(true) + expect(['workflowName', 'instance.name', 'range'].some(locked)).toBe(false) + + menuItem('[data-testid="benchmark-column-range"]')!.click() + menuItem('[data-testid="benchmark-column-workflowName"]')!.click() await flushPromises() + await closeMenu() - const columnIds = () => - wrapper - .findAll('[data-testid^="benchmark-comparison-column-title-"]') - .map((header) => header.attributes('data-testid')?.split('-').at(-1)) + expect(texts(wrapper, 'thead th')).toEqual([ + '', + 'Session', + 'Build', + 'Median ↑', + 'Vs baseline', + '' + ]) + expect(wrapper.find('.benchmarks__bar-cell').exists()).toBe(false) + }) - expect(columnIds()).toEqual(['13', '12', '11']) + it('adds picked columns to the table and the export, best value in green', async () => { + const wrapper = await mountGrouped('Hardware') - const dataTransfer = new DataTransfer() - await wrapper - .get('[data-testid="benchmark-comparison-column-title-13"] .benchmarks__column-grip') - .trigger('dragstart', { dataTransfer }) - await wrapper.get('[data-testid="benchmark-comparison-column-title-11"]').trigger('dragover', { - dataTransfer - }) + await openMenu(wrapper, '[data-testid="benchmarks-columns"]') + expect(menuItem('[data-testid="benchmark-column-systemInfo.cpu_model"]')).not.toBeNull() + menuItem('[data-testid="benchmark-column-averageJobDurationSeconds"]')!.click() + await flushPromises() + expect(openMenuElement()).not.toBeNull() + menuItem('[data-testid="benchmark-column-customScore"]')!.click() + menuItem('[data-testid="benchmark-column-instance.name"]')!.click() + await flushPromises() expect( - wrapper - .get('[data-testid="benchmark-comparison-column-title-11"]') - .classes('benchmarks__matrix-column--drop-target') - ).toBe(true) - await wrapper.get('[data-testid="benchmark-comparison-column-title-11"]').trigger('drop', { - dataTransfer - }) + menuItem('[data-testid="benchmark-column-customScore"]')!.getAttribute('aria-checked') + ).toBe('true') + await closeMenu() - expect(columnIds()).toEqual(['12', '13', '11']) + expect(texts(wrapper, 'thead th')).toEqual([ + '', + 'Session', + 'Workflow', + 'Median and range', + 'Average', + 'Custom Score', + 'Median ↑', + 'Vs baseline', + '' + ]) + expect(texts(wrapper, '.benchmarks__best')).toEqual(['2 s']) + expect(row(wrapper, '12').get('[data-testid="benchmark-cell-12-customScore"]').text()).toBe( + '12' + ) - await wrapper - .get('[data-testid="benchmark-comparison-column-title-11"] .benchmarks__column-grip') - .trigger('dragstart', { dataTransfer }) - await wrapper.get('[data-testid="benchmark-comparison-column-title-12"]').trigger('drop', { - dataTransfer - }) + await groupAction(wrapper, '.benchmarks__export-group') + const svg = createResultsPngMock.mock.calls.at(-1)![0] + expect(svg).toContain('Custom Score: 13') + expect(svg).toContain('GPU / Hardware: NVIDIA RTX 4090') + expect(svg).toContain('Duration range') + }) + + it('compares every run in one table by default and groups on demand', async () => { + const wrapper = mountView() + await flushPromises() + + expect(wrapper.findAll('.benchmarks__group')).toHaveLength(1) + expect(group(wrapper, 'benchmark-group-all').get('.benchmarks__group-toggle').text()).toBe( + 'All runs' + ) + expect(rowIds(wrapper, 'benchmark-group-all')).toHaveLength(5) + expect(texts(wrapper, '.benchmarks__workflow', 'benchmark-group-all')).toContain( + 'landscape.json' + ) + expect(wrapper.find('.benchmarks__single-note').exists()).toBe(false) + expect(await menuOptions(wrapper, 'group-by')).toEqual(['None', 'Workflow', 'Hardware']) - expect(columnIds()).toEqual(['11', '12', '13']) - expect(wrapper.findAll('.benchmarks__chart-row strong').map((title) => title.text())).toEqual([ + await chooseOption(wrapper, 'group-by', 'Workflow') + expect(wrapper.findAll('.benchmarks__group-toggle').map((toggle) => toggle.text())).toEqual([ 'portrait.json', - 'product.json', - 'portrait.json' + 'landscape.json' ]) + expect(wrapper.find('.benchmarks__workflow').exists()).toBe(false) + expect(texts(wrapper, 'thead th', 'benchmark-group-portrait.json')).toContain('Vs baseline') + + await chooseOption(wrapper, 'group-by', 'None') + expect(wrapper.findAll('.benchmarks__group')).toHaveLength(1) }) - it('sorts comparison columns by duration metrics in either direction', async () => { + it('truncates long workflow names and keeps the full name in a tooltip', async () => { + const longName = 'my_client_project_final_v3_portrait_upscale_with_face_detailer_2x.json' + useBenchmarks([run('5', longName, 2), run('6', longName, 3, { build: 'Stable 0.3.60' })]) const wrapper = mountView() await flushPromises() - const columnIds = () => - wrapper - .findAll('[data-testid^="benchmark-comparison-column-title-"]') - .map((header) => header.attributes('data-testid')?.split('-').at(-1)) - const sortSelect = wrapper - .findAllComponents(BaseSelect) - .find((select) => select.props('ariaLabel') === 'Sort comparison') - - expect(sortSelect?.props('options').map((option) => option.label)).toEqual([ - 'Manual sort', - 'Fastest', - 'Average', - 'Median', - 'Slowest' - ]) + const cell = row(wrapper, '5').get('.benchmarks__workflow') + expect(cell.classes()).toContain('truncate') + expect(cell.attributes('title')).toBe(longName) - sortSelect?.vm.$emit('update:modelValue', 'fastestJobDurationSeconds') - await wrapper.vm.$nextTick() - expect(columnIds()).toEqual(['11', '13', '12']) + await chooseOption(wrapper, 'group-by', 'Workflow') + const toggle = group(wrapper, `benchmark-group-${longName}`).get('.benchmarks__group-toggle') + expect(toggle.attributes('title')).toBe(longName) + expect(toggle.get('.benchmarks__group-label').classes()).toContain('truncate') + }) - await wrapper.get('.benchmarks__sort-direction').trigger('click') - expect(columnIds()).toEqual(['12', '13', '11']) - expect(wrapper.get('.benchmarks__sort-direction').attributes('aria-label')).toBe( - 'Switch to ascending sort' - ) + it('sets and resets the baseline from the flag on a row', async () => { + const wrapper = await mountGrouped('Hardware') + + expect(row(wrapper, '13').find('[data-testid="benchmark-baseline-13"]').exists()).toBe(false) + const flag = row(wrapper, '12').get('[data-testid="benchmark-baseline-12"]') + expect(flag.attributes('aria-pressed')).toBe('false') + expect(flag.attributes('aria-label')).toBe('Set as baseline') + + await flag.trigger('click') + const chosen = row(wrapper, '12').get('[data-testid="benchmark-baseline-12"]') + expect(chosen.attributes('aria-pressed')).toBe('true') + expect(chosen.attributes('aria-label')).toBe('Reset baseline') + expect(group(wrapper).find('.benchmarks__reset-baseline').exists()).toBe(true) + expect(row(wrapper, '13').find('[data-testid="benchmark-baseline-13"]').exists()).toBe(true) + + await chosen.trigger('click') + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.11× slower', '1.53× slower']) + expect(group(wrapper).find('.benchmarks__reset-baseline').exists()).toBe(false) }) - it('defaults the baseline to the first displayed column until one is picked', async () => { + it('reorders picked columns by dragging headers and with Alt+arrow keys', async () => { const wrapper = mountView() await flushPromises() - await wrapper.findAll('.benchmarks__visualization-tabs button')[1]!.trigger('click') - const baselineColumnId = () => + await openMenu(wrapper, '[data-testid="benchmarks-columns"]') + menuItem('[data-testid="benchmark-column-averageJobDurationSeconds"]')!.click() + menuItem('[data-testid="benchmark-column-customScore"]')!.click() + await flushPromises() + await closeMenu() + const heads = () => wrapper - .get('.benchmarks__matrix-column--baseline') - .attributes('data-testid') - ?.split('-') - .at(-1) - const sortSelect = wrapper - .findAllComponents(BaseSelect) - .find((select) => select.props('ariaLabel') === 'Sort comparison') - - expect(baselineColumnId()).toBe('13') - sortSelect?.vm.$emit('update:modelValue', 'fastestJobDurationSeconds') - await wrapper.vm.$nextTick() - expect(baselineColumnId()).toBe('11') - - await wrapper - .get('[data-testid="benchmark-comparison-column-title-12"] .benchmarks__matrix-title') - .trigger('click') - await wrapper.get('.benchmarks__sort-direction').trigger('click') - expect(baselineColumnId()).toBe('12') - }) - - it('places endpoint labels immediately outside their data point markers', async () => { - const edgeBenchmark = { - ...benchmark('14', 'qwen_image_2.1_int8_bf16.json', 45.83), - fastestJobDurationSeconds: 44.7, - averageJobDurationSeconds: 45.83, - medianJobDurationSeconds: 45.61, - slowestJobDurationSeconds: 47.24 + .findAll('[data-testid^="benchmark-column-head-"]') + .map((head) => head.attributes('data-testid')!.replace('benchmark-column-head-', '')) + const head = (key: string) => wrapper.get(`[data-testid="benchmark-column-head-${key}"]`) + expect(heads()).toEqual(['averageJobDurationSeconds', 'customScore']) + + const dataTransfer = new DataTransfer() + await head('customScore').trigger('dragstart', { dataTransfer }) + await head('averageJobDurationSeconds').trigger('dragover', { dataTransfer }) + expect(head('averageJobDurationSeconds').classes()).toContain( + 'benchmarks__column-head--drop-target' + ) + await head('averageJobDurationSeconds').trigger('drop', { dataTransfer }) + await head('customScore').trigger('dragend', { dataTransfer }) + expect(heads()).toEqual(['customScore', 'averageJobDurationSeconds']) + expect(wrapper.find('.benchmarks__column-head--drop-target').exists()).toBe(false) + + const columnHint = 'Drag to reorder. You can also press Alt+Left or Alt+Right.' + const sortButton = (key: string) => head(key).get('button').element + expect(head('customScore').attributes('title')).toBe(columnHint) + const describedBy = sortButton('customScore').getAttribute('aria-describedby')! + expect(document.getElementById(describedBy)?.textContent?.trim()).toBe(columnHint) + + const pressOnHead = async (key: string, arrow: string) => { + sortButton(key).focus() + await head(key).get('button').trigger('keydown', { key: arrow, altKey: true }) + await flushPromises() } - vi.mocked(window.api.listPerformanceTestBenchmarks).mockResolvedValueOnce({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: [edgeBenchmark] - }) + await pressOnHead('averageJobDurationSeconds', 'ArrowLeft') + expect(heads()).toEqual(['averageJobDurationSeconds', 'customScore']) + expect(document.activeElement).toBe(sortButton('averageJobDurationSeconds')) + await pressOnHead('averageJobDurationSeconds', 'ArrowRight') + expect(heads()).toEqual(['customScore', 'averageJobDurationSeconds']) + expect(document.activeElement).toBe(sortButton('averageJobDurationSeconds')) + await pressOnHead('customScore', 'ArrowLeft') + expect(heads()).toEqual(['customScore', 'averageJobDurationSeconds']) + }) + + it('filters by search and the toolbar menus', async () => { const wrapper = mountView() await flushPromises() - const fastestLabel = wrapper.get('[data-testid="benchmark-chart-14-fastest-label"]') - expect(fastestLabel.classes()).toContain('benchmarks__chart-point-label--fastest') - expect(wrapper.get('[data-testid="benchmark-chart-14-slowest-label"]').classes()).toContain( - 'benchmarks__chart-point-label--slowest' - ) - expect(wrapper.get('[data-testid="benchmark-chart-14-average-label"]').classes()).toContain( - 'benchmarks__chart-point-label--center' + await search(wrapper, 'stable 0.3.55') + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) + expect(wrapper.find('[data-testid="benchmark-row-9"]').exists()).toBe(true) + + await search(wrapper, 'nothing here') + expect(wrapper.get('.benchmarks__no-results').text()).toContain('No runs match these filters') + + await search(wrapper, '') + await chooseOption(wrapper, 'hardware', 'Apple M3 Max') + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) + expect(wrapper.find('[data-testid="benchmark-row-10"]').exists()).toBe(true) + await chooseOption(wrapper, 'hardware', 'All') + await chooseOption(wrapper, 'workflow', 'landscape.json') + expect(wrapper.findAll('.benchmarks__group')).toHaveLength(1) + }) + + it('labels unmanaged workspaces and unknown hardware', async () => { + useBenchmarks([ + { ...run('31', 'mystery.json', 1, { gpu: null }), workspace: { id: null, name: null } } + ]) + const wrapper = await mountGrouped('Hardware') + + expect(wrapper.get('.benchmarks__group-toggle').text()).toBe('Unknown hardware') + expect(wrapper.get('.benchmarks__group-meta').text()).toBe('1 session · 5 runs') + expect(await menuOptions(wrapper, 'workspace')).toEqual(['All', 'Unmanaged']) + await chooseOption(wrapper, 'workspace', 'Unmanaged') + expect(wrapper.find('[data-testid="benchmark-row-31"]').exists()).toBe(true) + }) + + it('exports the selected runs that are on screen', async () => { + const wrapper = mountView() + await flushPromises() + + await selectRow(wrapper, '12') + await selectRow(wrapper, '13') + expect(row(wrapper, '12').classes()).toContain('benchmarks__row--selected') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('2 selected') + let svg = await exportSelection(wrapper) + const [png, imageType, defaultPath] = exportResultsImageMock.mock.calls[0] as unknown as [ + ArrayBuffer, + string, + string + ] + expect(png).toBeInstanceOf(ArrayBuffer) + expect(imageType).toBe('benchmark-comparison') + expect(defaultPath).toBe(DEFAULT_FOLDER) + expect(svg).toContain('Session: 13') + expect(svg).toContain('Session: 12') + expect(svg.indexOf('Session: 13')).toBeLessThan(svg.indexOf('Session: 12')) + expect(svg).toContain('Each line shows the fastest and slowest run, with the average marked.') + expect(svg.match(/fill="#e8e8ee"/g)?.length).toBeGreaterThan(0) + expect(svg.match(/fill="#7aa2f7"/g)?.length).toBeGreaterThan(0) + + await search(wrapper, 'nightly') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('1 selected') + svg = await exportSelection(wrapper) + expect(svg).toContain('Session: 13') + expect(svg).not.toContain('Session: 12') + + await search(wrapper, '') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('2 selected') + await wrapper.get('.benchmarks__clear-selection').trigger('click') + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) + }) + + it('exports relative speed only for one group with a chosen baseline', async () => { + const wrapper = await mountGrouped('Hardware') + + await rowAction(wrapper, '12', '.benchmarks__set-baseline') + await groupAction(wrapper, '.benchmarks__select-group') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('3 selected') + let svg = await exportSelection(wrapper) + expect(svg).toContain('Relative speed factor = baseline duration / run duration.') + expect(svg).toContain('1.11× Faster') + expect(svg).toContain('1.00× Baseline') + + await selectRow(wrapper, '11') + svg = await exportSelection(wrapper) + expect(svg).not.toContain('Relative speed') + expect(svg).toContain('Duration range (min → max)') + }) + + it('shows export errors in the selection bar until the list reloads', async () => { + exportResultsImageMock.mockResolvedValueOnce({ ok: false, message: 'Disk full' }) + const wrapper = await mountGrouped('Hardware') + + await groupAction(wrapper, '.benchmarks__export-group') + expect(wrapper.get('.benchmarks__export-error').text()).toBe('Disk full') + expect(wrapper.find('.benchmarks__selected-count').exists()).toBe(false) + + await wrapper.get('[data-testid="benchmarks-refresh"]').trigger('click') + await flushPromises() + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) + }) + + it('limits exports to twenty runs', async () => { + useBenchmarks( + Array.from({ length: 21 }, (_, index) => + run(String(index + 1), 'bulk.json', 2 + index / 10, { gpu: 'GPU', build: `Build ${index}` }) + ) ) + const wrapper = mountView() + await flushPromises() + + await openMenu(wrapper, '.benchmarks__group-menu') + expect(menuItem('.benchmarks__export-group')?.hasAttribute('data-disabled')).toBe(true) + await chooseMenuItem('.benchmarks__select-group') - await wrapper.get('.benchmarks__export-results').trigger('click') + expect(wrapper.get('.benchmarks__selected-count').text()).toBe('21 selected') + expect(wrapper.get('.benchmarks__export-results').attributes()).toHaveProperty('disabled') + await wrapper.get('.benchmarks__selection-bar .tooltip-wrap').trigger('focusin') await flushPromises() - const svg = createResultsPngMock.mock.calls[0]![0] - const markerMatches = [...svg.matchAll(/]*\/>/g)] - const fastestLabelMatch = svg.match( - /44\.7 s<\/text>/ + expect(document.body.textContent).toContain('Up to 20 runs') + + await selectRow(wrapper, '21', false) + expect(wrapper.get('.benchmarks__export-results').attributes()).not.toHaveProperty('disabled') + }) + + it('renames a session in place and keeps its selection, order and baseline', async () => { + const wrapper = await mountGrouped('Hardware') + await rowAction(wrapper, '12', '.benchmarks__set-baseline') + await selectRow(wrapper, '12') + await drag(wrapper, '9', '13') + + await rowAction(wrapper, '12', '.benchmarks__rename-session') + await vi.waitFor(() => expect(document.activeElement).toBe(sessionField(wrapper, '12'))) + await typeSessionName(sessionField(wrapper, '12'), 'gpu-baseline') + + expect(renamePerformanceTestBenchmarkMock).toHaveBeenCalledWith( + DEFAULT_FOLDER, + '12', + 'gpu-baseline' ) - const slowestLabelMatch = svg.match( - /47\.24 s<\/text>/ + expect(rowIds(wrapper)).toEqual(['9', '13', 'gpu-baseline']) + expect(vsLabels(wrapper)).toEqual(['1.38× slower', '1.11× faster', 'Baseline']) + expect(row(wrapper, 'gpu-baseline').get('[role="checkbox"]').attributes('aria-checked')).toBe( + 'true' ) - const averageMarkerX = svg.match( - /]+r="5"[^>]+class="chart-average"/ - )?.[1] - expect(svg).toContain('r="5" fill="#55e0d1" class="chart-average"') - const averageLabelX = svg.match( - /]+class="chart-point-label" text-anchor="middle">45\.83 s<\/text>/ - )?.[1] - expect(Number(fastestLabelMatch?.[1])).toBe(Number(markerMatches[0]?.[1]) - 4) - expect(Number(slowestLabelMatch?.[1])).toBe(Number(markerMatches[1]?.[1]) + 4) - expect(Number(fastestLabelMatch?.[2])).toBe(Number(markerMatches[0]?.[2]) + 4) - expect(Number(slowestLabelMatch?.[2])).toBe(Number(markerMatches[1]?.[2]) + 4) - expect(averageLabelX).toBe(averageMarkerX) - }) - - it('confirms before deleting all files for a benchmark record', async () => { - const wrapper = mountView() + }) + + it('puts the old name back when a rename is refused', async () => { + renamePerformanceTestBenchmarkMock.mockResolvedValueOnce({ ok: false, message: 'Name taken' }) + const wrapper = await mountGrouped('Hardware') + const dialogs = useDialogs() + + expect(sessionField(wrapper, '13').getAttribute('aria-label')).toBe('Edit session 13') + await typeSessionName(sessionField(wrapper, '13'), '12') + + expect(dialogs.state.kind).toBe('alert') + expect(dialogs.state.alert.title).toBe('Could not rename session') + expect(dialogs.state.alert.message).toBe('Name taken') + dialogs.cancel() await flushPromises() + expect(sessionField(wrapper, '13').textContent).toBe('13') + expect(rowIds(wrapper)).toEqual(['13', '12', '9']) + }) + + it('confirms before deleting all files for a session', async () => { + const wrapper = await mountGrouped('Hardware') const dialogs = useDialogs() + await selectRow(wrapper, '13') - expect(wrapper.findAll('.benchmarks__delete-record')).toHaveLength(4) - await wrapper - .get('[data-testid="benchmark-row-13"] .benchmarks__delete-record') - .trigger('click') - expect(dialogs.state.open).toBe(true) + await rowAction(wrapper, '13', '.benchmarks__delete-record') expect(dialogs.state.kind).toBe('confirm') expect(dialogs.state.confirm.title).toBe('Delete portrait.json?') expect(dialogs.state.confirm.message).toContain('permanently delete all files for session 13') @@ -575,25 +1026,19 @@ describe('BenchmarksView', () => { dialogs.confirmPrimary() await flushPromises() - expect(deletePerformanceTestBenchmarkMock).toHaveBeenCalledWith( - 'C:\\results\\performance-tests', - '13' - ) + expect(deletePerformanceTestBenchmarkMock).toHaveBeenCalledWith(DEFAULT_FOLDER, '13') expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(false) - expect(wrapper.find('[data-testid="benchmark-comparison-column-title-13"]').exists()).toBe( - false - ) + expect(vsLabels(wrapper)).toEqual(['Fastest', '1.38× slower']) + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) }) - it('shows an error when the benchmark delete bridge rejects', async () => { + it('shows an error when deleting a session fails', async () => { deletePerformanceTestBenchmarkMock.mockRejectedValueOnce(new Error('Delete IPC unavailable')) const wrapper = mountView() await flushPromises() const dialogs = useDialogs() - await wrapper - .get('[data-testid="benchmark-row-13"] .benchmarks__delete-record') - .trigger('click') + await rowAction(wrapper, '13', '.benchmarks__delete-record') dialogs.confirmPrimary() await flushPromises() @@ -603,85 +1048,62 @@ describe('BenchmarksView', () => { expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(true) }) - it('renames a session inline and preserves its comparison selection', async () => { - const wrapper = mountView() - await flushPromises() - await wrapper - .findAll('.benchmarks__visualization-tabs button') - .find((tab) => tab.text() === 'Speed')! - .trigger('click') - await wrapper - .get('[data-testid="benchmark-comparison-column-title-13"] .benchmarks__matrix-title') - .trigger('click') - - const sessionButton = wrapper.get('[data-testid="benchmark-row-13"] .benchmarks__session-name') - expect(sessionButton.text()).toBe('13') - expect(sessionButton.attributes('aria-label')).toBe('Edit session 13') - - await sessionButton.trigger('click') - const input = wrapper.get('.benchmarks__session-name-input') - expect(input.element.value).toBe('13') - expect( - wrapper.get('[data-testid="benchmark-row-13"] .benchmarks__delete-record').attributes() - ).toHaveProperty('disabled') - await input.setValue('gpu-baseline') - await input.trigger('keydown', { key: 'Enter' }) - await flushPromises() - - expect(renamePerformanceTestBenchmarkMock).toHaveBeenCalledWith( - 'C:\\results\\performance-tests', - '13', - 'gpu-baseline' - ) - expect(wrapper.find('[data-testid="benchmark-row-13"]').exists()).toBe(false) - expect(wrapper.get('[data-testid="benchmark-row-gpu-baseline"]').text()).toContain( - 'gpu-baseline' - ) - expect( - wrapper.find('[data-testid="benchmark-comparison-column-title-gpu-baseline"]').exists() - ).toBe(true) - expect( - wrapper - .get('[data-testid="benchmark-comparison-column-title-gpu-baseline"]') - .classes('benchmarks__matrix-column--baseline') - ).toBe(true) - }) - it('opens the current results folder in the picker and loads a selected folder', async () => { const customFolder = 'D:\\shared-benchmarks' - const api = window.api - vi.mocked(api.browseFolder).mockResolvedValue(customFolder) - vi.mocked(api.listPerformanceTestBenchmarks) + vi.mocked(window.api.browseFolder).mockResolvedValue(customFolder) + listMock() .mockReset() - .mockResolvedValueOnce({ - folderPath: 'C:\\results\\performance-tests', - benchmarks: sampleBenchmarks - }) - .mockResolvedValueOnce({ - folderPath: customFolder, - benchmarks: [sampleBenchmarks[3]!] - }) + .mockResolvedValueOnce({ folderPath: DEFAULT_FOLDER, benchmarks: sampleBenchmarks }) + .mockResolvedValueOnce({ folderPath: customFolder, benchmarks: [sampleBenchmarks[3]!] }) const wrapper = mountView() await flushPromises() + await selectRow(wrapper, '10') + expect(wrapper.get('.benchmarks__open-folder').attributes('title')).toBe(DEFAULT_FOLDER) await wrapper.get('.benchmarks__open-folder').trigger('click') await flushPromises() - expect(api.browseFolder).toHaveBeenCalledWith('C:\\results\\performance-tests') - expect(api.listPerformanceTestBenchmarks).toHaveBeenLastCalledWith(customFolder) + expect(window.api.browseFolder).toHaveBeenCalledWith(DEFAULT_FOLDER) + expect(listMock()).toHaveBeenLastCalledWith(customFolder) expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(1) - expect(wrapper.get('[data-testid="benchmark-row-10"]').text()).toContain('landscape.json') + expect(row(wrapper, '10').text()).toContain('Stable 0.3.60') + expect(wrapper.find('.benchmarks__selection-bar').exists()).toBe(false) }) - it('refreshes the currently selected sessions folder', async () => { + it('shows a skeleton while loading', async () => { + listMock().mockReturnValue(new Promise(() => {})) const wrapper = mountView() await flushPromises() - await wrapper.get('[data-testid="benchmarks-refresh"]').trigger('click') + expect(wrapper.find('.benchmarks__state--loading').exists()).toBe(true) + expect(wrapper.get('[data-testid="benchmarks-refresh"]').attributes()).toHaveProperty( + 'disabled' + ) + }) + + it('offers a refresh when loading fails', async () => { + listMock().mockRejectedValueOnce(new Error('broken')) + const wrapper = mountView() + await flushPromises() + + expect(wrapper.get('.benchmarks__state--error').text()).toContain( + 'Could not load performance test results' + ) + useBenchmarks(sampleBenchmarks) + await wrapper.get('.benchmarks__retry').trigger('click') + await flushPromises() + expect(wrapper.findAll('[data-testid^="benchmark-row-"]')).toHaveLength(5) + }) + + it('points to the Performance Test when there are no results', async () => { + useBenchmarks([]) + const wrapper = mountView() await flushPromises() - expect(window.api.listPerformanceTestBenchmarks).toHaveBeenLastCalledWith( - 'C:\\results\\performance-tests' + expect(wrapper.get('.benchmarks__state--empty').text()).toContain( + 'No performance test results yet' ) + await wrapper.get('.benchmarks__open-performance-test').trigger('click') + expect(navigatePanelMock).toHaveBeenCalledExactlyOnceWith('performance-test') }) }) diff --git a/src/renderer/src/views/BenchmarksView.vue b/src/renderer/src/views/BenchmarksView.vue index 85cb6358f..bcffec9dc 100644 --- a/src/renderer/src/views/BenchmarksView.vue +++ b/src/renderer/src/views/BenchmarksView.vue @@ -1,193 +1,119 @@ + + diff --git a/src/renderer/src/views/benchmarks/BenchmarkSessionName.vue b/src/renderer/src/views/benchmarks/BenchmarkSessionName.vue new file mode 100644 index 000000000..36ad9cdf6 --- /dev/null +++ b/src/renderer/src/views/benchmarks/BenchmarkSessionName.vue @@ -0,0 +1,43 @@ + + + diff --git a/src/types/ipc.ts b/src/types/ipc.ts index cd7bdbcea..9bbab6f45 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -748,10 +748,17 @@ export interface PerformanceTestBenchmark { averageJobDurationSeconds: number | null medianJobDurationSeconds: number | null measuredJobCount: number - hardwareName: string | null + machine: PerformanceTestMachine result: Record } +export interface PerformanceTestMachine { + gpu: string | null + cpu: string | null + memoryGb: number | null + platform: string | null +} + export interface SnapshotDiffEntry { createdAt: string trigger: string