From b19a1b187c21eae480dea320acc6b680c5e4f1b3 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 30 Jul 2026 14:34:52 -0700 Subject: [PATCH 001/119] refactor(schemas): minify generated JSON schemas (#1210) Signed-off-by: Anthony Casagrande (cherry picked from commit 2f413f0dec833e17a5fd3c5307ed0a23a5d7bd7a) --- src/aiperf/common/loop_scheduler.py | 3 +- .../plugin/schema/categories.schema.json | 62 +- src/aiperf/plugin/schema/plugins.schema.json | 2160 +---------------- tests/unit/common/test_loop_scheduler.py | 26 + tools/generate_config_schema.py | 5 +- tools/generate_plugin_artifacts.py | 9 +- 6 files changed, 37 insertions(+), 2228 deletions(-) diff --git a/src/aiperf/common/loop_scheduler.py b/src/aiperf/common/loop_scheduler.py index 4a1588efe5..66074aac49 100644 --- a/src/aiperf/common/loop_scheduler.py +++ b/src/aiperf/common/loop_scheduler.py @@ -84,6 +84,7 @@ def set_drain_observer(self, observer: Callable[[], None] | None) -> None: def _done_callback(self, task: asyncio.Task) -> None: """Remove completed task from tracking; invoke exception handler if failed.""" + was_tracked = task in self._tasks self._tasks.discard(task) # discard() is safe if already removed by cancel_all # Must check cancelled() first - calling exception() on a cancelled task raises CancelledError if ( @@ -92,7 +93,7 @@ def _done_callback(self, task: asyncio.Task) -> None: and self._exception_handler is not None ): self._exception_handler(task) - if not self._tasks and self._drain_observer is not None: + if was_tracked and not self._tasks and self._drain_observer is not None: try: self._drain_observer() except Exception as exc: # noqa: BLE001 diff --git a/src/aiperf/plugin/schema/categories.schema.json b/src/aiperf/plugin/schema/categories.schema.json index edf159fb15..da22a3df03 100644 --- a/src/aiperf/plugin/schema/categories.schema.json +++ b/src/aiperf/plugin/schema/categories.schema.json @@ -1,61 +1 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "categories.schema.json", - "additionalProperties": { - "description": "Specification for a plugin category (extension point).\n\nCategories define extension points in AIPerf where plugins can provide\ncustom implementations. The protocol defines the interface, and the enum\nprovides type-safe selection in configuration files and APIs.\n\nExample::\n\n endpoint:\n protocol: aiperf.endpoints.protocols:EndpointProtocol\n metadata_class: aiperf.plugin.schema.schemas:EndpointMetadata\n enum: EndpointType\n description: HTTP endpoint handlers for different LLM APIs.", - "properties": { - "protocol": { - "description": "The interface that plugins in this category must implement. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.protocols:EndpointProtocol'.", - "title": "Protocol", - "type": "string" - }, - "metadata_class": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional class path for category-specific metadata. When set, plugins can include typed metadata fields validated against this schema. Use 'module.path:ClassName' format.", - "title": "Metadata Class" - }, - "enum": { - "description": "Name of the enum that will be auto-generated from registered plugins. This enum is used in config files and APIs to select plugin types, e.g., 'EndpointType' generates EndpointType.CHAT, EndpointType.COMPLETIONS, etc.", - "title": "Enum", - "type": "string" - }, - "description": { - "description": "Brief explanation of what this category is for and when to use it.", - "title": "Description", - "type": "string" - }, - "internal": { - "default": false, - "description": "Set to true for infrastructure categories not meant for end users. Internal categories are hidden from documentation and plugin listings.", - "title": "Internal", - "type": "boolean" - } - }, - "required": [ - "protocol", - "enum", - "description" - ], - "title": "CategorySpec", - "type": "object" - }, - "description": "Root model for categories.yaml file.\n\nThis file defines all plugin extension points in AIPerf. Each category\nspecifies a protocol interface and generates an enum for type selection.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n protocol: aiperf.endpoints.protocols:EndpointProtocol\n metadata_class: aiperf.plugin.schema.schemas:EndpointMetadata\n enum: EndpointType\n description: HTTP endpoint handlers for LLM APIs\n\nservice:\n protocol: aiperf.common.protocols:ServiceProtocol\n metadata_class: aiperf.plugin.schema.schemas:ServiceMetadata\n enum: ServiceType\n description: Service plugins for AIPerf.\n\n...\n```", - "properties": { - "schema_version": { - "default": "1.0", - "description": "Version of the categories.yaml schema format. Used for backwards compatibility.", - "title": "Schema Version", - "type": "string" - } - }, - "title": "CategoriesManifest", - "type": "object" -} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"categories.schema.json","additionalProperties":{"description":"Specification for a plugin category (extension point).\n\nCategories define extension points in AIPerf where plugins can provide\ncustom implementations. The protocol defines the interface, and the enum\nprovides type-safe selection in configuration files and APIs.\n\nExample::\n\n endpoint:\n protocol: aiperf.endpoints.protocols:EndpointProtocol\n metadata_class: aiperf.plugin.schema.schemas:EndpointMetadata\n enum: EndpointType\n description: HTTP endpoint handlers for different LLM APIs.","properties":{"protocol":{"description":"The interface that plugins in this category must implement. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.protocols:EndpointProtocol'.","title":"Protocol","type":"string"},"metadata_class":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Optional class path for category-specific metadata. When set, plugins can include typed metadata fields validated against this schema. Use 'module.path:ClassName' format.","title":"Metadata Class"},"enum":{"description":"Name of the enum that will be auto-generated from registered plugins. This enum is used in config files and APIs to select plugin types, e.g., 'EndpointType' generates EndpointType.CHAT, EndpointType.COMPLETIONS, etc.","title":"Enum","type":"string"},"description":{"description":"Brief explanation of what this category is for and when to use it.","title":"Description","type":"string"},"internal":{"default":false,"description":"Set to true for infrastructure categories not meant for end users. Internal categories are hidden from documentation and plugin listings.","title":"Internal","type":"boolean"}},"required":["protocol","enum","description"],"title":"CategorySpec","type":"object"},"description":"Root model for categories.yaml file.\n\nThis file defines all plugin extension points in AIPerf. Each category\nspecifies a protocol interface and generates an enum for type selection.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n protocol: aiperf.endpoints.protocols:EndpointProtocol\n metadata_class: aiperf.plugin.schema.schemas:EndpointMetadata\n enum: EndpointType\n description: HTTP endpoint handlers for LLM APIs\n\nservice:\n protocol: aiperf.common.protocols:ServiceProtocol\n metadata_class: aiperf.plugin.schema.schemas:ServiceMetadata\n enum: ServiceType\n description: Service plugins for AIPerf.\n\n...\n```","properties":{"schema_version":{"default":"1.0","description":"Version of the categories.yaml schema format. Used for backwards compatibility.","title":"Schema Version","type":"string"}},"title":"CategoriesManifest","type":"object"} diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 7f46115fb9..2552f42a90 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -1,2159 +1 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "plugins.schema.json", - "additionalProperties": false, - "description": "Root model for plugins.yaml file.\n\nThis file registers plugin implementations for AIPerf. Each top-level section\ncorresponds to a category from categories.yaml and maps type names to their\nimplementing classes.\n\nNote: Package metadata (name, version, author) comes from pyproject.toml\nvia importlib.metadata, not from this file.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n my_custom:\n class: my_package.endpoints.my_custom:MyCustomEndpoint\n description: Custom endpoint for my API.\n metadata:\n endpoint_path: /v1/generate\n supports_streaming: true\n produces_tokens: true\n tokenizes_input: true\n metrics_title: My Custom Metrics\n\ncustom_dataset_loader:\n my_jsonl:\n class: my_package.dataset_loaders.my_jsonl:MyJSONLDatasetLoader\n description: Custom dataset loader for my API.\n```", - "properties": { - "schema_version": { - "default": "1.0", - "description": "Version of the plugins.yaml schema format. Use '1.0' for current format.", - "title": "Schema Version", - "type": "string" - }, - "api_router": { - "title": "API Router Plugins", - "type": "object", - "description": "API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded.", - "additionalProperties": { - "$ref": "#/$defs/ApiRouterPlugin" - } - }, - "timing_strategy": { - "title": "Timing Strategy Plugins", - "type": "object", - "description": "Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run.", - "additionalProperties": { - "$ref": "#/$defs/TimingStrategyPlugin" - } - }, - "arrival_pattern": { - "title": "Arrival Pattern Plugins", - "type": "object", - "description": "Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode.", - "additionalProperties": { - "$ref": "#/$defs/ArrivalPatternPlugin" - } - }, - "ramp": { - "title": "Ramp Plugins", - "type": "object", - "description": "Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns.", - "additionalProperties": { - "$ref": "#/$defs/RampPlugin" - } - }, - "dataset_backing_store": { - "title": "Dataset Backing Store Plugins", - "type": "object", - "description": "Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access.", - "additionalProperties": { - "$ref": "#/$defs/DatasetBackingStorePlugin" - } - }, - "dataset_client_store": { - "title": "Dataset Client Store Plugins", - "type": "object", - "description": "Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance.", - "additionalProperties": { - "$ref": "#/$defs/DatasetClientStorePlugin" - } - }, - "dataset_sampler": { - "title": "Dataset Sampler Plugins", - "type": "object", - "description": "Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration.", - "additionalProperties": { - "$ref": "#/$defs/DatasetSamplerPlugin" - } - }, - "dataset_composer": { - "title": "Dataset Composer Plugins", - "type": "object", - "description": "Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration.", - "additionalProperties": { - "$ref": "#/$defs/DatasetComposerPlugin" - } - }, - "custom_dataset_loader": { - "title": "Custom Dataset Loader Plugins", - "type": "object", - "description": "Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load().", - "additionalProperties": { - "$ref": "#/$defs/CustomDatasetLoaderPlugin" - } - }, - "public_dataset_loader": { - "title": "Public Dataset Loader Plugins", - "type": "object", - "description": "Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration.", - "additionalProperties": { - "$ref": "#/$defs/PublicDatasetLoaderPlugin" - } - }, - "endpoint": { - "title": "Endpoint Plugins", - "type": "object", - "description": "Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration.", - "additionalProperties": { - "$ref": "#/$defs/EndpointPlugin" - } - }, - "transport": { - "title": "Transport Plugins", - "type": "object", - "description": "Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration.", - "additionalProperties": { - "$ref": "#/$defs/TransportPlugin" - } - }, - "record_processor": { - "title": "Record Processor Plugins", - "type": "object", - "description": "Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/RecordProcessorPlugin" - } - }, - "record_observer": { - "title": "Record Observer Plugins", - "type": "object", - "description": "Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/RecordObserverPlugin" - } - }, - "accumulator": { - "title": "Accumulator Plugins", - "type": "object", - "description": "Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/AccumulatorPlugin" - } - }, - "stream_exporter": { - "title": "Stream Exporter Plugins", - "type": "object", - "description": "Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/StreamExporterPlugin" - } - }, - "analyzer": { - "title": "Analyzer Plugins", - "type": "object", - "description": "Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/AnalyzerPlugin" - } - }, - "spec_decode_adapter": { - "title": "Spec Decode Adapter Plugins", - "type": "object", - "description": "Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins.", - "additionalProperties": { - "$ref": "#/$defs/SpecDecodeAdapterPlugin" - } - }, - "accuracy_grader": { - "title": "Accuracy Grader Plugins", - "type": "object", - "description": "Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default.", - "additionalProperties": { - "$ref": "#/$defs/AccuracyGraderPlugin" - } - }, - "accuracy_benchmark": { - "title": "Accuracy Benchmark Plugins", - "type": "object", - "description": "Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration.", - "additionalProperties": { - "$ref": "#/$defs/AccuracyBenchmarkPlugin" - } - }, - "data_exporter": { - "title": "Data Exporter Plugins", - "type": "object", - "description": "Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/DataExporterPlugin" - } - }, - "console_exporter": { - "title": "Console Exporter Plugins", - "type": "object", - "description": "Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/ConsoleExporterPlugin" - } - }, - "ui": { - "title": "UI Plugins", - "type": "object", - "description": "UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration.", - "additionalProperties": { - "$ref": "#/$defs/UiPlugin" - } - }, - "url_selection_strategy": { - "title": "Url Selection Strategy Plugins", - "type": "object", - "description": "URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration.", - "additionalProperties": { - "$ref": "#/$defs/UrlSelectionStrategyPlugin" - } - }, - "service": { - "title": "Service Plugins", - "type": "object", - "description": "Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services.", - "additionalProperties": { - "$ref": "#/$defs/ServicePlugin" - } - }, - "service_manager": { - "title": "Service Manager Plugins", - "type": "object", - "description": "Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration.", - "additionalProperties": { - "$ref": "#/$defs/ServiceManagerPlugin" - } - }, - "communication": { - "title": "Communication Plugins", - "type": "object", - "description": "Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration.", - "additionalProperties": { - "$ref": "#/$defs/CommunicationPlugin" - } - }, - "communication_client": { - "title": "Communication Client Plugins", - "type": "object", - "description": "Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage.", - "additionalProperties": { - "$ref": "#/$defs/CommunicationClientPlugin" - } - }, - "zmq_proxy": { - "title": "ZMQ Proxy Plugins", - "type": "object", - "description": "ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration.", - "additionalProperties": { - "$ref": "#/$defs/ZmqProxyPlugin" - } - }, - "plot": { - "title": "Plot Plugins", - "type": "object", - "description": "Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection.", - "additionalProperties": { - "$ref": "#/$defs/PlotPlugin" - } - }, - "gpu_telemetry_collector": { - "title": "GPU Telemetry Collector Plugins", - "type": "object", - "description": "GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration.", - "additionalProperties": { - "$ref": "#/$defs/GpuTelemetryCollectorPlugin" - } - }, - "search_recipe": { - "title": "Search Recipe Plugins", - "type": "object", - "description": "Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run.", - "additionalProperties": { - "$ref": "#/$defs/SearchRecipePlugin" - } - }, - "search_recipe_post_process": { - "title": "Search Recipe Post Process Plugins", - "type": "object", - "description": "Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec.", - "additionalProperties": { - "$ref": "#/$defs/SearchRecipePostProcessPlugin" - } - }, - "convergence_criterion": { - "title": "Convergence Criterion Plugins", - "type": "object", - "description": "Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run.", - "additionalProperties": { - "$ref": "#/$defs/ConvergenceCriterionPlugin" - } - }, - "search_planner": { - "title": "Search Planner Plugins", - "type": "object", - "description": "Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run.", - "additionalProperties": { - "$ref": "#/$defs/SearchPlannerPlugin" - } - } - }, - "title": "PluginsManifest", - "type": "object", - "$defs": { - "ApiRouterPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "API Router Plugin", - "description": "API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded." - }, - "TimingStrategyPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Timing Strategy Plugin", - "description": "Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run." - }, - "ArrivalPatternPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Arrival Pattern Plugin", - "description": "Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode." - }, - "RampPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Ramp Plugin", - "description": "Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns." - }, - "DatasetBackingStorePlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Dataset Backing Store Plugin", - "description": "Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access." - }, - "DatasetClientStorePlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Dataset Client Store Plugin", - "description": "Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance." - }, - "DatasetSamplerPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Dataset Sampler Plugin", - "description": "Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration." - }, - "DatasetComposerPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Dataset Composer Plugin", - "description": "Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration." - }, - "CustomDatasetLoaderPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for custom dataset loader plugins.\n\nDefines format-specific defaults for dataset loaders. When a loader specifies\n``block_size``, it overrides the user's ``--isl-block-size`` config default,\nensuring hash-based prompt generation uses the correct token block size for the\ntrace format (e.g. 16 for Bailian, 512 for Mooncake).\n\nReferenced by: categories.yaml custom_dataset_loader.metadata_class\nUsed in: plugins.yaml custom_dataset_loader entries", - "properties": { - "is_trace": { - "default": false, - "description": "Whether this loader handles trace-format datasets. Trace datasets use hash_ids-based prompt generation, support synthesis options, and prefer sequential sampling with fixed_schedule timing.", - "title": "Is Trace", - "type": "boolean" - }, - "default_block_size": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids (e.g. 16 for Bailian, 512 for Mooncake).", - "title": "Default Block Size" - }, - "default_prompt_corpus": { - "default": "sonnet", - "description": "Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces (e.g. weka_trace) override to 'coding' so reconstructed prompts resemble real tool-use content.", - "title": "Default Prompt Corpus", - "type": "string" - }, - "category": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter dataset rows to a specific category value. Used by custom loaders that support per-category subsets (e.g. SPEED-Bench).", - "title": "Category" - }, - "multi_turn": { - "default": false, - "description": "When true, each row becomes one Conversation with multiple Turn objects.", - "title": "Multi Turn", - "type": "boolean" - } - }, - "title": "CustomDatasetLoaderMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Custom Dataset Loader Plugin", - "description": "Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load()." - }, - "PublicDatasetLoaderPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for public dataset loader plugins.\n\nReferenced by: categories.yaml public_dataset_loader.metadata_class\nUsed in: plugins.yaml public_dataset_loader entries", - "properties": { - "hf_dataset_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace dataset identifier (e.g. 'AI-MO/NuminaMath-TIR'). Required for HF-backed loaders.", - "title": "Hf Dataset Name" - }, - "hf_split": { - "default": "train", - "description": "HuggingFace dataset split to load (e.g. 'train', 'test', 'validation').", - "title": "Hf Split", - "type": "string" - }, - "hf_subset": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace dataset subset/config name. Only needed for datasets with multiple configs.", - "title": "Hf Subset" - }, - "is_trace": { - "default": false, - "description": "Whether this loader handles trace-format datasets. Trace public datasets reuse hash_ids-based prompt generation, require a tokenizer, and prefer sequential sampling. Mirrors the field of the same name on CustomDatasetLoaderMetadata so trace loaders can live in either pipeline.", - "title": "Is Trace", - "type": "boolean" - }, - "default_block_size": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids.", - "title": "Default Block Size" - }, - "default_prompt_corpus": { - "default": "sonnet", - "description": "Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces override to 'coding'.", - "title": "Default Prompt Corpus", - "type": "string" - }, - "prompt_column": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Column name containing the prompt/instruction text. Required for HFInstructionResponseDatasetLoader.", - "title": "Prompt Column" - }, - "image_column": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Column name containing the image data (PIL Image). Used for multimodal datasets.", - "title": "Image Column" - }, - "video_column": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Column name containing the video data (URL string or bytes dict). Used for video multimodal datasets.", - "title": "Video Column" - }, - "audio_column": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Column name containing audio data. Depending on loader/dataset, this can be an HF Audio dict with array/sampling_rate (decoded) or bytes/path (decode=False). Used for ASR/speech datasets.", - "title": "Audio Column" - }, - "conversation_column": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Column name containing the conversation messages array. Required for HFConversationDatasetLoader.", - "title": "Conversation Column" - }, - "message_content_key": { - "default": "content", - "description": "Key inside each message dict for the text content. Used with conversation_column (e.g. 'content', 'value').", - "title": "Message Content Key", - "type": "string" - }, - "multi_turn": { - "default": false, - "description": "When true, each row becomes one Conversation with multiple Turn objects. For HFConversationDatasetLoader: user\u2192assistant (OpenAI roles) or human\u2192gpt (ShareGPT-style ``from`` fields) pairs.", - "title": "Multi Turn", - "type": "boolean" - }, - "streaming": { - "default": false, - "description": "Whether to load the HuggingFace dataset in streaming mode. Use true for large datasets (>10 GB) to avoid downloading the full dataset. Use false (default) for small datasets to leverage HF caching and len() support.", - "title": "Streaming", - "type": "boolean" - }, - "has_timing_data": { - "default": false, - "description": "Whether the loader emits timestamps suitable for fixed-schedule replay.", - "title": "Has Timing Data", - "type": "boolean" - }, - "prompt_template": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.", - "title": "Prompt Template" - } - }, - "title": "PublicDatasetLoaderMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Public Dataset Loader Plugin", - "description": "Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration." - }, - "EndpointPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for endpoint plugins.\n\nDefines API capabilities, paths, and multimodal support for endpoint implementations.\nUsed by the framework to route requests, configure streaming, and enable\nmultimodal inputs/outputs (images, audio, video).\n\nReferenced by: categories.yaml endpoint.metadata_class\nUsed in: plugins.yaml endpoint entries", - "properties": { - "metrics_title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Display title for metrics dashboard.", - "title": "Metrics Title" - }, - "endpoint_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "API path (e.g., /v1/chat/completions).", - "title": "Endpoint Path" - }, - "streaming_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Streaming API path if different from the endpoint path (e.g., /generate_stream).", - "title": "Streaming Path" - }, - "service_kind": { - "default": "openai", - "description": "The service kind of the endpoint (used for artifact naming).", - "title": "Service Kind", - "type": "string" - }, - "supports_streaming": { - "description": "Whether endpoint supports streaming responses.", - "title": "Supports Streaming", - "type": "boolean" - }, - "tokenizes_input": { - "description": "Whether endpoint tokenizes text inputs.", - "title": "Tokenizes Input", - "type": "boolean" - }, - "produces_tokens": { - "description": "Whether endpoint produces token-based output.", - "title": "Produces Tokens", - "type": "boolean" - }, - "supports_audio": { - "default": false, - "description": "Whether endpoint accepts audio input.", - "title": "Supports Audio", - "type": "boolean" - }, - "supports_images": { - "default": false, - "description": "Whether endpoint accepts image input.", - "title": "Supports Images", - "type": "boolean" - }, - "supports_videos": { - "default": false, - "description": "Whether endpoint accepts video input.", - "title": "Supports Videos", - "type": "boolean" - }, - "produces_audio": { - "default": false, - "description": "Whether endpoint produces audio-based outputs.", - "title": "Produces Audio", - "type": "boolean" - }, - "produces_images": { - "default": false, - "description": "Whether endpoint produces image-based outputs.", - "title": "Produces Images", - "type": "boolean" - }, - "produces_videos": { - "default": false, - "description": "Whether endpoint produces video-based outputs.", - "title": "Produces Videos", - "type": "boolean" - }, - "requires_polling": { - "default": false, - "description": "Whether endpoint uses async job polling (submit job, poll for status, retrieve result).", - "title": "Requires Polling", - "type": "boolean" - }, - "requires_form_data": { - "default": false, - "description": "True for endpoints that require multipart/form-data (binary uploads), e.g., video_generation, image_edit. EndpointConfig consumes this flag to auto-select the request encoding.", - "title": "Requires Form Data", - "type": "boolean" - }, - "requires_inline_media": { - "default": false, - "description": "Whether endpoint requires media URLs to be downloaded and inlined as base64 data URLs.", - "title": "Requires Inline Media", - "type": "boolean" - }, - "consumes_system_message": { - "default": false, - "description": "Whether endpoint sends RequestInfo.system_message on the wire (e.g. a leading system role for chat, top-level instructions for responses). When False, the multi-turn loader does not hoist a leading system turn into the conversation system_message, since it would be silently dropped.", - "title": "Consumes System Message", - "type": "boolean" - } - }, - "required": [ - "metrics_title", - "endpoint_path", - "supports_streaming", - "tokenizes_input", - "produces_tokens" - ], - "title": "EndpointMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Endpoint Plugin", - "description": "Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration." - }, - "TransportPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for transport plugins.\n\nDefines network layer configuration including transport type and URL schemes.\nUsed by the framework to auto-detect appropriate transport based on URL scheme.\n\nReferenced by: categories.yaml transport.metadata_class\nUsed in: plugins.yaml transport entries", - "properties": { - "transport_type": { - "description": "Transport type identifier for this transport", - "title": "Transport Type", - "type": "string" - }, - "url_schemes": { - "description": "URL schemes this transport handles (for auto-detection and validation).", - "items": { - "type": "string" - }, - "title": "Url Schemes", - "type": "array" - } - }, - "required": [ - "transport_type" - ], - "title": "TransportMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Transport Plugin", - "description": "Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration." - }, - "RecordProcessorPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries", - "properties": { - "record_type": { - "description": "The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.", - "title": "Record Type", - "type": "string" - } - }, - "required": [ - "record_type" - ], - "title": "RecordProducerMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Record Processor Plugin", - "description": "Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously." - }, - "RecordObserverPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Record Observer Plugin", - "description": "Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously." - }, - "AccumulatorPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries", - "properties": { - "record_types": { - "description": "Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.", - "items": { - "type": "string" - }, - "title": "Record Types", - "type": "array" - } - }, - "required": [ - "record_types" - ], - "title": "RecordRoutingMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Accumulator Plugin", - "description": "Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously." - }, - "StreamExporterPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries", - "properties": { - "record_types": { - "description": "Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.", - "items": { - "type": "string" - }, - "title": "Record Types", - "type": "array" - } - }, - "required": [ - "record_types" - ], - "title": "RecordRoutingMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Stream Exporter Plugin", - "description": "Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously." - }, - "AnalyzerPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary \u2014 e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) \u2014 e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries", - "properties": { - "required_accumulators": { - "description": "AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.", - "items": { - "type": "string" - }, - "title": "Required Accumulators", - "type": "array" - }, - "required_summaries": { - "description": "AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.", - "items": { - "type": "string" - }, - "title": "Required Summaries", - "type": "array" - } - }, - "title": "AnalyzerMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Analyzer Plugin", - "description": "Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously." - }, - "SpecDecodeAdapterPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Spec Decode Adapter Plugin", - "description": "Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins." - }, - "AccuracyGraderPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Accuracy Grader Plugin", - "description": "Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default." - }, - "AccuracyBenchmarkPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Accuracy Benchmark Plugin", - "description": "Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration." - }, - "DataExporterPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Data Exporter Plugin", - "description": "Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously." - }, - "ConsoleExporterPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Console Exporter Plugin", - "description": "Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously." - }, - "UiPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "UI Plugin", - "description": "UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration." - }, - "UrlSelectionStrategyPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Url Selection Strategy Plugin", - "description": "URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration." - }, - "ServicePlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for service plugins.\n\nDefines lifecycle and runtime configuration for AIPerf distributed services.\nUsed by SystemController to manage service startup order and optimize\nlatency-sensitive services (timing, workers) by disabling garbage collection.\n\nReferenced by: categories.yaml service.metadata_class\nUsed in: plugins.yaml service entries", - "properties": { - "required": { - "description": "Whether the service is required for benchmark execution.", - "title": "Required", - "type": "boolean" - }, - "auto_start": { - "description": "Whether the service is automatically started by the system controller.", - "title": "Auto Start", - "type": "boolean" - }, - "disable_gc": { - "default": false, - "description": "Whether to disable garbage collection in the service for timing-critical operations.", - "title": "Disable Gc", - "type": "boolean" - }, - "replicable": { - "default": false, - "description": "Whether the service can have multiple instances running in parallel.", - "title": "Replicable", - "type": "boolean" - } - }, - "required": [ - "required", - "auto_start" - ], - "title": "ServiceMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Service Plugin", - "description": "Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services." - }, - "ServiceManagerPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Service Manager Plugin", - "description": "Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration." - }, - "CommunicationPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Communication Plugin", - "description": "Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration." - }, - "CommunicationClientPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Communication Client Plugin", - "description": "Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage." - }, - "ZmqProxyPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "ZMQ Proxy Plugin", - "description": "ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration." - }, - "PlotPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for plot plugins.\n\nDefines display properties and categorization for visualization handlers.\nUsed by the framework to group plots and generate selection interfaces.\n\nReferenced by: categories.yaml plot.metadata_class\nUsed in: plugins.yaml plot entries", - "properties": { - "display_name": { - "description": "Human-readable name for UI display.", - "title": "Display Name", - "type": "string" - }, - "category": { - "description": "Plot category (per_request, aggregated, combined, comparison).", - "title": "Category", - "type": "string" - } - }, - "required": [ - "display_name", - "category" - ], - "title": "PlotMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Plot Plugin", - "description": "Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection." - }, - "GpuTelemetryCollectorPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Config-time metadata for GPU telemetry collector plugins.\n\nLocal collectors set ``is_local`` so ``--gpu-telemetry `` is treated\nas an in-process collector keyword instead of a DCGM URL. Native binding\nvalidation lives on the collector class itself via ``validate_environment``\nso each implementation owns its own dependency check.\n\nReferenced by: categories.yaml gpu_telemetry_collector.metadata_class\nUsed in: plugins.yaml gpu_telemetry_collector entries", - "properties": { - "is_local": { - "default": false, - "description": "Whether this collector runs in-process against the local host.", - "title": "Is Local", - "type": "boolean" - } - }, - "title": "GPUTelemetryCollectorMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "GPU Telemetry Collector Plugin", - "description": "GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration." - }, - "SearchRecipePlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Search Recipe Plugin", - "description": "Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run." - }, - "SearchRecipePostProcessPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Search Recipe Post Process Plugin", - "description": "Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec." - }, - "ConvergenceCriterionPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for convergence criterion plugins.\n\nDeclares statistical-method capabilities so the CLI/config layer can\nvalidate `--convergence-metric` / `--convergence-stat` against the chosen\ncriterion before the plugin is imported.\n\nReferenced by: categories.yaml convergence_criterion.metadata_class\nUsed in: plugins.yaml convergence_criterion entries", - "properties": { - "min_samples": { - "description": "Minimum number of successful runs required before convergence can trigger.", - "minimum": 1, - "title": "Min Samples", - "type": "integer" - }, - "requires_confidence_level": { - "default": false, - "description": "Whether the criterion consumes plan.confidence_level (e.g. CI-width does, CV doesn't).", - "title": "Requires Confidence Level", - "type": "boolean" - }, - "requires_jsonl_export": { - "default": false, - "description": "Whether the criterion reads per-request metrics from JSONL exports (e.g. distribution does).", - "title": "Requires Jsonl Export", - "type": "boolean" - }, - "metric_kinds": { - "description": "Kinds of metrics this criterion handles. One or more of 'continuous', 'counts', 'categorical'.", - "items": { - "type": "string" - }, - "title": "Metric Kinds", - "type": "array" - } - }, - "required": [ - "min_samples" - ], - "title": "ConvergenceCriterionMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Convergence Criterion Plugin", - "description": "Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run." - }, - "SearchPlannerPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "description": "Metadata schema for search planner plugins.\n\nDeclares dimension-type and objective-direction support so the CLI/config\nlayer can validate `--search-space` shape against the chosen planner\nbefore the planner (and its heavy soft-dep imports) is loaded.\n\nReferenced by: categories.yaml search_planner.metadata_class\nUsed in: plugins.yaml search_planner entries", - "properties": { - "supports_continuous": { - "description": "Whether the planner accepts Real-valued search-space dimensions.", - "title": "Supports Continuous", - "type": "boolean" - }, - "supports_discrete": { - "description": "Whether the planner accepts Integer-valued search-space dimensions.", - "title": "Supports Discrete", - "type": "boolean" - }, - "supports_categorical": { - "default": false, - "description": "Whether the planner accepts Categorical search-space dimensions.", - "title": "Supports Categorical", - "type": "boolean" - }, - "requires_initial_samples": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Minimum number of initial random/Sobol samples required before the planner's model is fit. None when the planner has no warm-up phase.", - "title": "Requires Initial Samples" - }, - "compatible_objective_directions": { - "description": "Objective directions the planner can optimize. Lower-case strings.", - "items": { - "type": "string" - }, - "title": "Compatible Objective Directions", - "type": "array" - }, - "requires_extras": { - "description": "Names of pyproject.toml extras (e.g. ['optuna']) required to install this planner's heavy dependencies. Informational; the planner class itself owns the ImportError surface.", - "items": { - "type": "string" - }, - "title": "Requires Extras", - "type": "array" - } - }, - "required": [ - "supports_continuous", - "supports_discrete" - ], - "title": "SearchPlannerMetadata", - "type": "object" - } - }, - "required": [ - "class" - ], - "title": "Search Planner Plugin", - "description": "Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run." - } - } -} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"plugins.schema.json","additionalProperties":false,"description":"Root model for plugins.yaml file.\n\nThis file registers plugin implementations for AIPerf. Each top-level section\ncorresponds to a category from categories.yaml and maps type names to their\nimplementing classes.\n\nNote: Package metadata (name, version, author) comes from pyproject.toml\nvia importlib.metadata, not from this file.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n my_custom:\n class: my_package.endpoints.my_custom:MyCustomEndpoint\n description: Custom endpoint for my API.\n metadata:\n endpoint_path: /v1/generate\n supports_streaming: true\n produces_tokens: true\n tokenizes_input: true\n metrics_title: My Custom Metrics\n\ncustom_dataset_loader:\n my_jsonl:\n class: my_package.dataset_loaders.my_jsonl:MyJSONLDatasetLoader\n description: Custom dataset loader for my API.\n```","properties":{"schema_version":{"default":"1.0","description":"Version of the plugins.yaml schema format. Use '1.0' for current format.","title":"Schema Version","type":"string"},"api_router":{"title":"API Router Plugins","type":"object","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded.","additionalProperties":{"$ref":"#/$defs/ApiRouterPlugin"}},"timing_strategy":{"title":"Timing Strategy Plugins","type":"object","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/TimingStrategyPlugin"}},"arrival_pattern":{"title":"Arrival Pattern Plugins","type":"object","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode.","additionalProperties":{"$ref":"#/$defs/ArrivalPatternPlugin"}},"ramp":{"title":"Ramp Plugins","type":"object","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns.","additionalProperties":{"$ref":"#/$defs/RampPlugin"}},"dataset_backing_store":{"title":"Dataset Backing Store Plugins","type":"object","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access.","additionalProperties":{"$ref":"#/$defs/DatasetBackingStorePlugin"}},"dataset_client_store":{"title":"Dataset Client Store Plugins","type":"object","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance.","additionalProperties":{"$ref":"#/$defs/DatasetClientStorePlugin"}},"dataset_sampler":{"title":"Dataset Sampler Plugins","type":"object","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration.","additionalProperties":{"$ref":"#/$defs/DatasetSamplerPlugin"}},"dataset_composer":{"title":"Dataset Composer Plugins","type":"object","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration.","additionalProperties":{"$ref":"#/$defs/DatasetComposerPlugin"}},"custom_dataset_loader":{"title":"Custom Dataset Loader Plugins","type":"object","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load().","additionalProperties":{"$ref":"#/$defs/CustomDatasetLoaderPlugin"}},"public_dataset_loader":{"title":"Public Dataset Loader Plugins","type":"object","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration.","additionalProperties":{"$ref":"#/$defs/PublicDatasetLoaderPlugin"}},"endpoint":{"title":"Endpoint Plugins","type":"object","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration.","additionalProperties":{"$ref":"#/$defs/EndpointPlugin"}},"transport":{"title":"Transport Plugins","type":"object","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration.","additionalProperties":{"$ref":"#/$defs/TransportPlugin"}},"record_processor":{"title":"Record Processor Plugins","type":"object","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordProcessorPlugin"}},"record_observer":{"title":"Record Observer Plugins","type":"object","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordObserverPlugin"}},"accumulator":{"title":"Accumulator Plugins","type":"object","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AccumulatorPlugin"}},"stream_exporter":{"title":"Stream Exporter Plugins","type":"object","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/StreamExporterPlugin"}},"analyzer":{"title":"Analyzer Plugins","type":"object","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AnalyzerPlugin"}},"spec_decode_adapter":{"title":"Spec Decode Adapter Plugins","type":"object","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins.","additionalProperties":{"$ref":"#/$defs/SpecDecodeAdapterPlugin"}},"accuracy_grader":{"title":"Accuracy Grader Plugins","type":"object","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default.","additionalProperties":{"$ref":"#/$defs/AccuracyGraderPlugin"}},"accuracy_benchmark":{"title":"Accuracy Benchmark Plugins","type":"object","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration.","additionalProperties":{"$ref":"#/$defs/AccuracyBenchmarkPlugin"}},"data_exporter":{"title":"Data Exporter Plugins","type":"object","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/DataExporterPlugin"}},"console_exporter":{"title":"Console Exporter Plugins","type":"object","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/ConsoleExporterPlugin"}},"ui":{"title":"UI Plugins","type":"object","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration.","additionalProperties":{"$ref":"#/$defs/UiPlugin"}},"url_selection_strategy":{"title":"Url Selection Strategy Plugins","type":"object","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration.","additionalProperties":{"$ref":"#/$defs/UrlSelectionStrategyPlugin"}},"service":{"title":"Service Plugins","type":"object","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services.","additionalProperties":{"$ref":"#/$defs/ServicePlugin"}},"service_manager":{"title":"Service Manager Plugins","type":"object","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration.","additionalProperties":{"$ref":"#/$defs/ServiceManagerPlugin"}},"communication":{"title":"Communication Plugins","type":"object","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration.","additionalProperties":{"$ref":"#/$defs/CommunicationPlugin"}},"communication_client":{"title":"Communication Client Plugins","type":"object","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage.","additionalProperties":{"$ref":"#/$defs/CommunicationClientPlugin"}},"zmq_proxy":{"title":"ZMQ Proxy Plugins","type":"object","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration.","additionalProperties":{"$ref":"#/$defs/ZmqProxyPlugin"}},"plot":{"title":"Plot Plugins","type":"object","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection.","additionalProperties":{"$ref":"#/$defs/PlotPlugin"}},"gpu_telemetry_collector":{"title":"GPU Telemetry Collector Plugins","type":"object","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration.","additionalProperties":{"$ref":"#/$defs/GpuTelemetryCollectorPlugin"}},"search_recipe":{"title":"Search Recipe Plugins","type":"object","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchRecipePlugin"}},"search_recipe_post_process":{"title":"Search Recipe Post Process Plugins","type":"object","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec.","additionalProperties":{"$ref":"#/$defs/SearchRecipePostProcessPlugin"}},"convergence_criterion":{"title":"Convergence Criterion Plugins","type":"object","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/ConvergenceCriterionPlugin"}},"search_planner":{"title":"Search Planner Plugins","type":"object","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchPlannerPlugin"}}},"title":"PluginsManifest","type":"object","$defs":{"ApiRouterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"API Router Plugin","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded."},"TimingStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Timing Strategy Plugin","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run."},"ArrivalPatternPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Arrival Pattern Plugin","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode."},"RampPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Ramp Plugin","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns."},"DatasetBackingStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Backing Store Plugin","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access."},"DatasetClientStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Client Store Plugin","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance."},"DatasetSamplerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Sampler Plugin","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration."},"DatasetComposerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Composer Plugin","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration."},"CustomDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for custom dataset loader plugins.\n\nDefines format-specific defaults for dataset loaders. When a loader specifies\n``block_size``, it overrides the user's ``--isl-block-size`` config default,\nensuring hash-based prompt generation uses the correct token block size for the\ntrace format (e.g. 16 for Bailian, 512 for Mooncake).\n\nReferenced by: categories.yaml custom_dataset_loader.metadata_class\nUsed in: plugins.yaml custom_dataset_loader entries","properties":{"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace datasets use hash_ids-based prompt generation, support synthesis options, and prefer sequential sampling with fixed_schedule timing.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids (e.g. 16 for Bailian, 512 for Mooncake).","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces (e.g. weka_trace) override to 'coding' so reconstructed prompts resemble real tool-use content.","title":"Default Prompt Corpus","type":"string"},"category":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Filter dataset rows to a specific category value. Used by custom loaders that support per-category subsets (e.g. SPEED-Bench).","title":"Category"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects.","title":"Multi Turn","type":"boolean"}},"title":"CustomDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Custom Dataset Loader Plugin","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load()."},"PublicDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for public dataset loader plugins.\n\nReferenced by: categories.yaml public_dataset_loader.metadata_class\nUsed in: plugins.yaml public_dataset_loader entries","properties":{"hf_dataset_name":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset identifier (e.g. 'AI-MO/NuminaMath-TIR'). Required for HF-backed loaders.","title":"Hf Dataset Name"},"hf_split":{"default":"train","description":"HuggingFace dataset split to load (e.g. 'train', 'test', 'validation').","title":"Hf Split","type":"string"},"hf_subset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name. Only needed for datasets with multiple configs.","title":"Hf Subset"},"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace public datasets reuse hash_ids-based prompt generation, require a tokenizer, and prefer sequential sampling. Mirrors the field of the same name on CustomDatasetLoaderMetadata so trace loaders can live in either pipeline.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids.","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces override to 'coding'.","title":"Default Prompt Corpus","type":"string"},"prompt_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the prompt/instruction text. Required for HFInstructionResponseDatasetLoader.","title":"Prompt Column"},"image_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the image data (PIL Image). Used for multimodal datasets.","title":"Image Column"},"video_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the video data (URL string or bytes dict). Used for video multimodal datasets.","title":"Video Column"},"audio_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing audio data. Depending on loader/dataset, this can be an HF Audio dict with array/sampling_rate (decoded) or bytes/path (decode=False). Used for ASR/speech datasets.","title":"Audio Column"},"conversation_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the conversation messages array. Required for HFConversationDatasetLoader.","title":"Conversation Column"},"message_content_key":{"default":"content","description":"Key inside each message dict for the text content. Used with conversation_column (e.g. 'content', 'value').","title":"Message Content Key","type":"string"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects. For HFConversationDatasetLoader: user→assistant (OpenAI roles) or human→gpt (ShareGPT-style ``from`` fields) pairs.","title":"Multi Turn","type":"boolean"},"streaming":{"default":false,"description":"Whether to load the HuggingFace dataset in streaming mode. Use true for large datasets (>10 GB) to avoid downloading the full dataset. Use false (default) for small datasets to leverage HF caching and len() support.","title":"Streaming","type":"boolean"},"has_timing_data":{"default":false,"description":"Whether the loader emits timestamps suitable for fixed-schedule replay.","title":"Has Timing Data","type":"boolean"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.","title":"Prompt Template"}},"title":"PublicDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Public Dataset Loader Plugin","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration."},"EndpointPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for endpoint plugins.\n\nDefines API capabilities, paths, and multimodal support for endpoint implementations.\nUsed by the framework to route requests, configure streaming, and enable\nmultimodal inputs/outputs (images, audio, video).\n\nReferenced by: categories.yaml endpoint.metadata_class\nUsed in: plugins.yaml endpoint entries","properties":{"metrics_title":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"Display title for metrics dashboard.","title":"Metrics Title"},"endpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"API path (e.g., /v1/chat/completions).","title":"Endpoint Path"},"streaming_path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Streaming API path if different from the endpoint path (e.g., /generate_stream).","title":"Streaming Path"},"service_kind":{"default":"openai","description":"The service kind of the endpoint (used for artifact naming).","title":"Service Kind","type":"string"},"supports_streaming":{"description":"Whether endpoint supports streaming responses.","title":"Supports Streaming","type":"boolean"},"tokenizes_input":{"description":"Whether endpoint tokenizes text inputs.","title":"Tokenizes Input","type":"boolean"},"produces_tokens":{"description":"Whether endpoint produces token-based output.","title":"Produces Tokens","type":"boolean"},"supports_audio":{"default":false,"description":"Whether endpoint accepts audio input.","title":"Supports Audio","type":"boolean"},"supports_images":{"default":false,"description":"Whether endpoint accepts image input.","title":"Supports Images","type":"boolean"},"supports_videos":{"default":false,"description":"Whether endpoint accepts video input.","title":"Supports Videos","type":"boolean"},"produces_audio":{"default":false,"description":"Whether endpoint produces audio-based outputs.","title":"Produces Audio","type":"boolean"},"produces_images":{"default":false,"description":"Whether endpoint produces image-based outputs.","title":"Produces Images","type":"boolean"},"produces_videos":{"default":false,"description":"Whether endpoint produces video-based outputs.","title":"Produces Videos","type":"boolean"},"requires_polling":{"default":false,"description":"Whether endpoint uses async job polling (submit job, poll for status, retrieve result).","title":"Requires Polling","type":"boolean"},"requires_form_data":{"default":false,"description":"True for endpoints that require multipart/form-data (binary uploads), e.g., video_generation, image_edit. EndpointConfig consumes this flag to auto-select the request encoding.","title":"Requires Form Data","type":"boolean"},"requires_inline_media":{"default":false,"description":"Whether endpoint requires media URLs to be downloaded and inlined as base64 data URLs.","title":"Requires Inline Media","type":"boolean"},"consumes_system_message":{"default":false,"description":"Whether endpoint sends RequestInfo.system_message on the wire (e.g. a leading system role for chat, top-level instructions for responses). When False, the multi-turn loader does not hoist a leading system turn into the conversation system_message, since it would be silently dropped.","title":"Consumes System Message","type":"boolean"}},"required":["metrics_title","endpoint_path","supports_streaming","tokenizes_input","produces_tokens"],"title":"EndpointMetadata","type":"object"}},"required":["class"],"title":"Endpoint Plugin","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration."},"TransportPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for transport plugins.\n\nDefines network layer configuration including transport type and URL schemes.\nUsed by the framework to auto-detect appropriate transport based on URL scheme.\n\nReferenced by: categories.yaml transport.metadata_class\nUsed in: plugins.yaml transport entries","properties":{"transport_type":{"description":"Transport type identifier for this transport","title":"Transport Type","type":"string"},"url_schemes":{"description":"URL schemes this transport handles (for auto-detection and validation).","items":{"type":"string"},"title":"Url Schemes","type":"array"}},"required":["transport_type"],"title":"TransportMetadata","type":"object"}},"required":["class"],"title":"Transport Plugin","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration."},"RecordProcessorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries","properties":{"record_type":{"description":"The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.","title":"Record Type","type":"string"}},"required":["record_type"],"title":"RecordProducerMetadata","type":"object"}},"required":["class"],"title":"Record Processor Plugin","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously."},"RecordObserverPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Record Observer Plugin","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously."},"AccumulatorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Accumulator Plugin","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously."},"StreamExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Stream Exporter Plugin","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously."},"AnalyzerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary — e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) — e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries","properties":{"required_accumulators":{"description":"AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.","items":{"type":"string"},"title":"Required Accumulators","type":"array"},"required_summaries":{"description":"AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.","items":{"type":"string"},"title":"Required Summaries","type":"array"}},"title":"AnalyzerMetadata","type":"object"}},"required":["class"],"title":"Analyzer Plugin","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously."},"SpecDecodeAdapterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Spec Decode Adapter Plugin","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins."},"AccuracyGraderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Grader Plugin","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default."},"AccuracyBenchmarkPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Benchmark Plugin","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration."},"DataExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Data Exporter Plugin","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously."},"ConsoleExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Console Exporter Plugin","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously."},"UiPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"UI Plugin","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration."},"UrlSelectionStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Url Selection Strategy Plugin","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration."},"ServicePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for service plugins.\n\nDefines lifecycle and runtime configuration for AIPerf distributed services.\nUsed by SystemController to manage service startup order and optimize\nlatency-sensitive services (timing, workers) by disabling garbage collection.\n\nReferenced by: categories.yaml service.metadata_class\nUsed in: plugins.yaml service entries","properties":{"required":{"description":"Whether the service is required for benchmark execution.","title":"Required","type":"boolean"},"auto_start":{"description":"Whether the service is automatically started by the system controller.","title":"Auto Start","type":"boolean"},"disable_gc":{"default":false,"description":"Whether to disable garbage collection in the service for timing-critical operations.","title":"Disable Gc","type":"boolean"},"replicable":{"default":false,"description":"Whether the service can have multiple instances running in parallel.","title":"Replicable","type":"boolean"}},"required":["required","auto_start"],"title":"ServiceMetadata","type":"object"}},"required":["class"],"title":"Service Plugin","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services."},"ServiceManagerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Service Manager Plugin","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration."},"CommunicationPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Plugin","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration."},"CommunicationClientPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Client Plugin","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage."},"ZmqProxyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"ZMQ Proxy Plugin","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration."},"PlotPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for plot plugins.\n\nDefines display properties and categorization for visualization handlers.\nUsed by the framework to group plots and generate selection interfaces.\n\nReferenced by: categories.yaml plot.metadata_class\nUsed in: plugins.yaml plot entries","properties":{"display_name":{"description":"Human-readable name for UI display.","title":"Display Name","type":"string"},"category":{"description":"Plot category (per_request, aggregated, combined, comparison).","title":"Category","type":"string"}},"required":["display_name","category"],"title":"PlotMetadata","type":"object"}},"required":["class"],"title":"Plot Plugin","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection."},"GpuTelemetryCollectorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Config-time metadata for GPU telemetry collector plugins.\n\nLocal collectors set ``is_local`` so ``--gpu-telemetry `` is treated\nas an in-process collector keyword instead of a DCGM URL. Native binding\nvalidation lives on the collector class itself via ``validate_environment``\nso each implementation owns its own dependency check.\n\nReferenced by: categories.yaml gpu_telemetry_collector.metadata_class\nUsed in: plugins.yaml gpu_telemetry_collector entries","properties":{"is_local":{"default":false,"description":"Whether this collector runs in-process against the local host.","title":"Is Local","type":"boolean"}},"title":"GPUTelemetryCollectorMetadata","type":"object"}},"required":["class"],"title":"GPU Telemetry Collector Plugin","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration."},"SearchRecipePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Plugin","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run."},"SearchRecipePostProcessPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Post Process Plugin","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec."},"ConvergenceCriterionPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for convergence criterion plugins.\n\nDeclares statistical-method capabilities so the CLI/config layer can\nvalidate `--convergence-metric` / `--convergence-stat` against the chosen\ncriterion before the plugin is imported.\n\nReferenced by: categories.yaml convergence_criterion.metadata_class\nUsed in: plugins.yaml convergence_criterion entries","properties":{"min_samples":{"description":"Minimum number of successful runs required before convergence can trigger.","minimum":1,"title":"Min Samples","type":"integer"},"requires_confidence_level":{"default":false,"description":"Whether the criterion consumes plan.confidence_level (e.g. CI-width does, CV doesn't).","title":"Requires Confidence Level","type":"boolean"},"requires_jsonl_export":{"default":false,"description":"Whether the criterion reads per-request metrics from JSONL exports (e.g. distribution does).","title":"Requires Jsonl Export","type":"boolean"},"metric_kinds":{"description":"Kinds of metrics this criterion handles. One or more of 'continuous', 'counts', 'categorical'.","items":{"type":"string"},"title":"Metric Kinds","type":"array"}},"required":["min_samples"],"title":"ConvergenceCriterionMetadata","type":"object"}},"required":["class"],"title":"Convergence Criterion Plugin","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run."},"SearchPlannerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for search planner plugins.\n\nDeclares dimension-type and objective-direction support so the CLI/config\nlayer can validate `--search-space` shape against the chosen planner\nbefore the planner (and its heavy soft-dep imports) is loaded.\n\nReferenced by: categories.yaml search_planner.metadata_class\nUsed in: plugins.yaml search_planner entries","properties":{"supports_continuous":{"description":"Whether the planner accepts Real-valued search-space dimensions.","title":"Supports Continuous","type":"boolean"},"supports_discrete":{"description":"Whether the planner accepts Integer-valued search-space dimensions.","title":"Supports Discrete","type":"boolean"},"supports_categorical":{"default":false,"description":"Whether the planner accepts Categorical search-space dimensions.","title":"Supports Categorical","type":"boolean"},"requires_initial_samples":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Minimum number of initial random/Sobol samples required before the planner's model is fit. None when the planner has no warm-up phase.","title":"Requires Initial Samples"},"compatible_objective_directions":{"description":"Objective directions the planner can optimize. Lower-case strings.","items":{"type":"string"},"title":"Compatible Objective Directions","type":"array"},"requires_extras":{"description":"Names of pyproject.toml extras (e.g. ['optuna']) required to install this planner's heavy dependencies. Informational; the planner class itself owns the ImportError surface.","items":{"type":"string"},"title":"Requires Extras","type":"array"}},"required":["supports_continuous","supports_discrete"],"title":"SearchPlannerMetadata","type":"object"}},"required":["class"],"title":"Search Planner Plugin","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run."}}} diff --git a/tests/unit/common/test_loop_scheduler.py b/tests/unit/common/test_loop_scheduler.py index 473dc0b30b..d3bdc24509 100644 --- a/tests/unit/common/test_loop_scheduler.py +++ b/tests/unit/common/test_loop_scheduler.py @@ -271,6 +271,32 @@ async def noop(): assert observed == [(0, 0)] + async def test_drain_observer_does_not_fire_for_bulk_cancelled_tasks( + self, scheduler: LoopScheduler + ): + observed: list[tuple[int, int]] = [] + started = asyncio.Event() + started_count = 0 + scheduler.set_drain_observer( + lambda: observed.append((scheduler.pending_count, scheduler.running_count)) + ) + + async def wait_forever(): + nonlocal started_count + started_count += 1 + if started_count == 2: + started.set() + await asyncio.Event().wait() + + scheduler.execute_async(wait_forever()) + scheduler.execute_async(wait_forever()) + await started.wait() + + tasks = scheduler.cancel_all_running() + await asyncio.gather(*tasks, return_exceptions=True) + + assert observed == [] + class TestExecuteAsync: """Tests for execute_async (immediate task execution).""" diff --git a/tools/generate_config_schema.py b/tools/generate_config_schema.py index 35bfaecaca..20dbc1208f 100644 --- a/tools/generate_config_schema.py +++ b/tools/generate_config_schema.py @@ -33,9 +33,10 @@ from __future__ import annotations import copy -import json from pathlib import Path +import orjson + from tools._core import ( GeneratedFile, Generator, @@ -278,7 +279,7 @@ def generate(self) -> GeneratorResult: ) # Serialize with proper formatting - content = json.dumps(enhanced_schema, indent=2, ensure_ascii=False) + "\n" + content = orjson.dumps(enhanced_schema).decode("utf-8") + "\n" if self.verbose: print_step(f"Schema size: {len(content):,} bytes") diff --git a/tools/generate_plugin_artifacts.py b/tools/generate_plugin_artifacts.py index 19925c8a42..0ca3815cf1 100755 --- a/tools/generate_plugin_artifacts.py +++ b/tools/generate_plugin_artifacts.py @@ -25,12 +25,12 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) import argparse -import json import re import time from collections import defaultdict from typing import Any +import orjson import yaml from rich.traceback import Traceback @@ -245,14 +245,13 @@ def generate_schemas(check: bool = False) -> int: ("plugins.schema.json", plug_schema), ]: content = ( - json.dumps( + orjson.dumps( { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": filename, **schema, - }, - indent=2, - ) + } + ).decode("utf-8") + "\n" ) path = SCHEMA_DIR / filename From 09eb2b0f54c3fd1e07988282a5af5b32356974df Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:45:59 -0700 Subject: [PATCH 002/119] docs(standards): allow multi-line docstrings in project code (#1234) Signed-off-by: Elias Bermudez (cherry picked from commit 20eb25626afad54bf51bc6096263ea8887520e8e) --- .cursor/rules/python.mdc | 1 + .github/copilot-instructions.md | 1 + AGENTS.md | 1 + CLAUDE.md | 1 + 4 files changed, 4 insertions(+) diff --git a/.cursor/rules/python.mdc b/.cursor/rules/python.mdc index d1888e864d..bdf5cce17f 100644 --- a/.cursor/rules/python.mdc +++ b/.cursor/rules/python.mdc @@ -42,6 +42,7 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo - Use mermaid diagrams instead of ASCII art in markdown files. - Do not create markdown files to document code changes or decisions. - Do not over-comment code. Removing code is fine without adding comments to explain why. +- Multi-line docstrings are permitted. Use them when the behavior is non-obvious. - No emojis in code or comments. - Hide a metric from the console table with `console_group = MetricConsoleGroup.NONE`; group it into a separate section with `MetricConsoleGroup.{USAGE,CACHE,PREDICTION,AUDIO,REASONING,SPEC_DECODE,GPU_POWER_EFFICIENCY_NVIDIA,GPU_POWER_EFFICIENCY_AMD}`. Default is `DEFAULT`. See `docs/metrics-reference.md` "Metric Console Group Reference". - Platform-conditional code MUST branch on `IS_WINDOWS` / `IS_MACOS` / `IS_LINUX` from `aiperf.common.constants`, never on `platform.system()` directly. The constants are evaluated once at import time, are uniformly greppable, and produce smaller diffs. See `src/aiperf/common/bootstrap.py` and `src/aiperf/config/comm/ipc.py` for canonical examples. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4479626529..35afb07d47 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -37,6 +37,7 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo - Use mermaid diagrams instead of ASCII art in markdown files. - Do not create markdown files to document code changes or decisions. - Do not over-comment code. Removing code is fine without adding comments to explain why. +- Multi-line docstrings are permitted. Use them when the behavior is non-obvious. - No emojis in code or comments. - Hide a metric from the console table with `console_group = MetricConsoleGroup.NONE`; group it into a separate section with `MetricConsoleGroup.{USAGE,CACHE,PREDICTION,AUDIO,REASONING,SPEC_DECODE,GPU_POWER_EFFICIENCY_NVIDIA,GPU_POWER_EFFICIENCY_AMD}`. Default is `DEFAULT`. See `docs/metrics-reference.md` "Metric Console Group Reference". - Platform-conditional code MUST branch on `IS_WINDOWS` / `IS_MACOS` / `IS_LINUX` from `aiperf.common.constants`, never on `platform.system()` directly. The constants are evaluated once at import time, are uniformly greppable, and produce smaller diffs. See `src/aiperf/common/bootstrap.py` and `src/aiperf/config/comm/ipc.py` for canonical examples. diff --git a/AGENTS.md b/AGENTS.md index 12f2fcb735..59643c536e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo - Use mermaid diagrams instead of ASCII art in markdown files. - Do not create markdown files to document code changes or decisions. - Do not over-comment code. Removing code is fine without adding comments to explain why. +- Multi-line docstrings are permitted. Use them when the behavior is non-obvious. - No emojis in code or comments. - Hide a metric from the console table with `console_group = MetricConsoleGroup.NONE`; group it into a separate section with `MetricConsoleGroup.{USAGE,CACHE,PREDICTION,AUDIO,REASONING,SPEC_DECODE,GPU_POWER_EFFICIENCY_NVIDIA,GPU_POWER_EFFICIENCY_AMD}`. Default is `DEFAULT`. See `docs/metrics-reference.md` "Metric Console Group Reference". - Platform-conditional code MUST branch on `IS_WINDOWS` / `IS_MACOS` / `IS_LINUX` from `aiperf.common.constants`, never on `platform.system()` directly. The constants are evaluated once at import time, are uniformly greppable, and produce smaller diffs. See `src/aiperf/common/bootstrap.py` and `src/aiperf/config/comm/ipc.py` for canonical examples. diff --git a/CLAUDE.md b/CLAUDE.md index 4de349a738..d8f9360b7b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,6 +36,7 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo - Use mermaid diagrams instead of ASCII art in markdown files. - Do not create markdown files to document code changes or decisions. - Do not over-comment code. Removing code is fine without adding comments to explain why. +- Multi-line docstrings are permitted. Use them when the behavior is non-obvious. - No emojis in code or comments. - Hide a metric from the console table with `console_group = MetricConsoleGroup.NONE`; group it into a separate section with `MetricConsoleGroup.{USAGE,CACHE,PREDICTION,AUDIO,REASONING,SPEC_DECODE,GPU_POWER_EFFICIENCY_NVIDIA,GPU_POWER_EFFICIENCY_AMD}`. Default is `DEFAULT`. See `docs/metrics-reference.md` "Metric Console Group Reference". - Platform-conditional code MUST branch on `IS_WINDOWS` / `IS_MACOS` / `IS_LINUX` from `aiperf.common.constants`, never on `platform.system()` directly. The constants are evaluated once at import time, are uniformly greppable, and produce smaller diffs. See `src/aiperf/common/bootstrap.py` and `src/aiperf/config/comm/ipc.py` for canonical examples. From 19c9ed4886b1c20c245af1d98707d1cd35784f22 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 31 Jul 2026 07:55:13 -0700 Subject: [PATCH 003/119] feat(cache-bust): support all structured workloads (#1233) Signed-off-by: Anthony Casagrande (cherry picked from commit 1d1829540b5d64c89832f877b87043eaaee687e5) --- docs/reference/isl-tokenization.md | 2 +- docs/tutorials/anthropic-messages-endpoint.md | 1 + docs/tutorials/weka-trace.md | 2 +- src/aiperf/config/config.py | 62 ++----- src/aiperf/plugin/plugins.yaml | 3 + src/aiperf/plugin/schema/plugins.schema.json | 2 +- src/aiperf/plugin/schema/schemas.py | 7 + src/aiperf/timing/branch_orchestrator.py | 15 +- src/aiperf/timing/conversation_source.py | 121 ++++++++------ src/aiperf/timing/phase_orchestrator.py | 11 +- .../timing/strategies/fixed_schedule.py | 16 +- .../timing/strategies/user_centric_rate.py | 5 + src/aiperf/workers/worker.py | 59 ++++++- .../timing/test_dag_timing_pathology.py | 37 +++-- ..._config_cache_bust_lockdown_adversarial.py | 154 +++++++----------- .../unit/endpoints/test_messages_endpoint.py | 47 +++++- .../unit/timing/strategies/_shared_helpers.py | 9 +- .../timing/strategies/test_agentic_replay.py | 9 +- .../test_agentic_replay_marker_uniqueness.py | 9 +- ...test_agentic_replay_recycle_adversarial.py | 9 +- tests/unit/timing/test_conversation_source.py | 60 ++++++- tests/unit/timing/test_dag_cross_component.py | 29 ++++ .../test_worker_cache_bust_injection.py | 128 +++++++++++++++ 23 files changed, 551 insertions(+), 246 deletions(-) diff --git a/docs/reference/isl-tokenization.md b/docs/reference/isl-tokenization.md index 46714b96bd..988e0e8770 100644 --- a/docs/reference/isl-tokenization.md +++ b/docs/reference/isl-tokenization.md @@ -26,7 +26,7 @@ Hello<|im_end|> That wrapping — the role markers, the headers, the trailing "now generate the assistant response" suffix — is the **chat template overhead**. It's not part of "Hello", but the model still has to process it. A single user turn typically adds 5–15 wrapper tokens depending on the model. -There's also a second source of extra tokens: AIPerf's **cache-bust marker**, a short hex string injected into the wire payload to prevent the server's KV cache from short-circuiting load tests. It's only present when `--cache-bust` is set, and only on the message it's targeted at. `BenchmarkConfig.validate_cache_bust_compatibility` in `src/aiperf/config/config.py` refuses incompatible combinations when it can tell at construction time: an explicit non-`agentic_replay` profiling `timing_mode`, or an endpoint type other than `chat`/`responses`. It skips when a `scenario` is set (the scenario's own locks apply later) and defers when no profiling phase has an explicit `timing_mode` yet — so not every non-agentic combo fails at config construction. When the lockdown does fire, failing loudly is preferable to a benchmark that looks correct but exercises no cache-busting. +There's also a second source of extra tokens: AIPerf's **cache-bust marker**, a short hex string injected into the wire payload to prevent the server's KV cache from short-circuiting load tests. It's only present when `--cache-bust` is set, and only on the message it's targeted at. Cache-bust works with every timing mode and synthetic, file, and public datasets when endpoint plugin metadata sets `supports_cache_bust=true` (currently `chat`, `responses`, and Anthropic `messages`); unsupported endpoint shapes and pre-encoded `PAYLOAD_BYTES` workloads are rejected rather than silently sending unchanged bytes. When AIPerf computes ISL, it has a choice: count the bare prompt text, or count what the model actually processes. By default AIPerf counts the **bare prompt** — the value you asked for in `--isl` is exactly what gets reported and what the composer aims to generate, so the metric is intuitive and matches the user's mental model. Pass `--apply-chat-template` to opt into the wire-payload total instead, which makes the metric **comparable to `--use-server-token-count`** (which reads ISL from the server's `usage.prompt_tokens` field) and matches the model's view of the workload. diff --git a/docs/tutorials/anthropic-messages-endpoint.md b/docs/tutorials/anthropic-messages-endpoint.md index 3eb26608db..2eeea16b9e 100644 --- a/docs/tutorials/anthropic-messages-endpoint.md +++ b/docs/tutorials/anthropic-messages-endpoint.md @@ -371,6 +371,7 @@ Notes: - The field is Anthropic-specific. Other endpoints (`chat`, `responses`, ...) ignore it. - Block contents flow into ISL accounting via the endpoint's system-prompt walk; see [Input token accounting (ISL)](#input-token-accounting-isl) below. - `raw_system` is currently populated by trace-replay loaders that ingest Anthropic-shaped traces; programmatic callers building `Turn` objects directly can set it as well. +- `--cache-bust system-prefix` / `system-suffix` targets `raw_system` when it is set, since that is what actually ships. The marker is appended or prepended as its own `{"type": "text", ...}` block rather than being spliced into an existing block's text, so a neighbouring `cache_control` breakpoint stays attached to the bytes it was authored against. ### Audio and video are unsupported diff --git a/docs/tutorials/weka-trace.md b/docs/tutorials/weka-trace.md index 1a5cecbb18..8c259853e9 100644 --- a/docs/tutorials/weka-trace.md +++ b/docs/tutorials/weka-trace.md @@ -178,7 +178,7 @@ This is locked on for the AgentX MVP scenario — auto-injected as `first_turn_p A few details worth knowing if you're using `--cache-bust` outside the scenario: -- **Compatibility is checked at startup.** `--cache-bust` requires the `agentic_replay` timing mode (set by `--scenario inferencex-agentx-mvp`) and a chat-shaped endpoint (`--endpoint-type chat` or `responses`). Other combinations error before the run starts with a message naming the offending flag, not silently mid-run. +- **Compatibility is checked at startup.** `--cache-bust` works with every timing mode when endpoint plugin metadata sets `supports_cache_bust=true` (currently `--endpoint-type chat`, `responses`, or Anthropic `messages`). Other endpoint types error before the run starts with a message naming the offending flag, rather than silently dropping the marker. - **Multimodal turns are supported.** When a turn carries images or audio alongside text, the marker is added as a new `{type: "text", text: ""}` content part at the start (prefix) or end (suffix) of the parts list; existing text/image/audio parts pass through untouched. - **`system_*` falls back to the first user turn when there's no system message.** If a trace has no system role anywhere (neither a conversation-level system message nor a `raw_messages[0].role=='system'`), `--cache-bust system_prefix` and `system_suffix` route the marker to the first user turn with the same orientation (prefix stays prefix, suffix stays suffix). Like `first_turn_*`, that injection runs on every credit and is idempotent (it will not stack the marker). A one-shot WARN is logged when the session has an empty `turn_list` and the marker cannot be placed at all. - **Incompatible with `payload_bytes` workloads.** AIPerf's pre-encoded mmap fast path bypasses the per-request rendering that injection needs. If your dataset would otherwise pick the `PAYLOAD_BYTES` format, AIPerf refuses the run with a clear error rather than silently dropping markers. Either drop `--cache-bust` or use a workload that goes through the normal compose path. diff --git a/src/aiperf/config/config.py b/src/aiperf/config/config.py index 331b847621..20fc2c489a 100644 --- a/src/aiperf/config/config.py +++ b/src/aiperf/config/config.py @@ -543,64 +543,24 @@ def validate_phase_dataset_compatibility(self) -> Self: @model_validator(mode="after") def validate_cache_bust_compatibility(self) -> Self: - """Refuse cache-bust on incompatible timing modes / endpoint types. - - Marker minting only fires in the agentic-replay strategy and only the - chat / responses endpoint formatters consume the system-message field - that hosts the marker. Any other combination silently drops the marker - and would produce a benchmark that looks normal but exercises no cache- - busting at all — refuse loudly at config time. - - Pure validation: this RAISES only, never mutates (per the v2 resolver - convention that cross-cutting config mutation belongs in a - ConfigResolver / scenario step, not a model_validator). - - Deferral cases (the lockdown does NOT fire — the agentic timing mode is - resolved later, not at config-construction time): - - * a config carrying a ``scenario`` is governed by that scenario's own - invariant locks (``apply_scenario`` stamps the agentic ``timing_mode`` - onto the profiling phases and enforces ``require_cache_bust``) - POST-construction via field assignment, which does not re-trigger this - validator (e.g. inferencex-agentx-mvp = agentic timing + chat endpoint - + first_turn_prefix is valid once the scenario applies); - * no profiling phase carries an EXPLICIT ``timing_mode`` override. The - effective mode is then derived at runtime from the phase type (or set - by the agentic-replay engine itself), so the agentic-ness is not yet - knowable here. The lockdown only fires once a phase has explicitly - declared a (non-agentic) timing mode that is incompatible with the - requested cache-bust. + """Refuse cache-bust when endpoint metadata does not support it. + + Cache-bust markers are minted by every timing strategy and injected by + the worker into structured endpoint turns. Endpoint plugins opt in via + ``supports_cache_bust`` metadata; timing mode is not a restriction. """ from aiperf.common.enums import CacheBustTarget - from aiperf.plugin.enums import EndpointType, TimingMode - - if self.scenario is not None: - return self + from aiperf.plugin import plugins if self.get_cache_bust_target() == CacheBustTarget.NONE: return self - profiling_phases = self.get_profiling_phases() - explicit_timing_modes = [ - phase.timing_mode - for phase in profiling_phases - if phase.timing_mode is not None - ] - if not explicit_timing_modes: - return self - if TimingMode.AGENTIC_REPLAY not in explicit_timing_modes: - raise ValueError( - "cache-bust requires the agentic_replay timing mode " - "(set today by --scenario inferencex-agentx-mvp); the profiling " - "phase(s) are not agentic_replay. Cache-bust marker minting is " - "only implemented for agentic_replay." - ) - - if self.endpoint.type not in {EndpointType.CHAT, EndpointType.RESPONSES}: + endpoint_metadata = plugins.get_endpoint_metadata(self.endpoint.type) + if not endpoint_metadata.supports_cache_bust: raise ValueError( - f"cache-bust requires --endpoint-type chat or responses; " - f"got {self.endpoint.type}. Other endpoint formatters do not " - f"consume the system message field that hosts the marker." + "cache-bust is not supported by endpoint " + f"{self.endpoint.type}; select an endpoint whose plugin metadata " + "sets supports_cache_bust=true" ) return self diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index b77dcdd0eb..62df3f01ec 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -196,6 +196,7 @@ endpoint: supports_images: true supports_videos: true consumes_system_message: true + supports_cache_bust: true metrics_title: LLM Metrics cohere_rankings: @@ -236,6 +237,7 @@ endpoint: supports_images: true supports_audio: true consumes_system_message: true + supports_cache_bust: true metrics_title: LLM Metrics messages: @@ -253,6 +255,7 @@ endpoint: tokenizes_input: true supports_images: true consumes_system_message: true + supports_cache_bust: true metrics_title: LLM Metrics chat_embeddings: diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 2552f42a90..0b68712e6a 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"plugins.schema.json","additionalProperties":false,"description":"Root model for plugins.yaml file.\n\nThis file registers plugin implementations for AIPerf. Each top-level section\ncorresponds to a category from categories.yaml and maps type names to their\nimplementing classes.\n\nNote: Package metadata (name, version, author) comes from pyproject.toml\nvia importlib.metadata, not from this file.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n my_custom:\n class: my_package.endpoints.my_custom:MyCustomEndpoint\n description: Custom endpoint for my API.\n metadata:\n endpoint_path: /v1/generate\n supports_streaming: true\n produces_tokens: true\n tokenizes_input: true\n metrics_title: My Custom Metrics\n\ncustom_dataset_loader:\n my_jsonl:\n class: my_package.dataset_loaders.my_jsonl:MyJSONLDatasetLoader\n description: Custom dataset loader for my API.\n```","properties":{"schema_version":{"default":"1.0","description":"Version of the plugins.yaml schema format. Use '1.0' for current format.","title":"Schema Version","type":"string"},"api_router":{"title":"API Router Plugins","type":"object","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded.","additionalProperties":{"$ref":"#/$defs/ApiRouterPlugin"}},"timing_strategy":{"title":"Timing Strategy Plugins","type":"object","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/TimingStrategyPlugin"}},"arrival_pattern":{"title":"Arrival Pattern Plugins","type":"object","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode.","additionalProperties":{"$ref":"#/$defs/ArrivalPatternPlugin"}},"ramp":{"title":"Ramp Plugins","type":"object","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns.","additionalProperties":{"$ref":"#/$defs/RampPlugin"}},"dataset_backing_store":{"title":"Dataset Backing Store Plugins","type":"object","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access.","additionalProperties":{"$ref":"#/$defs/DatasetBackingStorePlugin"}},"dataset_client_store":{"title":"Dataset Client Store Plugins","type":"object","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance.","additionalProperties":{"$ref":"#/$defs/DatasetClientStorePlugin"}},"dataset_sampler":{"title":"Dataset Sampler Plugins","type":"object","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration.","additionalProperties":{"$ref":"#/$defs/DatasetSamplerPlugin"}},"dataset_composer":{"title":"Dataset Composer Plugins","type":"object","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration.","additionalProperties":{"$ref":"#/$defs/DatasetComposerPlugin"}},"custom_dataset_loader":{"title":"Custom Dataset Loader Plugins","type":"object","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load().","additionalProperties":{"$ref":"#/$defs/CustomDatasetLoaderPlugin"}},"public_dataset_loader":{"title":"Public Dataset Loader Plugins","type":"object","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration.","additionalProperties":{"$ref":"#/$defs/PublicDatasetLoaderPlugin"}},"endpoint":{"title":"Endpoint Plugins","type":"object","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration.","additionalProperties":{"$ref":"#/$defs/EndpointPlugin"}},"transport":{"title":"Transport Plugins","type":"object","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration.","additionalProperties":{"$ref":"#/$defs/TransportPlugin"}},"record_processor":{"title":"Record Processor Plugins","type":"object","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordProcessorPlugin"}},"record_observer":{"title":"Record Observer Plugins","type":"object","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordObserverPlugin"}},"accumulator":{"title":"Accumulator Plugins","type":"object","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AccumulatorPlugin"}},"stream_exporter":{"title":"Stream Exporter Plugins","type":"object","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/StreamExporterPlugin"}},"analyzer":{"title":"Analyzer Plugins","type":"object","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AnalyzerPlugin"}},"spec_decode_adapter":{"title":"Spec Decode Adapter Plugins","type":"object","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins.","additionalProperties":{"$ref":"#/$defs/SpecDecodeAdapterPlugin"}},"accuracy_grader":{"title":"Accuracy Grader Plugins","type":"object","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default.","additionalProperties":{"$ref":"#/$defs/AccuracyGraderPlugin"}},"accuracy_benchmark":{"title":"Accuracy Benchmark Plugins","type":"object","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration.","additionalProperties":{"$ref":"#/$defs/AccuracyBenchmarkPlugin"}},"data_exporter":{"title":"Data Exporter Plugins","type":"object","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/DataExporterPlugin"}},"console_exporter":{"title":"Console Exporter Plugins","type":"object","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/ConsoleExporterPlugin"}},"ui":{"title":"UI Plugins","type":"object","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration.","additionalProperties":{"$ref":"#/$defs/UiPlugin"}},"url_selection_strategy":{"title":"Url Selection Strategy Plugins","type":"object","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration.","additionalProperties":{"$ref":"#/$defs/UrlSelectionStrategyPlugin"}},"service":{"title":"Service Plugins","type":"object","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services.","additionalProperties":{"$ref":"#/$defs/ServicePlugin"}},"service_manager":{"title":"Service Manager Plugins","type":"object","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration.","additionalProperties":{"$ref":"#/$defs/ServiceManagerPlugin"}},"communication":{"title":"Communication Plugins","type":"object","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration.","additionalProperties":{"$ref":"#/$defs/CommunicationPlugin"}},"communication_client":{"title":"Communication Client Plugins","type":"object","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage.","additionalProperties":{"$ref":"#/$defs/CommunicationClientPlugin"}},"zmq_proxy":{"title":"ZMQ Proxy Plugins","type":"object","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration.","additionalProperties":{"$ref":"#/$defs/ZmqProxyPlugin"}},"plot":{"title":"Plot Plugins","type":"object","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection.","additionalProperties":{"$ref":"#/$defs/PlotPlugin"}},"gpu_telemetry_collector":{"title":"GPU Telemetry Collector Plugins","type":"object","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration.","additionalProperties":{"$ref":"#/$defs/GpuTelemetryCollectorPlugin"}},"search_recipe":{"title":"Search Recipe Plugins","type":"object","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchRecipePlugin"}},"search_recipe_post_process":{"title":"Search Recipe Post Process Plugins","type":"object","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec.","additionalProperties":{"$ref":"#/$defs/SearchRecipePostProcessPlugin"}},"convergence_criterion":{"title":"Convergence Criterion Plugins","type":"object","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/ConvergenceCriterionPlugin"}},"search_planner":{"title":"Search Planner Plugins","type":"object","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchPlannerPlugin"}}},"title":"PluginsManifest","type":"object","$defs":{"ApiRouterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"API Router Plugin","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded."},"TimingStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Timing Strategy Plugin","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run."},"ArrivalPatternPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Arrival Pattern Plugin","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode."},"RampPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Ramp Plugin","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns."},"DatasetBackingStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Backing Store Plugin","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access."},"DatasetClientStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Client Store Plugin","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance."},"DatasetSamplerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Sampler Plugin","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration."},"DatasetComposerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Composer Plugin","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration."},"CustomDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for custom dataset loader plugins.\n\nDefines format-specific defaults for dataset loaders. When a loader specifies\n``block_size``, it overrides the user's ``--isl-block-size`` config default,\nensuring hash-based prompt generation uses the correct token block size for the\ntrace format (e.g. 16 for Bailian, 512 for Mooncake).\n\nReferenced by: categories.yaml custom_dataset_loader.metadata_class\nUsed in: plugins.yaml custom_dataset_loader entries","properties":{"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace datasets use hash_ids-based prompt generation, support synthesis options, and prefer sequential sampling with fixed_schedule timing.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids (e.g. 16 for Bailian, 512 for Mooncake).","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces (e.g. weka_trace) override to 'coding' so reconstructed prompts resemble real tool-use content.","title":"Default Prompt Corpus","type":"string"},"category":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Filter dataset rows to a specific category value. Used by custom loaders that support per-category subsets (e.g. SPEED-Bench).","title":"Category"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects.","title":"Multi Turn","type":"boolean"}},"title":"CustomDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Custom Dataset Loader Plugin","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load()."},"PublicDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for public dataset loader plugins.\n\nReferenced by: categories.yaml public_dataset_loader.metadata_class\nUsed in: plugins.yaml public_dataset_loader entries","properties":{"hf_dataset_name":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset identifier (e.g. 'AI-MO/NuminaMath-TIR'). Required for HF-backed loaders.","title":"Hf Dataset Name"},"hf_split":{"default":"train","description":"HuggingFace dataset split to load (e.g. 'train', 'test', 'validation').","title":"Hf Split","type":"string"},"hf_subset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name. Only needed for datasets with multiple configs.","title":"Hf Subset"},"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace public datasets reuse hash_ids-based prompt generation, require a tokenizer, and prefer sequential sampling. Mirrors the field of the same name on CustomDatasetLoaderMetadata so trace loaders can live in either pipeline.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids.","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces override to 'coding'.","title":"Default Prompt Corpus","type":"string"},"prompt_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the prompt/instruction text. Required for HFInstructionResponseDatasetLoader.","title":"Prompt Column"},"image_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the image data (PIL Image). Used for multimodal datasets.","title":"Image Column"},"video_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the video data (URL string or bytes dict). Used for video multimodal datasets.","title":"Video Column"},"audio_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing audio data. Depending on loader/dataset, this can be an HF Audio dict with array/sampling_rate (decoded) or bytes/path (decode=False). Used for ASR/speech datasets.","title":"Audio Column"},"conversation_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the conversation messages array. Required for HFConversationDatasetLoader.","title":"Conversation Column"},"message_content_key":{"default":"content","description":"Key inside each message dict for the text content. Used with conversation_column (e.g. 'content', 'value').","title":"Message Content Key","type":"string"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects. For HFConversationDatasetLoader: user→assistant (OpenAI roles) or human→gpt (ShareGPT-style ``from`` fields) pairs.","title":"Multi Turn","type":"boolean"},"streaming":{"default":false,"description":"Whether to load the HuggingFace dataset in streaming mode. Use true for large datasets (>10 GB) to avoid downloading the full dataset. Use false (default) for small datasets to leverage HF caching and len() support.","title":"Streaming","type":"boolean"},"has_timing_data":{"default":false,"description":"Whether the loader emits timestamps suitable for fixed-schedule replay.","title":"Has Timing Data","type":"boolean"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.","title":"Prompt Template"}},"title":"PublicDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Public Dataset Loader Plugin","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration."},"EndpointPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for endpoint plugins.\n\nDefines API capabilities, paths, and multimodal support for endpoint implementations.\nUsed by the framework to route requests, configure streaming, and enable\nmultimodal inputs/outputs (images, audio, video).\n\nReferenced by: categories.yaml endpoint.metadata_class\nUsed in: plugins.yaml endpoint entries","properties":{"metrics_title":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"Display title for metrics dashboard.","title":"Metrics Title"},"endpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"API path (e.g., /v1/chat/completions).","title":"Endpoint Path"},"streaming_path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Streaming API path if different from the endpoint path (e.g., /generate_stream).","title":"Streaming Path"},"service_kind":{"default":"openai","description":"The service kind of the endpoint (used for artifact naming).","title":"Service Kind","type":"string"},"supports_streaming":{"description":"Whether endpoint supports streaming responses.","title":"Supports Streaming","type":"boolean"},"tokenizes_input":{"description":"Whether endpoint tokenizes text inputs.","title":"Tokenizes Input","type":"boolean"},"produces_tokens":{"description":"Whether endpoint produces token-based output.","title":"Produces Tokens","type":"boolean"},"supports_audio":{"default":false,"description":"Whether endpoint accepts audio input.","title":"Supports Audio","type":"boolean"},"supports_images":{"default":false,"description":"Whether endpoint accepts image input.","title":"Supports Images","type":"boolean"},"supports_videos":{"default":false,"description":"Whether endpoint accepts video input.","title":"Supports Videos","type":"boolean"},"produces_audio":{"default":false,"description":"Whether endpoint produces audio-based outputs.","title":"Produces Audio","type":"boolean"},"produces_images":{"default":false,"description":"Whether endpoint produces image-based outputs.","title":"Produces Images","type":"boolean"},"produces_videos":{"default":false,"description":"Whether endpoint produces video-based outputs.","title":"Produces Videos","type":"boolean"},"requires_polling":{"default":false,"description":"Whether endpoint uses async job polling (submit job, poll for status, retrieve result).","title":"Requires Polling","type":"boolean"},"requires_form_data":{"default":false,"description":"True for endpoints that require multipart/form-data (binary uploads), e.g., video_generation, image_edit. EndpointConfig consumes this flag to auto-select the request encoding.","title":"Requires Form Data","type":"boolean"},"requires_inline_media":{"default":false,"description":"Whether endpoint requires media URLs to be downloaded and inlined as base64 data URLs.","title":"Requires Inline Media","type":"boolean"},"consumes_system_message":{"default":false,"description":"Whether endpoint sends RequestInfo.system_message on the wire (e.g. a leading system role for chat, top-level instructions for responses). When False, the multi-turn loader does not hoist a leading system turn into the conversation system_message, since it would be silently dropped.","title":"Consumes System Message","type":"boolean"}},"required":["metrics_title","endpoint_path","supports_streaming","tokenizes_input","produces_tokens"],"title":"EndpointMetadata","type":"object"}},"required":["class"],"title":"Endpoint Plugin","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration."},"TransportPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for transport plugins.\n\nDefines network layer configuration including transport type and URL schemes.\nUsed by the framework to auto-detect appropriate transport based on URL scheme.\n\nReferenced by: categories.yaml transport.metadata_class\nUsed in: plugins.yaml transport entries","properties":{"transport_type":{"description":"Transport type identifier for this transport","title":"Transport Type","type":"string"},"url_schemes":{"description":"URL schemes this transport handles (for auto-detection and validation).","items":{"type":"string"},"title":"Url Schemes","type":"array"}},"required":["transport_type"],"title":"TransportMetadata","type":"object"}},"required":["class"],"title":"Transport Plugin","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration."},"RecordProcessorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries","properties":{"record_type":{"description":"The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.","title":"Record Type","type":"string"}},"required":["record_type"],"title":"RecordProducerMetadata","type":"object"}},"required":["class"],"title":"Record Processor Plugin","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously."},"RecordObserverPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Record Observer Plugin","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously."},"AccumulatorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Accumulator Plugin","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously."},"StreamExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Stream Exporter Plugin","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously."},"AnalyzerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary — e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) — e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries","properties":{"required_accumulators":{"description":"AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.","items":{"type":"string"},"title":"Required Accumulators","type":"array"},"required_summaries":{"description":"AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.","items":{"type":"string"},"title":"Required Summaries","type":"array"}},"title":"AnalyzerMetadata","type":"object"}},"required":["class"],"title":"Analyzer Plugin","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously."},"SpecDecodeAdapterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Spec Decode Adapter Plugin","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins."},"AccuracyGraderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Grader Plugin","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default."},"AccuracyBenchmarkPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Benchmark Plugin","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration."},"DataExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Data Exporter Plugin","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously."},"ConsoleExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Console Exporter Plugin","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously."},"UiPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"UI Plugin","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration."},"UrlSelectionStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Url Selection Strategy Plugin","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration."},"ServicePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for service plugins.\n\nDefines lifecycle and runtime configuration for AIPerf distributed services.\nUsed by SystemController to manage service startup order and optimize\nlatency-sensitive services (timing, workers) by disabling garbage collection.\n\nReferenced by: categories.yaml service.metadata_class\nUsed in: plugins.yaml service entries","properties":{"required":{"description":"Whether the service is required for benchmark execution.","title":"Required","type":"boolean"},"auto_start":{"description":"Whether the service is automatically started by the system controller.","title":"Auto Start","type":"boolean"},"disable_gc":{"default":false,"description":"Whether to disable garbage collection in the service for timing-critical operations.","title":"Disable Gc","type":"boolean"},"replicable":{"default":false,"description":"Whether the service can have multiple instances running in parallel.","title":"Replicable","type":"boolean"}},"required":["required","auto_start"],"title":"ServiceMetadata","type":"object"}},"required":["class"],"title":"Service Plugin","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services."},"ServiceManagerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Service Manager Plugin","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration."},"CommunicationPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Plugin","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration."},"CommunicationClientPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Client Plugin","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage."},"ZmqProxyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"ZMQ Proxy Plugin","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration."},"PlotPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for plot plugins.\n\nDefines display properties and categorization for visualization handlers.\nUsed by the framework to group plots and generate selection interfaces.\n\nReferenced by: categories.yaml plot.metadata_class\nUsed in: plugins.yaml plot entries","properties":{"display_name":{"description":"Human-readable name for UI display.","title":"Display Name","type":"string"},"category":{"description":"Plot category (per_request, aggregated, combined, comparison).","title":"Category","type":"string"}},"required":["display_name","category"],"title":"PlotMetadata","type":"object"}},"required":["class"],"title":"Plot Plugin","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection."},"GpuTelemetryCollectorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Config-time metadata for GPU telemetry collector plugins.\n\nLocal collectors set ``is_local`` so ``--gpu-telemetry `` is treated\nas an in-process collector keyword instead of a DCGM URL. Native binding\nvalidation lives on the collector class itself via ``validate_environment``\nso each implementation owns its own dependency check.\n\nReferenced by: categories.yaml gpu_telemetry_collector.metadata_class\nUsed in: plugins.yaml gpu_telemetry_collector entries","properties":{"is_local":{"default":false,"description":"Whether this collector runs in-process against the local host.","title":"Is Local","type":"boolean"}},"title":"GPUTelemetryCollectorMetadata","type":"object"}},"required":["class"],"title":"GPU Telemetry Collector Plugin","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration."},"SearchRecipePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Plugin","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run."},"SearchRecipePostProcessPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Post Process Plugin","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec."},"ConvergenceCriterionPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for convergence criterion plugins.\n\nDeclares statistical-method capabilities so the CLI/config layer can\nvalidate `--convergence-metric` / `--convergence-stat` against the chosen\ncriterion before the plugin is imported.\n\nReferenced by: categories.yaml convergence_criterion.metadata_class\nUsed in: plugins.yaml convergence_criterion entries","properties":{"min_samples":{"description":"Minimum number of successful runs required before convergence can trigger.","minimum":1,"title":"Min Samples","type":"integer"},"requires_confidence_level":{"default":false,"description":"Whether the criterion consumes plan.confidence_level (e.g. CI-width does, CV doesn't).","title":"Requires Confidence Level","type":"boolean"},"requires_jsonl_export":{"default":false,"description":"Whether the criterion reads per-request metrics from JSONL exports (e.g. distribution does).","title":"Requires Jsonl Export","type":"boolean"},"metric_kinds":{"description":"Kinds of metrics this criterion handles. One or more of 'continuous', 'counts', 'categorical'.","items":{"type":"string"},"title":"Metric Kinds","type":"array"}},"required":["min_samples"],"title":"ConvergenceCriterionMetadata","type":"object"}},"required":["class"],"title":"Convergence Criterion Plugin","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run."},"SearchPlannerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for search planner plugins.\n\nDeclares dimension-type and objective-direction support so the CLI/config\nlayer can validate `--search-space` shape against the chosen planner\nbefore the planner (and its heavy soft-dep imports) is loaded.\n\nReferenced by: categories.yaml search_planner.metadata_class\nUsed in: plugins.yaml search_planner entries","properties":{"supports_continuous":{"description":"Whether the planner accepts Real-valued search-space dimensions.","title":"Supports Continuous","type":"boolean"},"supports_discrete":{"description":"Whether the planner accepts Integer-valued search-space dimensions.","title":"Supports Discrete","type":"boolean"},"supports_categorical":{"default":false,"description":"Whether the planner accepts Categorical search-space dimensions.","title":"Supports Categorical","type":"boolean"},"requires_initial_samples":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Minimum number of initial random/Sobol samples required before the planner's model is fit. None when the planner has no warm-up phase.","title":"Requires Initial Samples"},"compatible_objective_directions":{"description":"Objective directions the planner can optimize. Lower-case strings.","items":{"type":"string"},"title":"Compatible Objective Directions","type":"array"},"requires_extras":{"description":"Names of pyproject.toml extras (e.g. ['optuna']) required to install this planner's heavy dependencies. Informational; the planner class itself owns the ImportError surface.","items":{"type":"string"},"title":"Requires Extras","type":"array"}},"required":["supports_continuous","supports_discrete"],"title":"SearchPlannerMetadata","type":"object"}},"required":["class"],"title":"Search Planner Plugin","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run."}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"plugins.schema.json","additionalProperties":false,"description":"Root model for plugins.yaml file.\n\nThis file registers plugin implementations for AIPerf. Each top-level section\ncorresponds to a category from categories.yaml and maps type names to their\nimplementing classes.\n\nNote: Package metadata (name, version, author) comes from pyproject.toml\nvia importlib.metadata, not from this file.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n my_custom:\n class: my_package.endpoints.my_custom:MyCustomEndpoint\n description: Custom endpoint for my API.\n metadata:\n endpoint_path: /v1/generate\n supports_streaming: true\n produces_tokens: true\n tokenizes_input: true\n metrics_title: My Custom Metrics\n\ncustom_dataset_loader:\n my_jsonl:\n class: my_package.dataset_loaders.my_jsonl:MyJSONLDatasetLoader\n description: Custom dataset loader for my API.\n```","properties":{"schema_version":{"default":"1.0","description":"Version of the plugins.yaml schema format. Use '1.0' for current format.","title":"Schema Version","type":"string"},"api_router":{"title":"API Router Plugins","type":"object","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded.","additionalProperties":{"$ref":"#/$defs/ApiRouterPlugin"}},"timing_strategy":{"title":"Timing Strategy Plugins","type":"object","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/TimingStrategyPlugin"}},"arrival_pattern":{"title":"Arrival Pattern Plugins","type":"object","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode.","additionalProperties":{"$ref":"#/$defs/ArrivalPatternPlugin"}},"ramp":{"title":"Ramp Plugins","type":"object","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns.","additionalProperties":{"$ref":"#/$defs/RampPlugin"}},"dataset_backing_store":{"title":"Dataset Backing Store Plugins","type":"object","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access.","additionalProperties":{"$ref":"#/$defs/DatasetBackingStorePlugin"}},"dataset_client_store":{"title":"Dataset Client Store Plugins","type":"object","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance.","additionalProperties":{"$ref":"#/$defs/DatasetClientStorePlugin"}},"dataset_sampler":{"title":"Dataset Sampler Plugins","type":"object","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration.","additionalProperties":{"$ref":"#/$defs/DatasetSamplerPlugin"}},"dataset_composer":{"title":"Dataset Composer Plugins","type":"object","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration.","additionalProperties":{"$ref":"#/$defs/DatasetComposerPlugin"}},"custom_dataset_loader":{"title":"Custom Dataset Loader Plugins","type":"object","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load().","additionalProperties":{"$ref":"#/$defs/CustomDatasetLoaderPlugin"}},"public_dataset_loader":{"title":"Public Dataset Loader Plugins","type":"object","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration.","additionalProperties":{"$ref":"#/$defs/PublicDatasetLoaderPlugin"}},"endpoint":{"title":"Endpoint Plugins","type":"object","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration.","additionalProperties":{"$ref":"#/$defs/EndpointPlugin"}},"transport":{"title":"Transport Plugins","type":"object","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration.","additionalProperties":{"$ref":"#/$defs/TransportPlugin"}},"record_processor":{"title":"Record Processor Plugins","type":"object","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordProcessorPlugin"}},"record_observer":{"title":"Record Observer Plugins","type":"object","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordObserverPlugin"}},"accumulator":{"title":"Accumulator Plugins","type":"object","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AccumulatorPlugin"}},"stream_exporter":{"title":"Stream Exporter Plugins","type":"object","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/StreamExporterPlugin"}},"analyzer":{"title":"Analyzer Plugins","type":"object","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AnalyzerPlugin"}},"spec_decode_adapter":{"title":"Spec Decode Adapter Plugins","type":"object","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins.","additionalProperties":{"$ref":"#/$defs/SpecDecodeAdapterPlugin"}},"accuracy_grader":{"title":"Accuracy Grader Plugins","type":"object","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default.","additionalProperties":{"$ref":"#/$defs/AccuracyGraderPlugin"}},"accuracy_benchmark":{"title":"Accuracy Benchmark Plugins","type":"object","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration.","additionalProperties":{"$ref":"#/$defs/AccuracyBenchmarkPlugin"}},"data_exporter":{"title":"Data Exporter Plugins","type":"object","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/DataExporterPlugin"}},"console_exporter":{"title":"Console Exporter Plugins","type":"object","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/ConsoleExporterPlugin"}},"ui":{"title":"UI Plugins","type":"object","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration.","additionalProperties":{"$ref":"#/$defs/UiPlugin"}},"url_selection_strategy":{"title":"Url Selection Strategy Plugins","type":"object","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration.","additionalProperties":{"$ref":"#/$defs/UrlSelectionStrategyPlugin"}},"service":{"title":"Service Plugins","type":"object","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services.","additionalProperties":{"$ref":"#/$defs/ServicePlugin"}},"service_manager":{"title":"Service Manager Plugins","type":"object","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration.","additionalProperties":{"$ref":"#/$defs/ServiceManagerPlugin"}},"communication":{"title":"Communication Plugins","type":"object","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration.","additionalProperties":{"$ref":"#/$defs/CommunicationPlugin"}},"communication_client":{"title":"Communication Client Plugins","type":"object","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage.","additionalProperties":{"$ref":"#/$defs/CommunicationClientPlugin"}},"zmq_proxy":{"title":"ZMQ Proxy Plugins","type":"object","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration.","additionalProperties":{"$ref":"#/$defs/ZmqProxyPlugin"}},"plot":{"title":"Plot Plugins","type":"object","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection.","additionalProperties":{"$ref":"#/$defs/PlotPlugin"}},"gpu_telemetry_collector":{"title":"GPU Telemetry Collector Plugins","type":"object","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration.","additionalProperties":{"$ref":"#/$defs/GpuTelemetryCollectorPlugin"}},"search_recipe":{"title":"Search Recipe Plugins","type":"object","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchRecipePlugin"}},"search_recipe_post_process":{"title":"Search Recipe Post Process Plugins","type":"object","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec.","additionalProperties":{"$ref":"#/$defs/SearchRecipePostProcessPlugin"}},"convergence_criterion":{"title":"Convergence Criterion Plugins","type":"object","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/ConvergenceCriterionPlugin"}},"search_planner":{"title":"Search Planner Plugins","type":"object","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchPlannerPlugin"}}},"title":"PluginsManifest","type":"object","$defs":{"ApiRouterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"API Router Plugin","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded."},"TimingStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Timing Strategy Plugin","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run."},"ArrivalPatternPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Arrival Pattern Plugin","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode."},"RampPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Ramp Plugin","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns."},"DatasetBackingStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Backing Store Plugin","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access."},"DatasetClientStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Client Store Plugin","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance."},"DatasetSamplerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Sampler Plugin","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration."},"DatasetComposerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Composer Plugin","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration."},"CustomDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for custom dataset loader plugins.\n\nDefines format-specific defaults for dataset loaders. When a loader specifies\n``block_size``, it overrides the user's ``--isl-block-size`` config default,\nensuring hash-based prompt generation uses the correct token block size for the\ntrace format (e.g. 16 for Bailian, 512 for Mooncake).\n\nReferenced by: categories.yaml custom_dataset_loader.metadata_class\nUsed in: plugins.yaml custom_dataset_loader entries","properties":{"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace datasets use hash_ids-based prompt generation, support synthesis options, and prefer sequential sampling with fixed_schedule timing.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids (e.g. 16 for Bailian, 512 for Mooncake).","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces (e.g. weka_trace) override to 'coding' so reconstructed prompts resemble real tool-use content.","title":"Default Prompt Corpus","type":"string"},"category":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Filter dataset rows to a specific category value. Used by custom loaders that support per-category subsets (e.g. SPEED-Bench).","title":"Category"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects.","title":"Multi Turn","type":"boolean"}},"title":"CustomDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Custom Dataset Loader Plugin","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load()."},"PublicDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for public dataset loader plugins.\n\nReferenced by: categories.yaml public_dataset_loader.metadata_class\nUsed in: plugins.yaml public_dataset_loader entries","properties":{"hf_dataset_name":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset identifier (e.g. 'AI-MO/NuminaMath-TIR'). Required for HF-backed loaders.","title":"Hf Dataset Name"},"hf_split":{"default":"train","description":"HuggingFace dataset split to load (e.g. 'train', 'test', 'validation').","title":"Hf Split","type":"string"},"hf_subset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name. Only needed for datasets with multiple configs.","title":"Hf Subset"},"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace public datasets reuse hash_ids-based prompt generation, require a tokenizer, and prefer sequential sampling. Mirrors the field of the same name on CustomDatasetLoaderMetadata so trace loaders can live in either pipeline.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids.","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces override to 'coding'.","title":"Default Prompt Corpus","type":"string"},"prompt_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the prompt/instruction text. Required for HFInstructionResponseDatasetLoader.","title":"Prompt Column"},"image_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the image data (PIL Image). Used for multimodal datasets.","title":"Image Column"},"video_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the video data (URL string or bytes dict). Used for video multimodal datasets.","title":"Video Column"},"audio_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing audio data. Depending on loader/dataset, this can be an HF Audio dict with array/sampling_rate (decoded) or bytes/path (decode=False). Used for ASR/speech datasets.","title":"Audio Column"},"conversation_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the conversation messages array. Required for HFConversationDatasetLoader.","title":"Conversation Column"},"message_content_key":{"default":"content","description":"Key inside each message dict for the text content. Used with conversation_column (e.g. 'content', 'value').","title":"Message Content Key","type":"string"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects. For HFConversationDatasetLoader: user→assistant (OpenAI roles) or human→gpt (ShareGPT-style ``from`` fields) pairs.","title":"Multi Turn","type":"boolean"},"streaming":{"default":false,"description":"Whether to load the HuggingFace dataset in streaming mode. Use true for large datasets (>10 GB) to avoid downloading the full dataset. Use false (default) for small datasets to leverage HF caching and len() support.","title":"Streaming","type":"boolean"},"has_timing_data":{"default":false,"description":"Whether the loader emits timestamps suitable for fixed-schedule replay.","title":"Has Timing Data","type":"boolean"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.","title":"Prompt Template"}},"title":"PublicDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Public Dataset Loader Plugin","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration."},"EndpointPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for endpoint plugins.\n\nDefines API capabilities, paths, and multimodal support for endpoint implementations.\nUsed by the framework to route requests, configure streaming, and enable\nmultimodal inputs/outputs (images, audio, video).\n\nReferenced by: categories.yaml endpoint.metadata_class\nUsed in: plugins.yaml endpoint entries","properties":{"metrics_title":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"Display title for metrics dashboard.","title":"Metrics Title"},"endpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"API path (e.g., /v1/chat/completions).","title":"Endpoint Path"},"streaming_path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Streaming API path if different from the endpoint path (e.g., /generate_stream).","title":"Streaming Path"},"service_kind":{"default":"openai","description":"The service kind of the endpoint (used for artifact naming).","title":"Service Kind","type":"string"},"supports_streaming":{"description":"Whether endpoint supports streaming responses.","title":"Supports Streaming","type":"boolean"},"tokenizes_input":{"description":"Whether endpoint tokenizes text inputs.","title":"Tokenizes Input","type":"boolean"},"produces_tokens":{"description":"Whether endpoint produces token-based output.","title":"Produces Tokens","type":"boolean"},"supports_cache_bust":{"default":false,"description":"Whether the endpoint formatter can apply a cache-bust marker to the structured request payload.","title":"Supports Cache Bust","type":"boolean"},"supports_audio":{"default":false,"description":"Whether endpoint accepts audio input.","title":"Supports Audio","type":"boolean"},"supports_images":{"default":false,"description":"Whether endpoint accepts image input.","title":"Supports Images","type":"boolean"},"supports_videos":{"default":false,"description":"Whether endpoint accepts video input.","title":"Supports Videos","type":"boolean"},"produces_audio":{"default":false,"description":"Whether endpoint produces audio-based outputs.","title":"Produces Audio","type":"boolean"},"produces_images":{"default":false,"description":"Whether endpoint produces image-based outputs.","title":"Produces Images","type":"boolean"},"produces_videos":{"default":false,"description":"Whether endpoint produces video-based outputs.","title":"Produces Videos","type":"boolean"},"requires_polling":{"default":false,"description":"Whether endpoint uses async job polling (submit job, poll for status, retrieve result).","title":"Requires Polling","type":"boolean"},"requires_form_data":{"default":false,"description":"True for endpoints that require multipart/form-data (binary uploads), e.g., video_generation, image_edit. EndpointConfig consumes this flag to auto-select the request encoding.","title":"Requires Form Data","type":"boolean"},"requires_inline_media":{"default":false,"description":"Whether endpoint requires media URLs to be downloaded and inlined as base64 data URLs.","title":"Requires Inline Media","type":"boolean"},"consumes_system_message":{"default":false,"description":"Whether endpoint sends RequestInfo.system_message on the wire (e.g. a leading system role for chat, top-level instructions for responses). When False, the multi-turn loader does not hoist a leading system turn into the conversation system_message, since it would be silently dropped.","title":"Consumes System Message","type":"boolean"}},"required":["metrics_title","endpoint_path","supports_streaming","tokenizes_input","produces_tokens"],"title":"EndpointMetadata","type":"object"}},"required":["class"],"title":"Endpoint Plugin","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration."},"TransportPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for transport plugins.\n\nDefines network layer configuration including transport type and URL schemes.\nUsed by the framework to auto-detect appropriate transport based on URL scheme.\n\nReferenced by: categories.yaml transport.metadata_class\nUsed in: plugins.yaml transport entries","properties":{"transport_type":{"description":"Transport type identifier for this transport","title":"Transport Type","type":"string"},"url_schemes":{"description":"URL schemes this transport handles (for auto-detection and validation).","items":{"type":"string"},"title":"Url Schemes","type":"array"}},"required":["transport_type"],"title":"TransportMetadata","type":"object"}},"required":["class"],"title":"Transport Plugin","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration."},"RecordProcessorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries","properties":{"record_type":{"description":"The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.","title":"Record Type","type":"string"}},"required":["record_type"],"title":"RecordProducerMetadata","type":"object"}},"required":["class"],"title":"Record Processor Plugin","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously."},"RecordObserverPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Record Observer Plugin","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously."},"AccumulatorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Accumulator Plugin","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously."},"StreamExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Stream Exporter Plugin","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously."},"AnalyzerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary — e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) — e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries","properties":{"required_accumulators":{"description":"AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.","items":{"type":"string"},"title":"Required Accumulators","type":"array"},"required_summaries":{"description":"AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.","items":{"type":"string"},"title":"Required Summaries","type":"array"}},"title":"AnalyzerMetadata","type":"object"}},"required":["class"],"title":"Analyzer Plugin","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously."},"SpecDecodeAdapterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Spec Decode Adapter Plugin","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins."},"AccuracyGraderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Grader Plugin","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default."},"AccuracyBenchmarkPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Benchmark Plugin","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration."},"DataExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Data Exporter Plugin","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously."},"ConsoleExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Console Exporter Plugin","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously."},"UiPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"UI Plugin","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration."},"UrlSelectionStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Url Selection Strategy Plugin","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration."},"ServicePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for service plugins.\n\nDefines lifecycle and runtime configuration for AIPerf distributed services.\nUsed by SystemController to manage service startup order and optimize\nlatency-sensitive services (timing, workers) by disabling garbage collection.\n\nReferenced by: categories.yaml service.metadata_class\nUsed in: plugins.yaml service entries","properties":{"required":{"description":"Whether the service is required for benchmark execution.","title":"Required","type":"boolean"},"auto_start":{"description":"Whether the service is automatically started by the system controller.","title":"Auto Start","type":"boolean"},"disable_gc":{"default":false,"description":"Whether to disable garbage collection in the service for timing-critical operations.","title":"Disable Gc","type":"boolean"},"replicable":{"default":false,"description":"Whether the service can have multiple instances running in parallel.","title":"Replicable","type":"boolean"}},"required":["required","auto_start"],"title":"ServiceMetadata","type":"object"}},"required":["class"],"title":"Service Plugin","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services."},"ServiceManagerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Service Manager Plugin","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration."},"CommunicationPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Plugin","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration."},"CommunicationClientPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Client Plugin","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage."},"ZmqProxyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"ZMQ Proxy Plugin","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration."},"PlotPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for plot plugins.\n\nDefines display properties and categorization for visualization handlers.\nUsed by the framework to group plots and generate selection interfaces.\n\nReferenced by: categories.yaml plot.metadata_class\nUsed in: plugins.yaml plot entries","properties":{"display_name":{"description":"Human-readable name for UI display.","title":"Display Name","type":"string"},"category":{"description":"Plot category (per_request, aggregated, combined, comparison).","title":"Category","type":"string"}},"required":["display_name","category"],"title":"PlotMetadata","type":"object"}},"required":["class"],"title":"Plot Plugin","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection."},"GpuTelemetryCollectorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Config-time metadata for GPU telemetry collector plugins.\n\nLocal collectors set ``is_local`` so ``--gpu-telemetry `` is treated\nas an in-process collector keyword instead of a DCGM URL. Native binding\nvalidation lives on the collector class itself via ``validate_environment``\nso each implementation owns its own dependency check.\n\nReferenced by: categories.yaml gpu_telemetry_collector.metadata_class\nUsed in: plugins.yaml gpu_telemetry_collector entries","properties":{"is_local":{"default":false,"description":"Whether this collector runs in-process against the local host.","title":"Is Local","type":"boolean"}},"title":"GPUTelemetryCollectorMetadata","type":"object"}},"required":["class"],"title":"GPU Telemetry Collector Plugin","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration."},"SearchRecipePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Plugin","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run."},"SearchRecipePostProcessPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Post Process Plugin","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec."},"ConvergenceCriterionPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for convergence criterion plugins.\n\nDeclares statistical-method capabilities so the CLI/config layer can\nvalidate `--convergence-metric` / `--convergence-stat` against the chosen\ncriterion before the plugin is imported.\n\nReferenced by: categories.yaml convergence_criterion.metadata_class\nUsed in: plugins.yaml convergence_criterion entries","properties":{"min_samples":{"description":"Minimum number of successful runs required before convergence can trigger.","minimum":1,"title":"Min Samples","type":"integer"},"requires_confidence_level":{"default":false,"description":"Whether the criterion consumes plan.confidence_level (e.g. CI-width does, CV doesn't).","title":"Requires Confidence Level","type":"boolean"},"requires_jsonl_export":{"default":false,"description":"Whether the criterion reads per-request metrics from JSONL exports (e.g. distribution does).","title":"Requires Jsonl Export","type":"boolean"},"metric_kinds":{"description":"Kinds of metrics this criterion handles. One or more of 'continuous', 'counts', 'categorical'.","items":{"type":"string"},"title":"Metric Kinds","type":"array"}},"required":["min_samples"],"title":"ConvergenceCriterionMetadata","type":"object"}},"required":["class"],"title":"Convergence Criterion Plugin","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run."},"SearchPlannerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for search planner plugins.\n\nDeclares dimension-type and objective-direction support so the CLI/config\nlayer can validate `--search-space` shape against the chosen planner\nbefore the planner (and its heavy soft-dep imports) is loaded.\n\nReferenced by: categories.yaml search_planner.metadata_class\nUsed in: plugins.yaml search_planner entries","properties":{"supports_continuous":{"description":"Whether the planner accepts Real-valued search-space dimensions.","title":"Supports Continuous","type":"boolean"},"supports_discrete":{"description":"Whether the planner accepts Integer-valued search-space dimensions.","title":"Supports Discrete","type":"boolean"},"supports_categorical":{"default":false,"description":"Whether the planner accepts Categorical search-space dimensions.","title":"Supports Categorical","type":"boolean"},"requires_initial_samples":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Minimum number of initial random/Sobol samples required before the planner's model is fit. None when the planner has no warm-up phase.","title":"Requires Initial Samples"},"compatible_objective_directions":{"description":"Objective directions the planner can optimize. Lower-case strings.","items":{"type":"string"},"title":"Compatible Objective Directions","type":"array"},"requires_extras":{"description":"Names of pyproject.toml extras (e.g. ['optuna']) required to install this planner's heavy dependencies. Informational; the planner class itself owns the ImportError surface.","items":{"type":"string"},"title":"Requires Extras","type":"array"}},"required":["supports_continuous","supports_discrete"],"title":"SearchPlannerMetadata","type":"object"}},"required":["class"],"title":"Search Planner Plugin","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run."}}} diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index 06bc590592..462374529e 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -256,6 +256,13 @@ class EndpointMetadata(BaseModel): produces_tokens: bool = Field( ..., description="Whether endpoint produces token-based output." ) + supports_cache_bust: bool = Field( + default=False, + description=( + "Whether the endpoint formatter can apply a cache-bust marker to " + "the structured request payload." + ), + ) supports_audio: bool = Field( default=False, description="Whether endpoint accepts audio input." ) diff --git a/src/aiperf/timing/branch_orchestrator.py b/src/aiperf/timing/branch_orchestrator.py index 923504d968..cd0f321e75 100644 --- a/src/aiperf/timing/branch_orchestrator.py +++ b/src/aiperf/timing/branch_orchestrator.py @@ -936,16 +936,25 @@ async def _spawn_children_and_register_gates( for child_conv_id in branch.child_conversation_ids: try: + credit_marker = getattr(credit, "cache_bust_marker", None) + marker = ( + credit_marker + if isinstance(credit_marker, str) + else self._marker_for_root(credit.effective_root_correlation_id) + ) + credit_target = getattr(credit, "cache_bust_target", None) child = self._cs.start_branch_child( parent_correlation_id=parent_corr, child_conversation_id=child_conv_id, agent_depth=parent_depth + 1, root_correlation_id=credit.effective_root_correlation_id, branch_mode=branch.mode, - cache_bust_marker=self._marker_for_root( - credit.effective_root_correlation_id + cache_bust_marker=marker, + cache_bust_target=( + credit_target + if isinstance(credit_target, CacheBustTarget) + else self._cache_bust_target ), - cache_bust_target=self._cache_bust_target, ) except Exception: logger.exception("start_branch_child failed for %s", child_conv_id) diff --git a/src/aiperf/timing/conversation_source.py b/src/aiperf/timing/conversation_source.py index 6d89f1b401..9b263caf74 100644 --- a/src/aiperf/timing/conversation_source.py +++ b/src/aiperf/timing/conversation_source.py @@ -22,6 +22,7 @@ from aiperf.common.models import ConversationMetadata, DatasetMetadata, TurnMetadata from aiperf.credit.structs import Credit, TurnToSend from aiperf.dataset.protocols import DatasetSamplingStrategyProtocol +from aiperf.timing.strategies.cache_bust import build_cache_bust_marker @dataclass(slots=True) @@ -152,6 +153,8 @@ def build_turn_at_index(self, turn_index: int) -> TurnToSend: is_session_start=True, has_forks=meta.has_forks if meta is not None else False, branch_mode=self.branch_mode, + cache_bust_marker=self.cache_bust_marker, + cache_bust_target=self.cache_bust_target, ) @@ -166,10 +169,17 @@ def __init__( self, dataset_metadata: DatasetMetadata, dataset_sampler: DatasetSamplingStrategyProtocol, - ): + *, + benchmark_id: str = "unknown", + cache_bust_target: CacheBustTarget = CacheBustTarget.NONE, + ) -> None: """Initialize conversation source.""" self._dataset_metadata = dataset_metadata self._dataset_sampler = dataset_sampler + self._benchmark_id = benchmark_id + self._cache_bust_target = cache_bust_target + self._cache_bust_pass = 0 + self._cache_bust_markers: dict[str, str | None] = {} self._metadata_lookup: dict[str, ConversationMetadata] = { conv.conversation_id: conv for conv in dataset_metadata.conversations } @@ -179,15 +189,51 @@ def dataset_metadata(self) -> DatasetMetadata: """Dataset metadata.""" return self._dataset_metadata + def _marker_for_session( + self, conversation_id: str, x_correlation_id: str, *, retain: bool + ) -> str | None: + """Mint one deterministic marker for an ordinary session instance. + + ``retain`` gates the ``_cache_bust_markers`` cache, which exists only so + a caller-supplied correlation id resolves to the SAME marker on every + lookup (user-centric reuses ``str(user_id)``; ``marker_for_correlation_id`` + reads it back). A correlation id minted internally here is a fresh uuid + no caller can name, so retaining it would grow the dict without bound — + one dead entry per sampled session for the length of the run. + """ + if self._cache_bust_target == CacheBustTarget.NONE: + return None + if x_correlation_id in self._cache_bust_markers: + return self._cache_bust_markers[x_correlation_id] + marker = build_cache_bust_marker( + self._benchmark_id, + self._cache_bust_pass, + self._cache_bust_pass, + conversation_id or x_correlation_id, + target=self._cache_bust_target, + ) + self._cache_bust_pass += 1 + if retain: + self._cache_bust_markers[x_correlation_id] = marker + return marker + + def release_marker_for_correlation_id(self, correlation_id: str) -> None: + """Drop a retained marker once its session reaches a terminal credit.""" + self._cache_bust_markers.pop(correlation_id, None) + def next(self, x_correlation_id: str | None = None) -> SampledSession: """Sample next conversation and return a new session instance.""" conversation_id = self._dataset_sampler.next_conversation_id() metadata = self._metadata_lookup[conversation_id] - + correlation_id = x_correlation_id or str(uuid.uuid4()) return SampledSession( conversation_id=conversation_id, metadata=metadata, - x_correlation_id=x_correlation_id or str(uuid.uuid4()), + x_correlation_id=correlation_id, + cache_bust_marker=self._marker_for_session( + conversation_id, correlation_id, retain=x_correlation_id is not None + ), + cache_bust_target=self._cache_bust_target, ) def get_metadata(self, conversation_id: str) -> ConversationMetadata: @@ -196,6 +242,26 @@ def get_metadata(self, conversation_id: str) -> ConversationMetadata: raise KeyError(f"No metadata for conversation {conversation_id}") return self._metadata_lookup[conversation_id] + def session_for_conversation( + self, conversation_id: str, *, x_correlation_id: str | None = None + ) -> SampledSession: + """Build a session for a known conversation, preserving marker policy.""" + metadata = self.get_metadata(conversation_id) + correlation_id = x_correlation_id or str(uuid.uuid4()) + return SampledSession( + conversation_id=conversation_id, + metadata=metadata, + x_correlation_id=correlation_id, + cache_bust_marker=self._marker_for_session( + conversation_id, correlation_id, retain=x_correlation_id is not None + ), + cache_bust_target=self._cache_bust_target, + ) + + def marker_for_correlation_id(self, correlation_id: str) -> str | None: + """Return the marker minted for a live ordinary session, if any.""" + return self._cache_bust_markers.get(correlation_id) + def start_branch_child( self, parent_correlation_id: str, @@ -207,30 +273,7 @@ def start_branch_child( cache_bust_marker: str | None = None, cache_bust_target: CacheBustTarget = CacheBustTarget.NONE, ) -> SampledSession: - """Build a SampledSession for a DAG child conversation. - - Under FORK mode, the returned session inherits sticky-routing from its - parent via ``parent_correlation_id``; the credit router pins the child - to the parent's worker, where ``UserSessionManager.create_and_store`` - seeds ``turn_list`` by cloning the parent's in-memory session. - SPAWN-mode children start with a fresh context, but still carry - ``parent_correlation_id`` so the sticky router co-locates them on the - parent's worker while that entry is live. The orchestrator skips - SPAWN sticky refcount bumps; least-loaded routing applies only once - the parent sticky entry is gone. - - ``root_correlation_id`` is the depth-0 root of the spawning parent's - tree; the child inherits it so all descendants of one root share a - single per-tree session-slot key. Defaults to ``parent_correlation_id`` - when not supplied (the live spawn path's parent is always the root). - - ``cache_bust_marker`` / ``cache_bust_target`` are resolved by the caller - (BranchOrchestrator via ``_marker_for_root``) to the marker of the - child's trajectory TREE, so every SPAWN descendant of one root shares - the root's marker (the tree is one prefix-cache domain) while subagents - in different traces get distinct markers — preventing cross-trace - sharing of a server KV-cache prefix and artificially inflated hit rates. - """ + """Build a SampledSession for a DAG child conversation.""" metadata = self._metadata_lookup[child_conversation_id] return SampledSession( conversation_id=child_conversation_id, @@ -250,23 +293,7 @@ def start_pre_session_child( cache_bust_marker: str | None = None, cache_bust_target: CacheBustTarget = CacheBustTarget.NONE, ) -> SampledSession: - """Build a SampledSession for a pre-session (turn-0) background SPAWN child. - - Used by ``BranchOrchestrator.dispatch_pre_session_branches`` to fire - a child before its parent's turn 0 is issued. The child gets a fresh - correlation id, ``agent_depth=1``, and ``parent_correlation_id=None`` - (no real parent session exists yet). Because ``parent_correlation_id`` - is None, the child's ``routing_key`` naturally equals its own - ``x_correlation_id`` — the child routes freely (no sticky pin). - - Restricted to SPAWN mode with ``dispatch_timing="pre"`` at the validator - level; FORK pre-dispatch would require inheriting a non-existent - parent session and is rejected at load time. - - ``cache_bust_marker`` / ``cache_bust_target`` are minted by the caller - so background SPAWN children get the same per-session unique-marker - treatment as parents. - """ + """Build a SampledSession for a pre-session background SPAWN child.""" metadata = self._metadata_lookup[child_conversation_id] return SampledSession( conversation_id=child_conversation_id, @@ -280,11 +307,7 @@ def start_pre_session_child( ) def get_next_turn_metadata(self, credit: Credit) -> TurnMetadata: - """Get metadata for next turn after completed credit. - - Raises: - ValueError: If next turn doesn't exist (credit is final turn). - """ + """Get metadata for next turn after completed credit.""" metadata = self.get_metadata(credit.conversation_id) next_index = credit.turn_index + 1 diff --git a/src/aiperf/timing/phase_orchestrator.py b/src/aiperf/timing/phase_orchestrator.py index 5a43c21480..7edf28f3bd 100644 --- a/src/aiperf/timing/phase_orchestrator.py +++ b/src/aiperf/timing/phase_orchestrator.py @@ -17,7 +17,7 @@ from collections.abc import Callable from typing import TYPE_CHECKING -from aiperf.common.enums import CreditPhase +from aiperf.common.enums import CacheBustTarget, CreditPhase from aiperf.common.hooks import on_init, on_start, on_stop from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.credit.callback_handler import CreditCallbackHandler @@ -145,6 +145,10 @@ def __init__( is_agentic_replay = any( pc.timing_mode == TimingMode.AGENTIC_REPLAY for pc in config.phase_configs ) + benchmark_id = run.benchmark_id if run is not None else "unknown" + cache_bust_target = ( + run.cfg.get_cache_bust_target() if run is not None else CacheBustTarget.NONE + ) if is_agentic_replay: if config.concurrency is None: raise ValueError( @@ -180,7 +184,10 @@ def __init__( ) else: self._conversation_source = ConversationSource( - self._dataset_metadata, self._dataset_sampler + self._dataset_metadata, + self._dataset_sampler, + benchmark_id=benchmark_id, + cache_bust_target=cache_bust_target, ) self._concurrency_manager = ConcurrencyManager() self._session_tree_registry = ( diff --git a/src/aiperf/timing/strategies/fixed_schedule.py b/src/aiperf/timing/strategies/fixed_schedule.py index 53898aa2d3..f72d4ab1c4 100644 --- a/src/aiperf/timing/strategies/fixed_schedule.py +++ b/src/aiperf/timing/strategies/fixed_schedule.py @@ -9,7 +9,6 @@ from __future__ import annotations -import uuid from typing import TYPE_CHECKING, NamedTuple from aiperf.common.constants import MILLIS_PER_SECOND @@ -52,7 +51,7 @@ def __init__( lifecycle: PhaseLifecycle, stop_checker: StopConditionChecker, **kwargs, - ): + ) -> None: """Initialize fixed schedule timing strategy with all dependencies.""" super().__init__(logger_name="FixedScheduleTiming") self._config = config @@ -92,15 +91,16 @@ async def setup_phase(self) -> None: f"First turn of {conv.conversation_id} missing timestamp_ms" ) + # Route through the source rather than building TurnToSend inline so + # the session carries the cache-bust marker minted for it. + sampled = self._conversation_source.session_for_conversation( + conv.conversation_id, + x_correlation_id=f"fixed-schedule-{conv.conversation_id}", + ) self._absolute_schedule.append( ScheduleEntry( timestamp_ms=conv.turns[0].timestamp_ms, - turn=TurnToSend( - conversation_id=conv.conversation_id, - x_correlation_id=str(uuid.uuid4()), - turn_index=0, - num_turns=len(conv.turns), - ), + turn=sampled.build_first_turn(), ) ) diff --git a/src/aiperf/timing/strategies/user_centric_rate.py b/src/aiperf/timing/strategies/user_centric_rate.py index 7ac03709ec..ceba537f7d 100644 --- a/src/aiperf/timing/strategies/user_centric_rate.py +++ b/src/aiperf/timing/strategies/user_centric_rate.py @@ -403,6 +403,11 @@ async def handle_credit_return( if credit.is_final_turn: # User finished all their turns. New users continue spawning in execute_phase. self._session_to_user.pop(credit.x_correlation_id, None) + # user_ids are never reused, so the retained cache-bust marker for + # this correlation id can never be looked up again. + self._conversation_source.release_marker_for_correlation_id( + credit.x_correlation_id + ) return current_sec = time.perf_counter() diff --git a/src/aiperf/workers/worker.py b/src/aiperf/workers/worker.py index 2a8efbcb6a..a062073cc6 100644 --- a/src/aiperf/workers/worker.py +++ b/src/aiperf/workers/worker.py @@ -250,6 +250,45 @@ def _inject_marker_into_first_user_turn( return +def _find_effective_raw_system(turn_list: list[Turn]) -> list[dict] | None: + """Return the ``raw_system`` block list the endpoint will actually send. + + Mirrors ``base_endpoint._latest_turn_attr``: walks ``turn_list`` from the + end and returns the first non-None ``raw_system``. Only ``MessagesEndpoint`` + reads this field, and there it OUTRANKS both the conversation-level + ``system_message`` string and any system-role ``raw_messages`` entry — so a + ``SYSTEM_*`` marker written to either of those would be silently dropped + from the wire whenever a turn carries ``raw_system``. + """ + for turn in reversed(turn_list): + if turn.raw_system is not None: + return turn.raw_system + return None + + +def _inject_marker_into_raw_system( + raw_system: list[dict], marker: str, *, is_prefix: bool +) -> None: + """Insert the marker as a text block at the edge of a ``raw_system`` list. + + ``raw_system`` is a list of vendor content blocks (not role/content + messages), so the marker becomes its own ``{"type": "text", ...}`` block + rather than being spliced into an existing block's text — that leaves any + neighbouring block's ``cache_control`` breakpoint attached to the same + bytes it was authored against. Idempotent via + :func:`_content_has_marker_at_edge`. + """ + if not marker: + return + if _content_has_marker_at_edge(raw_system, marker, is_prefix=is_prefix): + return + marker_block = {"type": "text", "text": marker.strip()} + if is_prefix: + raw_system.insert(0, marker_block) + else: + raw_system.append(marker_block) + + def _find_first_system_message(turn_list: list[Turn]) -> list[dict] | None: """Return the raw_messages list whose first dict has ``role == "system"``, or None. @@ -399,6 +438,7 @@ def _apply_cache_bust( if target in (CacheBustTarget.SYSTEM_PREFIX, CacheBustTarget.SYSTEM_SUFFIX): return _apply_system_target_cache_bust( prefix_turns, + all_turns=session.turn_list, system_message=system_message, marker=marker, target=target, @@ -416,6 +456,7 @@ def _apply_cache_bust( def _apply_system_target_cache_bust( prefix_turns: list[Turn], *, + all_turns: list[Turn], system_message: str | None, marker: str, target: CacheBustTarget, @@ -424,20 +465,30 @@ def _apply_system_target_cache_bust( """Inject a ``SYSTEM_PREFIX`` / ``SYSTEM_SUFFIX`` marker for one credit. ``prefix_turns`` is the effective wire prefix slice (see - :func:`_effective_prefix_turns`). Three sub-paths: - 1. Conversation-level ``system_message`` present: marker applied every + :func:`_effective_prefix_turns`). Four sub-paths, ordered to match how the + endpoint resolves the system field on the wire — marking a lower-precedence + carrier would leave the bytes that actually ship unchanged: + 1. ``raw_system`` present on any turn: it outranks both ``system_message`` + and system-role ``raw_messages`` in ``MessagesEndpoint``, so the marker + goes there. Resolved over ``all_turns`` (not the prefix slice) because + ``_latest_turn_attr`` ignores ``reset_context``. + 2. Conversation-level ``system_message`` present: marker applied every turn (string mutation re-applied per credit). Unaffected by ``reset_context`` — the ``system_message`` rides on ``RequestInfo`` and is re-emitted every turn independent of ``build_messages``' reset. - 2. ``raw_messages`` first dict has ``role=="system"``: marker injected + 3. ``raw_messages`` first dict has ``role=="system"``: marker injected into the first system message of the prefix slice. - 3. No system in the slice -> first-user-turn fallback: marker injected + 4. No system anywhere -> first-user-turn fallback: marker injected into the first user turn every credit (idempotent), matching ``FIRST_TURN_*`` semantics so a seeded mid-trajectory resume still marks the prefix. Returns the (possibly modified) ``system_message``. """ + raw_system_blocks = _find_effective_raw_system(all_turns) + if raw_system_blocks is not None: + _inject_marker_into_raw_system(raw_system_blocks, marker, is_prefix=is_prefix) + return system_message if system_message is not None: return _apply_cache_bust_to_system_message(system_message, marker, target) raw_system = _find_first_system_message(prefix_turns) diff --git a/tests/component_integration/timing/test_dag_timing_pathology.py b/tests/component_integration/timing/test_dag_timing_pathology.py index 581c37c73d..cc2a171c46 100644 --- a/tests/component_integration/timing/test_dag_timing_pathology.py +++ b/tests/component_integration/timing/test_dag_timing_pathology.py @@ -38,13 +38,16 @@ TurnPrerequisite, ) from aiperf.credit.structs import Credit +from aiperf.plugin import plugins from aiperf.plugin.enums import ( ArrivalPattern, DatasetSamplingStrategy, + PluginType, TimingMode, ) from aiperf.timing.branch_orchestrator import BranchOrchestrator from aiperf.timing.config import CreditPhaseConfig +from aiperf.timing.conversation_source import ConversationSource from aiperf.timing.intervals import IntervalGeneratorConfig from aiperf.timing.strategies.fixed_schedule import FixedScheduleStrategy from aiperf.timing.strategies.request_rate import RequestRateStrategy @@ -57,6 +60,20 @@ # ============================================================================= +def _mk_conversation_source(ds: DatasetMetadata) -> ConversationSource: + """Real ConversationSource over ``ds``. + + FixedScheduleStrategy builds its schedule through + ``session_for_conversation``, so a bare MagicMock source would yield mock + turns and make schedule assertions vacuous. + """ + SamplerClass = plugins.get_class(PluginType.DATASET_SAMPLER, ds.sampling_strategy) + sampler = SamplerClass( + conversation_ids=[c.conversation_id for c in ds.conversations] + ) + return ConversationSource(ds, sampler) + + def _mk_credit( conv_id: str, x_corr: str, @@ -164,9 +181,7 @@ async def test_fixed_schedule_out_of_order_timestamps_within_conversation() -> N ds = DatasetMetadata( conversations=[conv], sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL ) - src = MagicMock() - src.dataset_metadata = ds - src.get_next_turn_metadata = lambda credit: turns[credit.turn_index + 1] + src = _mk_conversation_source(ds) scheduler = MagicMock() issuer = MagicMock() @@ -236,9 +251,7 @@ async def test_fixed_schedule_very_large_timestamp_no_overflow() -> None: ds = DatasetMetadata( conversations=[conv], sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL ) - src = MagicMock() - src.dataset_metadata = ds - src.get_next_turn_metadata = lambda credit: turns[credit.turn_index + 1] + src = _mk_conversation_source(ds) scheduler = MagicMock() issuer = MagicMock() @@ -285,8 +298,7 @@ async def test_fixed_schedule_setup_sorts_identical_timestamps_stably() -> None: ds = DatasetMetadata( conversations=convs, sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL ) - src = MagicMock() - src.dataset_metadata = ds + src = _mk_conversation_source(ds) scheduler = MagicMock() issuer = MagicMock() @@ -327,8 +339,7 @@ async def test_fixed_schedule_zero_timestamp_fires_at_perf_start() -> None: ds = DatasetMetadata( conversations=convs, sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL ) - src = MagicMock() - src.dataset_metadata = ds + src = _mk_conversation_source(ds) scheduler = MagicMock() issuer = MagicMock() @@ -432,8 +443,7 @@ async def test_request_rate_dag_child_continuation_bypasses_continuation_queue() ds = DatasetMetadata( conversations=[conv], sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL ) - src = MagicMock() - src.dataset_metadata = ds + src = _mk_conversation_source(ds) src.get_next_turn_metadata = lambda credit: turns[credit.turn_index + 1] issuer = MagicMock() @@ -486,8 +496,7 @@ async def test_request_rate_dag_child_with_delay_uses_scheduler() -> None: ds = DatasetMetadata( conversations=[conv], sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL ) - src = MagicMock() - src.dataset_metadata = ds + src = _mk_conversation_source(ds) src.get_next_turn_metadata = lambda credit: turns[credit.turn_index + 1] issuer = MagicMock() diff --git a/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py b/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py index 21aff561fd..d3488390d4 100644 --- a/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py +++ b/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""The cache-bust compatibility lockdown rejects a non-NONE cache_bust target paired with a non-agentic timing mode or a non-chat/responses endpoint.""" +"""Cache-bust endpoint capability validation.""" from __future__ import annotations @@ -12,8 +12,6 @@ from aiperf.config.config import BenchmarkConfig from aiperf.plugin.enums import EndpointType, TimingMode -# Endpoint URL paths are cosmetic here; the lockdown only cares about the -# endpoint type and the cache_bust target/timing combination. _URL = "http://localhost:8000/v1/chat/completions" @@ -23,7 +21,7 @@ def _build( endpoint_type: EndpointType = EndpointType.CHAT, timing_mode: TimingMode | None = None, ) -> BenchmarkConfig: - """Construct a v2 BenchmarkConfig from a (cache_bust target, endpoint type, timing_mode) trio.""" + """Construct a v2 BenchmarkConfig for an endpoint capability combination.""" phase: dict[str, Any] = { "name": "profiling", "type": "concurrency", @@ -35,7 +33,6 @@ def _build( endpoint: dict[str, Any] = {"urls": [_URL], "type": endpoint_type} if endpoint_type == EndpointType.TEMPLATE: - # Template endpoint needs a payload template to construct at all. endpoint["template"] = { "body": '{"prompt": "{{prompt}}"}', "response_field": "text", @@ -63,37 +60,33 @@ def _build( _NON_NONE_CACHE_BUST_TARGETS: list[CacheBustTarget] = [ t for t in CacheBustTarget if t != CacheBustTarget.NONE ] - -_NON_AGENTIC_TIMING_MODES: list[TimingMode] = [ - m for m in TimingMode if m != TimingMode.AGENTIC_REPLAY -] - +_TIMING_MODES: list[TimingMode] = list(TimingMode) _INCOMPATIBLE_ENDPOINT_TYPES: list[EndpointType] = [ - e for e in EndpointType if e not in {EndpointType.CHAT, EndpointType.RESPONSES} + e + for e in EndpointType + if e not in {EndpointType.CHAT, EndpointType.RESPONSES, EndpointType.MESSAGES} ] -@pytest.mark.parametrize("timing_mode", _NON_AGENTIC_TIMING_MODES) +@pytest.mark.parametrize("timing_mode", _TIMING_MODES) @pytest.mark.parametrize("target", _NON_NONE_CACHE_BUST_TARGETS) -def test_cache_bust_rejected_with_every_non_agentic_timing_mode( +def test_cache_bust_accepted_with_every_timing_mode( timing_mode: TimingMode, target: CacheBustTarget ) -> None: - """Every non-agentic TimingMode paired with a non-NONE cache_bust target raises.""" - with pytest.raises(ValueError, match="agentic_replay"): - _build( - target=target, - endpoint_type=EndpointType.CHAT, - timing_mode=timing_mode, - ) + """Every timing mode accepts cache-bust with a structured chat endpoint.""" + cfg = _build( + target=target, endpoint_type=EndpointType.CHAT, timing_mode=timing_mode + ) + assert cfg.get_cache_bust_target() == target @pytest.mark.parametrize("endpoint_type", _INCOMPATIBLE_ENDPOINT_TYPES) @pytest.mark.parametrize("target", _NON_NONE_CACHE_BUST_TARGETS) -def test_cache_bust_rejected_with_every_non_chat_endpoint_type( +def test_cache_bust_rejected_with_unsupported_endpoint( endpoint_type: EndpointType, target: CacheBustTarget ) -> None: - """Every non-chat/responses endpoint paired with a non-NONE cache_bust target raises.""" - with pytest.raises(ValueError, match="chat or responses"): + """Endpoints without the capability metadata reject cache-bust.""" + with pytest.raises(ValueError, match=r"not supported|chat or responses"): _build( target=target, endpoint_type=endpoint_type, @@ -101,39 +94,48 @@ def test_cache_bust_rejected_with_every_non_chat_endpoint_type( ) -def test_unsafe_override_does_not_bypass_cache_bust_validation() -> None: - """``unsafe_override`` is a scenario-lock escape hatch and does not bypass the cache-bust compatibility lockdown.""" - body: dict[str, Any] = { - "models": ["test-model"], - "endpoint": {"urls": [_URL], "type": EndpointType.CHAT}, - "unsafe_override": True, - "datasets": [ - { - "name": "main", - "type": "synthetic", - "prompts": { - "isl": 128, - "osl": 16, - "cache_bust": {"target": CacheBustTarget.SYSTEM_PREFIX}, - }, - } - ], - "phases": [ - { - "name": "profiling", - "type": "concurrency", - "concurrency": 1, - "requests": 10, - "timing_mode": TimingMode.REQUEST_RATE, - } - ], - } - with pytest.raises(ValueError, match="agentic_replay"): - BenchmarkConfig.model_validate(body) +@pytest.mark.parametrize("target", _NON_NONE_CACHE_BUST_TARGETS) +@pytest.mark.parametrize( + "endpoint_type", [EndpointType.CHAT, EndpointType.RESPONSES, EndpointType.MESSAGES] +) +def test_cache_bust_accepted_with_supported_endpoint( + endpoint_type: EndpointType, target: CacheBustTarget +) -> None: + """Every supported structured endpoint accepts every cache-bust target.""" + cfg = _build( + target=target, + endpoint_type=endpoint_type, + timing_mode=TimingMode.REQUEST_RATE, + ) + assert cfg.get_cache_bust_target() == target + + +@pytest.mark.parametrize("timing_mode", list(TimingMode)) +def test_cache_bust_none_passes_all_timing_modes(timing_mode: TimingMode) -> None: + """target=NONE never trips construction regardless of timing_mode.""" + cfg = _build( + target=CacheBustTarget.NONE, + endpoint_type=EndpointType.CHAT, + timing_mode=timing_mode, + ) + assert cfg.get_cache_bust_target() == CacheBustTarget.NONE + assert cfg.get_profiling_phases()[0].timing_mode == timing_mode + + +@pytest.mark.parametrize("endpoint_type", list(EndpointType)) +def test_cache_bust_none_passes_all_endpoint_types(endpoint_type: EndpointType) -> None: + """target=NONE never trips construction regardless of endpoint type.""" + cfg = _build( + target=CacheBustTarget.NONE, + endpoint_type=endpoint_type, + timing_mode=TimingMode.AGENTIC_REPLAY, + ) + assert cfg.get_cache_bust_target() == CacheBustTarget.NONE + assert cfg.endpoint.type == endpoint_type def test_unsafe_override_does_not_bypass_cache_bust_endpoint_validation() -> None: - """Same as above but for the endpoint-type branch of the lockdown.""" + """unsafe_override does not bypass endpoint capability validation.""" body: dict[str, Any] = { "models": ["test-model"], "endpoint": { @@ -158,51 +160,9 @@ def test_unsafe_override_does_not_bypass_cache_bust_endpoint_validation() -> Non "type": "concurrency", "concurrency": 1, "requests": 10, - "timing_mode": TimingMode.AGENTIC_REPLAY, + "timing_mode": TimingMode.REQUEST_RATE, } ], } - with pytest.raises(ValueError, match="chat or responses"): + with pytest.raises(ValueError, match=r"not supported|chat or responses"): BenchmarkConfig.model_validate(body) - - -@pytest.mark.parametrize("timing_mode", list(TimingMode)) -def test_cache_bust_none_passes_all_timing_modes(timing_mode: TimingMode) -> None: - """target=NONE never trips construction regardless of timing_mode.""" - cfg = _build( - target=CacheBustTarget.NONE, - endpoint_type=EndpointType.CHAT, - timing_mode=timing_mode, - ) - assert cfg.get_cache_bust_target() == CacheBustTarget.NONE - assert cfg.get_profiling_phases()[0].timing_mode == timing_mode - - -@pytest.mark.parametrize("endpoint_type", list(EndpointType)) -def test_cache_bust_none_passes_all_endpoint_types( - endpoint_type: EndpointType, -) -> None: - """target=NONE never trips construction -- regardless of endpoint_type.""" - cfg = _build( - target=CacheBustTarget.NONE, - endpoint_type=endpoint_type, - timing_mode=TimingMode.AGENTIC_REPLAY, - ) - assert cfg.get_cache_bust_target() == CacheBustTarget.NONE - assert cfg.endpoint.type == endpoint_type - - -@pytest.mark.parametrize("target", _NON_NONE_CACHE_BUST_TARGETS) -@pytest.mark.parametrize("endpoint_type", [EndpointType.CHAT, EndpointType.RESPONSES]) -def test_cache_bust_all_targets_construct_with_chat_endpoint_and_agentic_replay( - target: CacheBustTarget, endpoint_type: EndpointType -) -> None: - """Every non-NONE CacheBustTarget constructs with the compatible agentic_replay + chat/responses combo.""" - cfg = _build( - target=target, - endpoint_type=endpoint_type, - timing_mode=TimingMode.AGENTIC_REPLAY, - ) - assert cfg.get_cache_bust_target() == target - assert cfg.endpoint.type == endpoint_type - assert cfg.get_profiling_phases()[0].timing_mode == TimingMode.AGENTIC_REPLAY diff --git a/tests/unit/endpoints/test_messages_endpoint.py b/tests/unit/endpoints/test_messages_endpoint.py index 278e5c36bc..7df4de526d 100644 --- a/tests/unit/endpoints/test_messages_endpoint.py +++ b/tests/unit/endpoints/test_messages_endpoint.py @@ -7,6 +7,7 @@ from pytest import param from aiperf.common.models import ExtractedPayload, Text, Turn +from aiperf.common.models.model_endpoint_info import ModelEndpointInfo from aiperf.common.models.record_models import ( ReasoningResponseData, SSEField, @@ -171,7 +172,9 @@ def test_empty_turns_raises(self, endpoint, model_endpoint): with pytest.raises(ValueError, match="requires at least one turn"): endpoint.format_payload(request_info) - def test_user_context_message_prepended(self, endpoint, model_endpoint): + def test_user_context_message_prepended( + self, endpoint: MessagesEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: turn = Turn(texts=[Text(contents=["Hello"])], model="claude-sonnet-4-20250514") request_info = create_request_info( model_endpoint=model_endpoint, @@ -185,6 +188,48 @@ def test_user_context_message_prepended(self, endpoint, model_endpoint): assert payload["messages"][0]["content"] == "Context info" assert payload["messages"][0]["role"] == "user" + def test_cache_bust_system_prefix_is_top_level_system( + self, endpoint: MessagesEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + turn = Turn(texts=[Text(contents=["Hello"])]) + request_info = create_request_info( + model_endpoint=model_endpoint, + turns=[turn], + system_message="[rid:abc123]\n\nYou are helpful.", + ) + + payload = endpoint.format_payload(request_info) + + assert payload["system"] == "[rid:abc123]\n\nYou are helpful." + assert payload["messages"] == [{"role": "user", "content": "Hello"}] + + def test_cache_bust_first_user_prefix_renders_anthropic_text_block( + self, endpoint: MessagesEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + turn = Turn( + texts=[ + Text(contents=["[rid:abc123]", "\n\nHello"]), + ] + ) + request_info = create_request_info(model_endpoint=model_endpoint, turns=[turn]) + + payload = endpoint.format_payload(request_info) + + assert payload["messages"][0]["content"] == [ + {"type": "text", "text": "[rid:abc123]"}, + {"type": "text", "text": "\n\nHello"}, + ] + + def test_cache_bust_first_user_suffix_preserves_text( + self, endpoint: MessagesEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + turn = Turn(texts=[Text(contents=["Hello\n\n[rid:abc123]"])]) + request_info = create_request_info(model_endpoint=model_endpoint, turns=[turn]) + + payload = endpoint.format_payload(request_info) + + assert payload["messages"][0]["content"] == "Hello\n\n[rid:abc123]" + class TestAnthropicMessagesHeaders: """Tests for MessagesEndpoint get_endpoint_headers.""" diff --git a/tests/unit/timing/strategies/_shared_helpers.py b/tests/unit/timing/strategies/_shared_helpers.py index 3349e59ccd..018baced9b 100644 --- a/tests/unit/timing/strategies/_shared_helpers.py +++ b/tests/unit/timing/strategies/_shared_helpers.py @@ -12,6 +12,7 @@ DatasetMetadata, TurnMetadata, ) +from aiperf.config import BenchmarkRun from aiperf.credit.structs import Credit from aiperf.dataset.dataset_samplers import SequentialSampler from aiperf.plugin.enums import DatasetSamplingStrategy @@ -73,7 +74,9 @@ def _make_credit( ) -def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench-fixed"): +def _make_run( + *, target: CacheBustTarget, benchmark_id: str = "bench-fixed" +) -> BenchmarkRun: """Build a v2 ``BenchmarkRun`` exposing the cache-bust target (on the synthetic dataset's ``prompts.cache_bust.target``) and ``benchmark_id`` the strategy reads.""" from aiperf.config import BenchmarkConfig, BenchmarkRun @@ -81,8 +84,8 @@ def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench-fixed"): { "models": ["test-model"], "endpoint": { - "type": "completions", - "urls": ["http://localhost:8000/v1"], + "type": "chat", + "urls": ["http://localhost:8000/v1/chat/completions"], "streaming": False, }, "datasets": [ diff --git a/tests/unit/timing/strategies/test_agentic_replay.py b/tests/unit/timing/strategies/test_agentic_replay.py index 44215ba556..32d1f4e5bc 100644 --- a/tests/unit/timing/strategies/test_agentic_replay.py +++ b/tests/unit/timing/strategies/test_agentic_replay.py @@ -22,6 +22,7 @@ TurnMetadata, ) from aiperf.common.scenario.base import TrajectoryWarmupFailedError +from aiperf.config import BenchmarkRun from aiperf.credit.dispatch import ChildDispatchResult, TurnAdmission from aiperf.credit.structs import Credit, TurnToSend from aiperf.dataset.dataset_samplers import SequentialSampler @@ -3122,7 +3123,9 @@ async def test_warmup_skips_mark_sending_complete_when_already_complete(): _RID_RE = re.compile(r"\[rid:[0-9a-f]{12}\]") -def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench-fixed"): +def _make_run( + *, target: CacheBustTarget, benchmark_id: str = "bench-fixed" +) -> BenchmarkRun: """Build a v2 ``BenchmarkRun`` exposing the cache-bust target (on the synthetic dataset's ``prompts.cache_bust.target``) and ``benchmark_id`` the strategy reads.""" from aiperf.config import BenchmarkConfig, BenchmarkRun @@ -3130,8 +3133,8 @@ def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench-fixed"): { "models": ["test-model"], "endpoint": { - "type": "completions", - "urls": ["http://localhost:8000/v1"], + "type": "chat", + "urls": ["http://localhost:8000/v1/chat/completions"], "streaming": False, }, "datasets": [ diff --git a/tests/unit/timing/strategies/test_agentic_replay_marker_uniqueness.py b/tests/unit/timing/strategies/test_agentic_replay_marker_uniqueness.py index 292512eb3d..f9e3593351 100644 --- a/tests/unit/timing/strategies/test_agentic_replay_marker_uniqueness.py +++ b/tests/unit/timing/strategies/test_agentic_replay_marker_uniqueness.py @@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock from aiperf.common.enums import CacheBustTarget, CreditPhase +from aiperf.config import BenchmarkRun from aiperf.timing.strategies.agentic_replay import AgenticReplayStrategy from aiperf.timing.trajectory_source import Trajectory, TrajectorySource from tests.unit.timing.strategies._shared_helpers import _make_dataset @@ -34,7 +35,9 @@ def _build_real_trajectory_source( return src -def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench-uniqueness"): +def _make_run( + *, target: CacheBustTarget, benchmark_id: str = "bench-uniqueness" +) -> BenchmarkRun: """Build a v2 ``BenchmarkRun`` exposing the values the strategy reads.""" from aiperf.config import BenchmarkConfig, BenchmarkRun @@ -42,8 +45,8 @@ def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench-uniqueness" { "models": ["test-model"], "endpoint": { - "type": "completions", - "urls": ["http://localhost:8000/v1"], + "type": "chat", + "urls": ["http://localhost:8000/v1/chat/completions"], "streaming": False, }, "datasets": [ diff --git a/tests/unit/timing/strategies/test_agentic_replay_recycle_adversarial.py b/tests/unit/timing/strategies/test_agentic_replay_recycle_adversarial.py index 0c70ba05c7..e75e6993a5 100644 --- a/tests/unit/timing/strategies/test_agentic_replay_recycle_adversarial.py +++ b/tests/unit/timing/strategies/test_agentic_replay_recycle_adversarial.py @@ -15,6 +15,7 @@ DatasetMetadata, TurnMetadata, ) +from aiperf.config import BenchmarkRun from aiperf.credit.structs import Credit from aiperf.plugin.enums import DatasetSamplingStrategy from aiperf.timing.strategies.agentic_replay import AgenticReplayStrategy @@ -67,7 +68,9 @@ def _make_strategy( return strategy, issuer, stop_checker -def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench_test"): +def _make_run( + *, target: CacheBustTarget, benchmark_id: str = "bench_test" +) -> BenchmarkRun: """Build a v2 ``BenchmarkRun`` exposing the values the strategy reads.""" from aiperf.config import BenchmarkConfig, BenchmarkRun @@ -75,8 +78,8 @@ def _make_run(*, target: CacheBustTarget, benchmark_id: str = "bench_test"): { "models": ["test-model"], "endpoint": { - "type": "completions", - "urls": ["http://localhost:8000/v1"], + "type": "chat", + "urls": ["http://localhost:8000/v1/chat/completions"], "streaming": False, }, "datasets": [ diff --git a/tests/unit/timing/test_conversation_source.py b/tests/unit/timing/test_conversation_source.py index 619c6dce99..d63554586d 100644 --- a/tests/unit/timing/test_conversation_source.py +++ b/tests/unit/timing/test_conversation_source.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from aiperf.common.enums import CacheBustTarget from aiperf.common.models import ConversationMetadata, DatasetMetadata, TurnMetadata from aiperf.plugin import plugins from aiperf.plugin.enums import DatasetSamplingStrategy, PluginType @@ -9,12 +10,21 @@ from tests.unit.timing.conftest import make_credit -def _mk_source(ds: DatasetMetadata) -> ConversationSource: +def _mk_source( + ds: DatasetMetadata, + *, + cache_bust_target: CacheBustTarget = CacheBustTarget.NONE, +) -> ConversationSource: SamplerClass = plugins.get_class(PluginType.DATASET_SAMPLER, ds.sampling_strategy) sampler = SamplerClass( conversation_ids=[c.conversation_id for c in ds.conversations], ) - return ConversationSource(ds, sampler) + return ConversationSource( + ds, + sampler, + benchmark_id="test-benchmark", + cache_bust_target=cache_bust_target, + ) @pytest.fixture @@ -70,6 +80,52 @@ def test_get_metadata_raises_for_invalid(self, src): with pytest.raises(KeyError, match="No metadata for conversation bad"): src.get_metadata("bad") + def test_cache_bust_marker_is_unique_for_consecutive_sessions( + self, ds: DatasetMetadata + ) -> None: + src = _mk_source(ds, cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX) + first = src.next(x_correlation_id="session-1") + second = src.next(x_correlation_id="session-2") + + assert first.cache_bust_marker is not None + assert first.cache_bust_marker != second.cache_bust_marker + + def test_cache_bust_marker_propagates_through_build_first_turn( + self, ds: DatasetMetadata + ) -> None: + src = _mk_source(ds, cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX) + session = src.next(x_correlation_id="session-1") + + assert session.build_first_turn().cache_bust_marker == session.cache_bust_marker + + def test_cache_bust_target_propagates_through_build_turn_at_index( + self, ds: DatasetMetadata + ) -> None: + src = _mk_source(ds, cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX) + session = src.next(x_correlation_id="session-1") + + assert ( + session.build_turn_at_index(0).cache_bust_target + == CacheBustTarget.FIRST_TURN_PREFIX + ) + + def test_cache_bust_marker_is_stable_for_explicit_session_id( + self, ds: DatasetMetadata + ) -> None: + src = _mk_source(ds, cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX) + first = src.session_for_conversation("c1", x_correlation_id="session-1") + second = src.session_for_conversation("c2", x_correlation_id="session-1") + assert first.cache_bust_marker == second.cache_bust_marker + + def test_cache_bust_disabled_does_not_retain_session_markers( + self, ds: DatasetMetadata + ) -> None: + src = _mk_source(ds) + + src.next(x_correlation_id="session-1") + + assert src._cache_bust_markers == {} + class TestMultiTurn: @pytest.fixture diff --git a/tests/unit/timing/test_dag_cross_component.py b/tests/unit/timing/test_dag_cross_component.py index 85bb92655e..0f662c9000 100644 --- a/tests/unit/timing/test_dag_cross_component.py +++ b/tests/unit/timing/test_dag_cross_component.py @@ -11,6 +11,7 @@ import pytest from aiperf.common.enums import ( + CacheBustTarget, ConversationBranchMode, CreditPhase, PrerequisiteKind, @@ -557,6 +558,34 @@ async def test_spawn_child_does_not_call_register_child_routing(): sticky.release_child_routing.assert_not_called() +@pytest.mark.asyncio +async def test_spawn_child_inherits_parent_cache_bust_state(): + """A non-agentic SPAWN child preserves its parent's cache-bust state.""" + branch = ConversationBranchInfo( + branch_id="root:0", + child_conversation_ids=["child"], + mode=ConversationBranchMode.SPAWN, + ) + root = _mk_conv_meta("root", [TurnMetadata(branch_ids=["root:0"])], [branch]) + cs = _mk_source([root, _mk_conv_meta("child", [TurnMetadata()], [])]) + issuer = _mk_issuer() + orch = BranchOrchestrator( + conversation_source=cs, + credit_issuer=issuer, + cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX, + ) + credit = _mk_credit("root", "corr-root", 0) + credit.effective_root_correlation_id = "corr-root" + credit.cache_bust_marker = "[rid:parent]" + credit.cache_bust_target = CacheBustTarget.FIRST_TURN_PREFIX + + await orch.intercept(credit) + + kwargs = cs.start_branch_child.call_args.kwargs + assert kwargs["cache_bust_marker"] == "[rid:parent]" + assert kwargs["cache_bust_target"] == CacheBustTarget.FIRST_TURN_PREFIX + + def test_pre_session_child_routing_key_falls_back_to_own_correlation(): """Pre-session children have parent_correlation_id=None, so routing_key falls back to x_correlation_id (no sticky pin to a non-existent parent).""" from aiperf.timing.conversation_source import SampledSession diff --git a/tests/unit/workers/test_worker_cache_bust_injection.py b/tests/unit/workers/test_worker_cache_bust_injection.py index 74dd984e54..de6e47972c 100644 --- a/tests/unit/workers/test_worker_cache_bust_injection.py +++ b/tests/unit/workers/test_worker_cache_bust_injection.py @@ -251,6 +251,134 @@ def test_system_prefix_uses_existing_raw_system_role_when_no_conversation_system assert msgs[1]["content"] == "hi" +def _make_raw_system_session( + raw_system: list[dict], + *, + system_message: str | None = None, + raw_messages: list[dict] | None = None, +) -> UserSession: + """Session whose dispatch turn carries a vendor-shaped ``raw_system``.""" + turn = Turn(raw_system=raw_system, raw_messages=raw_messages) + conversation = Conversation( + session_id="conv_test", turns=[turn], system_message=system_message + ) + return UserSession( + x_correlation_id="xcorr_test", + num_turns=1, + conversation=conversation, + turn_list=[turn], + ) + + +@pytest.mark.parametrize( + "target,marker,expected", + [ + param( + CacheBustTarget.SYSTEM_PREFIX, + _PREFIX_MARKER, + [ + {"type": "text", "text": _PREFIX_MARKER.strip()}, + {"type": "text", "text": "sys"}, + ], + id="prefix", + ), + param( + CacheBustTarget.SYSTEM_SUFFIX, + _SUFFIX_MARKER, + [ + {"type": "text", "text": "sys"}, + {"type": "text", "text": _SUFFIX_MARKER.strip()}, + ], + id="suffix", + ), + ], +) # fmt: skip +def test_system_target_injects_into_raw_system_blocks(target, marker, expected): + """``raw_system`` is what MessagesEndpoint ships, so the marker lands there.""" + session = _make_raw_system_session([{"type": "text", "text": "sys"}]) + credit = _make_credit(target=target, marker=marker, turn_index=0) + + _apply_cache_bust(session, credit, system_message=None) + + assert session.turn_list[-1].raw_system == expected + + +def test_raw_system_outranks_conversation_system_message(): + """``raw_system`` wins over ``system_message`` on the wire, so marking the + string instead would ship unchanged system bytes.""" + session = _make_raw_system_session( + [{"type": "text", "text": "sys"}], system_message="ignored-on-wire" + ) + credit = _make_credit( + target=CacheBustTarget.SYSTEM_PREFIX, marker=_PREFIX_MARKER, turn_index=0 + ) + + out = _apply_cache_bust(session, credit, system_message="ignored-on-wire") + + assert out == "ignored-on-wire" + assert session.turn_list[-1].raw_system[0] == { + "type": "text", + "text": _PREFIX_MARKER.strip(), + } + + +def test_raw_system_outranks_system_role_raw_messages(): + session = _make_raw_system_session( + [{"type": "text", "text": "sys"}], + raw_messages=[ + {"role": "system", "content": "sys-msg"}, + {"role": "user", "content": "hi"}, + ], + ) + credit = _make_credit( + target=CacheBustTarget.SYSTEM_PREFIX, marker=_PREFIX_MARKER, turn_index=0 + ) + + _apply_cache_bust(session, credit, system_message=None) + + assert session.turn_list[-1].raw_system[0]["text"] == _PREFIX_MARKER.strip() + msgs = session.turn_list[-1].raw_messages + assert msgs[0]["content"] == "sys-msg" + assert msgs[1]["content"] == "hi" + + +def test_raw_system_injection_is_idempotent_across_credits(): + """The turn object is shared across a session's credits; re-injecting the + constant per-session marker must not stack it.""" + session = _make_raw_system_session([{"type": "text", "text": "sys"}]) + credit = _make_credit( + target=CacheBustTarget.SYSTEM_PREFIX, marker=_PREFIX_MARKER, turn_index=0 + ) + + _apply_cache_bust(session, credit, system_message=None) + _apply_cache_bust(session, credit, system_message=None) + + assert session.turn_list[-1].raw_system == [ + {"type": "text", "text": _PREFIX_MARKER.strip()}, + {"type": "text", "text": "sys"}, + ] + + +def test_raw_system_injection_preserves_cache_control_blocks(): + """The marker becomes its own block, so an authored ``cache_control`` + breakpoint stays attached to the bytes it was written against.""" + blocks = [ + {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}, + ] + session = _make_raw_system_session(blocks) + credit = _make_credit( + target=CacheBustTarget.SYSTEM_PREFIX, marker=_PREFIX_MARKER, turn_index=0 + ) + + _apply_cache_bust(session, credit, system_message=None) + + assert session.turn_list[-1].raw_system[1] == { + "type": "text", + "text": "sys", + "cache_control": {"type": "ephemeral"}, + } + + def test_system_prefix_fallback_marks_first_user_on_turn_index_gt_zero(): """SYSTEM_PREFIX with no system anywhere falls back to the first user turn and injects every credit (seeded-resume fix).""" raw = [{"role": "user", "content": "hi"}] From 2eb30b4091c9c19438c7b26d2f96c05ad04314d6 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:54:20 -0700 Subject: [PATCH 004/119] fix(dag): update BranchStats test expectations for graphs_admitted/graphs_completed_to_end (#1238) Signed-off-by: Elias Bermudez (cherry picked from commit f8c8e365335952174a5036dcb59202dbd1b7e0ee) --- tests/unit/records/test_dag_metadata_tagging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/records/test_dag_metadata_tagging.py b/tests/unit/records/test_dag_metadata_tagging.py index 6b875f2d82..699ee04236 100644 --- a/tests/unit/records/test_dag_metadata_tagging.py +++ b/tests/unit/records/test_dag_metadata_tagging.py @@ -137,6 +137,8 @@ def test_branch_stats_defaults_all_zero(self): "joins_suppressed": 0, "children_truncated": 0, "children_delayed": 0, + "graphs_admitted": 0, + "graphs_completed_to_end": 0, } def test_branch_stats_dict_helper(self): @@ -157,6 +159,8 @@ def test_branch_stats_dict_helper(self): "joins_suppressed": 0, "children_truncated": 0, "children_delayed": 0, + "graphs_admitted": 0, + "graphs_completed_to_end": 0, } def test_branch_stats_roundtrip_through_profile_results(self): From 86bd64bac71631ae74c6059031fe7e1c9ca0ef14 Mon Sep 17 00:00:00 2001 From: Shiwei Ma <80623564+shm197@users.noreply.github.com> Date: Sat, 1 Aug 2026 08:57:53 +0800 Subject: [PATCH 005/119] fix(search): evaluate error-rate SLA on average percentage (#1155) Signed-off-by: Shiwei Ma Co-authored-by: Shiwei Ma (cherry picked from commit 5a24e0d7c791b2468db595ec8490551105d0e488) --- docs/api/search-history.md | 4 +- docs/cli-options.md | 8 ++-- docs/sweeping/bayesian-optimization.md | 12 +++-- src/aiperf/config/flags/cli_config.py | 11 +++-- .../types/request_error_rate_metric.py | 22 ++++----- .../orchestrator/search_planner/parsing.py | 6 +-- .../_max_concurrency_under_sla.py | 6 ++- tests/integration/test_search_recipes.py | 46 +++++++++++++++++++ .../metrics/test_request_error_rate_metric.py | 8 +++- .../test_max_concurrency_under_sla.py | 14 +++++- 10 files changed, 103 insertions(+), 34 deletions(-) diff --git a/docs/api/search-history.md b/docs/api/search-history.md index 719d066d2f..e4681ae704 100644 --- a/docs/api/search-history.md +++ b/docs/api/search-history.md @@ -73,7 +73,7 @@ A snapshot of the adaptive-search configuration fields that the v1 writer persis {"metric": "time_to_first_token", "stat": "p99", "direction": "MINIMIZE", "threshold": 250.0} ], "outcome_constraints": [ - {"metric": "request_error_rate", "op": "<=", "bound": 0.01} + {"metric": "request_error_rate", "op": "<=", "bound": 1.0} ], "max_iterations": 30, "n_initial_points": 5, @@ -104,7 +104,7 @@ A snapshot of the adaptive-search configuration fields that the v1 writer persis | `outcome_constraints` | array<object> | yes | Feasibility gates on metrics the optimizer is **not** optimizing. Empty list = no constraints. Distinct from `objectives[].threshold` (Pareto reference point) and from `sla_filters` (post-hoc benchmark eligibility): outcome constraints down-weight infeasible candidates inside BoTorch's acquisition function. | | `outcome_constraints[].metric` | string | yes | Metric tag to constrain. | | `outcome_constraints[].op` | string | yes | Comparison operator. One of `"<="`, `">="`, `"=="`. **Distinct from `sla_filters[].op`**, which uses the lowercase mnemonics `lt`/`le`/`gt`/`ge` — outcome constraints feed BoTorch's acquisition mask, SLA filters feed post-hoc feasibility ranking. | -| `outcome_constraints[].bound` | float | yes | Threshold value. | +| `outcome_constraints[].bound` | float | yes | Threshold in the constrained metric's native unit. For `request_error_rate`, `1.0` means 1%. | | `max_iterations` | int | yes | Iteration budget. The loop also stops earlier on convergence. | | `n_initial_points` | int | yes | Sobol-random points before BoTorch fits the GP. Validator enforces `n_initial_points < max_iterations` for `bayesian` and `optuna` planners only; `monotonic_sla` / `smooth_isotonic` planners drive their own probe sequence and ignore this field. | | `random_seed` | int \| null | yes | Reproducibility seed passed to the planner backend. `null` when the run was unseeded. | diff --git a/docs/cli-options.md b/docs/cli-options.md index 29214644f4..ba6a6274c1 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -1516,7 +1516,7 @@ Repeatable: each occurrence describes one sweep variation. Format: '[name:] key= #### `--search-sla` `` -SLA filter to attach to the adaptive-search or grid path. Format: 'metric_tag:stat:op:threshold'. Stat in {avg, min, max, p1, p5, p10, p25, p50, p75, p90, p95, p99}; op in {lt, le, gt, ge}; threshold is a float. Repeatable. Example: --search-sla 'time_to_first_token:p95:lt:200' --search-sla 'request_error_rate:p99:lt:0.05'. Composes with recipe-named SLA flags (--ttft-sla-ms etc.); the final filter list is recipe filters first, then --search-sla filters in CLI order. +SLA filter to attach to the adaptive-search or grid path. Format: 'metric_tag:stat:op:threshold'. Stat in {avg, min, max, p1, p5, p10, p25, p50, p75, p90, p95, p99}; op in {lt, le, gt, ge}; threshold is a float in the metric's native unit. For request_error_rate, 1 means 1% (percentage points), unlike --error-rate-sla, which accepts a fraction. Repeatable. Example: --search-sla 'time_to_first_token:p95:lt:200' --search-sla 'request_error_rate:avg:lt:1'. Composes with recipe-named SLA flags (--ttft-sla-ms etc.); the final filter list is recipe filters first, then --search-sla filters in CLI order. #### `--search-sla-tier` `` @@ -1552,7 +1552,7 @@ End-to-end request-latency SLA threshold in milliseconds (p99). Maps to the `req #### `--error-rate-sla` `` -Maximum acceptable request error rate as a fraction in (0, 1) (e.g. 0.05 = 5%). Maps to the `request_error_rate` metric tag (p99). Consumed by the max-concurrency-under-sla recipe; ignored otherwise. Available without streaming. +Maximum acceptable request error rate as a fraction in (0, 1) (e.g. 0.05 = 5%). Maps to the `request_error_rate` metric tag (avg), converting the fraction to the metric's percentage-point scale. Consumed by the max-concurrency-under-sla recipe; ignored otherwise. Available without streaming.
_Constraints: > 0, < 1_ #### `--slo-attainment-fraction` `` @@ -3051,7 +3051,7 @@ Repeatable: each occurrence describes one sweep variation. Format: '[name:] key= #### `--search-sla` `` -SLA filter to attach to the adaptive-search or grid path. Format: 'metric_tag:stat:op:threshold'. Stat in {avg, min, max, p1, p5, p10, p25, p50, p75, p90, p95, p99}; op in {lt, le, gt, ge}; threshold is a float. Repeatable. Example: --search-sla 'time_to_first_token:p95:lt:200' --search-sla 'request_error_rate:p99:lt:0.05'. Composes with recipe-named SLA flags (--ttft-sla-ms etc.); the final filter list is recipe filters first, then --search-sla filters in CLI order. +SLA filter to attach to the adaptive-search or grid path. Format: 'metric_tag:stat:op:threshold'. Stat in {avg, min, max, p1, p5, p10, p25, p50, p75, p90, p95, p99}; op in {lt, le, gt, ge}; threshold is a float in the metric's native unit. For request_error_rate, 1 means 1% (percentage points), unlike --error-rate-sla, which accepts a fraction. Repeatable. Example: --search-sla 'time_to_first_token:p95:lt:200' --search-sla 'request_error_rate:avg:lt:1'. Composes with recipe-named SLA flags (--ttft-sla-ms etc.); the final filter list is recipe filters first, then --search-sla filters in CLI order. #### `--search-sla-tier` `` @@ -3087,7 +3087,7 @@ End-to-end request-latency SLA threshold in milliseconds (p99). Maps to the `req #### `--error-rate-sla` `` -Maximum acceptable request error rate as a fraction in (0, 1) (e.g. 0.05 = 5%). Maps to the `request_error_rate` metric tag (p99). Consumed by the max-concurrency-under-sla recipe; ignored otherwise. Available without streaming. +Maximum acceptable request error rate as a fraction in (0, 1) (e.g. 0.05 = 5%). Maps to the `request_error_rate` metric tag (avg), converting the fraction to the metric's percentage-point scale. Consumed by the max-concurrency-under-sla recipe; ignored otherwise. Available without streaming.
_Constraints: > 0, < 1_ #### `--slo-attainment-fraction` `` diff --git a/docs/sweeping/bayesian-optimization.md b/docs/sweeping/bayesian-optimization.md index f448706941..f94ddb8b1b 100644 --- a/docs/sweeping/bayesian-optimization.md +++ b/docs/sweeping/bayesian-optimization.md @@ -277,7 +277,7 @@ sweep: - {metric: output_token_throughput, stat: avg, direction: maximize} - {metric: time_to_first_token, stat: p99, direction: minimize, threshold: 250.0} outcome_constraints: - - {metric: request_error_rate, op: "<=", bound: 0.01} + - {metric: request_error_rate, op: "<=", bound: 1.0} max_iterations: 30 ``` @@ -445,16 +445,18 @@ The four issue-named SLA flags are sugar over the generic `--search-sla` syntax. | `--ttft-sla-ms` | `time_to_first_token` | `p95` | `lt` | Streaming required | | `--tpot-sla-ms` (a.k.a. `--itl-sla-ms`) | `inter_token_latency` | `p95` | `lt` | Streaming required; TPOT == ITL in AIPerf metric tags | | `--e2e-sla-ms` | `request_latency` | `p99` | `lt` | | -| `--error-rate-sla` | `request_error_rate` | `p99` | `lt` | Fraction in `[0, 1]` | -| `--search-sla "TAG:STAT:OP:THRESHOLD"` | any | any of `{avg, p50, p90, p95, p99}` | any of `{lt, le, gt, ge}` | Repeatable; format is strict colon-delimited 4-tuple | +| `--error-rate-sla` | `request_error_rate` | `avg` | `lt` | Fraction in `(0, 1)`; converted to percentage points for comparison | +| `--search-sla "TAG:STAT:OP:THRESHOLD"` | any | any of `{avg, p50, p90, p95, p99}` | any of `{lt, le, gt, ge}` | Repeatable; threshold uses the metric's native unit; format is strict colon-delimited 4-tuple | + +Generic `--search-sla` thresholds use each metric's native unit. In particular, `request_error_rate` is exported in percentage points, so `request_error_rate:avg:lt:1` means an average error rate below 1%. This differs from the recipe-specific `--error-rate-sla`, which accepts a fraction (`0.01` means 1%) and converts it to percentage points. ```bash -# Compose: TTFT p95 < 200ms AND error rate p99 < 1%, on the explicit +# Compose: TTFT p95 < 200ms AND average error rate < 1%, on the explicit # --search-space path (no recipe). aiperf profile --model my-model --streaming \ --search-space "concurrency:1,1000:int" \ --search-sla "time_to_first_token:p95:lt:200" \ - --search-sla "request_error_rate:p99:lt:0.01" \ + --search-sla "request_error_rate:avg:lt:1" \ --search-metric output_token_throughput --search-direction maximize \ --search-max-iterations 30 ``` diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 6fe39b5765..706f02bb9a 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -3493,10 +3493,12 @@ def url(self) -> str: "SLA filter to attach to the adaptive-search or grid path. " "Format: 'metric_tag:stat:op:threshold'. Stat in " "{avg, min, max, p1, p5, p10, p25, p50, p75, p90, p95, p99}; " - "op in {lt, le, gt, ge}; threshold is " - "a float. Repeatable. Example: --search-sla " + "op in {lt, le, gt, ge}; threshold is a float in the metric's " + "native unit. For request_error_rate, 1 means 1% (percentage " + "points), unlike --error-rate-sla, which accepts a fraction. " + "Repeatable. Example: --search-sla " "'time_to_first_token:p95:lt:200' --search-sla " - "'request_error_rate:p99:lt:0.05'. Composes with recipe-named " + "'request_error_rate:avg:lt:1'. Composes with recipe-named " "SLA flags (--ttft-sla-ms etc.); the final filter list is " "recipe filters first, then --search-sla filters in CLI order." ), @@ -3643,7 +3645,8 @@ def url(self) -> str: description=( "Maximum acceptable request error rate as a fraction in (0, 1) " "(e.g. 0.05 = 5%). Maps to the `request_error_rate` metric tag " - "(p99). Consumed by the max-concurrency-under-sla recipe; ignored " + "(avg), converting the fraction to the metric's percentage-point " + "scale. Consumed by the max-concurrency-under-sla recipe; ignored " "otherwise. Available without streaming." ), ), diff --git a/src/aiperf/metrics/types/request_error_rate_metric.py b/src/aiperf/metrics/types/request_error_rate_metric.py index 27dae4c67c..717201ac15 100644 --- a/src/aiperf/metrics/types/request_error_rate_metric.py +++ b/src/aiperf/metrics/types/request_error_rate_metric.py @@ -12,11 +12,16 @@ class RequestErrorRateMetric(BaseDerivedMetric[float]): """Percentage of completed requests that ended in error. - Reads :class:`ErrorRequestCountMetric` and :class:`RequestCountMetric` - so latency percentiles (computed on successes only) can be read alongside - the operational error rate. Pair with the ``adj_*`` percentile band on - flagged latency metrics (see ``MetricFlags.PERCENTILE_INCLUDES_FAILED_REQUESTS``) - for a full picture of failure-contaminated tail behavior. + Uses :class:`RequestCountMetric` for valid requests and + :class:`ErrorRequestCountMetric` for failed requests. Because + ``ErrorRequestCountMetric`` carries ``MetricFlags.ERROR_ONLY``, clean runs + omit it instead of emitting zero; an omitted error count is therefore treated + as zero. + + Latency percentiles are computed on successful requests only. Read this metric + alongside the ``adj_*`` percentile band on metrics flagged with + ``MetricFlags.PERCENTILE_INCLUDES_FAILED_REQUESTS`` for a fuller picture of + failure-contaminated tail behavior. See https://github.com/ai-dynamo/aiperf/issues/688. """ @@ -28,12 +33,7 @@ class RequestErrorRateMetric(BaseDerivedMetric[float]): unit = GenericMetricUnit.PERCENT display_order = 1080 flags = MetricFlags.NO_INDIVIDUAL_RECORDS - required_metrics = frozenset( - { - RequestCountMetric.tag, - ErrorRequestCountMetric.tag, - } - ) + required_metrics = frozenset({RequestCountMetric.tag}) def _derive_value(self, metric_results: MetricResultsDict) -> float: successes = int(metric_results.get_or_raise(RequestCountMetric)) diff --git a/src/aiperf/orchestrator/search_planner/parsing.py b/src/aiperf/orchestrator/search_planner/parsing.py index dc05db12b7..99faaa251c 100644 --- a/src/aiperf/orchestrator/search_planner/parsing.py +++ b/src/aiperf/orchestrator/search_planner/parsing.py @@ -108,9 +108,9 @@ def parse_sla_filter(value: str) -> SLAFilter: parse_sla_filter("time_to_first_token:p95:lt:200") # -> SLAFilter(metric_tag='time_to_first_token', stat='p95', # op='lt', threshold=200.0) - parse_sla_filter("request_error_rate:p99:lt:0.05") - # -> SLAFilter(metric_tag='request_error_rate', stat='p99', - # op='lt', threshold=0.05) + parse_sla_filter("request_error_rate:avg:lt:1") + # -> SLAFilter(metric_tag='request_error_rate', stat='avg', + # op='lt', threshold=1.0) Raises: TypeError: when the value has the wrong colon count, an unknown stat diff --git a/src/aiperf/search_recipes/_max_concurrency_under_sla.py b/src/aiperf/search_recipes/_max_concurrency_under_sla.py index 2fe8463166..e7bbf88a41 100644 --- a/src/aiperf/search_recipes/_max_concurrency_under_sla.py +++ b/src/aiperf/search_recipes/_max_concurrency_under_sla.py @@ -182,9 +182,11 @@ def _build_sla_filters(self, ctx: SearchRecipeContext) -> list[SLAFilter]: filters.append( SLAFilter( metric_tag="request_error_rate", - stat="p99", + # request_error_rate is a run-level percentage scalar, so + # it only populates avg. The CLI flag remains a fraction. + stat="avg", op="lt", - threshold=float(err), + threshold=float(err) * 100.0, ) ) return filters diff --git a/tests/integration/test_search_recipes.py b/tests/integration/test_search_recipes.py index d4a1d01da6..f5ecf7e05c 100644 --- a/tests/integration/test_search_recipes.py +++ b/tests/integration/test_search_recipes.py @@ -286,6 +286,52 @@ async def test_max_concurrency_under_sla_finds_boundary( if feasible_max is not None: assert feasible_max["value"] >= 1, history + @pytest.mark.slow + async def test_max_concurrency_error_rate_sla_accepts_zero_error_runs( + self, + cli: AIPerfCLI, + mock_server_factory, + temp_output_dir: Path, + ) -> None: + async with mock_server_factory(error_rate=0.0, fast=True, workers=1) as server: + await cli.run( + f""" + aiperf profile + --model mock-model + --url {server.url} + --endpoint-type chat + --search-recipe max-concurrency-under-sla + --search-style monotonic + --error-rate-sla 0.15 + --concurrency-min 1 + --concurrency-max 2 + --search-max-iterations 4 + --request-count 4 + --warmup-request-count 1 + --synthetic-input-tokens-mean 8 + --synthetic-input-tokens-stddev 0 + --output-tokens-mean 4 + --output-tokens-stddev 0 + --extra-inputs ignore_eos:true + --ui none + """, + timeout=300.0, + ) + + exports = list(temp_output_dir.glob("**/profile_export_aiperf.json")) + assert exports, "adaptive search did not emit profile JSON artifacts" + for export_path in exports: + export = orjson.loads(export_path.read_bytes()) + assert export["request_error_rate"]["avg"] == 0.0, export_path + + history_path = temp_output_dir / "search_history.json" + assert history_path.exists(), "adaptive search did not emit search_history.json" + history = orjson.loads(history_path.read_bytes()) + assert history["iterations"], history + assert all(iteration["feasible"] is True for iteration in history["iterations"]) + assert history["boundary_summary"]["feasible_max"] is not None, history + assert history["convergence_reason"] != "monotonic_no_pass_in_range", history + @pytest.mark.slow async def test_max_goodput_under_slo_lands_near_collapse_point( self, diff --git a/tests/unit/metrics/test_request_error_rate_metric.py b/tests/unit/metrics/test_request_error_rate_metric.py index f1ba2da97b..ce34a8786a 100644 --- a/tests/unit/metrics/test_request_error_rate_metric.py +++ b/tests/unit/metrics/test_request_error_rate_metric.py @@ -26,6 +26,12 @@ def test_error_rate_zero_errors(self): value = RequestErrorRateMetric().derive_value(results) assert value == approx(0.0) + def test_error_rate_missing_error_count_treated_as_zero(self): + results = MetricResultsDict() + results[RequestCountMetric.tag] = 100 + value = RequestErrorRateMetric().derive_value(results) + assert value == approx(0.0) + def test_error_rate_none_error_value_treated_as_zero(self): """``.get(..., 0) or 0`` defends against an explicit None value.""" results = MetricResultsDict() @@ -57,5 +63,5 @@ def test_error_rate_missing_request_count_raises(self): def test_error_rate_required_metrics_declared(self): assert RequestErrorRateMetric.required_metrics == frozenset( - {RequestCountMetric.tag, ErrorRequestCountMetric.tag} + {RequestCountMetric.tag} ) diff --git a/tests/unit/search_recipes/test_max_concurrency_under_sla.py b/tests/unit/search_recipes/test_max_concurrency_under_sla.py index 76680473f8..5127fd73b0 100644 --- a/tests/unit/search_recipes/test_max_concurrency_under_sla.py +++ b/tests/unit/search_recipes/test_max_concurrency_under_sla.py @@ -133,7 +133,11 @@ def test_max_concurrency_under_sla_multi_filter_composition() -> None: by_tag = {f.metric_tag: f for f in filters} assert by_tag["time_to_first_token"].threshold == 200.0 assert by_tag["request_latency"].threshold == 1000.0 - assert by_tag["request_error_rate"].threshold == 0.05 + error_rate = by_tag["request_error_rate"] + assert error_rate.stat == "avg" + # --error-rate-sla is a fraction, while request_error_rate is exported as + # percentage points. + assert error_rate.threshold == 5.0 def test_max_concurrency_under_sla_streaming_required_for_ttft_filter() -> None: @@ -151,7 +155,13 @@ def test_max_concurrency_under_sla_streaming_not_required_for_e2e_only() -> None ) ) assert out.adaptive_search is not None - assert len(out.adaptive_search.sla_filters) == 2 + filters = {f.metric_tag: f for f in out.adaptive_search.sla_filters} + assert len(filters) == 2 + assert filters["request_latency"].threshold == 1000.0 + error_rate = filters["request_error_rate"] + assert error_rate.stat == "avg" + assert error_rate.op == "lt" + assert error_rate.threshold == 1.0 def test_max_concurrency_under_sla_unknown_style_raises() -> None: From eca4c7cbe6813ef1c273d35506261564e7656d91 Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:10:58 -0700 Subject: [PATCH 006/119] fix(endpoints): handle top-level JSON array responses (#1195) Signed-off-by: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 093038afab9a97ccee5fb1ec21171aa9c02ca1f5) --- src/aiperf/endpoints/base_endpoint.py | 13 ++++++- tests/unit/endpoints/test_raw_endpoint.py | 45 ++++++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index 18c9b30682..5405d65c43 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -435,18 +435,27 @@ def make_text_response_data(text: str | None) -> TextResponseData | None: """Make a TextResponseData object from a string or return None if the text is empty.""" return TextResponseData(text=text) if text else None - def auto_detect_and_extract(self, json_obj: dict) -> BaseResponseData | None: + def auto_detect_and_extract(self, json_obj: Any) -> BaseResponseData | None: """Optional utility: Auto-detect response type and extract relevant data. Tries to extract data in this order: embeddings, rankings, text. Endpoints can use this as a fallback or for flexible response handling. + A JSON body is not necessarily an object: a top-level array (e.g. a + reranker returning ``[{"index": 0, "score": 0.98}, ...]``) is valid + JSON and reaches this method as a ``list``. Such values are routed + through ``convert_to_response_data`` instead of the ``dict``-only + probes below, which would raise ``AttributeError``. + Args: - json_obj: JSON response object + json_obj: JSON response body Returns: Typed response data object or None if not found """ + if not isinstance(json_obj, dict): + return self.convert_to_response_data(json_obj) + if data := self.try_extract_embeddings(json_obj): return data diff --git a/tests/unit/endpoints/test_raw_endpoint.py b/tests/unit/endpoints/test_raw_endpoint.py index 49759438fe..ace48a588f 100644 --- a/tests/unit/endpoints/test_raw_endpoint.py +++ b/tests/unit/endpoints/test_raw_endpoint.py @@ -2,8 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from pytest import param -from aiperf.common.models import Turn +from aiperf.common.models import ( + BaseResponseData, + EmbeddingResponseData, + RankingsResponseData, + TextResponse, + Turn, +) from aiperf.endpoints.raw_endpoint import RawEndpoint from aiperf.plugin.enums import EndpointType from tests.unit.endpoints.conftest import ( @@ -14,7 +21,7 @@ @pytest.fixture -def raw_endpoint(): +def raw_endpoint() -> RawEndpoint: ep_info = create_model_endpoint(EndpointType.RAW) return create_endpoint_with_mock_transport(RawEndpoint, ep_info) @@ -44,3 +51,37 @@ def test_no_turns_raises(self, raw_endpoint): request_info = create_request_info(model_endpoint=ep_info, turns=[]) with pytest.raises(NotImplementedError): raw_endpoint.format_payload(request_info) + + +class TestRawEndpointParseResponse: + @pytest.mark.parametrize( + "text, expected", + [ + param( + '[{"index": 0, "score": 0.98}]', + RankingsResponseData(rankings=[{"index": 0, "score": 0.98}]), + id="list_of_dict_rankings", + ), + param( + "[0.1, 0.2, 0.3]", + EmbeddingResponseData(embeddings=[[0.1, 0.2, 0.3]]), + id="list_of_float_embedding", + ), + param( + "[[0.1, 0.2], [0.3, 0.4]]", + EmbeddingResponseData(embeddings=[[0.1, 0.2], [0.3, 0.4]]), + id="list_of_list_float_batch_embeddings", + ), + ], + ) # fmt: skip + def test_top_level_json_array_is_parsed( + self, + raw_endpoint: RawEndpoint, + text: str, + expected: BaseResponseData, + ) -> None: + """A top-level JSON array (e.g. a reranker response) must not crash.""" + response = TextResponse(perf_ns=1, text=text) + parsed = raw_endpoint.parse_response(response) + assert parsed is not None + assert parsed.data == expected From 78c7b18499d28da3fee35aef2cd6b3b77759199e Mon Sep 17 00:00:00 2001 From: AlexHuang Date: Sat, 1 Aug 2026 11:30:30 +0800 Subject: [PATCH 007/119] fix(docs): correct output directory flag in arrival-patterns tutorial (#1239) Signed-off-by: Alex (cherry picked from commit aaaa72e69d91f457df2f6dd180e06c9947af0edd) --- docs/tutorials/arrival-patterns.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/arrival-patterns.md b/docs/tutorials/arrival-patterns.md index 899324d757..940c317ae8 100644 --- a/docs/tutorials/arrival-patterns.md +++ b/docs/tutorials/arrival-patterns.md @@ -163,7 +163,7 @@ aiperf profile \ --request-rate 100 \ --arrival-pattern constant \ --benchmark-duration 60 \ - --output-dir results/constant + --output-artifact-dir results/constant **Expected Output (Run 1):** ``` @@ -197,7 +197,7 @@ aiperf profile \ --request-rate 100 \ --arrival-pattern poisson \ --benchmark-duration 60 \ - --output-dir results/poisson + --output-artifact-dir results/poisson ``` **Expected Output (Run 2):** @@ -325,7 +325,7 @@ for smoothness in 2.0 1.0 0.7 0.5 0.3; do --arrival-pattern gamma \ --arrival-smoothness $smoothness \ --benchmark-duration 60 \ - --output-dir results/smoothness_$smoothness + --output-artifact-dir results/smoothness_$smoothness done ``` From 687f23a8bee93d6cb721ddabef4ea6a3d546c7ac Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Sat, 1 Aug 2026 08:14:53 -0700 Subject: [PATCH 008/119] perf: cache normalized value/hash on hot-path string enums (#1184) Signed-off-by: Anthony Casagrande (cherry picked from commit c2f5e9d459005d362457716bbd865d247232fa30) --- src/aiperf/common/enums/base_enums.py | 39 ++++++++++-- src/aiperf/plugin/extensible_enums.py | 27 +++++++-- tests/unit/common/enums/test_base_enums.py | 69 ++++++++++++++++++++++ tests/unit/plugin/test_extensible_enums.py | 34 ++++++++++- 4 files changed, 157 insertions(+), 12 deletions(-) diff --git a/src/aiperf/common/enums/base_enums.py b/src/aiperf/common/enums/base_enums.py index c256954701..62b36bcf4e 100644 --- a/src/aiperf/common/enums/base_enums.py +++ b/src/aiperf/common/enums/base_enums.py @@ -22,21 +22,50 @@ class CaseInsensitiveStrEnum(str, Enum): lookup functionality for its members. """ + def __init__(self: Self, *args: object) -> None: + # Comparisons and hashing sit on hot paths (per-SSE-chunk field checks + # at >1M/s under load); normalize once per member instead of per call. + self._norm_value_cache = _normalize_name(self.value) + self._norm_hash_cache = hash(self._norm_value_cache) + + def _norm_value(self: Self) -> str: + # Lazy fallback for members created outside Enum construction + # (e.g. dynamically registered custom members), which skip __init__. + norm = self.__dict__.get("_norm_value_cache") + if norm is None: + norm = _normalize_name(self.value) + self._norm_value_cache = norm + return norm + def __str__(self) -> str: return self.value def __repr__(self) -> str: return f"{self.__class__.__name__}.{self.name}" - def __eq__(self, other: object) -> bool: + def __eq__(self: Self, other: object) -> bool: + if self is other: + return True + if isinstance(other, CaseInsensitiveStrEnum): + return self._norm_value() == other._norm_value() if isinstance(other, str): - return _normalize_name(self.value) == _normalize_name(other) + # Exact match first: skips normalization for the common case where + # the raw wire string already equals the member value. + return str.__eq__( + self, other + ) is True or self._norm_value() == _normalize_name(other) if isinstance(other, Enum): - return _normalize_name(self.value) == _normalize_name(other.value) + return isinstance(other.value, str) and self._norm_value() == ( + _normalize_name(other.value) + ) return super().__eq__(other) - def __hash__(self) -> int: - return hash(_normalize_name(self.value)) + def __hash__(self: Self) -> int: + norm_hash = self.__dict__.get("_norm_hash_cache") + if norm_hash is None: + norm_hash = hash(self._norm_value()) + self._norm_hash_cache = norm_hash + return norm_hash @classmethod def _missing_(cls, value): diff --git a/src/aiperf/plugin/extensible_enums.py b/src/aiperf/plugin/extensible_enums.py index 164bd11754..271e4e8964 100644 --- a/src/aiperf/plugin/extensible_enums.py +++ b/src/aiperf/plugin/extensible_enums.py @@ -107,15 +107,32 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}.{self.name}" - def __eq__(self, other: object) -> bool: + def _norm_value(self: Self) -> str: + # Lazily cached: members can be created dynamically via register(), + # so there is no single construction point to precompute this. + norm = self.__dict__.get("_norm_value_cache") + if norm is None: + norm = _normalize_name(self.value) + self._norm_value_cache = norm + return norm + + def __eq__(self: Self, other: object) -> bool: + if self is other: + return True if isinstance(other, str): - return _normalize_name(self.value) == _normalize_name(other) + return str.__eq__( + self, other + ) is True or self._norm_value() == _normalize_name(other) if hasattr(other, "value") and isinstance(other.value, str): - return _normalize_name(self.value) == _normalize_name(other.value) + return self._norm_value() == _normalize_name(other.value) return super().__eq__(other) - def __hash__(self) -> int: - return hash(_normalize_name(self.value)) + def __hash__(self: Self) -> int: + norm_hash = self.__dict__.get("_norm_hash_cache") + if norm_hash is None: + norm_hash = hash(self._norm_value()) + self._norm_hash_cache = norm_hash + return norm_hash @property def name(self) -> str: diff --git a/tests/unit/common/enums/test_base_enums.py b/tests/unit/common/enums/test_base_enums.py index c5f6f86dd1..090d18fb18 100644 --- a/tests/unit/common/enums/test_base_enums.py +++ b/tests/unit/common/enums/test_base_enums.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Unit tests for CaseInsensitiveStrEnum and dash/underscore normalization.""" +from typing import Self + import pytest from pydantic import BaseModel, ValidationError @@ -285,3 +287,70 @@ class MultiDashEnum(CaseInsensitiveStrEnum): assert MultiDashEnum("foo__bar") == MultiDashEnum.MULTI assert MultiDashEnum.MULTI == "foo__bar" + + +# ============================================================================= +# Normalization Caching Tests +# ============================================================================= + + +class TestNormalizationCaching: + """Tests for the per-member cached normalized value/hash hot-path.""" + + def test_norm_value_cached_on_construction(self: Self) -> None: + """Members precompute the normalized value once during construction.""" + assert SampleEnum.FOO_BAR.__dict__.get("_norm_value_cache") == "foo_bar" + assert DashValueEnum.MY_VALUE.__dict__.get("_norm_value_cache") == "my_value" + + def test_norm_hash_cached_on_construction(self: Self) -> None: + """Members precompute the normalized hash once during construction.""" + member = DashValueEnum.MY_VALUE + assert member.__dict__.get("_norm_hash_cache") == hash("my_value") + assert hash(member) == hash("my_value") + + def test_exact_match_fast_path(self: Self) -> None: + """An exact string match compares equal without normalization work.""" + assert SampleEnum.FOO_BAR == "foo_bar" + # Non-exact but normalized-equal still matches via the fallback. + assert SampleEnum.FOO_BAR == "FOO-BAR" + + def test_identity_fast_path(self: Self) -> None: + """A member is equal to itself via the identity short-circuit.""" + member = SampleEnum.ALPHA + assert member == member + + def test_lazy_norm_value_fallback(self: Self) -> None: + """A member missing the cached attr recomputes and caches lazily. + + A non-exact (normalized-equal) compare misses the exact-match fast path + and forces the normalization fallback, which repopulates the cache. + """ + member = SampleEnum.BETA + # Simulate a member that skipped __init__ (e.g. dynamic creation). + member.__dict__.pop("_norm_value_cache", None) + member.__dict__.pop("_norm_hash_cache", None) + assert member == "BETA" # non-exact -> normalization path + assert member.__dict__.get("_norm_value_cache") == "beta" + assert hash(member) == hash("beta") + assert member.__dict__.get("_norm_hash_cache") == hash("beta") + + def test_enum_vs_enum_uses_cached_norm(self: Self) -> None: + """Cross-enum equality still holds using cached normalized values.""" + + class EnumA(CaseInsensitiveStrEnum): + ITEM = "foo_bar" + + class EnumB(CaseInsensitiveStrEnum): + ITEM = "foo-bar" + + assert EnumA.ITEM == EnumB.ITEM + assert hash(EnumA.ITEM) == hash(EnumB.ITEM) + + def test_non_str_enum_not_equal(self: Self) -> None: + """An Enum whose value is not a str does not compare equal.""" + from enum import Enum + + class IntEnum(Enum): + X = 1 + + assert SampleEnum.ALPHA != IntEnum.X diff --git a/tests/unit/plugin/test_extensible_enums.py b/tests/unit/plugin/test_extensible_enums.py index 3c0c8a977a..5b4aa81662 100644 --- a/tests/unit/plugin/test_extensible_enums.py +++ b/tests/unit/plugin/test_extensible_enums.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Unit tests for ExtensibleStrEnum and related utilities.""" +from typing import Self + import pytest from pydantic import BaseModel, ValidationError @@ -24,7 +26,7 @@ class SampleEnum(ExtensibleStrEnum): @pytest.fixture -def fresh_enum(): +def fresh_enum() -> type[ExtensibleStrEnum]: """Create a fresh enum class for tests that mutate state.""" class TestEnum(ExtensibleStrEnum): @@ -34,7 +36,9 @@ class TestEnum(ExtensibleStrEnum): @pytest.fixture -def enum_with_extension(fresh_enum): +def enum_with_extension( + fresh_enum: type[ExtensibleStrEnum], +) -> type[ExtensibleStrEnum]: """Create enum with one registered extension.""" fresh_enum.register("EXT", "ext") return fresh_enum @@ -219,6 +223,32 @@ def test_str_subclass(self): assert SampleEnum.ALPHA.upper() == "ALPHA" assert SampleEnum.ALPHA.startswith("al") + def test_exact_match_fast_path(self: Self) -> None: + """An exact string compares equal; normalized-equal still matches.""" + assert SampleEnum.ALPHA == "alpha" + assert SampleEnum.ALPHA == "ALPHA" + + def test_identity_fast_path(self: Self) -> None: + """A member is equal to itself via the identity short-circuit.""" + member = SampleEnum.ALPHA + assert member == member + + def test_norm_value_cached_lazily( + self: Self, enum_with_extension: type[ExtensibleStrEnum] + ) -> None: + """Normalized value/hash are cached lazily on first use. + + Extension members are created via str.__new__ and skip __init__, so the + cache is populated on the first comparison/hash rather than eagerly. + """ + ext = enum_with_extension.EXT + ext.__dict__.pop("_norm_value_cache", None) + ext.__dict__.pop("_norm_hash_cache", None) + assert ext == "EXT" # non-exact -> normalization path caches + assert ext.__dict__.get("_norm_value_cache") == "ext" + assert hash(ext) == hash("ext") + assert ext.__dict__.get("_norm_hash_cache") == hash("ext") + # ============================================================================= # Registration Tests From 5f09805ba3d35f498704cd4b92783d7831bb9b56 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:01:03 -0700 Subject: [PATCH 009/119] test(amdsmi): add e2e integration tests for AMD mock telemetry path (#1192) (cherry picked from commit 9be59a963607ab5c2a251f47c3a9013c62d442da) --- CONTRIBUTING.md | 4 +- Makefile | 4 +- .../metrics/energy_efficiency_analyzer.py | 11 +- tests/integration/test_gpu_telemetry.py | 350 ++++++++++++++++++ 4 files changed, 364 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c61db124b8..f5e08e0bf6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ make first-time-setup # Or manually: make setup-venv # Create virtual environment -make install # Install project + mock server in editable mode +make install # Install project + mock server + fake amdsmi bindings pre-commit install # Install pre-commit hooks ``` @@ -65,7 +65,7 @@ pre-commit install # Install pre-commit hooks | Command | Description | |---------|-------------| | `make first-time-setup` | Full environment setup (venv + install + hooks) | -| `make install` | Install project and mock server in editable mode | +| `make install` | Install project, mock server, and fake amdsmi bindings in editable mode | | `make install-app` | Install project only | | `make install-mock-server` | Install mock server only | | `make install-mock-amdsmi` | Install fake `amdsmi` bindings to exercise the AMD telemetry path on non-AMD hardware (see [Mocking a ROCm Environment](docs/reference/mock-amdsmi.md)) | diff --git a/Makefile b/Makefile index 6ecae22c8e..724f7e5cd9 100644 --- a/Makefile +++ b/Makefile @@ -148,7 +148,7 @@ check-agent-files-sync: #? verify AGENTS.md, CLAUDE.md, .github/copilot-instruct coverage: #? run the tests and generate an html coverage report. $(activate_venv) && pytest tests/unit -n auto --cov=src/aiperf --cov-branch --cov-report=html --cov-report=xml --cov-report=term -m 'not integration and not performance and not component_integration and not slow' $(args) -install: install-app install-mock-server #? install the project and mock server in editable mode. +install: install-app install-mock-server install-mock-amdsmi #? install the project, mock server, and fake amdsmi bindings in editable mode. install-app: #? install the project in editable mode. $(activate_venv) && uv pip install -e ".[dev]" @@ -165,7 +165,7 @@ version: #? print the version of the project. install-mock-server: #? install the mock server in editable mode. $(activate_venv) && uv pip install -e "tests/aiperf_mock_server[dev]" -install-mock-amdsmi: #? install the fake amdsmi bindings for testing the AMD telemetry path (not part of default install). +install-mock-amdsmi: #? install the fake amdsmi bindings for testing the AMD telemetry path. $(activate_venv) && uv pip install -e "tests/aiperf_mock_amdsmi[dev]" check-mock-server-install: #? verify the mock server package and CLI entry point are installed. diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py index d13b6ed2a3..fe13f582ec 100644 --- a/src/aiperf/metrics/energy_efficiency_analyzer.py +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -238,7 +238,16 @@ def _analyze_vendor( out.append(_result(vendor_metrics["total_power"], power[0])) if source is EnergySource.UNAVAILABLE or total_energy_j <= 0: if source is EnergySource.UNAVAILABLE: - _logger.warning( + # Only warn when the user explicitly configured GPU telemetry. + # Default DCGM probing (no --gpu-telemetry flag) auto-connects to + # localhost:9400/9401; if those are reachable but have no power/energy + # fields the warning is noise, not actionable. + explicitly_configured = self.run is not None and ( + bool(self.run.cfg.gpu_telemetry.urls) + or str(self.run.cfg.gpu_telemetry.collector) != "dcgm" + ) + log = _logger.warning if explicitly_configured else _logger.debug + log( lambda p=platform: ( f"EnergyEfficiencyAnalyzer: no power or energy data available " f"for platform '{p}' — GPU power efficiency metrics will not be " diff --git a/tests/integration/test_gpu_telemetry.py b/tests/integration/test_gpu_telemetry.py index 347fa78c29..ff8ae96894 100644 --- a/tests/integration/test_gpu_telemetry.py +++ b/tests/integration/test_gpu_telemetry.py @@ -6,8 +6,10 @@ import orjson import pytest +from pytest import param from aiperf.common.models.telemetry_models import TelemetryRecord +from aiperf.gpu_telemetry.constants import AMDSMI_SOURCE_IDENTIFIER from tests.harness.utils import AIPerfCLI, AIPerfMockServer @@ -195,3 +197,351 @@ async def test_gpu_telemetry_disabled( # Verify no GPU telemetry files were created jsonl_files = list(result.artifacts_dir.glob("*gpu_telemetry*.jsonl")) assert len(jsonl_files) == 0, f"Unexpected GPU telemetry files: {jsonl_files}" + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestAMDSMITelemetry: + """Tests for AMD GPU telemetry collection using the fake amdsmi bindings. + + These tests activate ``aiperf-mock-amdsmi`` via ``AIPERF_MOCK_AMDSMI=1`` so + the full AMD telemetry path exercises on any host — no AMD hardware required. + """ + + @pytest.fixture(autouse=True) + def _activate_mock_amdsmi(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AIPERF_MOCK_AMDSMI", "1") + + async def test_amd_gpu_telemetry( + self, cli: AIPerfCLI, aiperf_mock_server: AIPerfMockServer + ) -> None: + """AMD telemetry collection populates amd_* metrics end-to-end.""" + result = await cli.run( + f""" + aiperf profile \ + --model nvidia/llama-3.1-nemotron-70b-instruct \ + --url {aiperf_mock_server.url} \ + --tokenizer builtin \ + --endpoint-type chat \ + --gpu-telemetry amdsmi \ + --streaming \ + --request-count 10 \ + --concurrency 2 \ + --workers-max 2 + """ + ) + assert result.request_count == 10 + assert result.has_gpu_telemetry + assert result.json.telemetry_data.endpoints is not None + assert len(result.json.telemetry_data.endpoints) > 0 + + # amd_energy_consumption and amd_ecc_uncorrectable are counters (delta only) + counter_metrics = {"amd_energy_consumption", "amd_ecc_uncorrectable"} + for endpoint_data in result.json.telemetry_data.endpoints.values(): + assert endpoint_data.gpus is not None + assert len(endpoint_data.gpus) > 0 + for gpu_data in endpoint_data.gpus.values(): + assert gpu_data.metrics is not None + assert len(gpu_data.metrics) > 0 + amd_metrics = {k for k in gpu_data.metrics if k.startswith("amd_")} + assert amd_metrics, "No amd_* metrics collected" + # amd_mm_activity returns 'N/A' on Instinct GPUs — the collector + # must drop it rather than surfacing it as 0.0 or None. + assert "amd_mm_activity" not in gpu_data.metrics, ( + "amd_mm_activity should be absent: mock returns 'N/A' for " + "Instinct GPUs and the collector must filter it out" + ) + for metric_name, metric_value in gpu_data.metrics.items(): + assert metric_value.avg is not None + assert metric_value.unit is not None + if metric_name not in counter_metrics: + assert metric_value.min is not None + assert metric_value.max is not None + + async def test_amd_gpu_telemetry_export( + self, cli: AIPerfCLI, aiperf_mock_server: AIPerfMockServer + ) -> None: + """AMD telemetry JSONL export contains valid TelemetryRecord entries.""" + result = await cli.run( + f""" + aiperf profile \ + --model nvidia/llama-3.1-nemotron-70b-instruct \ + --url {aiperf_mock_server.url} \ + --tokenizer builtin \ + --endpoint-type chat \ + --gpu-telemetry amdsmi \ + --streaming \ + --request-count 10 \ + --concurrency 2 \ + --workers-max 2 + """ + ) + assert result.request_count == 10 + assert result.has_gpu_telemetry + + export_file = result.artifacts_dir / "gpu_telemetry_export.jsonl" + assert export_file.exists(), "GPU telemetry export file should exist" + + lines = export_file.read_text(encoding="utf-8").splitlines() + assert len(lines) > 0, "Export file should contain telemetry records" + + gpu_uuids: set[str] = set() + for line in lines: + record = TelemetryRecord.model_validate(orjson.loads(line)) + assert record.timestamp_ns > 0 + assert record.telemetry_source_url == AMDSMI_SOURCE_IDENTIFIER + assert record.gpu_index >= 0 + assert record.gpu_uuid is not None + assert record.gpu_model_name is not None + assert record.telemetry_data is not None + gpu_uuids.add(record.gpu_uuid) + + # Default mock exposes 2 GPUs + assert len(gpu_uuids) >= 2, "Should have records from at least two GPUs" + + async def test_amd_gpu_telemetry_export_with_custom_prefix( + self, cli: AIPerfCLI, aiperf_mock_server: AIPerfMockServer + ) -> None: + """AMD telemetry JSONL export respects --profile-export-prefix.""" + result = await cli.run( + f""" + aiperf profile \ + --model nvidia/llama-3.1-nemotron-70b-instruct \ + --url {aiperf_mock_server.url} \ + --tokenizer builtin \ + --endpoint-type chat \ + --gpu-telemetry amdsmi \ + --streaming \ + --request-count 10 \ + --concurrency 1 \ + --workers-max 1 \ + --profile-export-prefix custom_amd + """ + ) + + export_file = result.artifacts_dir / "custom_amd_gpu_telemetry.jsonl" + assert export_file.exists(), ( + "Custom-prefix GPU telemetry export file should exist" + ) + lines = export_file.read_text(encoding="utf-8").splitlines() + assert len(lines) > 0, "Export file should contain telemetry records" + first = TelemetryRecord.model_validate_json(lines[0]) + assert first.timestamp_ns > 0 + assert first.telemetry_source_url == AMDSMI_SOURCE_IDENTIFIER + + async def test_amd_gpu_telemetry_disabled( + self, cli: AIPerfCLI, aiperf_mock_server: AIPerfMockServer + ) -> None: + """--no-gpu-telemetry suppresses AMD collection even when mock is active.""" + result = await cli.run( + f""" + aiperf profile \ + --model nvidia/llama-3.1-nemotron-70b-instruct \ + --url {aiperf_mock_server.url} \ + --tokenizer builtin \ + --endpoint-type chat \ + --streaming \ + --request-count 10 \ + --concurrency 1 \ + --workers-max 1 \ + --no-gpu-telemetry + """ + ) + assert result.request_count == 10 + assert not result.has_gpu_telemetry, "GPU telemetry should not be collected" + jsonl_files = list(result.artifacts_dir.glob("*gpu_telemetry*.jsonl")) + assert len(jsonl_files) == 0, f"Unexpected GPU telemetry files: {jsonl_files}" + + async def test_amd_derived_efficiency_metrics( + self, cli: AIPerfCLI, aiperf_mock_server: AIPerfMockServer + ) -> None: + """AMD energy-efficiency derived metrics are present and internally consistent. + + Validates three cross-metric relationships that must hold by construction: + energy_per_output_token (mJ/tok) = total_energy (J) * 1000 / total_osl + energy_per_request (J/req) = total_energy (J) / request_count + output_tokens_per_joule (tok/J) = total_osl / total_energy (J) + """ + result = await cli.run( + f""" + aiperf profile \ + --model nvidia/llama-3.1-nemotron-70b-instruct \ + --url {aiperf_mock_server.url} \ + --tokenizer builtin \ + --endpoint-type chat \ + --gpu-telemetry amdsmi \ + --streaming \ + --request-count 10 \ + --concurrency 1 \ + --workers-max 1 + """ + ) + assert result.request_count == 10 + assert result.has_gpu_telemetry + + from aiperf.common.models.export_models import JsonMetricResult + + j = result.json + extra = j.model_extra or {} + + def _m(name: str) -> JsonMetricResult | None: + val = getattr(j, name, None) or extra.get(name) + if val is None: + return None + if isinstance(val, dict): + return JsonMetricResult.model_validate(val) + return val + + total_energy = _m("amd_total_gpu_energy") + energy_per_output_token = _m("amd_energy_per_output_token") + energy_per_request = _m("amd_energy_per_request") + output_tokens_per_joule = _m("amd_output_tokens_per_joule") + + assert total_energy is not None, "amd_total_gpu_energy missing from JSON export" + assert energy_per_output_token is not None, ( + "amd_energy_per_output_token missing" + ) + assert energy_per_request is not None, "amd_energy_per_request missing" + + total_energy_j: float = total_energy.avg + assert total_energy_j > 0, "Total AMD GPU energy must be positive" + assert energy_per_output_token.avg > 0 + assert energy_per_request.avg > 0 + + # energy_per_output_token (mJ/tok) = total_energy_j * 1000 / total_osl + if j.total_osl and j.total_osl.avg and j.total_osl.avg > 0: + expected_mj_per_tok = total_energy_j * 1000.0 / j.total_osl.avg + assert energy_per_output_token.avg == pytest.approx( + expected_mj_per_tok, rel=1e-3 + ), ( + f"energy_per_output_token={energy_per_output_token.avg:.4f} mJ/tok " + f"!= total_energy*1000/total_osl={expected_mj_per_tok:.4f} mJ/tok" + ) + + # output_tokens_per_joule = 1 / (energy_per_output_token / 1000) + if output_tokens_per_joule is not None: + expected_tpj = j.total_osl.avg / total_energy_j + assert output_tokens_per_joule.avg == pytest.approx( + expected_tpj, rel=1e-3 + ) + + # energy_per_request (J/req) = total_energy_j / request_count + if j.request_count and j.request_count.avg and j.request_count.avg > 0: + expected_j_per_req = total_energy_j / j.request_count.avg + assert energy_per_request.avg == pytest.approx( + expected_j_per_req, rel=1e-3 + ), ( + f"energy_per_request={energy_per_request.avg:.4f} J/req " + f"!= total_energy/request_count={expected_j_per_req:.4f} J/req" + ) + + # energy_per_total_token <= energy_per_output_token (more tokens in denominator) + energy_per_total_token = _m("amd_energy_per_total_token") + if ( + energy_per_total_token is not None + and energy_per_total_token.avg is not None + ): + assert energy_per_total_token.avg <= energy_per_output_token.avg, ( + "energy_per_total_token should be <= energy_per_output_token " + "(total tokens >= output tokens)" + ) + + +_PLATFORM_EXPECTED_PREFIX: dict[str, str] = { + "nvidia": "nvidia_", + "amd": "amd_", +} +_PLATFORM_FORBIDDEN_PREFIX: dict[str, str] = { + "nvidia": "amd_", + "amd": "nvidia_", +} + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestTelemetryVendorIsolation: + """Per-GPU vendor namespace isolation. + + Each GPU's metrics must only contain the prefix matching its platform tag. + This design is forward-compatible with heterogeneous systems: validation is + localised to each GPU using GpuSummary.platform so a mixed NVIDIA+AMD node + passes as long as every individual GPU is internally consistent. + """ + + @pytest.mark.parametrize( + "collector", + [ + param( + "dcgm", + marks=pytest.mark.skipif( + platform.system() in ("Darwin", "Windows"), + reason="DCGM telemetry requires Linux", + ), + id="dcgm", + ), + param("amdsmi", id="amdsmi"), + ], + ) # fmt: skip + async def test_vendor_metric_isolation( + self, + collector: str, + cli: AIPerfCLI, + aiperf_mock_server: AIPerfMockServer, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Each GPU only carries metrics prefixed for its own platform. + + For every GPU in the result: + - at least one metric with the expected vendor prefix is present + (the vendor URL produced data) + - no metric with the other vendor's prefix is present + (namespaces are not crossed) + """ + if collector == "amdsmi": + monkeypatch.setenv("AIPERF_MOCK_AMDSMI", "1") + telemetry_arg = "amdsmi" + else: + telemetry_arg = " ".join(aiperf_mock_server.dcgm_urls) + + result = await cli.run( + f""" + aiperf profile \ + --model nvidia/llama-3.1-nemotron-70b-instruct \ + --url {aiperf_mock_server.url} \ + --tokenizer builtin \ + --endpoint-type chat \ + --gpu-telemetry {telemetry_arg} \ + --streaming \ + --request-count 10 \ + --concurrency 1 \ + --workers-max 1 + """ + ) + assert result.has_gpu_telemetry + assert result.json.telemetry_data.endpoints is not None + + for source_url, endpoint_data in result.json.telemetry_data.endpoints.items(): + assert endpoint_data.gpus, f"No GPUs reported for {source_url}" + for gpu_uuid, gpu_data in endpoint_data.gpus.items(): + gpu_platform = gpu_data.platform + assert gpu_platform in _PLATFORM_EXPECTED_PREFIX, ( + f"GPU {gpu_uuid[:12]} at {source_url!r} has unexpected " + f"platform={gpu_platform!r}; expected one of " + f"{sorted(_PLATFORM_EXPECTED_PREFIX)}" + ) + expected_prefix = _PLATFORM_EXPECTED_PREFIX[gpu_platform] + forbidden_prefix = _PLATFORM_FORBIDDEN_PREFIX[gpu_platform] + + present = [k for k in gpu_data.metrics if k.startswith(expected_prefix)] + assert present, ( + f"GPU {gpu_uuid[:12]} (platform={gpu_platform!r}) at " + f"{source_url!r} has no '{expected_prefix}' metrics — " + f"vendor URL present but no matching data collected" + ) + + leaked = [k for k in gpu_data.metrics if k.startswith(forbidden_prefix)] + assert not leaked, ( + f"GPU {gpu_uuid[:12]} (platform={gpu_platform!r}) at " + f"{source_url!r} contains forbidden '{forbidden_prefix}' " + f"metrics: {leaked}" + ) From aa88837ac780e4c880e859a0a1d2568c031a6021 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:33:05 -0700 Subject: [PATCH 010/119] feat(mock-server): add ResponsesRequest model with full dispatch plumbing (#1000) (cherry picked from commit 59cd43e43afa9aaba1eb7d84a6cb5f8e02f5bea8) --- docs/tutorials/openai-responses.md | 39 ++++++++++ tests/aiperf_mock_server/README.md | 7 +- tests/aiperf_mock_server/app.py | 35 +-------- tests/aiperf_mock_server/models.py | 72 +++++++++++++++++- tests/aiperf_mock_server/request_recorder.py | 3 + tests/aiperf_mock_server/tokens.py | 13 ++++ tests/aiperf_mock_server/utils.py | 3 + .../test_request_recorder.py | 45 +++++++++++ tests/unit/server/test_api_compliance.py | 20 +++++ tests/unit/server/test_models.py | 66 +++++++++++++++++ tests/unit/server/test_tokens.py | 74 +++++++++++++++++++ 11 files changed, 342 insertions(+), 35 deletions(-) diff --git a/docs/tutorials/openai-responses.md b/docs/tutorials/openai-responses.md index 941c487108..383dc67e07 100644 --- a/docs/tutorials/openai-responses.md +++ b/docs/tutorials/openai-responses.md @@ -292,6 +292,45 @@ aiperf profile \ --- +## Verifying ISL/OSL Distribution with the Mock Server + +Use the mock server's `--record-requests` flag to capture the exact token lengths +AIPerf sends on the wire before running against a real server: + +```bash +aiperf-mock-server --record-requests /tmp/responses-req.jsonl --fast & +mock_server_pid=$! + +aiperf profile \ + --model Qwen/Qwen3-0.6B \ + --endpoint-type responses \ + --endpoint /v1/responses \ + --synthetic-input-tokens-mean 512 \ + --output-tokens-mean 256 \ + --url http://localhost:8000 \ + --request-count 100 + +kill $mock_server_pid # graceful shutdown flushes the JSONL and prints a summary +``` + +The recorder writes one JSON line per request. For `/v1/responses` requests the +`request_id` carries a `resp-` prefix and `max_output_tokens` is canonicalized +into the `max_completion_tokens` column so the schema stays uniform with chat +and completions rows: + +```json +{"ts": 1714000000.123, "request_id": "resp-3b8568cb-...", "endpoint": "/v1/responses", + "model": "Qwen/Qwen3-0.6B", "isl": 512, + "requested_osl": 256, "max_tokens": null, "max_completion_tokens": 256, + "min_tokens": null, "ignore_eos": false, "reasoning_effort": null, + "stream": false, "tokenization_mode": "tokenizer_call"} +``` + +See the [mock server README](https://github.com/ai-dynamo/aiperf/blob/main/tests/aiperf_mock_server/README.md#request-recording) +for the full output format and summary schema. + +--- + ## Key Differences from Chat Completions When migrating AIPerf benchmarks from `--endpoint-type chat` to `--endpoint-type responses`: diff --git a/tests/aiperf_mock_server/README.md b/tests/aiperf_mock_server/README.md index 767c90af57..e5988994f0 100644 --- a/tests/aiperf_mock_server/README.md +++ b/tests/aiperf_mock_server/README.md @@ -29,6 +29,7 @@ A mock server for integration testing and performance benchmarking of LLM applic | Endpoint | Description | |----------|-------------| | [`/v1/chat/completions`](#chat-completions) | OpenAI chat completions (streaming supported) | +| `/v1/responses` | OpenAI Responses API (non-streaming; `max_output_tokens` recorded as `max_completion_tokens` in JSONL) | | `/v1/messages` | Anthropic Messages API (streaming, thinking, tool_use, disjoint cache accounting; requires `anthropic-version` header and `max_tokens`) | | [`/v1/completions`](#text-completions) | OpenAI text completions (streaming supported) | | [`/v1/embeddings`](#embeddings) | OpenAI embeddings (768-dim) | @@ -405,7 +406,7 @@ Multiple endpoints simulate different LLM backends: ### Latency Models -**LLM Endpoints** (`/v1/chat/completions`, `/v1/completions`, `/v1/custom-multimodal`): +**LLM Endpoints** (`/v1/chat/completions`, `/v1/responses`, `/v1/completions`, `/v1/custom-multimodal`): - Streaming: TTFT delay before first token, ITL between subsequent tokens - Non-streaming: TTFT + (ITL × token_count) @@ -581,8 +582,8 @@ flowchart LR | Field | Meaning | |---|---| | `isl` | Real tokenized length of the assembled prompt (always populated). | -| `requested_osl` | Resolved OSL cap: `max_completion_tokens or max_tokens` for chat, `max_tokens` for completions, `parameters.max_new_tokens` for TGI. `null` for embeddings/ranking/image retrieval. | -| `max_tokens` / `max_completion_tokens` | The raw fields the client sent — useful to see which API name-space they used. TGI's `max_new_tokens` is recorded under `max_tokens` so the schema stays uniform. | +| `requested_osl` | Resolved OSL cap: `max_completion_tokens or max_tokens` for chat, `max_output_tokens` for responses, `max_tokens` for completions, `parameters.max_new_tokens` for TGI. `null` for embeddings/ranking/image retrieval. | +| `max_tokens` / `max_completion_tokens` | The raw fields the client sent — useful to see which API name-space they used. TGI's `max_new_tokens` is recorded under `max_tokens`; the Responses API's `max_output_tokens` is recorded under `max_completion_tokens`. Both keep the JSONL schema uniform across endpoint types. | | `min_tokens` | Floor on generated tokens (vllm/SGLang). | | `ignore_eos` | If `true`, server generates exactly `max_tokens`. | | `reasoning_effort` | `low`/`medium`/`high` for reasoning models — adds a reasoning budget on top of the main output. | diff --git a/tests/aiperf_mock_server/app.py b/tests/aiperf_mock_server/app.py index 0a8fa1c3d5..33b10d2a4d 100644 --- a/tests/aiperf_mock_server/app.py +++ b/tests/aiperf_mock_server/app.py @@ -53,6 +53,7 @@ ImageGenerationRequest, ImageRetrievalRequest, RankingRequest, + ResponsesRequest, SolidoRAGRequest, TGIGenerateRequest, ) @@ -625,29 +626,6 @@ async def _text_stream_wrapper(ctx: RequestCtx, req: CompletionRequest, endpoint # ============================================================================ -def _extract_responses_prompt(payload: dict[str, Any]) -> str: - input_value = payload.get("input", "") - if isinstance(input_value, str): - return input_value - if isinstance(input_value, list): - parts: list[str] = [] - for item in input_value: - if isinstance(item, str): - parts.append(item) - elif isinstance(item, dict): - content = item.get("content", "") - if isinstance(content, str): - parts.append(content) - elif isinstance(content, list): - parts.extend( - str(part.get("text", "")) - for part in content - if isinstance(part, dict) - ) - return "\n".join(part for part in parts if part) - return str(input_value) - - def _build_responses_response_data(ctx: RequestCtx) -> dict[str, Any]: return { "id": ctx.request_id, @@ -668,17 +646,12 @@ def _build_responses_response_data(ctx: RequestCtx) -> dict[str, Any]: @app.post("/v1/responses", response_model=None) @with_error_injection -async def responses(req: dict[str, Any], request: Request) -> ORJSONResponse: +async def responses(req: ResponsesRequest, request: Request) -> ORJSONResponse: """Mock OpenAI Responses endpoint.""" endpoint = "/v1/responses" - model = str(req.get("model") or "test-model") - mock_req = ChatCompletionRequest( - model=model, - messages=[{"role": "user", "content": _extract_responses_prompt(req)}], - ) - ctx = make_ctx(mock_req, endpoint, request.state.start_time) + ctx = make_ctx(req, endpoint, request.state.start_time) - with track_llm_request(ctx, model, endpoint): + with track_llm_request(ctx, ctx.model, endpoint): await ctx.latency_sim.wait_for_tokens(len(ctx.tokens)) response_data = _build_responses_response_data(ctx) record_request_bytes( diff --git a/tests/aiperf_mock_server/models.py b/tests/aiperf_mock_server/models.py index c2537b707a..7072a330c0 100644 --- a/tests/aiperf_mock_server/models.py +++ b/tests/aiperf_mock_server/models.py @@ -3,7 +3,7 @@ from typing import Any, Literal from pydantic import BaseModel as PydanticBaseModel -from pydantic import ConfigDict +from pydantic import ConfigDict, Field # ============================================================================ # Base Models (for request parsing only) @@ -260,6 +260,75 @@ def max_output_tokens(self) -> int | None: return self.max_tokens +class ResponsesRequest(BaseModel): + """Request model for OpenAI's /v1/responses endpoint. + + The Responses API takes its prompt under `input` (which may be a string, + a list of strings, or a list of content-block dicts) and caps generation + via `max_output_tokens` rather than the chat API's `max_completion_tokens`. + Modeled here so the recorder can capture the real payload instead of the + synthetic ChatCompletionRequest the latency simulator drives off of. + """ + + model: str = Field(description="Model identifier.") + input: str | list[Any] = Field( + default="", + description="Prompt input — string, list of strings, or list of content-block dicts.", + ) + max_output_tokens: int | None = Field( + default=None, + description="Maximum number of output tokens to generate (the Responses-API name for the OSL cap).", + ) + stream: bool = Field(default=False, description="Whether to stream the response.") + reasoning_effort: Literal["low", "medium", "high"] | None = Field( + default=None, + description="Reasoning effort level for models that support extended thinking.", + ) + + # Mirrors BaseCompletionRequest so recorder/simulator share field semantics + # when the client supplies them via extras. + min_tokens: int | None = Field( + default=None, description="Minimum number of tokens to generate." + ) + ignore_eos: bool = Field( + default=False, + description="Whether to ignore the EOS token and continue generating up to max_output_tokens.", + ) + + @property + def prompt_text(self) -> str: + """Flatten `input` (str | list[str|dict]) into a single string. + + Matches the previous `_extract_responses_prompt` helper that lived in + the handler; moved here so the recorder and tokenizer dispatch sites + share one source of truth. + """ + return _flatten_responses_input(self.input) + + +def _flatten_responses_input(input_value: Any) -> str: + """Walk the Responses-API `input` shape and concatenate into text.""" + if isinstance(input_value, str): + return input_value + if isinstance(input_value, list): + parts: list[str] = [] + for item in input_value: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + content = item.get("content", "") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + parts.extend( + str(part.get("text", "")) + for part in content + if isinstance(part, dict) + ) + return "\n".join(part for part in parts if part) + return str(input_value) + + # ============================================================================ # Request Type Union # ============================================================================ @@ -276,4 +345,5 @@ def max_output_tokens(self) -> int | None: | ImageRetrievalRequest | SolidoRAGRequest | AnthropicMessagesRequest + | ResponsesRequest ) diff --git a/tests/aiperf_mock_server/request_recorder.py b/tests/aiperf_mock_server/request_recorder.py index 396817626c..88c12b16da 100644 --- a/tests/aiperf_mock_server/request_recorder.py +++ b/tests/aiperf_mock_server/request_recorder.py @@ -39,6 +39,7 @@ ImageGenerationRequest, RankingRequest, RequestT, + ResponsesRequest, SolidoRAGRequest, TGIGenerateRequest, ) @@ -303,6 +304,8 @@ def _encode_request_prompt_ids( return _encode_texts_without_special_tokens(tokenizer, [request.prompt]) if isinstance(request, SolidoRAGRequest): return _encode_texts_without_special_tokens(tokenizer, request.query) + if isinstance(request, ResponsesRequest): + return _encode_texts_with_tokenizer_call(tokenizer, [request.prompt_text]) return [], "unsupported_request" diff --git a/tests/aiperf_mock_server/tokens.py b/tests/aiperf_mock_server/tokens.py index bb9c8b1b38..d30e42a98a 100644 --- a/tests/aiperf_mock_server/tokens.py +++ b/tests/aiperf_mock_server/tokens.py @@ -18,6 +18,7 @@ ImageGenerationRequest, RankingRequest, RequestT, + ResponsesRequest, SolidoRAGRequest, TGIGenerateRequest, flatten_completion_prompt_token_ids, @@ -378,6 +379,8 @@ def _extract_request_content(request: RequestT) -> tuple[str, int | None]: return text, request.max_output_tokens elif isinstance(request, SolidoRAGRequest): return " ".join(request.query), None + elif isinstance(request, ResponsesRequest): + return request.prompt_text, request.max_output_tokens else: raise ValueError(f"Unsupported request type: {type(request)}") @@ -412,6 +415,16 @@ def _extract_osl_fingerprint(request: RequestT) -> dict[str, object]: fp["max_tokens"] = request.max_tokens fp["min_tokens"] = request.min_tokens fp["ignore_eos"] = request.ignore_eos + elif isinstance(request, ResponsesRequest): + # OpenAI Responses calls it `max_output_tokens` — recorded under + # `max_completion_tokens` because both name the same semantic (the + # OSL cap) and keeping the JSONL schema uniform is more useful than + # preserving the API name-space. The endpoint field on each row + # tells consumers which API the cap came from. + fp["max_completion_tokens"] = request.max_output_tokens + fp["min_tokens"] = request.min_tokens + fp["ignore_eos"] = request.ignore_eos + fp["reasoning_effort"] = request.reasoning_effort return fp diff --git a/tests/aiperf_mock_server/utils.py b/tests/aiperf_mock_server/utils.py index 7bf3f3db51..09c6aabaa1 100644 --- a/tests/aiperf_mock_server/utils.py +++ b/tests/aiperf_mock_server/utils.py @@ -36,6 +36,7 @@ ImageGenerationRequest, RankingRequest, RequestT, + ResponsesRequest, SolidoRAGRequest, TGIGenerateRequest, ) @@ -471,6 +472,8 @@ def _create_request_id(request: RequestT) -> str: return f"img-{uuid.uuid4()}" case SolidoRAGRequest(): return f"rag-{uuid.uuid4()}" + case ResponsesRequest(): + return f"resp-{uuid.uuid4()}" case _: raise ValueError(f"Invalid request type: {type(request)}") diff --git a/tests/unit/aiperf_mock_server/test_request_recorder.py b/tests/unit/aiperf_mock_server/test_request_recorder.py index 6c7a71d4b9..b29f80efc8 100644 --- a/tests/unit/aiperf_mock_server/test_request_recorder.py +++ b/tests/unit/aiperf_mock_server/test_request_recorder.py @@ -12,17 +12,20 @@ ChatCompletionRequest, CompletionRequest, Message, + ResponsesRequest, ) from aiperf_mock_server.request_recorder import ( RequestRecorder, _build_summary, _compute_shape_80, + _encode_request_prompt_ids, _histogram, _print_summary, _render_histogram, _render_vocab_lines, _vocab_distribution, ) +from aiperf_mock_server.utils import _create_request_id from pytest import param @@ -1269,3 +1272,45 @@ def test_blank_lines_between_blocks(self, capsys) -> None: assert lines[isl_hist_idx - 1] == "" assert lines[vocab_idx - 1] == "" assert lines[vocab_idx - 2] == "" + + +class TestResponsesRequestRecorderDispatch: + """`ResponsesRequest` flows through `_create_request_id` and + `_encode_request_prompt_ids`, so the recorder can produce a coherent + JSONL row for a `/v1/responses` request once handlers start passing it + in (subsequent commit). + """ + + def test_create_request_id_uses_resp_prefix(self): + req = ResponsesRequest(model="m", input="hi") + rid = _create_request_id(req) + assert rid.startswith("resp-") + + def test_encode_request_prompt_ids_tokenizes_flattened_prompt(self): + tok = _FakeTokenizer(vocab_size=100, encodings={"hello world": [1, 2, 3]}) + req = ResponsesRequest(model="m", input="hello world", max_output_tokens=64) + ids, mode = _encode_request_prompt_ids(tok, req) + assert ids == [1, 2, 3] + assert mode == "tokenizer_call" + + def test_encode_request_prompt_ids_walks_content_blocks(self): + """ResponsesRequest with a content-block list flows through + `prompt_text` → `_encode_texts_with_tokenizer_call`. The flattener + joins multi-part content with `\\n`, matching the original + `_extract_responses_prompt` behavior.""" + tok = _FakeTokenizer(vocab_size=100, encodings={"hello\nworld": [9, 9, 9]}) + req = ResponsesRequest( + model="m", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "hello"}, + {"type": "input_text", "text": "world"}, + ], + } + ], + ) + ids, mode = _encode_request_prompt_ids(tok, req) + assert ids == [9, 9, 9] + assert mode == "tokenizer_call" diff --git a/tests/unit/server/test_api_compliance.py b/tests/unit/server/test_api_compliance.py index 99039f78b4..d9ea6d103d 100644 --- a/tests/unit/server/test_api_compliance.py +++ b/tests/unit/server/test_api_compliance.py @@ -748,3 +748,23 @@ async def test_empty_prompt(self, client: AsyncClient): data = resp.json() # Empty prompt should result in empty output assert data["usage"]["completion_tokens"] == 0 + + +# ============================================================================ +# Responses API Compliance +# ============================================================================ + + +class TestResponses: + """Test /v1/responses endpoint dispatches the parsed ResponsesRequest.""" + + async def test_response_id_uses_resp_prefix(self, client: AsyncClient): + """The handler must route the real ResponsesRequest through make_ctx + so _create_request_id returns a `resp-` id, not a `chatcmpl-` id.""" + resp = await client.post( + "/v1/responses", + json={"model": "test-model", "input": "Hello world"}, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["id"].startswith("resp-") diff --git a/tests/unit/server/test_models.py b/tests/unit/server/test_models.py index eb4314dea5..02d1605c57 100644 --- a/tests/unit/server/test_models.py +++ b/tests/unit/server/test_models.py @@ -9,6 +9,7 @@ EmbeddingRequest, Message, RankingRequest, + ResponsesRequest, ) @@ -99,3 +100,68 @@ def test_passage_texts(self): ], ) assert req.passage_texts == ["passage 1", "passage 2"] + + +class TestResponsesRequest: + """Tests for ResponsesRequest model and its `prompt_text` shape-flattener.""" + + @pytest.mark.parametrize( + "input_value,expected", + [ + ("hello world", "hello world"), + (["alpha", "beta"], "alpha\nbeta"), + ( + [{"role": "user", "content": "single string content"}], + "single string content", + ), + ( + [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "first"}, + {"type": "input_text", "text": "second"}, + ], + } + ], + "first\nsecond", + ), + ( + [ + {"role": "system", "content": "policy"}, + {"role": "user", "content": "hi"}, + ], + "policy\nhi", + ), + ], + ) + def test_prompt_text_flattens_input_shapes(self, input_value, expected): + req = ResponsesRequest(model="m", input=input_value) + assert req.prompt_text == expected + + def test_unmodeled_fields_are_preserved_via_extras(self): + """`tools`, `instructions`, custom keys flow through the recorder + because BaseModel has `extra="allow"`.""" + req = ResponsesRequest.model_validate( + { + "model": "m", + "input": "hi", + "max_output_tokens": 64, + "stream": True, + "instructions": "be brief", + "tools": [{"type": "function", "name": "foo"}], + } + ) + assert req.max_output_tokens == 64 + assert req.stream is True + # Extras land on the model instance for downstream inspection. + assert req.instructions == "be brief" + assert req.tools == [{"type": "function", "name": "foo"}] + + def test_defaults_are_safe(self): + req = ResponsesRequest(model="m") + assert req.input == "" + assert req.max_output_tokens is None + assert req.stream is False + assert req.reasoning_effort is None + assert req.prompt_text == "" diff --git a/tests/unit/server/test_tokens.py b/tests/unit/server/test_tokens.py index 866acaa308..efa5e83dc8 100644 --- a/tests/unit/server/test_tokens.py +++ b/tests/unit/server/test_tokens.py @@ -7,9 +7,12 @@ ChatCompletionRequest, CompletionRequest, Message, + ResponsesRequest, ) from aiperf_mock_server.tokens import ( TokenizedText, + _extract_osl_fingerprint, + _extract_request_content, _generate_seed, _tokenize, count_tokens, @@ -406,3 +409,74 @@ def test_generate_seed_empty_prompt_is_zero(self): def test_generate_seed_uses_first_five_tokens_only(self): base = ["a", "b", "c", "d", "e"] assert _generate_seed(base) == _generate_seed(base + ["f", "g"]) + + +class TestResponsesRequestDispatch: + """Dispatch sites in `tokens.py` for the new `ResponsesRequest` type.""" + + def test_extract_request_content_returns_prompt_and_max_output_tokens(self): + req = ResponsesRequest( + model="m", + input="hello world", + max_output_tokens=64, + ) + text, max_tokens = _extract_request_content(req) + assert text == "hello world" + assert max_tokens == 64 + + def test_extract_request_content_walks_content_blocks(self): + req = ResponsesRequest( + model="m", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "hello"}, + {"type": "input_text", "text": "world"}, + ], + } + ], + ) + text, max_tokens = _extract_request_content(req) + assert text == "hello\nworld" + assert max_tokens is None + + def test_extract_osl_fingerprint_canonicalizes_max_output_tokens(self): + """Responses' `max_output_tokens` is recorded under the existing + `max_completion_tokens` JSONL field — same semantic (the OSL cap), + and the row's endpoint tells consumers which API name-space it came + from. `max_tokens` stays None so it's clear the client didn't use + that field name. + """ + req = ResponsesRequest( + model="m", + input="hi", + max_output_tokens=200, + reasoning_effort="high", + min_tokens=16, + ignore_eos=True, + ) + fp = _extract_osl_fingerprint(req) + assert fp["max_completion_tokens"] == 200 + assert fp["max_tokens"] is None + assert fp["reasoning_effort"] == "high" + assert fp["min_tokens"] == 16 + assert fp["ignore_eos"] is True + + def test_extract_osl_fingerprint_defaults(self): + fp = _extract_osl_fingerprint(ResponsesRequest(model="m")) + assert fp["max_completion_tokens"] is None + assert fp["max_tokens"] is None + assert fp["min_tokens"] is None + assert fp["ignore_eos"] is False + assert fp["reasoning_effort"] is None + + def test_tokenize_request_handles_responses_request(self): + """ResponsesRequest is in the generation path, not the no-output + path (embedding/ranking/image-gen). Output tokens should be > 0.""" + req = ResponsesRequest( + model="m", input="please summarize", max_output_tokens=32 + ) + result = tokenize_request(req) + assert result.prompt_token_count > 0 + assert 0 < result.count <= 32 From 65a0537507cd1fded5eebd813208279cade6458e Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:51:48 -0700 Subject: [PATCH 011/119] feat(endpoint): add AIPERF_ENDPOINT_FORCE_CONTENT_PARTS to control content serialization (#1259) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 37467dc38c1c46b5b985437584cfd14139de5680) --- docs/environment-variables.md | 8 ++++ src/aiperf/common/environment.py | 29 ++++++++++++++ src/aiperf/endpoints/base_endpoint.py | 4 +- tests/unit/endpoints/test_base_endpoint.py | 39 +++++++++++++++++++ tests/unit/property/test_finite_invariants.py | 3 ++ 5 files changed, 82 insertions(+), 1 deletion(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index babdfddb27..8a1ae23f29 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -118,6 +118,14 @@ Dataset loading and configuration. Controls timeouts and behavior for dataset lo | `AIPERF_DATASET_WEKA_AUX_REDUCTION_RATIO` | `20.0` | ≥ 0.0 | Auxiliary (sidecar) classification, reduction arm: the minimum input-to-output token ratio for a same-model single-request large-input chain to be treated as a reduction sidecar (see WEKA_AUX_REDUCTION_OSL_MAX). A reduction consumes a large body and emits a short summary, so input/output is high (corpus median ~120); 20 is a conservative floor that still excludes balanced request/response calls. Only applies when WEKA_AUX_REDUCTION_OSL_MAX > 0. | | `AIPERF_DATASET_WEKA_WORKER_GROUP_MIN` | `3` | ≥ 0 | Parallel worker-group tagging: a coordinated parallel fan-out must BOTH share a deep spawned context AND run concurrently. Workers that forked from shared context (fork depth > 0) are first scoped by their fork point (the parent request they branched off), then within each scope split into connected components of overlapping active [t0, t1) intervals; a component with at least this many members is emitted as ::wg:{group}_{member} (group = the concurrent fan-out, member = index by start time) instead of the generic ::fa: agent marker. The fork-point scope keeps unrelated fan-outs apart (pure interval overlap bridges a busy trace into one blob); the overlap split drops members that share the fork point but never run concurrently. This isolates genuine parallel sub-agent fan-out (the dominant agent population) from solo agents, unlike keying on the first context block (shared by ~every worker all session). Auxiliary chains are classified first, so a one-shot sidecar never becomes a worker-group member. Set to 0 to disable worker-group tagging (parallel workers keep the generic ::fa: tag). Only applies when WEKA_SPLIT_FLATTENED_AGENTS is True. | +## ENDPOINT + +Endpoint wire-format configuration. Controls how AIPerf serializes message content when building request payloads. The main knob is FORCE_CONTENT_PARTS, which overrides the single-text fast path that emits a plain string for simple turns. + +| Environment Variable | Default | Constraints | Description | +|----------------------|---------|-------------|-------------| +| `AIPERF_ENDPOINT_FORCE_CONTENT_PARTS` | `False` | — | When True, always emit the multi-part content array (e.g. [{"type": "text", "text": "..."}]) for synthetic turns, even when there is only a single text with no media. By default (False) single-text turns emit a plain string to stay compatible with servers that reject list-of-parts content for non-multimodal inputs (e.g. OpenAI Dynamo). Enable when the target server requires the structured content-parts shape unconditionally. | + ## GPU GPU telemetry collection configuration. Controls GPU metrics collection frequency, endpoint detection, and shutdown behavior. Metrics are collected from DCGM endpoints at the specified interval. diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 51cd5a5b5f..4484eb6f88 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -12,6 +12,7 @@ Environment.COMPRESSION.* - Compression settings for streaming file transfers Environment.DATASET.* - Dataset management Environment.DEV.* - Development and debugging settings + Environment.ENDPOINT.* - Endpoint wire-format settings Environment.GPU.* - GPU telemetry collection Environment.HTTP.* - HTTP client socket and connection settings Environment.LOGGING.* - Logging configuration @@ -541,6 +542,30 @@ class _DatasetSettings(BaseSettings): ) +class _EndpointSettings(BaseSettings): + """Endpoint wire-format configuration. + + Controls how AIPerf serializes message content when building request + payloads. The main knob is FORCE_CONTENT_PARTS, which overrides the + single-text fast path that emits a plain string for simple turns. + """ + + model_config = SettingsConfigDict( + env_prefix="AIPERF_ENDPOINT_", + ) + + FORCE_CONTENT_PARTS: bool = Field( + default=False, + description="When True, always emit the multi-part content array " + '(e.g. [{"type": "text", "text": "..."}]) for synthetic turns, ' + "even when there is only a single text with no media. By default " + "(False) single-text turns emit a plain string to stay compatible " + "with servers that reject list-of-parts content for non-multimodal " + "inputs (e.g. OpenAI Dynamo). Enable when the target server requires " + "the structured content-parts shape unconditionally.", + ) + + class _DagSettings(BaseSettings): """Settings for DAG benchmark mode (`dag_jsonl` input type).""" @@ -1758,6 +1783,10 @@ class _Environment(BaseSettings): default_factory=_DagSettings, description="DAG benchmark mode settings (dag_jsonl input type)", ) + ENDPOINT: _EndpointSettings = Field( + default_factory=_EndpointSettings, + description="Endpoint wire-format settings (content serialization, etc.)", + ) DEV: _DeveloperSettings = Field( default_factory=_DeveloperSettings, description="Development and debugging settings", diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index 5405d65c43..fadbfc4f9f 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -8,6 +8,7 @@ from typing import Any, ClassVar from aiperf.common.enums import MediaType +from aiperf.common.environment import Environment from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.common.models import ( BaseResponseData, @@ -316,7 +317,8 @@ def _render_turn_content(self, turn: Turn) -> str | list[dict[str, Any]]: part type names (e.g. ``text`` -> ``input_text`` for Responses API). """ if ( - len(turn.texts) == 1 + not Environment.ENDPOINT.FORCE_CONTENT_PARTS + and len(turn.texts) == 1 and len(turn.texts[0].contents) == 1 and not turn.images and not turn.audios diff --git a/tests/unit/endpoints/test_base_endpoint.py b/tests/unit/endpoints/test_base_endpoint.py index 22f40e95b7..9a9724437b 100644 --- a/tests/unit/endpoints/test_base_endpoint.py +++ b/tests/unit/endpoints/test_base_endpoint.py @@ -3,6 +3,7 @@ import pytest +from aiperf.common.environment import Environment from aiperf.common.models import ParsedResponse, TextResponse, TextResponseData from aiperf.common.models.record_models import ( InferenceServerResponse, @@ -329,3 +330,41 @@ def test_reset_context_without_raw_messages_is_noop(self, endpoint): ] messages = endpoint.build_messages(turns) assert [m["content"] for m in messages] == ["A", "B", "C"] + + +class TestRenderTurnContent: + """Tests for _render_turn_content scalar vs. parts selection.""" + + @pytest.fixture + def endpoint(self): + model_endpoint = create_model_endpoint( + EndpointType.CHAT, base_url="http://localhost:8000/v1/test" + ) + return create_endpoint_with_mock_transport(MockEndpoint, model_endpoint) + + def test_single_text_emits_scalar_by_default(self, endpoint): + """Default: single-text turn produces a plain string, not a parts list.""" + from aiperf.common.models import Text, Turn + + turn = Turn(role="user", texts=[Text(contents=["hello"])]) + content = endpoint._render_turn_content(turn) + assert content == "hello" + + def test_force_content_parts_emits_parts_array(self, endpoint, monkeypatch): + """FORCE_CONTENT_PARTS=True: single-text turn produces a parts list.""" + from aiperf.common.models import Text, Turn + + monkeypatch.setattr(Environment.ENDPOINT, "FORCE_CONTENT_PARTS", True) + turn = Turn(role="user", texts=[Text(contents=["hello"])]) + content = endpoint._render_turn_content(turn) + assert content == [{"type": "text", "text": "hello"}] + + def test_endpoint_settings_env_var(self, monkeypatch): + """AIPERF_ENDPOINT_FORCE_CONTENT_PARTS=true populates _EndpointSettings.""" + from aiperf.common.environment import ( + _EndpointSettings, # type: ignore[attr-defined] + ) + + monkeypatch.setenv("AIPERF_ENDPOINT_FORCE_CONTENT_PARTS", "true") + settings = _EndpointSettings() + assert settings.FORCE_CONTENT_PARTS is True diff --git a/tests/unit/property/test_finite_invariants.py b/tests/unit/property/test_finite_invariants.py index 807073a5a5..5841bcec3d 100644 --- a/tests/unit/property/test_finite_invariants.py +++ b/tests/unit/property/test_finite_invariants.py @@ -376,6 +376,9 @@ def _load_or_init_metric_baseline(current: list[str]) -> list[str]: # are non-negative by construction (summed from the neutral record's histogram). "ProfileResults.pooled_spec_decode_acceptance_histogram", "JsonExportData.pooled_spec_decode_acceptance_histogram", + # _Environment.ENDPOINT: nested _EndpointSettings object, not a numeric + # leaf. The heuristic fires because "int" appears in "_EndpointSettings". + "_Environment.ENDPOINT", } From adb24fa3c8966610f449cd023c03044723ddb269 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:18:28 -0700 Subject: [PATCH 012/119] feat(endpoints): add audio_transcription endpoint type (#1247) Signed-off-by: lkomali (cherry picked from commit 00ba1c5db3fbe5f7e9852fcba2f1f0e269d0f13a) --- docs/cli-options.md | 4 +- docs/plugins/plugin-system.md | 1 + src/aiperf/dataset/composer/synthetic.py | 20 +- src/aiperf/dataset/generator/audio.py | 7 + .../endpoints/openai_audio_transcription.py | 123 +++++++++ src/aiperf/plugin/enums.py | 2 +- src/aiperf/plugin/plugins.yaml | 17 ++ tests/aiperf_mock_server/app.py | 46 ++++ tests/aiperf_mock_server/config.py | 8 + .../test_audio_transcription_endpoint.py | 58 +++++ tests/harness/fake_transport.py | 23 ++ .../composer/test_synthetic_composer.py | 6 +- .../dataset/generator/test_audio_generator.py | 11 + .../test_audio_transcription_endpoint.py | 236 ++++++++++++++++++ 14 files changed, 553 insertions(+), 9 deletions(-) create mode 100644 src/aiperf/endpoints/openai_audio_transcription.py create mode 100644 tests/component_integration/endpoints/test_audio_transcription_endpoint.py create mode 100644 tests/unit/endpoints/test_audio_transcription_endpoint.py diff --git a/docs/cli-options.md b/docs/cli-options.md index ba6a6274c1..6cf50bf877 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -348,7 +348,7 @@ Set a custom API endpoint path (e.g., `/v1/custom`, `/my-api/chat`). By default, #### `--endpoint-type` `` The API endpoint type to benchmark. Determines request/response format and supported features. Common types: `chat` (multi-modal conversations), `embeddings` (vector generation), `completions` (text completion). See enum documentation for all supported endpoint types. -
_Choices: [`chat`, `cohere_rankings`, `completions`, `responses`, `messages`, `chat_embeddings`, `embeddings`, `hf_tei_rankings`, `huggingface_generate`, `image_generation`, `image_edit`, `video_generation`, `image_retrieval`, `nim_embeddings`, `nim_rankings`, `solido_rag`, `raw`, `template`]_ +
_Choices: [`audio_transcription`, `chat`, `cohere_rankings`, `completions`, `responses`, `messages`, `chat_embeddings`, `embeddings`, `hf_tei_rankings`, `huggingface_generate`, `image_generation`, `image_edit`, `video_generation`, `image_retrieval`, `nim_embeddings`, `nim_rankings`, `solido_rag`, `raw`, `template`]_
_Default: `chat`_ #### `--streaming` @@ -1883,7 +1883,7 @@ Set a custom API endpoint path (e.g., `/v1/custom`, `/my-api/chat`). By default, #### `--endpoint-type` `` The API endpoint type to benchmark. Determines request/response format and supported features. Common types: `chat` (multi-modal conversations), `embeddings` (vector generation), `completions` (text completion). See enum documentation for all supported endpoint types. -
_Choices: [`chat`, `cohere_rankings`, `completions`, `responses`, `messages`, `chat_embeddings`, `embeddings`, `hf_tei_rankings`, `huggingface_generate`, `image_generation`, `image_edit`, `video_generation`, `image_retrieval`, `nim_embeddings`, `nim_rankings`, `solido_rag`, `raw`, `template`]_ +
_Choices: [`audio_transcription`, `chat`, `cohere_rankings`, `completions`, `responses`, `messages`, `chat_embeddings`, `embeddings`, `hf_tei_rankings`, `huggingface_generate`, `image_generation`, `image_edit`, `video_generation`, `image_retrieval`, `nim_embeddings`, `nim_rankings`, `solido_rag`, `raw`, `template`]_
_Default: `chat`_ #### `--streaming` diff --git a/docs/plugins/plugin-system.md b/docs/plugins/plugin-system.md index 838ec32eb2..a2ce202f5a 100644 --- a/docs/plugins/plugin-system.md +++ b/docs/plugins/plugin-system.md @@ -381,6 +381,7 @@ pkg = plugins.get_package_metadata("aiperf") # PackageInfo(version, author, ... | Name | Class | Description | |------|-------|-------------| +| `audio_transcription` | `AudioTranscriptionEndpoint` | OpenAI Audio Transcription (Whisper-style) API; multipart upload of audio to `/v1/audio/transcriptions`, returns a plain-text transcript. Pairs with ASR datasets (e.g. `librispeech`). | | `chat` | `ChatEndpoint` | OpenAI Chat Completions API | | `chat_embeddings` | `ChatEmbeddingsEndpoint` | vLLM multimodal embeddings via chat API | | `completions` | `CompletionsEndpoint` | OpenAI Completions API | diff --git a/src/aiperf/dataset/composer/synthetic.py b/src/aiperf/dataset/composer/synthetic.py index fd6c39f1b4..e540f3468d 100644 --- a/src/aiperf/dataset/composer/synthetic.py +++ b/src/aiperf/dataset/composer/synthetic.py @@ -145,7 +145,11 @@ def _create_turn(self, is_first: bool) -> Turn: if self.include_image: turn.images.append(self._generate_image_payloads()) if self.include_audio: - turn.audios.append(self._generate_audio_payloads()) + audio, audio_duration_seconds = self._generate_audio_payloads() + turn.audios.append(audio) + # Hoist the sampled duration so ASR metrics (RTFx) work for + # synthetic audio, mirroring ASR dataset loaders (see hf_asr). + turn.audio_duration_seconds = audio_duration_seconds if self.include_video: turn.videos.append(self._generate_video_payloads()) @@ -233,18 +237,24 @@ def _generate_image_payloads(self) -> Image: image.contents.append(data) return image - def _generate_audio_payloads(self) -> Audio: + def _generate_audio_payloads(self) -> tuple[Audio, float]: """ Generate synthetic audios if the audio length is specified. Returns: - Audio: An audio payload object. + A tuple of the audio payload and the duration (seconds) of the FIRST + content -- the one ASR endpoints actually dispatch (``turn.audios[0] + .contents[0]``) -- so ``Turn.audio_duration_seconds`` matches the + audio sent and RTFx is accurate. """ audio = Audio(name="input_audio") - for _ in range(self._audio_batch_size): + first_duration = 0.0 + for i in range(self._audio_batch_size): data = self.audio_generator.generate() audio.contents.append(data) - return audio + if i == 0: + first_duration = self.audio_generator.last_audio_duration_seconds + return audio, first_duration def _generate_video_payloads(self) -> Video: """ diff --git a/src/aiperf/dataset/generator/audio.py b/src/aiperf/dataset/generator/audio.py index 8586f5c51f..f96ffc7d6d 100644 --- a/src/aiperf/dataset/generator/audio.py +++ b/src/aiperf/dataset/generator/audio.py @@ -82,6 +82,12 @@ def __init__(self, config: AudioConfig | None, **kwargs): self._format_rng = rng.derive("dataset.audio.format") self._data_rng = rng.derive("dataset.audio.data") + # Duration (seconds) sampled by the most recent ``generate()`` call. + # The composer hoists it onto ``Turn.audio_duration_seconds`` so ASR + # metrics (RTFx) work for synthetic audio, mirroring what ASR dataset + # loaders set (see ``hf_asr``). + self.last_audio_duration_seconds: float = 0.0 + def _validate_sampling_rate( self, sampling_rate_hz: int, audio_format: AudioFormat ) -> None: @@ -144,6 +150,7 @@ def generate(self, *args, **kwargs) -> str: audio_length = self._duration_rng.sample_normal( length_mean, length_stddev, lower=0.01 ) + self.last_audio_duration_seconds = audio_length # Randomly select sampling rate and bit depth sampling_rate_hz = int( diff --git a/src/aiperf/endpoints/openai_audio_transcription.py b/src/aiperf/endpoints/openai_audio_transcription.py new file mode 100644 index 0000000000..71f07e6ef1 --- /dev/null +++ b/src/aiperf/endpoints/openai_audio_transcription.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Any + +from aiperf.common.models import ( + InferenceServerResponse, + ParsedResponse, + RequestInfo, +) +from aiperf.endpoints.base_endpoint import BaseEndpoint + +# Covers OpenAI's supported transcription formats. mp3/mpga/mpeg all map to the +# IANA-registered audio/mpeg (RFC 3003); mpga/mpeg are file-extension aliases. +_MIME_BY_FORMAT: dict[str, str] = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "mpga": "audio/mpeg", + "mpeg": "audio/mpeg", + "flac": "audio/flac", + "ogg": "audio/ogg", + "m4a": "audio/mp4", + "mp4": "audio/mp4", + "webm": "audio/webm", +} + +# File field is managed by the endpoint; everything else can come from --extra-inputs. +_RESERVED_PAYLOAD_KEYS: frozenset[str] = frozenset({"file"}) + + +class AudioTranscriptionEndpoint(BaseEndpoint): + """OpenAI Audio Transcription endpoint (/v1/audio/transcriptions). + + Sends audio as a multipart file upload and returns a plain-text transcript. + Compatible with Whisper-style servers (OpenAI, NIM, SGLang). + + Audio is sourced from ``turn.audios`` in the ``","`` format + produced by HFASRDatasetLoader (e.g. ``"wav,..."``). + + Use ``--endpoint-type audio_transcription`` with an ASR dataset loader + (``--dataset librispeech``, ``--dataset voxpopuli``, etc.). + """ + + def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: + if not request_info.turns: + raise ValueError("Audio transcription endpoint requires at least one turn.") + + turn = request_info.turns[-1] + model_endpoint = request_info.model_endpoint + + if not turn.audios or not turn.audios[0].contents: + raise ValueError( + "Audio transcription endpoint requires audio in turn.audios[0]." + ) + + audio_content = turn.audios[0].contents[0] + if not audio_content: + raise ValueError("Audio content is empty.") + + payload: dict[str, Any] = { + "file": _build_audio_field(audio_content), + "model": turn.model or model_endpoint.primary_model_name, + } + + for key, value in model_endpoint.endpoint.extra or []: + if key in _RESERVED_PAYLOAD_KEYS: + self.warning( + f"--extra-inputs {key!r} is managed by the endpoint and was ignored." + ) + continue + payload[key] = value + + for key, value in (turn.extra_body or {}).items(): + if key in _RESERVED_PAYLOAD_KEYS: + self.warning( + f"extra_body {key!r} is managed by the endpoint and was ignored." + ) + continue + payload[key] = value + + self.trace( + lambda: f"Formatted audio transcription payload keys: {list(payload)}" + ) + return payload + + def parse_response( + self, response: InferenceServerResponse + ) -> ParsedResponse | None: + json_obj = response.get_json() + if json_obj is not None: + # response_format json / verbose_json: transcript is the "text" field. + # Use ``is not None`` (not truthiness) so an empty ``{}`` error body + # yields no transcript rather than falling through to get_text(). + text = json_obj.get("text") + usage = json_obj.get("usage") + else: + # response_format text / srt / vtt: the whole body IS the transcript + # (not JSON), so fall back to the raw text rather than dropping it. + text = response.get_text() + usage = None + if not text: + return None + return ParsedResponse( + perf_ns=response.perf_ns, + data=self.make_text_response_data(text), + usage=usage, + ) + + +def _build_audio_field(format_and_b64: str) -> dict[str, Any]: + """Convert a ``","`` audio string into a multipart file descriptor.""" + if "," not in format_and_b64: + raise ValueError( + f"audio content must be in the format ',' " + f"(got {format_and_b64[:40]!r}); " + f"expected e.g. 'wav,'" + ) + fmt, b64 = format_and_b64.split(",", 1) + mime = _MIME_BY_FORMAT.get(fmt.lower(), f"audio/{fmt.lower()}") + return { + "b64_data": b64, + "filename": f"audio.{fmt.lower()}", + "content_type": mime, + } diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index 7314477d14..3c702f8efe 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -67,7 +67,7 @@ EndpointTypeStr: TypeAlias = str EndpointType = plugins.create_enum(PluginType.ENDPOINT, "EndpointType", module=__name__) -"""Dynamic enum for endpoint. Example: EndpointType.CHAT, EndpointType.IMAGE_RETRIEVAL, EndpointType.VIDEO_GENERATION""" +"""Dynamic enum for endpoint. Example: EndpointType.AUDIO_TRANSCRIPTION, EndpointType.IMAGE_GENERATION, EndpointType.VIDEO_GENERATION""" TransportTypeStr: TypeAlias = str TransportType = plugins.create_enum(PluginType.TRANSPORT, "TransportType", module=__name__) diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 62df3f01ec..afe28d621b 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -181,6 +181,23 @@ service_manager: # One-to-one mapping: exactly one endpoint is selected based on endpoint_type config. # ============================================================================= endpoint: + audio_transcription: + class: aiperf.endpoints.openai_audio_transcription:AudioTranscriptionEndpoint + description: | + OpenAI Audio Transcription endpoint (Whisper-style). Uploads audio as a + multipart file and returns a plain-text transcript. Compatible with + OpenAI /v1/audio/transcriptions, NIM, and SGLang. Pair with an ASR + dataset loader (e.g. --dataset librispeech). Use + --endpoint-type audio_transcription. + metadata: + endpoint_path: /v1/audio/transcriptions + supports_streaming: false + produces_tokens: false + tokenizes_input: false + supports_audio: true + requires_form_data: true + metrics_title: Audio Transcription Metrics + chat: class: aiperf.endpoints.openai_chat:ChatEndpoint description: | diff --git a/tests/aiperf_mock_server/app.py b/tests/aiperf_mock_server/app.py index 33b10d2a4d..af6de7c309 100644 --- a/tests/aiperf_mock_server/app.py +++ b/tests/aiperf_mock_server/app.py @@ -212,6 +212,7 @@ async def __call__(self, scope, receive, send): "/generate_stream", "/rag/api/prompt", "/rerank", + "/v1/audio/transcriptions", "/v1/chat/completions", "/v1/chat/embeddings", "/v1/completions", @@ -1301,6 +1302,51 @@ async def image_edits( return ORJSONResponse(body) +# ============================================================================ +# AUDIO TRANSCRIPTION +# ============================================================================ + + +@app.post("/v1/audio/transcriptions", response_model=None) +@with_error_injection +async def audio_transcriptions( + request: Request, + file: UploadFile = File(...), # noqa: B008 + model: str = Form("mock-model"), # noqa: B008 + language: str | None = Form(None), # noqa: B008 + temperature: float | None = Form(None), # noqa: B008 + response_format: str = Form("json"), # noqa: B008 +) -> ORJSONResponse: + """Mock OpenAI Audio Transcription endpoint. + + Drains the uploaded audio file so multipart parsing is exercised, then + returns a deterministic transcript as ``{"text": ...}`` with usage. + """ + endpoint = "/v1/audio/transcriptions" + upload_bytes = await file.read() + + start_time = request.state.start_time + mock_req = ChatCompletionRequest( + model=model, messages=[{"role": "user", "content": "transcribe"}] + ) + ctx = make_ctx(mock_req, endpoint, start_time) + + with track_llm_request(ctx, model, endpoint): + await ctx.latency_sim.wait_for_tokens(len(ctx.tokens)) + body: dict[str, Any] = { + "text": ctx.content, + "input_audio_bytes": len(upload_bytes), + "usage": { + "prompt_tokens": 0, + "completion_tokens": len(ctx.tokens), + "total_tokens": len(ctx.tokens), + }, + } + if language is not None: + body["language"] = language + return ORJSONResponse(body) + + # ============================================================================ # SOLIDO RAG # ============================================================================ diff --git a/tests/aiperf_mock_server/config.py b/tests/aiperf_mock_server/config.py index a7cd3c627f..1d4c749a0d 100644 --- a/tests/aiperf_mock_server/config.py +++ b/tests/aiperf_mock_server/config.py @@ -65,6 +65,7 @@ def apply_flags(self) -> Self: self.ranking_per_passage_latency = 0.0 self.image_retrieval_base_latency = 0.0 self.image_retrieval_per_image_latency = 0.0 + self.audio_transcription_base_latency = 0.0 if self.record_requests is not None: if self.no_tokenizer: raise ValueError( @@ -355,6 +356,13 @@ def apply_flags(self) -> Self: Parameter(name="--image-retrieval-per-image-latency"), ] = 5.0 + # Audio transcription latency (base only; ASR responds once per request) + audio_transcription_base_latency: Annotated[ + float, + Field(description="Audio transcription base latency (ms)", ge=0.0), + Parameter(name="--audio-transcription-base-latency"), + ] = 10.0 + log_level: Annotated[ Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], Field(description="Logging level"), diff --git a/tests/component_integration/endpoints/test_audio_transcription_endpoint.py b/tests/component_integration/endpoints/test_audio_transcription_endpoint.py new file mode 100644 index 0000000000..b3b9a0ae38 --- /dev/null +++ b/tests/component_integration/endpoints/test_audio_transcription_endpoint.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Component integration tests for the audio_transcription endpoint type.""" + +import pytest + +from aiperf.common.constants import IS_WINDOWS_ARM +from tests.component_integration.conftest import ( + ComponentIntegrationTestDefaults as defaults, +) +from tests.harness.utils import AIPerfCLI + + +@pytest.mark.component_integration +@pytest.mark.skipif( + IS_WINDOWS_ARM, + reason="synthetic audio generation requires libsndfile, which has no " + "prebuilt Windows-on-ARM wheel (see audio.py::import_soundfile)", +) +class TestAudioTranscriptionEndpoint: + """Smoke tests for the audio_transcription endpoint against the mock server.""" + + def test_synthetic_audio_transcription(self, cli: AIPerfCLI) -> None: + """Audio transcription with synthetically generated WAV audio.""" + result = cli.run_sync( + f""" + aiperf profile \ + --model openai/whisper-large-v3 \ + --endpoint-type audio_transcription \ + --audio-batch-size 1 \ + --audio-length-mean 3.0 \ + --audio-length-stddev 0.0 \ + --request-count {defaults.request_count} \ + --concurrency {defaults.concurrency} \ + --workers-max {defaults.workers_max} \ + --ui {defaults.ui} + """ + ) + assert result.request_count == defaults.request_count + + def test_audio_transcription_with_extra_inputs(self, cli: AIPerfCLI) -> None: + """--extra-inputs (language, temperature) flow through to multipart form fields.""" + result = cli.run_sync( + f""" + aiperf profile \ + --model openai/whisper-large-v3 \ + --endpoint-type audio_transcription \ + --audio-batch-size 1 \ + --audio-length-mean 3.0 \ + --audio-length-stddev 0.0 \ + --extra-inputs language:en temperature:0.0 \ + --request-count {defaults.request_count} \ + --concurrency {defaults.concurrency} \ + --workers-max {defaults.workers_max} \ + --ui {defaults.ui} + """ + ) + assert result.request_count == defaults.request_count diff --git a/tests/harness/fake_transport.py b/tests/harness/fake_transport.py index 4b056bfb03..eabf35f956 100644 --- a/tests/harness/fake_transport.py +++ b/tests/harness/fake_transport.py @@ -365,6 +365,8 @@ async def _send_request_impl( ) case EndpointType.IMAGE_RETRIEVAL: return await self._do_image_retrieval(payload) + case EndpointType.AUDIO_TRANSCRIPTION: + return await self._do_audio_transcription(payload) case EndpointType.HUGGINGFACE_GENERATE: return await self._dispatch( payload, @@ -463,6 +465,27 @@ async def _do_image_retrieval(self, payload: RequestInputT) -> RequestRecord: _build_image_retrieval_response_data(req), ) + async def _do_audio_transcription(self, payload: RequestInputT) -> RequestRecord: + """Handle audio transcription requests (bypasses _dispatch; the audio + input is multipart, not a tokenized text prompt). Returns a plain-text + transcript as ``{"text": ...}``, matching the OpenAI response shape the + AudioTranscriptionEndpoint parses.""" + start_perf_ns = time.perf_counter_ns() + start_timestamp_ns = time.time_ns() + await _wait_for_processing(self.config.audio_transcription_base_latency, 0.0, 1) + return self._make_json_record( + start_perf_ns, + start_timestamp_ns, + { + "text": "the quick brown fox", + "usage": { + "prompt_tokens": 0, + "completion_tokens": 4, + "total_tokens": 4, + }, + }, + ) + # ============================================================================= # Plugin Registration - Hot-swap production implementations when imported diff --git a/tests/unit/dataset/composer/test_synthetic_composer.py b/tests/unit/dataset/composer/test_synthetic_composer.py index 7dbeb05690..9e16d26ac8 100644 --- a/tests/unit/dataset/composer/test_synthetic_composer.py +++ b/tests/unit/dataset/composer/test_synthetic_composer.py @@ -508,10 +508,13 @@ def test_generate_audio_payloads(self, mock_generate, audio_config, mock_tokeniz composer = SyntheticDatasetComposer( run=make_run(audio_config), tokenizer=mock_tokenizer ) + # The generator records the sampled duration; the composer hoists the + # first content's duration onto the turn for RTFx. + composer.audio_generator.last_audio_duration_seconds = 3.5 # Test audio payload generation turn = Turn() - audio = composer._generate_audio_payloads() + audio, audio_duration_seconds = composer._generate_audio_payloads() turn.audios.append(audio) # Test correct number of audio payloads based on batch_size @@ -520,6 +523,7 @@ def test_generate_audio_payloads(self, mock_generate, audio_config, mock_tokeniz audio_payload = turn.audios[0] assert audio_payload.name == "input_audio" assert audio_payload.contents == ["fake_audio_data"] + assert audio_duration_seconds == 3.5 # ============================================================================ # Configuration Variations Tests diff --git a/tests/unit/dataset/generator/test_audio_generator.py b/tests/unit/dataset/generator/test_audio_generator.py index 732dbc7969..c43bdab7b1 100644 --- a/tests/unit/dataset/generator/test_audio_generator.py +++ b/tests/unit/dataset/generator/test_audio_generator.py @@ -82,6 +82,17 @@ def test_different_audio_length(expected_audio_length): ) +@pytest.mark.parametrize("expected_audio_length", [1.0, 2.5]) +def test_generate_records_sampled_duration(expected_audio_length: float) -> None: + """generate() exposes the sampled duration via last_audio_duration_seconds + so the composer can hoist it onto Turn.audio_duration_seconds for RTFx.""" + config = make_config(mean=expected_audio_length, stddev=0.0) + gen = AudioGenerator(config) + assert gen.last_audio_duration_seconds == 0.0 # unset before any generate() + gen.generate() + assert abs(gen.last_audio_duration_seconds - expected_audio_length) < 0.1 + + def test_negative_length_raises_error(): """v2 NormalDistribution rejects negative mean via Pydantic validation.""" with pytest.raises((ValueError, ConfigurationError)): diff --git a/tests/unit/endpoints/test_audio_transcription_endpoint.py b/tests/unit/endpoints/test_audio_transcription_endpoint.py new file mode 100644 index 0000000000..8626a12034 --- /dev/null +++ b/tests/unit/endpoints/test_audio_transcription_endpoint.py @@ -0,0 +1,236 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for AudioTranscriptionEndpoint.""" + +import base64 + +import orjson +import pytest +from pydantic import TypeAdapter + +from aiperf.common.models import Audio, Text, Turn +from aiperf.common.models.model_endpoint_info import ModelEndpointInfo +from aiperf.endpoints.openai_audio_transcription import ( + AudioTranscriptionEndpoint, + _build_audio_field, +) +from aiperf.plugin.enums import EndpointType +from tests.unit.endpoints.conftest import ( + create_endpoint_with_mock_transport, + create_mock_response, + create_model_endpoint, + create_request_info, +) + +_WAV_B64 = base64.b64encode(b"RIFF\x00\x00\x00\x00WAVEfmt ").decode("ascii") +_WAV_CONTENT = f"wav,{_WAV_B64}" +_MP3_B64 = base64.b64encode(b"\xff\xfb\x90\x00" + b"\x00" * 16).decode("ascii") +_MP3_CONTENT = f"mp3,{_MP3_B64}" + + +class TestAudioTranscriptionEndpoint: + """Tests for AudioTranscriptionEndpoint format_payload + parse_response.""" + + @pytest.fixture + def model_endpoint(self) -> ModelEndpointInfo: + return create_model_endpoint( + EndpointType.AUDIO_TRANSCRIPTION, model_name="openai/whisper-large-v3" + ) + + @pytest.fixture + def endpoint(self, model_endpoint: ModelEndpointInfo) -> AudioTranscriptionEndpoint: + return create_endpoint_with_mock_transport( + AudioTranscriptionEndpoint, model_endpoint + ) + + # ===== format_payload ===== + + def test_format_payload_wav_produces_correct_file_field( + self, endpoint: AudioTranscriptionEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + turn = Turn( + audios=[Audio(contents=[_WAV_CONTENT])], model="openai/whisper-large-v3" + ) + request_info = create_request_info(model_endpoint=model_endpoint, turns=[turn]) + + payload = endpoint.format_payload(request_info) + + assert payload["model"] == "openai/whisper-large-v3" + file_field = payload["file"] + assert file_field["b64_data"] == _WAV_B64 + assert file_field["filename"] == "audio.wav" + assert file_field["content_type"] == "audio/wav" + + def test_format_payload_mp3_mime_and_filename( + self, endpoint: AudioTranscriptionEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + turn = Turn(audios=[Audio(contents=[_MP3_CONTENT])]) + request_info = create_request_info(model_endpoint=model_endpoint, turns=[turn]) + + payload = endpoint.format_payload(request_info) + + assert payload["file"]["filename"] == "audio.mp3" + assert payload["file"]["content_type"] == "audio/mpeg" + + def test_format_payload_is_json_serialisable( + self, endpoint: AudioTranscriptionEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + """Payload must survive model_dump(mode='json') + orjson round-trip.""" + turn = Turn(audios=[Audio(contents=[_WAV_CONTENT])]) + request_info = create_request_info(model_endpoint=model_endpoint, turns=[turn]) + + payload = endpoint.format_payload(request_info) + TypeAdapter(dict).dump_python(payload, mode="json") + orjson.dumps(payload) + + def test_format_payload_extra_inputs_forwarded( + self, model_endpoint: ModelEndpointInfo + ) -> None: + me = create_model_endpoint( + EndpointType.AUDIO_TRANSCRIPTION, + extra=[("language", "en"), ("temperature", "0.0")], + ) + endpoint = create_endpoint_with_mock_transport(AudioTranscriptionEndpoint, me) + turn = Turn(audios=[Audio(contents=[_WAV_CONTENT])]) + request_info = create_request_info(model_endpoint=me, turns=[turn]) + + payload = endpoint.format_payload(request_info) + + assert payload["language"] == "en" + assert payload["temperature"] == "0.0" + + def test_format_payload_reserved_key_in_extra_inputs_ignored( + self, model_endpoint: ModelEndpointInfo, caplog: pytest.LogCaptureFixture + ) -> None: + me = create_model_endpoint( + EndpointType.AUDIO_TRANSCRIPTION, + extra=[("file", "should-be-ignored")], + ) + endpoint = create_endpoint_with_mock_transport(AudioTranscriptionEndpoint, me) + turn = Turn(audios=[Audio(contents=[_WAV_CONTENT])]) + request_info = create_request_info(model_endpoint=me, turns=[turn]) + + payload = endpoint.format_payload(request_info) + + assert isinstance(payload["file"], dict) + assert "b64_data" in payload["file"] + + def test_format_payload_no_turns_raises( + self, endpoint: AudioTranscriptionEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + request_info = create_request_info(model_endpoint=model_endpoint, turns=[]) + + with pytest.raises(ValueError, match="requires at least one turn"): + endpoint.format_payload(request_info) + + def test_format_payload_no_audio_raises( + self, endpoint: AudioTranscriptionEndpoint, model_endpoint: ModelEndpointInfo + ) -> None: + turn = Turn(texts=[Text(contents=["transcribe this"])]) + request_info = create_request_info(model_endpoint=model_endpoint, turns=[turn]) + + with pytest.raises(ValueError, match="requires audio"): + endpoint.format_payload(request_info) + + # ===== parse_response ===== + + def test_parse_response_extracts_text( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + response = create_mock_response( + json_data={"text": "Hello world", "usage": {"prompt_tokens": 10}} + ) + + result = endpoint.parse_response(response) + + assert result is not None + assert result.usage == {"prompt_tokens": 10} + + def test_parse_response_no_json_returns_none( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + response = create_mock_response(json_data=None) + assert endpoint.parse_response(response) is None + + def test_parse_response_missing_text_field_returns_none( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + response = create_mock_response(json_data={"error": "bad request"}) + assert endpoint.parse_response(response) is None + + def test_parse_response_no_usage_is_fine( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + response = create_mock_response(json_data={"text": "Transcript here."}) + result = endpoint.parse_response(response) + assert result is not None + assert result.usage is None + + def test_parse_response_plain_text_body( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + """response_format text/srt/vtt returns a non-JSON body; the whole body + is the transcript and must not be dropped.""" + response = create_mock_response(json_data=None, text="a plain transcript") + result = endpoint.parse_response(response) + assert result is not None + assert result.usage is None + + def test_parse_response_empty_body_returns_none( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + response = create_mock_response(json_data=None, text="") + assert endpoint.parse_response(response) is None + + def test_parse_response_empty_json_object_returns_none( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + """An empty ``{}`` JSON body (e.g. an error path) must yield None, not + fall through to get_text() and return the raw ``"{}"`` as a transcript.""" + response = create_mock_response(json_data={}, text="{}") + assert endpoint.parse_response(response) is None + + def test_parse_response_preserves_empty_usage_object( + self, endpoint: AudioTranscriptionEndpoint + ) -> None: + """An explicit ``"usage": {}`` is preserved as-is (not coerced to None).""" + response = create_mock_response(json_data={"text": "hi", "usage": {}}) + result = endpoint.parse_response(response) + assert result is not None + assert result.usage == {} + + # ===== _build_audio_field ===== + + @pytest.mark.parametrize( + ("fmt", "expected_mime"), + [ + ("wav", "audio/wav"), + ("mp3", "audio/mpeg"), + ("mpga", "audio/mpeg"), + ("mpeg", "audio/mpeg"), + ("flac", "audio/flac"), + ("ogg", "audio/ogg"), + ("m4a", "audio/mp4"), + ("mp4", "audio/mp4"), + ("webm", "audio/webm"), + ], + ) + def test_build_audio_field_known_formats( + self, fmt: str, expected_mime: str + ) -> None: + b64 = base64.b64encode(b"fake").decode("ascii") + field = _build_audio_field(f"{fmt},{b64}") + assert field["content_type"] == expected_mime + assert field["filename"] == f"audio.{fmt}" + assert field["b64_data"] == b64 + + def test_build_audio_field_unknown_format_falls_back_to_audio_slash_fmt( + self, + ) -> None: + b64 = base64.b64encode(b"fake").decode("ascii") + field = _build_audio_field(f"aiff,{b64}") + assert field["content_type"] == "audio/aiff" + + def test_build_audio_field_missing_comma_raises(self) -> None: + with pytest.raises(ValueError, match="format"): + _build_audio_field("nob64herenocomma") From ac5348765a05166df35ffa0a25a387b1d3e953b8 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:40:33 -0700 Subject: [PATCH 013/119] feat(accuracy): restore codegen grade concurrency (AIP-1094) (#1237) Signed-off-by: Elias Bermudez (cherry picked from commit 03c9c6ddc5e6227782e53ded177f1227d332af48) --- .gitignore | 1 + .../accuracy/graders/_codegen_worker.py | 238 ++++++++--- .../graders/_codegen_worker_client.py | 348 ++++++++------- .../test_lcb_codegen_worker_e2e.py | 73 ++++ tests/unit/accuracy/test_codegen_worker.py | 365 +++++++++++++--- .../accuracy/test_codegen_worker_client.py | 397 ++++++++++++++++-- 6 files changed, 1141 insertions(+), 281 deletions(-) diff --git a/.gitignore b/.gitignore index bba2c25f5a..9f5e8db52b 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ src/aiperf/_build_info.py # Chaos/adversarial test run artifacts tests/scripts/.chaos_runs/ +.worktrees/ diff --git a/src/aiperf/accuracy/graders/_codegen_worker.py b/src/aiperf/accuracy/graders/_codegen_worker.py index 352f317cf1..fb885d5deb 100644 --- a/src/aiperf/accuracy/graders/_codegen_worker.py +++ b/src/aiperf/accuracy/graders/_codegen_worker.py @@ -5,11 +5,11 @@ Runs as ``python -m aiperf.accuracy.graders._codegen_worker``. A fresh, single-threaded interpreter that forces the ``fork`` start method once at -startup, then reads one JSONL grading request per line from stdin and writes one -JSONL response per line to a private protocol fd. Executing lighteval's -``codegen_metrics`` here (not in the multithreaded record-processor daemon) -avoids both the nested-function pickle failure under spawn/forkserver and the -multithreaded-fork hang. See issue #1145. +startup, then reads JSONL grading requests from stdin (batching queued +requests) and writes one JSONL response per request to a private protocol fd. +Executing lighteval's ``codegen_metrics`` here (not in the multithreaded +record-processor daemon) avoids both the nested-function pickle failure under +spawn/forkserver and the multithreaded-fork hang. See issue #1145. """ from __future__ import annotations @@ -24,6 +24,13 @@ from collections.abc import Callable from typing import Any, BinaryIO +try: + import fcntl as _fcntl + + _HAS_FCNTL = True +except ImportError: # Windows (worker unsupported there, but import must not crash) + _HAS_FCNTL = False + import orjson _LCB_PASS_AT_K = (1,) @@ -39,38 +46,6 @@ _MAX_ERROR_CHARS = 4096 -def handle_request( - req: Any, - codegen_fn: Callable[..., tuple[dict[str, Any], Any]], -) -> dict[str, Any]: - """Run one grading request. Never raises: all failures become an error - response so a single bad problem cannot kill the worker loop.""" - if not isinstance(req, dict): - # A valid-but-non-object JSON frame (e.g. ``[]``) would raise on the - # ``.get`` below and kill the loop; return the promised error instead. - return {"id": None, "ok": False, "error": "malformed request: expected object"} - req_id = req.get("id") - try: - evaluation_sample = req["evaluation_sample"] - generated_code = req["generated_code"] - except (KeyError, TypeError) as exc: - return {"id": req_id, "ok": False, "error": f"malformed request: {exc!r}"} - - try: - metrics, _ = codegen_fn( - evaluation_sample, - generated_code, - k_list=list(_LCB_PASS_AT_K), - num_process_evaluate=_LCB_NUM_PROCESSES, - ) - except Exception as exc: - # A single bad problem must never crash the worker loop. - error = f"{type(exc).__name__}: {exc}" - return {"id": req_id, "ok": False, "error": _truncate_error(error)} - - return {"id": req_id, "ok": True, "metrics": _coerce_metrics(metrics)} - - def _truncate_error(error: str) -> str: """Bound an error string so it cannot produce a multi-MB response line.""" if len(error) <= _MAX_ERROR_CHARS: @@ -102,27 +77,175 @@ def _is_number(value: Any) -> bool: return False +def handle_batch( + reqs: list[Any], + codegen_fn: Callable[..., tuple[dict[str, Any], Any]], + compute_metrics_fn: Callable[..., dict[str, Any]], +) -> list[dict[str, Any]]: + """Grade a batch of requests with a single codegen_fn call. + + Calls codegen_fn once with all well-formed requests batched together so + lighteval's ProcessPoolExecutor can process multiple problems in parallel. + Never raises: all failures become error responses so a bad batch cannot + kill the worker loop. + """ + all_samples: list[Any] = [] + all_generations: list[Any] = [] + # (req_idx, req_id, start, count) — start/count index into the flat lists. + # evaluation_sample and generated_code are already lists (one element each for + # the current single-problem-per-request grader), so extend, not append, to + # keep all_samples flat and avoid the double-nesting that causes pass@1 = 0.0. + id_map: list[tuple[int, Any, int, int]] = [] + responses: list[dict[str, Any] | None] = [None] * len(reqs) + + for i, req in enumerate(reqs): + if isinstance(req, dict) and "_parse_error" in req: + responses[i] = { + "id": None, + "ok": False, + "error": f"bad json: {req['_parse_error']}", + } + continue + if not isinstance(req, dict): + responses[i] = { + "id": None, + "ok": False, + "error": "malformed request: expected object", + } + continue + req_id = req.get("id") + try: + sample = req["evaluation_sample"] + generation = req["generated_code"] + start = len(all_samples) + all_samples.extend(sample) + all_generations.extend(generation) + except (KeyError, TypeError) as exc: + responses[i] = { + "id": req_id, + "ok": False, + "error": f"malformed request: {exc!r}", + } + continue + id_map.append((i, req_id, start, len(all_samples) - start)) + + if all_samples: + batch_error: str | None = None + raw_results: dict[int, Any] = {} + try: + _, raw_results = codegen_fn( + all_samples, + all_generations, + k_list=list(_LCB_PASS_AT_K), + num_process_evaluate=_LCB_NUM_PROCESSES, + ) + except Exception as exc: + batch_error = _truncate_error(f"{type(exc).__name__}: {exc}") + + for req_idx, req_id, start, count in id_map: + if batch_error is not None: + responses[req_idx] = {"id": req_id, "ok": False, "error": batch_error} + else: + try: + metrics = compute_metrics_fn( + {j: raw_results[start + j] for j in range(count)}, + k_list=list(_LCB_PASS_AT_K), + ) + responses[req_idx] = { + "id": req_id, + "ok": True, + "metrics": _coerce_metrics(metrics), + } + except Exception as exc: + responses[req_idx] = { + "id": req_id, + "ok": False, + "error": _truncate_error(f"{type(exc).__name__}: {exc}"), + } + + return [r for r in responses if r is not None] + + +def _drain_in_memory(stdin: BinaryIO) -> list[bytes]: + """Drain an in-memory stream (e.g. BytesIO in tests) by reading it all at once.""" + return [r for r in (ln.strip() for ln in stdin.read().split(b"\n")) if r] + + +def _drain_buffered(stdin: BinaryIO) -> list[bytes]: + """Drain all lines already buffered in stdin without blocking. + + For BufferedReader (sys.stdin.buffer in production), peek(0) issues a raw + read when the userspace buffer is empty, which blocks on a pipe until the + next request arrives. To avoid this, we temporarily set the underlying fd to + O_NONBLOCK so that peek() raises BlockingIOError (instead of blocking) when + the kernel pipe buffer is empty. readline() then operates on the BufferedReader + normally, consuming data already in its userspace buffer without extra raw reads. + Blocking mode is restored after the drain so the outer readline() in + run_worker_loop can block on the next cycle. + + For in-memory streams without a raw fd (e.g. BytesIO in tests), delegates to + _drain_in_memory. When a real fd exists but fcntl is unavailable, skips the + drain to avoid blocking on a real pipe. + """ + raw = getattr(stdin, "raw", None) + fd = raw.fileno() if raw is not None else -1 + + if not _HAS_FCNTL or fd < 0: + return _drain_in_memory(stdin) if fd < 0 else [] + + lines: list[bytes] = [] + flags = _fcntl.fcntl(fd, _fcntl.F_GETFL) + _fcntl.fcntl(fd, _fcntl.F_SETFL, flags | os.O_NONBLOCK) + try: + while True: + # BufferedReader.peek() catches BlockingIOError internally and returns + # b"" when the kernel pipe buffer is empty — so b"" means "no more data + # available now" (WouldBlock) or EOF. Both stop the drain. + available = stdin.peek(0) # type: ignore[union-attr] + if not available: + break + if b"\n" not in available: + # Only a partial line is buffered; leave it for the next cycle's + # blocking readline() to complete. + break + line = stdin.readline() + line = line.strip() + if line: + lines.append(line) + finally: + _fcntl.fcntl(fd, _fcntl.F_SETFL, flags) # Restore blocking for next cycle + return lines + + def run_worker_loop( stdin: BinaryIO, out: BinaryIO, codegen_fn: Callable[..., tuple[dict[str, Any], Any]], + compute_metrics_fn: Callable[..., dict[str, Any]], ) -> None: - """Serve JSONL grading requests until stdin EOF. One response per request.""" - for line in stdin: - line = line.strip() - if not line: + """Serve JSONL grading requests until stdin EOF. + + Blocks on the first request of each cycle, then non-blocking drains any + already-queued requests to form a batch. Calls codegen_fn once per batch so + lighteval's ProcessPoolExecutor can process multiple problems in parallel. + """ + while True: + first = stdin.readline() + if not first: + break # EOF: client closed stdin, clean exit + first = first.strip() + if not first: continue - try: - req = orjson.loads(line) - except orjson.JSONDecodeError as exc: - resp: dict[str, Any] = { - "id": None, - "ok": False, - "error": f"bad json: {exc}", - } - else: - resp = handle_request(req, codegen_fn) - out.write(orjson.dumps(resp) + b"\n") + batch_raw: list[bytes] = [first] + batch_raw.extend(_drain_buffered(stdin)) + reqs: list[Any] = [] + for raw in batch_raw: + try: + reqs.append(orjson.loads(raw)) + except orjson.JSONDecodeError as exc: + reqs.append({"_parse_error": str(exc)}) + for resp in handle_batch(reqs, codegen_fn, compute_metrics_fn): + out.write(orjson.dumps(resp) + b"\n") out.flush() @@ -200,9 +323,14 @@ def main() -> None: # reaps the worker even if the (heavy) import is still in flight. _start_death_watcher() _force_fork() - from lighteval.tasks.tasks.lcb.codegen_metrics import codegen_metrics - - run_worker_loop(sys.stdin.buffer, protocol_out, codegen_metrics) + from lighteval.tasks.tasks.lcb.codegen_metrics import ( + codegen_metrics, + compute_metrics_from_results, + ) + + run_worker_loop( + sys.stdin.buffer, protocol_out, codegen_metrics, compute_metrics_from_results + ) if __name__ == "__main__": diff --git a/src/aiperf/accuracy/graders/_codegen_worker_client.py b/src/aiperf/accuracy/graders/_codegen_worker_client.py index 1ba4074926..da75d3a94c 100644 --- a/src/aiperf/accuracy/graders/_codegen_worker_client.py +++ b/src/aiperf/accuracy/graders/_codegen_worker_client.py @@ -4,8 +4,8 @@ """In-process client for the out-of-process LCB codegen grading worker. Owned by ``CodeExecutionGrader``. Lazily spawns a single persistent worker -subprocess, serializes grading requests through an ``asyncio.Lock``, and -enforces per-grade timeouts with kill+restart. See issue #1145 and +subprocess, routes concurrent grading requests via an ``id → Future`` demux +table, and enforces per-grade timeouts with kill+restart. See issue #1094 and ``_codegen_worker.py``. """ @@ -72,7 +72,9 @@ def __init__( self._cmd = worker_cmd or _DEFAULT_WORKER_CMD self._max_start_failures = max_start_failures self._proc: asyncio.subprocess.Process | None = None - self._lock = asyncio.Lock() + self._spawn_lock = asyncio.Lock() + self._pending: dict[int, asyncio.Future[dict[str, Any]]] = {} + self._reader_task: asyncio.Task[None] | None = None self._next_id = 0 self._start_failures = 0 self._worker_proven = False @@ -86,68 +88,118 @@ async def grade_codegen( generated_code: list[list[str]], timeout: float, ) -> dict[str, Any]: - async with self._lock: - if self._start_failures >= self._max_start_failures: - raise CodegenWorkerError( - f"grading worker unavailable after {self._start_failures} start failures" - ) - await self._ensure_worker() - self._next_id += 1 - req = { - "id": self._next_id, - "evaluation_sample": evaluation_sample, - "generated_code": generated_code, - } - return await self._request(req, timeout) + if self._start_failures >= self._max_start_failures: + raise CodegenWorkerError( + f"grading worker unavailable after {self._start_failures} start failures" + ) + await self._ensure_worker() + self._next_id += 1 + req_id = self._next_id + req = { + "id": req_id, + "evaluation_sample": evaluation_sample, + "generated_code": generated_code, + } + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + fut: asyncio.Future[dict[str, Any]] = loop.create_future() + self._pending[req_id] = fut + proc = self._proc + if proc is None or proc.stdin is None: + self._pending.pop(req_id, None) + raise CodegenWorkerError("grading worker is not running") + # Guard drain() with the caller's deadline so a worker that stops consuming + # stdin (pipe buffer full) cannot block indefinitely. CancelledError during + # drain() is also handled here so the pending future is cleaned up. + try: + proc.stdin.write(orjson.dumps(req) + b"\n") + await asyncio.wait_for(proc.stdin.drain(), max(0.0, deadline - loop.time())) + except TimeoutError as exc: + # TimeoutError is an OSError subclass (PEP 3151), so it must be caught + # before the OSError clause below or it is silently misrouted there and + # _handle_fault() never runs, leaving the wedged worker alive. + self._pending.pop(req_id, None) + await self._handle_fault(count_start_failure=False) + raise CodegenWorkerError(f"grading worker timed out: {exc!r}") from exc + except (OSError, ConnectionError) as exc: + self._pending.pop(req_id, None) + raise CodegenWorkerError( + f"failed to submit grading request: {exc}" + ) from exc + except asyncio.CancelledError: + self._pending.pop(req_id, None) + raise + try: + return await asyncio.wait_for(fut, max(0.0, deadline - loop.time())) + except TimeoutError as exc: + self._pending.pop(req_id, None) + await self._handle_fault(count_start_failure=False) + raise CodegenWorkerError(f"grading worker timed out: {exc!r}") from exc + except asyncio.CancelledError: + # Cancellation does not desync the protocol — the request was already + # written and the late response will be dropped as stale by + # _dispatch_response. Kill the worker only on real faults. + self._pending.pop(req_id, None) + raise async def _ensure_worker(self) -> None: - if self._proc is not None and self._proc.returncode is None: - return - self._worker_proven = False - self._stderr_tail.clear() - self._close_death_pipe() - # Death pipe: the child inherits the read end; we keep the write end open - # for the worker's life so its close (even via the parent's os._exit) - # tells the worker to reap itself. os.pipe fds are non-inheritable by - # default, so mark the read end inheritable before passing it through. - # Windows has no process groups (nothing for the worker to killpg) and - # subprocess rejects pass_fds there, so skip the pipe and rely on the - # stdin-EOF teardown; the worker's death watcher likewise no-ops there. - death_r: int | None = None - death_w: int | None = None - pass_fds: tuple[int, ...] = () - death_env: dict[str, str] = {} - if not IS_WINDOWS: - death_r, death_w = os.pipe() - os.set_inheritable(death_r, True) - pass_fds = (death_r,) - death_env = {_DEATH_FD_ENV: str(death_r)} - try: - self._proc = await asyncio.create_subprocess_exec( - *self._cmd, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - limit=_STREAM_LIMIT, - # Own process group so _kill can reap lighteval's forked sandbox - # grandchildren, not just the worker (no-op on Windows). - start_new_session=True, - pass_fds=pass_fds, - env={**os.environ, **death_env}, - ) - except Exception as exc: + async with self._spawn_lock: + if self._proc is not None and self._proc.returncode is None: + return + # The old worker exited; tear down its reader/stderr tasks so a late + # EOF from the dead process cannot fault the replacement worker. + if self._proc is not None or self._reader_task is not None: + await self._kill() + self._worker_proven = False + self._stderr_tail.clear() + self._close_death_pipe() + # Death pipe: the child inherits the read end; we keep the write end open + # for the worker's life so its close (even via the parent's os._exit) + # tells the worker to reap itself. os.pipe fds are non-inheritable by + # default, so mark the read end inheritable before passing it through. + # Windows has no process groups (nothing for the worker to killpg) and + # subprocess rejects pass_fds there, so skip the pipe and rely on the + # stdin-EOF teardown; the worker's death watcher likewise no-ops there. + death_r: int | None = None + death_w: int | None = None + pass_fds: tuple[int, ...] = () + death_env: dict[str, str] = {} + if not IS_WINDOWS: + death_r, death_w = os.pipe() + os.set_inheritable(death_r, True) + pass_fds = (death_r,) + death_env = {_DEATH_FD_ENV: str(death_r)} + try: + self._proc = await asyncio.create_subprocess_exec( + *self._cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + limit=_STREAM_LIMIT, + # Own process group so _kill can reap lighteval's forked sandbox + # grandchildren, not just the worker (no-op on Windows). + start_new_session=True, + pass_fds=pass_fds, + env={**os.environ, **death_env}, + ) + except Exception as exc: + if death_r is not None: + os.close(death_r) + if death_w is not None: + os.close(death_w) + self._start_failures += 1 + raise CodegenWorkerError( + f"failed to spawn grading worker: {exc}" + ) from exc if death_r is not None: - os.close(death_r) - if death_w is not None: - os.close(death_w) - self._start_failures += 1 - raise CodegenWorkerError(f"failed to spawn grading worker: {exc}") from exc - if death_r is not None: - os.close(death_r) # the child holds it now; we keep only the write end - self._death_w = death_w - # Drain stderr continuously so the pipe never fills (which would block the - # worker) and the last output is retained to explain a fault. - self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr)) + os.close(death_r) # the child holds it now; we keep only the write end + self._death_w = death_w + # Drain stderr continuously so the pipe never fills (which would block the + # worker) and the last output is retained to explain a fault. + self._stderr_task = asyncio.create_task( + self._drain_stderr(self._proc.stderr) + ) + self._reader_task = asyncio.create_task(self._run_reader()) async def _drain_stderr(self, reader: asyncio.StreamReader | None) -> None: """Continuously copy worker stderr into a bounded tail. Best-effort: @@ -160,96 +212,91 @@ async def _drain_stderr(self, reader: asyncio.StreamReader | None) -> None: async for line in reader: self._stderr_tail.append(line.decode("utf-8", "replace").rstrip("\n")) - async def _request(self, req: dict[str, Any], timeout: float) -> dict[str, Any]: - assert self._proc is not None and self._proc.stdin and self._proc.stdout + def _dispatch_response(self, resp: dict[str, Any]) -> bool: + """Resolve the pending future for resp['id']. + + Returns True on success (including stale/timed-out ids, which are + silently skipped). Returns False when the worker has desynced and the + caller should fault: unhashable id, or ok=True with no metrics dict. + """ + req_id = resp.get("id") + if req_id is None: + # The client only writes integer-keyed requests, so a null id means + # the worker could not parse a request — the stream has desynced. + return False try: - self._proc.stdin.write(orjson.dumps(req) + b"\n") - await self._proc.stdin.drain() - line = await asyncio.wait_for(self._proc.stdout.readline(), timeout) - except asyncio.CancelledError: - # Shutdown/cancellation while awaiting the worker: kill it (and its - # sandbox group) so a pending request can't desync the next grade or - # leave orphaned children, then propagate. Not a worker failure. - await self._handle_fault(count_start_failure=False) - raise - except TimeoutError as exc: - # The worker is alive but this grade is too slow — a per-grade fault, - # not a worker-startup failure. Kill+respawn, but do NOT count it - # toward the readiness cap, or a few slow problems at the start of a - # run would trip the cap and disable all grading. - await self._handle_fault(count_start_failure=False) - raise CodegenWorkerError(f"grading worker timed out: {exc!r}") from exc - except (ConnectionError, BrokenPipeError, ValueError) as exc: - # Transport broke, or the response overran the StreamReader limit - # (ValueError covers asyncio.LimitOverrunError): the worker died or - # desynced, so this counts toward the startup/readiness cap. - await self._handle_fault() - raise CodegenWorkerError(f"grading worker fault: {exc!r}") from exc - - if not line: # EOF: worker died - await self._handle_fault() - raise CodegenWorkerError("grading worker exited before responding") - - try: - resp = orjson.loads(line) - except orjson.JSONDecodeError as exc: - # Garbage on stdout means the worker desynced; fault it like an EOF - # so it is killed and counted rather than silently reused. - await self._handle_fault() - raise CodegenWorkerError( - f"grading worker emitted non-JSON output: {line!r}" - ) from exc - - if not isinstance(resp, dict): - # Valid JSON that is not an object would make the ok/metrics lookups - # below raise; fault it like the other malformed-response classes. - await self._handle_fault() - raise CodegenWorkerError( - f"grading worker emitted a non-object response: {line!r}" - ) - - if resp.get("id") != req["id"]: - # A mismatched id means the response no longer correlates to the - # request; the stream is desynced, so fault it before trusting - # ok/metrics from a stale or wrong frame. - await self._handle_fault() - raise CodegenWorkerError( - f"grading worker response id mismatch: expected {req['id']}, " - f"got {resp.get('id')!r}" - ) - + fut = self._pending.pop(req_id, None) + except TypeError: + # req_id is unhashable (e.g., a list); the worker desynced + return False + if fut is None or fut.done(): + return True # stale id (caller already timed out or cancelled) if not resp.get("ok"): - # A clean error response is a proven worker; do not restart. - self._worker_proven = True - self._start_failures = 0 - raise CodegenWorkerError(resp.get("error", "unknown grading error")) - + fut.set_exception( + CodegenWorkerError(resp.get("error", "unknown grading error")) + ) + self._mark_proven() + return True metrics = resp.get("metrics") if not isinstance(metrics, dict): - # ok:true without a usable metrics dict is a broken worker, not a - # clean result; fault it rather than returning junk to the grader. - await self._handle_fault() - raise CodegenWorkerError( - "grading worker reported success without valid metrics" - ) + return False + fut.set_result(metrics) + self._mark_proven() + return True + + async def _run_reader(self) -> None: + """Read worker responses and resolve the corresponding pending futures by id.""" + if self._proc is None or self._proc.stdout is None: + return + reader = self._proc.stdout + try: + while True: + try: + line = await reader.readline() + except (ValueError, ConnectionError, BrokenPipeError): + # ValueError covers asyncio.LimitOverrunError (oversized line); + # the other two indicate a broken transport. All mean the worker + # desynced; fault it so callers are unblocked. + await self._handle_fault() + return + if not line: + await self._handle_fault() + return + try: + resp = orjson.loads(line) + except orjson.JSONDecodeError: + await self._handle_fault() + return + if not isinstance(resp, dict) or not self._dispatch_response(resp): + await self._handle_fault() + return + except asyncio.CancelledError: + pass + def _mark_proven(self) -> None: self._worker_proven = True self._start_failures = 0 - return metrics async def _handle_fault(self, count_start_failure: bool = True) -> None: - # Only failures that mean the worker never became usable (it died or - # emitted garbage before ever succeeding) count toward the startup cap. - # Per-grade timeouts and shutdown cancellation pass count_start_failure= - # False so a slow grade never disables the whole run. + if self._proc is None: + return # already handled; _handle_fault is idempotent if count_start_failure and not self._worker_proven: self._start_failures += 1 + # Kill and drain stderr before setting futures' exceptions. This ensures + # the debug log (which includes the stderr tail) is written before callers + # are unblocked — otherwise the event loop may schedule waiting coroutines + # between set_exception() and the log, causing the tail to appear empty in + # tests and diagnostics. tail = await self._kill() _log.debug( lambda: f"codegen worker fault (proven={self._worker_proven}, " f"start_failures={self._start_failures}); killed + respawning next grade" + (f"; stderr tail:\n{chr(10).join(tail)}" if tail else "") ) + for fut in list(self._pending.values()): + if not fut.done(): + fut.set_exception(CodegenWorkerError("grading worker fault")) + self._pending.clear() def _close_death_pipe(self) -> None: if self._death_w is not None: @@ -261,11 +308,24 @@ async def _kill(self) -> list[str]: """Kill the worker and return its captured stderr tail (for diagnostics).""" proc, self._proc = self._proc, None task, self._stderr_task = self._stderr_task, None + reader_task, self._reader_task = self._reader_task, None self._close_death_pipe() - if proc is not None and proc.returncode is None: + if proc is not None: + # Kill the process group regardless of returncode: the worker leader may + # have already exited while lighteval's forked sandbox grandchildren are + # still alive in the same dedicated process group. ProcessLookupError + # means the group is already gone — treat that as successful cleanup. _kill_process_group(proc) - with contextlib.suppress(ProcessLookupError): - await proc.wait() + if proc.returncode is None: + with contextlib.suppress(ProcessLookupError): + await proc.wait() + # Skip cancel/await if _reader_task is calling _kill from within itself to + # avoid self-awaiting deadlock when the reader detects a fault condition. + current = asyncio.current_task() + if reader_task is not None and reader_task is not current: + reader_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await reader_task tail: list[str] = [] if task is not None: # The dead worker's stderr hits EOF, so the drain task finishes; bound @@ -277,7 +337,13 @@ async def _kill(self) -> list[str]: return tail async def aclose(self) -> None: - # Acquire the lock so teardown cannot race with an in-flight _request - # clearing/reading _proc across an await. - async with self._lock: + for fut in list(self._pending.values()): + if not fut.done(): + fut.set_exception(CodegenWorkerError("grading worker closed")) + self._pending.clear() + # Acquire _spawn_lock so teardown cannot race with an in-flight + # _ensure_worker awaiting create_subprocess_exec. Without the lock, + # _kill() sees _proc=None and skips cleanup; the spawn then completes + # and leaves an orphaned start_new_session worker with no owner. + async with self._spawn_lock: await self._kill() diff --git a/tests/component_integration/test_lcb_codegen_worker_e2e.py b/tests/component_integration/test_lcb_codegen_worker_e2e.py index 581ed70439..a86071d7d8 100644 --- a/tests/component_integration/test_lcb_codegen_worker_e2e.py +++ b/tests/component_integration/test_lcb_codegen_worker_e2e.py @@ -9,6 +9,8 @@ from __future__ import annotations +import asyncio + import orjson import pytest @@ -32,6 +34,48 @@ def _sample_and_solution() -> tuple[list[dict[str, str]], list[list[str]]]: return sample, [["a, b = map(int, input().split())\nprint(a + b)"]] +def _make_problem( + io_pairs: list[tuple[str, str]], solution: str +) -> tuple[list[dict[str, str]], list[list[str]]]: + sample = [ + { + "input_output": orjson.dumps( + { + "inputs": [i for i, _ in io_pairs], + "outputs": [o for _, o in io_pairs], + "fn_name": None, + } + ).decode() + } + ] + return sample, [[solution]] + + +# Four distinct problems with distinct expected verdicts so per-problem +# misalignment (e.g. from a batching bug) can't pass all assertions. +_CONCURRENT_PROBLEMS: list[tuple[list[dict[str, str]], list[list[str]], float]] = [ + ( + *_make_problem( + [("1 2\n", "3\n"), ("10 20\n", "30\n")], + "a,b=map(int,input().split());print(a+b)", + ), + 1.0, + ), + ( + *_make_problem([("5\n", "25\n"), ("3\n", "9\n")], "n=int(input());print(n*n)"), + 1.0, + ), + ( + *_make_problem([("4\n", "16\n")], "print('wrong')"), # deliberately wrong + 0.0, + ), + ( + *_make_problem([("7\n", "49\n"), ("2\n", "4\n")], "n=int(input());print(n**2)"), + 1.0, + ), +] + + @pytest.mark.slow @pytest.mark.asyncio async def test_worker_grades_correct_stdin_solution() -> None: @@ -45,3 +89,32 @@ async def test_worker_grades_correct_stdin_solution() -> None: assert float(metrics["pass@1"]) == 1.0 finally: await worker.aclose() + + +@pytest.mark.slow +@pytest.mark.asyncio +async def test_worker_grades_multiple_problems_concurrently() -> None: + """Concurrent grade_codegen() calls with distinct problems all return the + correct per-problem verdict. + + Uses four problems with different expected pass@1 values (including one + deliberately wrong) so per-problem misalignment from a batching bug cannot + pass all assertions — identical problems would mask misattributed results. + """ + worker = CodegenGradingWorker() + try: + results = await asyncio.gather( + *[ + worker.grade_codegen(sample, code, timeout=240) + for sample, code, _ in _CONCURRENT_PROBLEMS + ] + ) + assert len(results) == len(_CONCURRENT_PROBLEMS) + for i, ((_, _, expected), result) in enumerate( + zip(_CONCURRENT_PROBLEMS, results, strict=True) + ): + assert float(result["pass@1"]) == expected, ( + f"problem {i}: expected pass@1={expected}, got {result}" + ) + finally: + await worker.aclose() diff --git a/tests/unit/accuracy/test_codegen_worker.py b/tests/unit/accuracy/test_codegen_worker.py index 019af1dbb9..d11ac067d7 100644 --- a/tests/unit/accuracy/test_codegen_worker.py +++ b/tests/unit/accuracy/test_codegen_worker.py @@ -6,6 +6,7 @@ import subprocess import sys import textwrap +from collections.abc import Callable from pathlib import Path from typing import Any @@ -17,84 +18,334 @@ _FORK_AVAILABLE = "fork" in mp.get_all_start_methods() -def _fake_codegen_ok(*_args: Any, **_kwargs: Any) -> tuple[dict[str, Any], Any]: - return {"pass@1": 1.0}, {} +def _fake_compute_metrics( + results: dict, k_list: list[int] | None = None +) -> dict[str, Any]: + # Mirrors compute_metrics_from_results: returns {"pass@1": } using + # the single-problem results dict {0: [[True, True, ...]]} passed by handle_batch. + result_list = results.get(0, [[-2]]) # [-2] = compile error + if result_list and all(x > 0 for x in result_list[0]): + return {"pass@1": 1.0} + return {"pass@1": 0.0} -def _fake_codegen_boom(*_args: Any, **_kwargs: Any) -> tuple[dict[str, Any], Any]: - raise RuntimeError("sandbox exploded") +def _fake_codegen_batch_ok( + samples: list, generations: list, **_kwargs: Any +) -> tuple[dict[str, Any], dict[int, list]]: + # Assert the contract: lighteval indexes samples[i] as a dict, so each + # element must be a dict, not a list. This catches the double-nesting bug + # where append() was used instead of extend() in handle_batch. + for s in samples: + assert isinstance(s, dict), ( + f"expected sample dict, got {type(s).__name__}: {s!r}" + ) + n = len(samples) + raw_results = {i: [[True]] for i in range(n)} # all pass + return {"pass@1": 1.0}, raw_results -def _fake_codegen_list_pass(*_args: Any, **_kwargs: Any) -> tuple[dict[str, Any], Any]: - return {"pass@1": [1.0]}, {} +def _fake_codegen_batch_boom( + samples: list, generations: list, **_kwargs: Any +) -> tuple[dict[str, Any], dict[int, list]]: + raise RuntimeError("pool exploded") -class TestHandleRequest: - def test_ok_request_returns_metrics_with_id(self) -> None: - req = { - "id": 7, +class TestHandleBatch: + def _req(self, req_id: int) -> dict[str, Any]: + return { + "id": req_id, "evaluation_sample": [{"input_output": "{}"}], "generated_code": [["x"]], } - resp = worker.handle_request(req, _fake_codegen_ok) - assert resp == {"id": 7, "ok": True, "metrics": {"pass@1": 1.0}} + + def test_single_request_returns_one_ok_response(self) -> None: + resps = worker.handle_batch( + [self._req(1)], _fake_codegen_batch_ok, _fake_compute_metrics + ) + assert len(resps) == 1 + assert resps[0] == {"id": 1, "ok": True, "metrics": {"pass@1": 1.0}} + + def test_batch_of_n_calls_codegen_fn_once(self) -> None: + call_count = 0 + + def counting_codegen(samples, generations, **kwargs): + nonlocal call_count + call_count += 1 + n = len(samples) + return {"pass@1": 1.0}, {i: [[True]] for i in range(n)} + + reqs = [self._req(i) for i in range(1, 5)] + resps = worker.handle_batch(reqs, counting_codegen, _fake_compute_metrics) + assert call_count == 1 + assert len(resps) == 4 + assert all(r["ok"] for r in resps) + assert [r["id"] for r in resps] == [1, 2, 3, 4] + + def test_response_order_matches_request_order(self) -> None: + reqs = [self._req(i) for i in [7, 3, 99]] + resps = worker.handle_batch(reqs, _fake_codegen_batch_ok, _fake_compute_metrics) + assert [r["id"] for r in resps] == [7, 3, 99] + + def test_batch_exception_returns_error_for_all(self) -> None: + reqs = [self._req(i) for i in range(1, 4)] + resps = worker.handle_batch( + reqs, _fake_codegen_batch_boom, _fake_compute_metrics + ) + assert len(resps) == 3 + assert all(not r["ok"] for r in resps) + assert all("pool exploded" in r["error"] for r in resps) + + def test_malformed_request_in_batch_does_not_affect_others(self) -> None: + reqs = [ + self._req(1), + {"id": 2}, # missing evaluation_sample + generated_code + self._req(3), + ] + resps = worker.handle_batch(reqs, _fake_codegen_batch_ok, _fake_compute_metrics) + assert len(resps) == 3 + assert resps[0] == {"id": 1, "ok": True, "metrics": {"pass@1": 1.0}} + assert resps[1]["id"] == 2 + assert not resps[1]["ok"] + assert resps[2] == {"id": 3, "ok": True, "metrics": {"pass@1": 1.0}} + + def test_non_object_request_in_batch_is_error(self) -> None: + reqs = [[1, 2, 3], self._req(5)] + resps = worker.handle_batch(reqs, _fake_codegen_batch_ok, _fake_compute_metrics) + assert len(resps) == 2 + assert resps[0]["id"] is None + assert not resps[0]["ok"] + assert resps[1] == {"id": 5, "ok": True, "metrics": {"pass@1": 1.0}} + + def test_parse_error_sentinel_produces_error_response(self) -> None: + # run_worker_loop encodes JSON decode errors as {"_parse_error": "..."}. + reqs = [{"_parse_error": "unexpected token"}, self._req(2)] + resps = worker.handle_batch(reqs, _fake_codegen_batch_ok, _fake_compute_metrics) + assert len(resps) == 2 + assert resps[0]["id"] is None + assert not resps[0]["ok"] + assert "unexpected token" in resps[0]["error"] + assert resps[1]["ok"] def test_list_shaped_pass_at_1_is_preserved(self) -> None: - # lighteval returns pass@1 as a list on some pins; it must survive - # coercion rather than be dropped as non-numeric (silent 0.000 bug). - req = { - "id": 9, - "evaluation_sample": [{"input_output": "{}"}], - "generated_code": [["x"]], - } - resp = worker.handle_request(req, _fake_codegen_list_pass) - assert resp["ok"] is True - assert resp["metrics"]["pass@1"] == [1.0] + # lighteval returns pass@1 as a list on some pins; _coerce_metrics must + # preserve it rather than drop it (silent 0.000 bug if dropped). + def list_metrics(_results: dict, **_kw: Any) -> dict[str, Any]: + return {"pass@1": [1.0]} + + resps = worker.handle_batch( + [self._req(1)], _fake_codegen_batch_ok, list_metrics + ) + assert resps[0]["ok"] is True + assert resps[0]["metrics"]["pass@1"] == [1.0] + + def test_compute_metrics_exception_produces_error_response(self) -> None: + def boom_compute(_results: dict, **_kw: Any) -> dict[str, Any]: + raise RuntimeError("metrics exploded") + + resps = worker.handle_batch( + [self._req(1)], _fake_codegen_batch_ok, boom_compute + ) + assert not resps[0]["ok"] + assert "metrics exploded" in resps[0]["error"] + + def test_truncate_error_long_string(self) -> None: + long_err = "x" * 5000 + truncated = worker._truncate_error(long_err) + assert len(truncated) < 5000 + assert truncated.endswith("...[truncated]") + + def test_is_number_non_numeric_returns_false(self) -> None: + assert not worker._is_number("notanumber") + assert not worker._is_number(None) + assert not worker._is_number([1, 2]) + + def test_non_finite_metric_values_are_dropped(self) -> None: + # NaN/Inf must not cross the JSONL boundary (repo NaN/Inf discipline). + def _nan_inf( + samples: list, generations: list, **_kwargs: Any + ) -> tuple[dict[str, Any], dict[int, list]]: + n = len(samples) + return {"pass@1": 1.0}, {i: [[True]] for i in range(n)} + + def _nan_compute(results: dict, **_kwargs: Any) -> dict[str, Any]: + return {"pass@1": float("nan"), "extra": float("inf"), "ok": 1.0} + + req = {"id": 9, "evaluation_sample": [{}], "generated_code": [["x"]]} + resps = worker.handle_batch([req], _nan_inf, _nan_compute) + assert resps[0]["ok"] is True + assert "pass@1" not in resps[0]["metrics"] + assert "extra" not in resps[0]["metrics"] + assert resps[0]["metrics"]["ok"] == 1.0 + - def test_codegen_exception_becomes_error_response(self) -> None: - req = { - "id": 3, +class TestRunWorkerLoopBatch: + def _run( + self, + payloads: list[dict[str, Any]], + codegen_fn: Callable[ + ..., tuple[dict[str, Any], dict[int, list]] + ] = _fake_codegen_batch_ok, + compute_metrics_fn: Callable[..., dict[str, Any]] = _fake_compute_metrics, + ) -> list[dict[str, Any]]: + # Write all payloads to a BytesIO pipe so they are already queued when + # run_worker_loop reads; this exercises the non-blocking drain path. + data = b"".join(orjson.dumps(p) + b"\n" for p in payloads) + stdin = io.BytesIO(data) + out = io.BytesIO() + worker.run_worker_loop(stdin, out, codegen_fn, compute_metrics_fn) + out.seek(0) + return [orjson.loads(line) for line in out if line.strip()] + + def _req(self, req_id: int) -> dict[str, Any]: + return { + "id": req_id, "evaluation_sample": [{"input_output": "{}"}], "generated_code": [["x"]], } - resp = worker.handle_request(req, _fake_codegen_boom) - assert resp["id"] == 3 - assert resp["ok"] is False - assert "sandbox exploded" in resp["error"] - - def test_malformed_request_missing_fields_is_error(self) -> None: - resp = worker.handle_request({"id": 5}, _fake_codegen_ok) - assert resp["id"] == 5 - assert resp["ok"] is False - assert resp["error"] - - def test_non_object_request_is_error_not_crash(self) -> None: - # A valid-but-non-object JSON frame must not raise (which would kill the - # worker loop); it returns the promised error response with id=None. - resp = worker.handle_request([1, 2, 3], _fake_codegen_ok) - assert resp["id"] is None - assert resp["ok"] is False - assert resp["error"] - def test_non_finite_metric_values_are_dropped(self) -> None: - # NaN/Inf must not cross the JSONL boundary (repo NaN/Inf discipline). - def _nan_inf(*_a: Any, **_k: Any) -> tuple[dict[str, Any], Any]: - return {"pass@1": float("nan"), "extra": float("inf"), "ok": 1.0}, {} + def test_pre_queued_requests_are_batched_in_one_call(self) -> None: + call_count = 0 + + def counting_codegen(samples, generations, **kwargs): + nonlocal call_count + call_count += 1 + n = len(samples) + return {"pass@1": 1.0}, {i: [[True]] for i in range(n)} + + reqs = [self._req(i) for i in range(1, 4)] + resps = self._run(reqs, counting_codegen) + assert call_count == 1 + assert len(resps) == 3 + + def test_responses_carry_correct_ids(self) -> None: + reqs = [self._req(i) for i in [10, 20, 30]] + resps = self._run(reqs) + assert {r["id"] for r in resps} == {10, 20, 30} + + def test_drain_in_memory_splits_on_newlines(self) -> None: + data = b'{"id":1}\n{"id":2}\n' + result = worker._drain_in_memory(io.BytesIO(data)) + assert result == [b'{"id":1}', b'{"id":2}'] + + def test_drain_buffered_no_fcntl_and_no_fd_uses_in_memory_path( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(worker, "_HAS_FCNTL", False) + data = io.BytesIO(b'{"id":1}\n{"id":2}\n') + result = worker._drain_buffered(data) + assert result == [b'{"id":1}', b'{"id":2}'] + + def test_drain_buffered_no_fcntl_with_real_fd_skips_drain( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(worker, "_HAS_FCNTL", False) + r_fd, w_fd = os.pipe() + try: + os.write(w_fd, b'{"id":1}\n') + with os.fdopen(r_fd, "rb") as reader: + r_fd = -1 # fdopen takes ownership + result = worker._drain_buffered(reader) + assert result == [] # skipped, not blocked + finally: + if r_fd >= 0: + os.close(r_fd) + os.close(w_fd) - resp = worker.handle_request( - {"id": 9, "evaluation_sample": [{}], "generated_code": [["x"]]}, _nan_inf + @pytest.mark.skipif( + not worker._HAS_FCNTL, reason="fcntl required for O_NONBLOCK drain" + ) + def test_drain_buffered_complete_line_then_empty_pipe_stops(self) -> None: + # BufferedReader.peek() returns b"" (not BlockingIOError) when the kernel + # pipe buffer is empty with write end still open. Verifies that the drain + # reads one complete line and stops cleanly on the next b"" peek. + req = self._req(77) + data = orjson.dumps(req) + b"\n" + r_fd, w_fd = os.pipe() + try: + os.write(w_fd, data) # one complete line, write end still open + with os.fdopen(r_fd, "rb") as reader: + r_fd = -1 + result = worker._drain_buffered(reader) + assert result == [data.strip()] + finally: + if r_fd >= 0: + os.close(r_fd) + os.close(w_fd) + + def test_drain_buffered_partial_line_breaks_without_readline(self) -> None: + # Covers lines 202-207: peek returns bytes without newline → break. + r_fd, w_fd = os.pipe() + try: + os.write(w_fd, b"partial-no-newline") # no \n, write end still open + with os.fdopen(r_fd, "rb") as reader: + r_fd = -1 + result = worker._drain_buffered(reader) + assert result == [] # nothing drained; no complete line + finally: + if r_fd >= 0: + os.close(r_fd) + os.close(w_fd) + + def test_run_worker_loop_skips_blank_lines(self) -> None: + # Covers line 237: blank lines between requests are skipped (continue). + reqs = [self._req(1), self._req(2)] + data = b"\n" + orjson.dumps(reqs[0]) + b"\n\n" + orjson.dumps(reqs[1]) + b"\n" + stdin = io.BytesIO(data) + out = io.BytesIO() + worker.run_worker_loop( + stdin, out, _fake_codegen_batch_ok, _fake_compute_metrics ) - assert resp["ok"] is True - assert "pass@1" not in resp["metrics"] - assert "extra" not in resp["metrics"] - assert resp["metrics"]["ok"] == 1.0 + out.seek(0) + resps = [orjson.loads(ln) for ln in out if ln.strip()] + assert {r["id"] for r in resps} == {1, 2} + + def test_partial_jsonl_line_is_deferred_to_next_cycle(self) -> None: + # Exercises the O_NONBLOCK peek(0) + partial-line guard in _drain_buffered. + # A partial write (no trailing newline) must not be submitted as a request; + # only after the newline arrives should the line be processed. + import threading + import time + + req = self._req(42) + line = orjson.dumps(req) + b"\n" + r_fd, w_fd = os.pipe() + out = io.BytesIO() + + with os.fdopen(r_fd, "rb") as reader: + t = threading.Thread( + target=worker.run_worker_loop, + args=(reader, out, _fake_codegen_batch_ok, _fake_compute_metrics), + daemon=True, + ) + t.start() + + # Write the request in two parts: body first, newline second + os.write(w_fd, line[:-1]) # partial — no newline yet + time.sleep(0.05) # give the loop a chance to drain + + # Worker must still be blocked (no complete line yet) + assert t.is_alive() + out.seek(0) + assert out.read() == b"" # nothing written yet + + # Complete the line and close stdin to trigger EOF after processing + os.write(w_fd, b"\n") + os.close(w_fd) + t.join(timeout=5) + + assert not t.is_alive() + out.seek(0) + resps = [orjson.loads(ln) for ln in out if ln.strip()] + assert len(resps) == 1 + assert resps[0]["id"] == 42 + assert resps[0]["ok"] is True class TestRunWorkerLoop: def _run(self, requests: list[bytes], codegen_fn) -> list[dict]: stdin = io.BytesIO(b"".join(r + b"\n" for r in requests)) out = io.BytesIO() - worker.run_worker_loop(stdin, out, codegen_fn) + worker.run_worker_loop(stdin, out, codegen_fn, _fake_compute_metrics) out.seek(0) return [orjson.loads(line) for line in out.read().splitlines() if line] @@ -107,16 +358,16 @@ def test_processes_each_request_in_order(self) -> None: {"id": 2, "evaluation_sample": [{}], "generated_code": [["b"]]} ), ] - resps = self._run(reqs, _fake_codegen_ok) + resps = self._run(reqs, _fake_codegen_batch_ok) assert [r["id"] for r in resps] == [1, 2] assert all(r["ok"] for r in resps) def test_eof_stops_the_loop(self) -> None: - resps = self._run([], _fake_codegen_ok) + resps = self._run([], _fake_codegen_batch_ok) assert resps == [] def test_garbled_line_yields_error_response(self) -> None: - resps = self._run([b"{not json"], _fake_codegen_ok) + resps = self._run([b"{not json"], _fake_codegen_batch_ok) assert len(resps) == 1 assert resps[0]["ok"] is False assert resps[0]["id"] is None diff --git a/tests/unit/accuracy/test_codegen_worker_client.py b/tests/unit/accuracy/test_codegen_worker_client.py index 718ac07ae4..78e68d153f 100644 --- a/tests/unit/accuracy/test_codegen_worker_client.py +++ b/tests/unit/accuracy/test_codegen_worker_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import contextlib import os import sys import textwrap @@ -22,39 +23,67 @@ def _write_worker(tmp_path: Path, body: str) -> list[str]: return [sys.executable, str(script)] -# Echoes pass@1=1.0 for every request, correlating id. -_ECHO_OK = """ - import sys, orjson +# Receives a request, signals receipt via a file, then blocks until a "go" +# file appears before responding. Used to guarantee fut is unresolved when +# we cancel, working around a Python 3.11 asyncio.wait_for bug where an +# already-resolved future silently absorbs the CancelledError. +_GATED_OK = """ + import sys, orjson, pathlib, time + gate_dir = pathlib.Path(sys.argv[1]) for line in sys.stdin.buffer: line = line.strip() if not line: continue req = orjson.loads(line) + (gate_dir / f"recv_{req['id']}").touch() + while not (gate_dir / "go").exists(): + time.sleep(0.005) resp = {"id": req["id"], "ok": True, "metrics": {"pass@1": 1.0}} sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") sys.stdout.buffer.flush() """ -# Worker that records overlap: writes "BUSY" markers around a small delay so a -# second concurrent request would interleave if not serialized. -_ECHO_TRACKED = """ - import sys, orjson, time - inflight = 0 +def _write_gated_worker(tmp_path: Path) -> tuple[list[str], Path]: + gate_dir = tmp_path / "gate" + gate_dir.mkdir() + script = tmp_path / "gated_worker.py" + script.write_text(textwrap.dedent(_GATED_OK)) + return [sys.executable, str(script), str(gate_dir)], gate_dir + + +# Echoes pass@1=1.0 for every request, correlating id. +_ECHO_OK = """ + import sys, orjson for line in sys.stdin.buffer: line = line.strip() if not line: continue req = orjson.loads(line) - inflight += 1 - overlap = inflight > 1 - time.sleep(0.05) - inflight -= 1 - resp = {"id": req["id"], "ok": True, "metrics": {"pass@1": 1.0}, "overlap": overlap} + resp = {"id": req["id"], "ok": True, "metrics": {"pass@1": 1.0}} sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") sys.stdout.buffer.flush() """ +# Buffers the first 4 requests and responds in REVERSE id order to exercise +# the demux table (correct demux requires id matching, not position matching). +_REVERSE_BATCH_OF_4 = """ + import sys, orjson + buf = [] + for line in sys.stdin.buffer: + line = line.strip() + if not line: + continue + req = orjson.loads(line) + buf.append(req) + if len(buf) == 4: + for r in reversed(buf): + resp = {"id": r["id"], "ok": True, "metrics": {"pass@1": r["id"] * 0.1}} + sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") + sys.stdout.buffer.flush() + buf = [] +""" + class TestHappyPath: async def test_grade_returns_metrics(self, tmp_path) -> None: @@ -78,19 +107,110 @@ async def test_second_grade_reuses_same_worker(self, tmp_path) -> None: await worker.aclose() -class TestSerialization: - async def test_concurrent_grades_do_not_interleave(self, tmp_path) -> None: - worker = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, _ECHO_TRACKED)) +class TestConcurrency: + async def test_concurrent_grades_all_complete(self, tmp_path) -> None: + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, _ECHO_OK)) + try: + results = await asyncio.gather( + *[ + w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=30) + for _ in range(5) + ] + ) + assert all(r == {"pass@1": 1.0} for r in results) + finally: + await w.aclose() + + async def test_concurrent_grades_demux_by_id_not_position(self, tmp_path) -> None: + # 4 concurrent grades; mock responds in reverse order. + # If demux were position-based, callers would get wrong metrics. + w = CodegenGradingWorker( + worker_cmd=_write_worker(tmp_path, _REVERSE_BATCH_OF_4) + ) + try: + results = await asyncio.gather( + *[ + w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=30) + for _ in range(4) + ] + ) + # IDs 1-4 → pass@1 values 0.1, 0.2, 0.3, 0.4 (one per caller) + values = sorted(r["pass@1"] for r in results) + assert values == pytest.approx([0.1, 0.2, 0.3, 0.4]) + finally: + await w.aclose() + + async def test_fault_cancels_all_pending_futures(self, tmp_path) -> None: + # Worker dies immediately after the first line — all concurrent callers + # should raise CodegenWorkerError, not hang. + w = CodegenGradingWorker( + worker_cmd=_write_worker( + tmp_path, + """ + import sys + sys.stdin.buffer.readline() # consume one line then exit + """, + ) + ) + try: + with pytest.raises(CodegenWorkerError): + await asyncio.gather( + *[ + w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=10) + for _ in range(3) + ], + return_exceptions=False, + ) + finally: + await w.aclose() + + async def test_stale_id_after_timeout_does_not_crash(self, tmp_path) -> None: + # Reader receives a response for an id that the caller already timed out on. + # The stale future was already removed from _pending; the reader must skip it. + # Use _ECHO_OK with a very short timeout so the grade times out, then send + # a second grade to prove the worker (if restarted) still works. + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, _ECHO_OK)) + try: + with pytest.raises(CodegenWorkerError): + await w.grade_codegen( + [{"input_output": "{}"}], [["x"]], timeout=0.000001 + ) + # If stale id handling is broken, the second grade would hang or crash. + result = await w.grade_codegen( + [{"input_output": "{}"}], [["x"]], timeout=10 + ) + assert result == {"pass@1": 1.0} + finally: + await w.aclose() + + async def test_aclose_with_pending_futures_does_not_hang(self, tmp_path) -> None: + hang_worker = """ + import sys, time + for line in sys.stdin.buffer: + time.sleep(3600) + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, hang_worker)) + grade_task = asyncio.create_task( + w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=60) + ) + await asyncio.sleep(0.05) # let grade_task start and block + await w.aclose() # must not hang even with grade_task pending + grade_task.cancel() + with contextlib.suppress(asyncio.CancelledError, CodegenWorkerError): + await grade_task + + async def test_concurrent_grades_return_correct_results(self, tmp_path) -> None: + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, _ECHO_OK)) try: results = await asyncio.gather( *[ - worker.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=30) + w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=30) for _ in range(4) ] ) assert all(r == {"pass@1": 1.0} for r in results) finally: - await worker.aclose() + await w.aclose() # The very first grade (client request id==1) hangs forever to trigger a @@ -180,22 +300,39 @@ async def test_timeout_on_unproven_worker_does_not_count_as_start_failure( class TestCancellation: - async def test_cancellation_kills_worker_and_propagates(self, tmp_path) -> None: - """A cancel while awaiting the worker (e.g. shutdown) kills the worker and - re-raises, rather than leaving it running with a pending request.""" - worker = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, _HANG_THEN_OK)) + async def test_cancellation_propagates_without_killing_worker( + self, tmp_path + ) -> None: + # Cancellation removes the request from _pending and re-raises; it does + # not fault the worker because the protocol is not desynced (the late + # response is dropped as a stale id by _dispatch_response). Concurrent + # grades continue unaffected. + # + # Uses a gated worker (not _ECHO_OK) so fut is guaranteed unresolved + # when grade.cancel() fires. On Python 3.11, asyncio.wait_for silently + # absorbs CancelledError if the inner future already has a result, so + # we must ensure the worker has not yet responded at cancel time. + worker_cmd, gate_dir = _write_gated_worker(tmp_path) + worker = CodegenGradingWorker(worker_cmd=worker_cmd) try: grade = asyncio.create_task( worker.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=30) ) - for _ in range(200): # wait until the request is in flight (worker up) - if worker._proc is not None: + for _ in range(200): # wait until the worker received the request + if (gate_dir / "recv_1").exists(): break await asyncio.sleep(0.01) + # Worker holds at the gate — fut is unresolved — cancel is safe. grade.cancel() with pytest.raises(asyncio.CancelledError): await grade - assert worker._proc is None # cancellation killed the worker + assert worker._proc is not None # worker stays alive after cancel + # Release the gate so the stale response drains and the next grade works. + (gate_dir / "go").touch() + result = await worker.grade_codegen( + [{"input_output": "{}"}], [["x"]], timeout=10 + ) + assert result == {"pass@1": 1.0} finally: await worker.aclose() @@ -269,7 +406,8 @@ async def test_worker_that_dies_on_start_hits_cap(self, tmp_path) -> None: # Always echoes a fixed WRONG id regardless of the request id, simulating a -# worker whose responses no longer correlate to requests (a desync). +# worker whose responses no longer correlate to requests (a desync). With the +# demux reader, the stale id is silently dropped and the caller times out. _ECHO_WRONG_ID = """ import sys, orjson for line in sys.stdin.buffer: @@ -300,13 +438,16 @@ async def test_oversized_line_faults_and_kills_worker(self, tmp_path) -> None: class TestResponseIdMismatch: async def test_wrong_id_faults_and_kills_worker(self, tmp_path) -> None: + # With the demux reader, a wrong id is treated as a stale response and + # silently dropped. The caller's future is never resolved, so it times out + # and calls _handle_fault, which kills the worker. worker = CodegenGradingWorker( worker_cmd=_write_worker(tmp_path, _ECHO_WRONG_ID) ) try: with pytest.raises(CodegenWorkerError): await worker.grade_codegen( - [{"input_output": "{}"}], [["x"]], timeout=30 + [{"input_output": "{}"}], [["x"]], timeout=0.5 ) assert worker._proc is None finally: @@ -403,4 +544,204 @@ async def test_death_pipe_held_for_worker_life_then_closed(self, tmp_path) -> No assert worker._death_w is not None finally: await worker.aclose() - assert worker._death_w is None + + +class TestCoverageGaps: + """Targeted tests for previously uncovered branches.""" + + async def test_grade_codegen_proc_is_none_raises(self) -> None: + # Covers the proc-is-None guard added after _ensure_worker (line 107-109). + w = CodegenGradingWorker.__new__(CodegenGradingWorker) + w._cmd = ["python", "-c", ""] + w._max_start_failures = 3 + w._proc = None + w._spawn_lock = asyncio.Lock() + w._pending = {} + w._reader_task = None + w._next_id = 0 + w._start_failures = 0 + w._worker_proven = False + from collections import deque + + w._stderr_tail = deque(maxlen=64) + w._stderr_task = None + w._death_w = None + # Manually set _proc to a sentinel with returncode=None so _ensure_worker returns, + # then null out stdin so the None-stdin branch is hit. + + class _FakeStdin: + pass + + class _FakeProc: + returncode = None + stdin = None + stdout = None + stderr = None + pid = 99999 + + w._proc = _FakeProc() # type: ignore[assignment] + with pytest.raises(CodegenWorkerError, match="not running"): + await w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=5) + + async def test_grade_codegen_write_error_raises_worker_error( + self, tmp_path: Path + ) -> None: + # Covers OSError from write/drain (lines 113-117). + # Use _ECHO_OK but close stdin immediately after spawn to force BrokenPipeError. + broken_worker = """ + import sys + sys.stdin.close() + import time; time.sleep(10) + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, broken_worker)) + try: + with pytest.raises(CodegenWorkerError): + await w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=5) + finally: + await w.aclose() + + async def test_ensure_worker_respawns_after_worker_exits( + self, tmp_path: Path + ) -> None: + # Worker serves one request then exits. The second grade_codegen() call + # must detect the dead worker via returncode, respawn, and succeed. + die_after_one = """ + import sys, orjson + line = sys.stdin.buffer.readline() + req = orjson.loads(line) + resp = {"id": req["id"], "ok": True, "metrics": {"pass@1": 1.0}} + sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") + sys.stdout.buffer.flush() + sys.exit(0) + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, die_after_one)) + try: + result1 = await w.grade_codegen( + [{"input_output": "{}"}], [["x"]], timeout=10 + ) + assert result1 == {"pass@1": 1.0} + first_proc = w._proc + # Let the worker finish exiting so _ensure_worker sees returncode != None. + if first_proc is not None: + await first_proc.wait() + result2 = await w.grade_codegen( + [{"input_output": "{}"}], [["x"]], timeout=10 + ) + assert result2 == {"pass@1": 1.0} + assert w._proc is not first_proc, ( + "expected a new worker process after respawn" + ) + finally: + await w.aclose() + + async def test_aclose_with_pending_futures_sets_exception( + self, tmp_path: Path + ) -> None: + # Covers lines 316-317: aclose() sets CodegenWorkerError on pending futures. + hang_worker = """ + import sys, time + for line in sys.stdin.buffer: + time.sleep(3600) + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, hang_worker)) + grade_task = asyncio.create_task( + w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=60) + ) + await asyncio.sleep(0.1) + await w.aclose() + with pytest.raises((CodegenWorkerError, asyncio.CancelledError)): + await grade_task + + async def test_dispatch_response_null_id_faults(self, tmp_path: Path) -> None: + # Covers the req_id is None guard in _dispatch_response (line 209-212). + null_id_worker = """ + import sys, orjson + for line in sys.stdin.buffer: + line = line.strip() + if not line: + continue + resp = {"id": None, "ok": False, "error": "bad parse"} + sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") + sys.stdout.buffer.flush() + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, null_id_worker)) + try: + with pytest.raises(CodegenWorkerError): + await w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=5) + finally: + await w.aclose() + + async def test_run_reader_json_decode_error_faults(self, tmp_path: Path) -> None: + # Covers the JSONDecodeError path in _run_reader (line 254). + garbage_worker = """ + import sys + for line in sys.stdin.buffer: + sys.stdout.buffer.write(b"not json at all\\n") + sys.stdout.buffer.flush() + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, garbage_worker)) + try: + with pytest.raises(CodegenWorkerError): + await w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=5) + finally: + await w.aclose() + + async def test_aclose_sets_exception_on_unfulfilled_pending_futures( + self, tmp_path: Path + ) -> None: + # Covers lines 316-317: aclose() calls set_exception on pending futures + # that are still waiting. The test injects a future manually to guarantee + # _pending is non-empty when aclose() runs (avoids looptime timing issues). + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, _ECHO_OK)) + loop = asyncio.get_running_loop() + fut: asyncio.Future[dict] = loop.create_future() + w._pending[99] = fut + await w.aclose() + assert fut.done() + with pytest.raises(CodegenWorkerError, match="closed"): + fut.result() + + async def test_dispatch_response_ok_false_sets_exception_and_marks_proven( + self, tmp_path: Path + ) -> None: + # Covers lines 221-225: ok=False dispatch path. + error_worker = """ + import sys, orjson + for line in sys.stdin.buffer: + req = orjson.loads(line.strip()) + resp = {"id": req["id"], "ok": False, "error": "deliberate error"} + sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") + sys.stdout.buffer.flush() + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, error_worker)) + try: + with pytest.raises(CodegenWorkerError, match="deliberate error"): + await w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=10) + assert w._worker_proven # ok=False counts as proven (worker responded) + finally: + await w.aclose() + + async def test_dispatch_response_unhashable_id_faults(self, tmp_path: Path) -> None: + # Covers lines 215-217: TypeError on _pending.pop(unhashable_id). + list_id_worker = """ + import sys, orjson + for line in sys.stdin.buffer: + resp = {"id": [1, 2], "ok": True, "metrics": {"pass@1": 1.0}} + sys.stdout.buffer.write(orjson.dumps(resp) + b"\\n") + sys.stdout.buffer.flush() + """ + w = CodegenGradingWorker(worker_cmd=_write_worker(tmp_path, list_id_worker)) + try: + with pytest.raises(CodegenWorkerError): + await w.grade_codegen([{"input_output": "{}"}], [["x"]], timeout=5) + finally: + await w.aclose() + + async def test_drain_stderr_none_reader_is_noop(self) -> None: + # Covers line 194: _drain_stderr returns immediately when reader is None. + w = CodegenGradingWorker.__new__(CodegenGradingWorker) + from collections import deque + + w._stderr_tail = deque(maxlen=64) + # Should complete without error + await w._drain_stderr(None) From ecc7aecb295a63689fce16d7717ed051dc30761d Mon Sep 17 00:00:00 2001 From: milesial Date: Mon, 10 Aug 2026 10:32:08 -0700 Subject: [PATCH 014/119] fix(config): apply synthesis CLI overrides to YAML (#1258) Signed-off-by: Alexandre Milesi (cherry picked from commit 6480e5467f0dd89af37114ebe7327bd58ddcc0dc) --- src/aiperf/config/flags/_converter_dataset.py | 21 ++- src/aiperf/config/flags/resolver.py | 53 +++++++ tests/unit/config/test_trace_flag_routing.py | 130 ++++++++++++++++++ 3 files changed, 198 insertions(+), 6 deletions(-) diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index 914e6c8b85..a201640e81 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from aiperf.config.flags import CLIConfig + from aiperf.plugin.enums import CustomDatasetType def _normalize_sample_rate_khz(value: float | int) -> float: @@ -647,7 +648,12 @@ def _reject_baseten_only_trace_flags(cli: CLIConfig) -> None: ) -def _reject_baseten_trace_unsupported_synthesis(cli: CLIConfig) -> None: +def _reject_baseten_trace_unsupported_synthesis( + cli: CLIConfig, + dataset_format: CustomDatasetType | str | None, + *, + dataset_format_source: str = "--custom-dataset-type baseten_trace", +) -> None: """Reject synthesis knobs that cannot apply to baseten_trace replay. baseten_trace replay is paced by --replay-speedup; synthesis speedup @@ -659,16 +665,19 @@ def _reject_baseten_trace_unsupported_synthesis(cli: CLIConfig) -> None: from the prompt. Output-length synthesis and the max-ISL/OSL filter/cap remain valid. The auto-detected dataset-type path is guarded at load time by the loader. + + ``dataset_format`` is the resolved loader identity: the CLI + ``--custom-dataset-type`` or the YAML ``format`` field being overlaid. + ``dataset_format_source`` is its user-facing spelling in error messages. """ from aiperf.plugin.enums import CustomDatasetType - if cli.custom_dataset_type != CustomDatasetType.BASETEN_TRACE: + if dataset_format != CustomDatasetType.BASETEN_TRACE: return if cli.synthesis_speedup_ratio != 1.0: raise ValueError( "--synthesis-speedup-ratio is not supported with " - "--custom-dataset-type baseten_trace; use --replay-speedup to " - "scale replay pacing." + f"{dataset_format_source}; use --replay-speedup to scale replay pacing." ) reshaping_flags = [ flag @@ -695,7 +704,7 @@ def _reject_baseten_trace_unsupported_synthesis(cli: CLIConfig) -> None: verb = "is" if len(reshaping_flags) == 1 else "are" raise ValueError( f"{', '.join(reshaping_flags)} {verb} not supported with " - "--custom-dataset-type baseten_trace: it replays recorded " + f"{dataset_format_source}: it replays recorded " "prompts verbatim, so hash-reshaping synthesis cannot change " "the sent prompt and would desync the forwarded hash_ids KV " "hints." @@ -1051,7 +1060,7 @@ def build_dataset(cli: CLIConfig) -> dict[str, Any]: needs_text = _determine_needs_text(cli) _reject_file_dataset_incompatible(cli) _reject_baseten_only_trace_flags(cli) - _reject_baseten_trace_unsupported_synthesis(cli) + _reject_baseten_trace_unsupported_synthesis(cli, cli.custom_dataset_type) _reject_baseten_trace_extra_input_collisions(cli) if cli.dataset_filters and not _implies_public_dataset(cli): raise ValueError("--dataset-filter requires --public-dataset") diff --git a/src/aiperf/config/flags/resolver.py b/src/aiperf/config/flags/resolver.py index aa80a46219..62000823a3 100644 --- a/src/aiperf/config/flags/resolver.py +++ b/src/aiperf/config/flags/resolver.py @@ -109,6 +109,7 @@ def resolve_config( yaml_dict = normalize_gpu_telemetry_base_for_override(yaml_dict, overrides) yaml_dict = normalize_server_metrics_base_for_override(yaml_dict, overrides) merged = deep_merge(yaml_dict, overrides) if overrides else yaml_dict + _apply_dataset_synthesis_overrides(merged, cli_config) _apply_dataset_filter_overrides(merged, cli_config) _apply_phase_loadgen_overrides(merged, cli_config) promote_benchmark_magic_lists( @@ -400,6 +401,58 @@ def _apply_dataset_filter_overrides(merged: dict[str, Any], cli: CLIConfig) -> N filters.update(_parse_dataset_filters(cli.dataset_filters)) +def _apply_dataset_synthesis_overrides(merged: dict[str, Any], cli: CLIConfig) -> None: + """Overlay explicit synthesis flags onto a YAML-supplied dataset.""" + if not any( + field == "allow_dataset_wrap" or field.startswith("synthesis_") + for field in cli.model_fields_set + ): + return + + from aiperf.config.dataset.trace import SynthesisConfig + from aiperf.config.flags._converter_dataset import ( + _apply_synthesis, + _reject_baseten_trace_unsupported_synthesis, + ) + + benchmark = merged.get("benchmark") + datasets = benchmark.get("datasets") if isinstance(benchmark, dict) else None + if not isinstance(datasets, list) or not datasets: + raise ValueError("synthesis flags require a file or public dataset") + if len(datasets) > 1: + logger.warning( + "Synthesis flags with multiple YAML datasets apply only to the first dataset" + ) + dataset = datasets[0] + if not isinstance(dataset, dict): + raise ValueError("synthesis flags require a file or public dataset") + if dataset.get("type") not in (DatasetType.FILE, DatasetType.PUBLIC): + logger.warning( + "Synthesis flags require a file or public dataset; ignoring them " + "for dataset type %r", + dataset.get("type"), + ) + return + + if dataset.get("type") == DatasetType.FILE: + _reject_baseten_trace_unsupported_synthesis( + cli, + dataset.get("format"), + dataset_format_source="YAML format: baseten_trace", + ) + + override = {"type": dataset.get("type")} + _apply_synthesis(override, cli) + if synthesis := override.get("synthesis"): + base = SynthesisConfig.model_validate( + dataset.get("synthesis") or {} + ).model_dump(by_alias=True, exclude_unset=True) + update = SynthesisConfig.model_validate(synthesis).model_dump( + by_alias=True, exclude_unset=True + ) + dataset["synthesis"] = deep_merge(base, update) + + # CLI loadgen flag -> phase field. Each entry is (loadgen_attr, phase_key). # The CLI help promises "CLI flags override values from the config file"; # this table makes that real for YAML-supplied phase shapes by overlaying diff --git a/tests/unit/config/test_trace_flag_routing.py b/tests/unit/config/test_trace_flag_routing.py index 3e9181237a..0b3ae2633e 100644 --- a/tests/unit/config/test_trace_flag_routing.py +++ b/tests/unit/config/test_trace_flag_routing.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio from pathlib import Path import pytest @@ -13,6 +14,10 @@ from aiperf.config.flags._converter_dataset import build_dataset from aiperf.config.flags.cli_config import CLIConfig from aiperf.config.flags.converter import convert_cli_to_aiperf +from aiperf.config.flags.resolver import ( + _apply_dataset_synthesis_overrides, + resolve_config, +) from aiperf.plugin.enums import CustomDatasetType, PublicDatasetType _WEKA_HF = PublicDatasetType.SEMIANALYSIS_CC_TRACES_WEKA_WITH_SUBAGENTS @@ -197,6 +202,131 @@ def test_routes_onto_public_weka_hf(self) -> None: assert ds.synthesis.max_osl == 512 +class TestSynthesisYamlOverride: + """Explicit synthesis flags overlay valid YAML datasets and report invalid shapes.""" + + async def test_cli_overrides_yaml_synthesis( + self, tmp_path: Path, trace_jsonl: Path + ) -> None: + config_file = tmp_path / "base.yaml" + await asyncio.to_thread( + config_file.write_text, + f""" +schemaVersion: "2.0" +benchmark: + model: target-model + endpoint: + url: http://localhost:8000 + type: chat + dataset: + type: file + path: {trace_jsonl} + format: mooncake_trace + synthesis: + speedupRatio: 2.0 + maxOsl: 16000 + profiling: + type: concurrency + requests: 1 + concurrency: 1 +""", + ) + + config = await asyncio.to_thread( + resolve_config, + CLIConfig(config_file=config_file, synthesis_max_osl=12000), + ) + dataset = config.benchmark.get_default_dataset() + + assert dataset.synthesis is not None + assert dataset.synthesis.max_osl == 12000 + assert dataset.synthesis.speedup_ratio == 2.0 + + @pytest.mark.parametrize( + "datasets", + [ + param([], id="empty"), + param([None], id="first-not-dict"), + ], + ) # fmt: skip + def test_invalid_yaml_datasets_raise(self, datasets: list[object]) -> None: + merged = {"benchmark": {"datasets": datasets}} + + with pytest.raises( + ValueError, match="synthesis flags require a file or public dataset" + ): + _apply_dataset_synthesis_overrides(merged, CLIConfig(synthesis_max_osl=512)) + + def test_multiple_yaml_datasets_warns_and_updates_only_first( + self, caplog: pytest.LogCaptureFixture + ) -> None: + first = { + "type": "file", + "synthesis": {"speedupRatio": 2.0, "maxOsl": 16000}, + } + second = {"type": "file", "synthesis": {"maxOsl": 8000}} + merged = {"benchmark": {"datasets": [first, second]}} + + with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): + _apply_dataset_synthesis_overrides( + merged, CLIConfig(synthesis_max_osl=12000) + ) + + assert "apply only to the first dataset" in caplog.text + assert first["synthesis"] == {"speedupRatio": 2.0, "maxOsl": 12000} + assert second["synthesis"] == {"maxOsl": 8000} + + def test_non_trace_yaml_dataset_warns_and_is_unchanged( + self, caplog: pytest.LogCaptureFixture + ) -> None: + dataset = {"type": "synthetic"} + merged = {"benchmark": {"datasets": [dataset]}} + + with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): + _apply_dataset_synthesis_overrides(merged, CLIConfig(synthesis_max_osl=512)) + + assert "require a file or public dataset" in caplog.text + assert dataset == {"type": "synthetic"} + + @pytest.mark.parametrize( + ("cli_kwargs", "match"), + [ + param( + {"synthesis_speedup_ratio": 2.0}, + "--synthesis-speedup-ratio is not supported", + id="speedup", + ), + param( + {"synthesis_prompt_len_multiplier": 2.0}, + "hash_ids", + id="prompt-reshaping", + ), + ], + ) # fmt: skip + def test_baseten_yaml_rejects_unsupported_synthesis( + self, cli_kwargs: dict[str, object], match: str + ) -> None: + merged = { + "benchmark": {"datasets": [{"type": "file", "format": "baseten_trace"}]} + } + + with pytest.raises(ValueError, match=match) as exc_info: + _apply_dataset_synthesis_overrides(merged, CLIConfig(**cli_kwargs)) + + assert "YAML format: baseten_trace" in str(exc_info.value) + assert "--custom-dataset-type" not in str(exc_info.value) + + def test_baseten_yaml_accepts_output_len_synthesis(self) -> None: + dataset = {"type": "file", "format": "baseten_trace"} + merged = {"benchmark": {"datasets": [dataset]}} + + _apply_dataset_synthesis_overrides( + merged, CLIConfig(synthesis_output_len_multiplier=2.0) + ) + + assert dataset["synthesis"] == {"outputLenMultiplier": 2.0} + + class TestOslFallbackRouting: """The --osl per-record fallback routes onto the flat osl field on either file or public datasets.""" From 603e6ef4730ed0d702e30406faac64668258809a Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:38:18 -0700 Subject: [PATCH 015/119] perf(ci): reduce test wall-clock time across platforms (#1261) Signed-off-by: Elias Bermudez (cherry picked from commit 23ed221c3d8cf94bc58a46696eadd335f2215e93) --- .github/workflows/run-unit-tests.yml | 24 ++--- Makefile | 25 +++-- tests/harness/optional_deps.py | 77 ++++++++++++++- tests/unit/harness/__init__.py | 0 tests/unit/harness/test_optional_deps.py | 116 +++++++++++++++++++++++ 5 files changed, 218 insertions(+), 24 deletions(-) create mode 100644 tests/unit/harness/__init__.py create mode 100644 tests/unit/harness/test_optional_deps.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 480a3b6814..71f5d6e8cb 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -108,24 +108,26 @@ jobs: # builder pod. -n auto without this OOM-kills the runner agent. PYTEST_XDIST_AUTO_NUM_WORKERS: "4" run: | - make test-ci PYTHON_VERSION=${{ matrix.python-version }} + # Only collect coverage on the linux/3.12 leg where Codecov upload runs. + COV=${{ (matrix.os == 'linux' && matrix.python-version == '3.12') && '1' || '0' }} + make test-ci PYTHON_VERSION=${{ matrix.python-version }} COV=$COV # Mirrors POSIX ``make test-ci`` semantics: run BOTH tiers and # aggregate exit codes so unit-test failures don't mask component- # integration failures (or vice versa). Without the explicit # ``always() && runner.os == 'Windows'`` guard on the second step, # GitHub Actions' implicit ``success()`` short-circuits the # component-integration step on any unit-test failure. - - name: Run unit tests (Windows) + - name: Run unit and zmq tests (Windows) + # Merged into one invocation to pay collection overhead once instead of + # twice. tests/unit/conftest.py's no_sleep patch applies only to + # tests/unit/**; tests/zmq/** uses real asyncio.sleep as before. + # --no-looptime is omitted: looptime is opt-in per test via marker/ + # fixture and zmq tests use neither, so they run with a real-time loop. + # --continue-on-collection-errors preserves the previous two-step + # behaviour where a unit collection error did not block the zmq suite. if: runner.os == 'Windows' run: | - uv run pytest tests/unit -n auto --tb=short -m 'not performance and not stress and not slow' - - name: Run zmq real-transport tests (Windows) - # Real-socket libzmq tests (no looptime). ``!cancelled()`` lets them run - # even if the unit step failed, matching the POSIX ``make test-ci`` - # exit-code aggregation across tiers. - if: ${{ !cancelled() && runner.os == 'Windows' }} - run: | - uv run pytest tests/zmq --tb=short --no-looptime -m 'not performance and not stress and not slow' + uv run pytest tests/unit tests/zmq -n auto --dist=worksteal --tb=short --continue-on-collection-errors -m 'not performance and not stress and not slow' - name: Run component integration tests (Windows) # ``!cancelled()`` overrides the implicit ``success()`` so component # integration still runs even if the unit-test step failed — matching @@ -134,7 +136,7 @@ jobs: # finishing after a manual stop. if: ${{ !cancelled() && runner.os == 'Windows' }} run: | - uv run pytest tests/component_integration -n auto --tb=short -v -m 'not performance and not stress and not slow' + uv run pytest tests/component_integration -n auto --dist=worksteal --tb=short -v -m 'not performance and not stress and not slow' - name: Upload results to Codecov if: matrix.os == 'linux' && matrix.python-version == '3.12' uses: codecov/codecov-action@v5 diff --git a/Makefile b/Makefile index 724f7e5cd9..8eeb81b23c 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,9 @@ SHELL := /bin/bash PROJECT_NAME ?= AIPerf +# Set COV=0 to skip coverage instrumentation (used in CI for non-coverage legs). +COV ?= 1 + # The path to the virtual environment VENV_PATH ?= .venv # The python version to use @@ -113,7 +116,7 @@ check-format check-fmt: #? check the formatting of the project using ruff. $(activate_venv) && ruff format . --check $(args) test: #? run the tests using pytest-xdist. - $(activate_venv) && pytest tests/unit -n auto -m 'not integration and not performance and not component_integration and not slow' $(args) + $(activate_venv) && pytest tests/unit -n auto --dist=worksteal -m 'not integration and not performance and not component_integration and not slow' $(args) test-zmq: #? run the real-socket zmq transport tests (real libzmq, no looptime). $(activate_venv) && pytest tests/zmq --no-looptime $(args) @@ -146,7 +149,7 @@ check-agent-files-sync: #? verify AGENTS.md, CLAUDE.md, .github/copilot-instruct $(activate_venv) && python tools/check_agent_files_sync.py coverage: #? run the tests and generate an html coverage report. - $(activate_venv) && pytest tests/unit -n auto --cov=src/aiperf --cov-branch --cov-report=html --cov-report=xml --cov-report=term -m 'not integration and not performance and not component_integration and not slow' $(args) + $(activate_venv) && pytest tests/unit -n auto --dist=worksteal --cov=src/aiperf --cov-branch --cov-report=html --cov-report=xml --cov-report=term -m 'not integration and not performance and not component_integration and not slow' $(args) install: install-app install-mock-server install-mock-amdsmi #? install the project, mock server, and fake amdsmi bindings in editable mode. @@ -248,15 +251,21 @@ test-all: #? run all tests (unit, component integration, and integration). test-ci: #? run the tests using pytest-xdist for CI. @printf "$(bold)$(blue)Running unit and component integration tests (CI mode)...$(reset)\n" - @# Run unit tests first with coverage + @# Run unit tests first, optionally with coverage @printf "$(bold)$(blue)Running unit tests...$(reset)\n" - @$(activate_venv) && pytest tests/unit -n auto --cov=src/aiperf --cov-branch --cov-report= -m 'not performance and not stress and not slow' --tb=short $(args) || exit_code=$$?; \ + @$(activate_venv) && pytest tests/unit -n auto --dist=worksteal \ + $$([ "$(COV)" = "1" ] && echo "--cov=src/aiperf --cov-branch --cov-report=" || true) \ + -m 'not performance and not stress and not slow' --tb=short $(args) || exit_code=$$?; \ # Run real-socket zmq transport tests (real time + real sockets, no looptime) regardless of unit result \ printf "$(bold)$(blue)Running zmq real-transport tests...$(reset)\n"; \ - $(activate_venv) && pytest tests/zmq --cov=src/aiperf --cov-branch --cov-append --cov-report= -m 'not performance and not stress and not slow' --no-looptime --tb=short $(args) || exit_code=$$((exit_code + $$?)); \ + $(activate_venv) && pytest tests/zmq \ + $$([ "$(COV)" = "1" ] && echo "--cov=src/aiperf --cov-branch --cov-append --cov-report=" || true) \ + -m 'not performance and not stress and not slow' --no-looptime --tb=short $(args) || exit_code=$$((exit_code + $$?)); \ # Run component integration tests with coverage append regardless of unit test result \ printf "$(bold)$(blue)Running component integration tests...$(reset)\n"; \ - $(activate_venv) && MALLOC_ARENA_MAX=2 pytest tests/component_integration -n auto --cov=src/aiperf --cov-branch --cov-append --cov-report=html --cov-report=xml --cov-report=term -m 'not performance and not stress and not slow' -v --tb=short $(args) || exit_code=$$((exit_code + $$?)); \ + $(activate_venv) && MALLOC_ARENA_MAX=2 pytest tests/component_integration -n auto --dist=worksteal \ + $$([ "$(COV)" = "1" ] && echo "--cov=src/aiperf --cov-branch --cov-append --cov-report=html --cov-report=xml --cov-report=term" || true) \ + -m 'not performance and not stress and not slow' -v --tb=short $(args) || exit_code=$$((exit_code + $$?)); \ if [[ $$exit_code -eq 0 ]]; then \ printf "$(bold)$(green)AIPerf unit and component integration tests (CI mode) passed!$(reset)\n"; \ else \ @@ -302,12 +311,12 @@ integration-tests-slow test-integration-slow: #? run only the slow-marked integr component-integration-tests test-component-integration: #? run component integration tests with with AIPerf Mock Server. @printf "$(bold)$(blue)Running Fake Component Integration tests...$(reset)\n" - $(activate_venv) && MALLOC_ARENA_MAX=2 pytest tests/component_integration/ -m 'component_integration and not stress and not performance and not slow' -n auto --tb=short $(args) + $(activate_venv) && MALLOC_ARENA_MAX=2 pytest tests/component_integration/ -m 'component_integration and not stress and not performance and not slow' -n auto --dist=worksteal --tb=short $(args) @printf "$(bold)$(green)AIPerf Fake Component Integration tests passed!$(reset)\n" component-integration-tests-ci test-component-integration-ci: #? run component integration tests with with AIPerf Mock Server for CI (parallel, verbose, no performance and no ffmpeg tests). @printf "$(bold)$(blue)Running Fake Component Integration tests (CI mode)...$(reset)\n" - $(activate_venv) && MALLOC_ARENA_MAX=2 pytest tests/component_integration/ -m 'component_integration and not performance and not ffmpeg and not stress and not slow' -n auto -v --tb=long $(args) + $(activate_venv) && MALLOC_ARENA_MAX=2 pytest tests/component_integration/ -m 'component_integration and not performance and not ffmpeg and not stress and not slow' -n auto --dist=worksteal -v --tb=long $(args) @printf "$(bold)$(green)AIPerf Fake Component Integration tests (CI mode) passed!$(reset)\n" component-integration-tests-verbose test-component-integration-verbose: #? run component integration tests with verbose output with AIPerf Mock Server. diff --git a/tests/harness/optional_deps.py b/tests/harness/optional_deps.py index 9b56e81935..c1a30076de 100644 --- a/tests/harness/optional_deps.py +++ b/tests/harness/optional_deps.py @@ -28,13 +28,25 @@ import ast import importlib.util +import os from pathlib import Path +import orjson + # Windows-on-ARM: native render/codec backends (kaleido's browser engine, etc.) # have no working ARM build and hard-crash (access violation) rather than # raising, so affected tests must be skipped by platform rather than probed. from aiperf.common.constants import IS_WINDOWS_ARM # noqa: F401 +# .pytest_cache lives next to the repo root conftest. All xdist workers share +# the same filesystem, so one worker populates the cache and the rest read it — +# no subprocess coordination needed. The cache is only written on platforms +# where unavailable_gated_deps() is non-empty (currently Windows-ARM); on all +# other platforms _test_files_needing_unavailable_deps() short-circuits before +# touching this path. +_CACHE_DIR = Path(__file__).parents[2] / ".pytest_cache" +_CACHE_FILE_NAME = "optional_deps_scan.json" + def is_installed(module: str) -> bool: """Whether ``module`` is present, without importing it. @@ -110,15 +122,70 @@ def _top_level_imports(path: Path) -> set[str]: def _test_files_needing_unavailable_deps(test_dir: Path) -> list[Path]: """Sorted test files under ``test_dir`` whose top-level imports need a dep that is unavailable on this platform. Empty when all gated deps are present. + + Results are cached in ``.pytest_cache/optional_deps_scan.json`` so that + multiple xdist worker processes pay the rglob+AST cost at most once per CI + job. The cache key includes the resolved ``test_dir`` path and the sorted + set of currently-unavailable deps; the entry also stores the total count of + ``test_*.py`` files so that adding a new file (which would not appear in + the cached skip list) invalidates the entry on the next call. """ unavailable = unavailable_gated_deps() if not unavailable: return [] - return [ - path - for path in sorted(test_dir.rglob("test_*.py")) - if _top_level_imports(path) & unavailable - ] + + cache_file = _CACHE_DIR / _CACHE_FILE_NAME + # Include the sorted unavailable set so a dep becoming available (e.g. a + # local install) also invalidates the cached skip list. + key = f"{test_dir.resolve()}|{','.join(sorted(unavailable))}" + + # List all test files once: used for count-based cache validation on a hit + # and for the AST scan on a miss. Directory listing (no reads) is cheap + # relative to 912 read+parse operations. + all_test_files = sorted(test_dir.rglob("test_*.py")) + total = len(all_test_files) + + # Try to read an existing cache entry. A hit is only valid when the total + # file count matches — any addition or removal of test_*.py files forces a + # re-scan so newly gated imports are never silently missed. + try: + raw = orjson.loads(cache_file.read_bytes()) + if not isinstance(raw, dict): + raise ValueError + entry = raw.get(key) + if isinstance(entry, dict) and entry.get("total") == total: + files = entry.get("files") + if isinstance(files, list): + return [Path(p) for p in files] + data = dict(raw) + except (OSError, orjson.JSONDecodeError, ValueError): + data = {} + + # Cache miss: full scan (read + AST parse per file). + found = [p for p in all_test_files if _top_level_imports(p) & unavailable] + + # Write result back (best-effort). Re-read first to merge any entries + # another worker wrote while we were scanning, reducing lost-update + # exposure. Use a per-process tmp path so concurrent workers don't stomp + # each other's writes on Windows (where os.replace on a shared .tmp path + # can raise PermissionError). + data[key] = {"files": [str(p) for p in found], "total": total} + try: + _CACHE_DIR.mkdir(parents=True, exist_ok=True) + try: + merged = orjson.loads(cache_file.read_bytes()) + if isinstance(merged, dict): + merged.update(data) + data = merged + except (OSError, orjson.JSONDecodeError): + pass + tmp = _CACHE_DIR / f"optional_deps_scan.{os.getpid()}.tmp" + tmp.write_bytes(orjson.dumps(data)) + os.replace(tmp, cache_file) + except OSError: + pass + + return found def collect_ignore_for_unavailable_deps(test_dir: str | Path) -> list[str]: diff --git a/tests/unit/harness/__init__.py b/tests/unit/harness/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/harness/test_optional_deps.py b/tests/unit/harness/test_optional_deps.py new file mode 100644 index 0000000000..4034546100 --- /dev/null +++ b/tests/unit/harness/test_optional_deps.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path +from unittest.mock import patch + +import orjson + +from tests.harness.optional_deps import _test_files_needing_unavailable_deps + + +def test__test_files_needing_unavailable_deps_first_call_populates_cache( + tmp_path: Path, +) -> None: + """Cache JSON is written to _CACHE_DIR on the first call when deps are absent.""" + fake_dep = "totally_absent_dep_xyz" + test_file = tmp_path / "test_fake.py" + test_file.write_text(f"import {fake_dep}\n") + + cache_dir = tmp_path / ".pytest_cache" + + with ( + patch( + "tests.harness.optional_deps.unavailable_gated_deps", + return_value={fake_dep}, + ), + patch("tests.harness.optional_deps._CACHE_DIR", cache_dir), + ): + result = _test_files_needing_unavailable_deps(tmp_path) + + assert result == [test_file] + cache_file = cache_dir / "optional_deps_scan.json" + assert cache_file.exists() + data = orjson.loads(cache_file.read_bytes()) + key = f"{tmp_path}|{fake_dep}" + assert str(test_file) in data[key]["files"] + assert data[key]["total"] == 1 + + +def test__test_files_needing_unavailable_deps_second_call_reads_from_cache( + tmp_path: Path, +) -> None: + """Second call with the same file tree returns the cached result without re-scanning.""" + fake_dep = "totally_absent_dep_xyz" + test_file = tmp_path / "test_fake.py" + test_file.write_text(f"import {fake_dep}\n") + + cache_dir = tmp_path / ".pytest_cache" + + with ( + patch( + "tests.harness.optional_deps.unavailable_gated_deps", + return_value={fake_dep}, + ), + patch("tests.harness.optional_deps._CACHE_DIR", cache_dir), + ): + # First call populates cache. + _test_files_needing_unavailable_deps(tmp_path) + # Overwrite the file so a fresh AST scan would return [] (import gone), + # but the file count is unchanged, so the cache should still be used. + test_file.write_text("# no imports\n") + result = _test_files_needing_unavailable_deps(tmp_path) + + assert result == [test_file] + + +def test__test_files_needing_unavailable_deps_new_file_invalidates_cache( + tmp_path: Path, +) -> None: + """Adding a test_*.py file invalidates the cache so the new file is not missed.""" + fake_dep = "totally_absent_dep_xyz" + test_file = tmp_path / "test_fake.py" + test_file.write_text(f"import {fake_dep}\n") + + cache_dir = tmp_path / ".pytest_cache" + + with ( + patch( + "tests.harness.optional_deps.unavailable_gated_deps", + return_value={fake_dep}, + ), + patch("tests.harness.optional_deps._CACHE_DIR", cache_dir), + ): + # First call: caches [test_fake.py], total=1. + _test_files_needing_unavailable_deps(tmp_path) + # Add a second test file with the gated import. + new_file = tmp_path / "test_new.py" + new_file.write_text(f"import {fake_dep}\n") + # Second call: total is now 2, so cache is stale → re-scan finds both. + result = _test_files_needing_unavailable_deps(tmp_path) + + # Both files must be present — the rescan must not lose the original file. + assert test_file in result + assert new_file in result + # Cache entry should be refreshed with total=2. + key = f"{tmp_path}|{fake_dep}" + data = orjson.loads((cache_dir / "optional_deps_scan.json").read_bytes()) + assert data[key]["total"] == 2 + + +def test__test_files_needing_unavailable_deps_all_deps_present_skips_cache( + tmp_path: Path, +) -> None: + """Cache is never written when all gated deps are present on this platform.""" + cache_dir = tmp_path / ".pytest_cache" + + with ( + patch( + "tests.harness.optional_deps.unavailable_gated_deps", + return_value=set(), + ), + patch("tests.harness.optional_deps._CACHE_DIR", cache_dir), + ): + result = _test_files_needing_unavailable_deps(tmp_path) + + assert result == [] + assert not cache_dir.exists() From 5ba8908ad4321c234302332b4866caefe945ec85 Mon Sep 17 00:00:00 2001 From: Michael Feil <63565275+michaelfeil@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:10:56 -0700 Subject: [PATCH 016/119] perf: speed up Baseten trace loading (#1248) Signed-off-by: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Signed-off-by: Michael Feil <63565275+michaelfeil@users.noreply.github.com> Co-authored-by: Harshini Komali <157742537+lkomali@users.noreply.github.com> (cherry picked from commit 1e32a51318af3474dc1672d225fd495471ab45df) --- docs/benchmark-modes/trace-replay.md | 15 +- docs/environment-variables.md | 5 +- docs/tutorials/baseten-trace.md | 4 +- src/aiperf/common/environment.py | 8 + .../config/flags/_converter_profiling.py | 40 +- src/aiperf/dataset/dataset_manager.py | 20 +- src/aiperf/dataset/loader/baseten_trace.py | 461 +++++++++++------- src/aiperf/plugin/plugins.yaml | 8 +- .../unit/config/test_baseten_replay_flags.py | 28 ++ .../unit/dataset/loader/test_baseten_trace.py | 407 ++++++++++++++-- .../test_baseten_trace_optional_pyarrow.py | 1 + ...dataset_manager_inputs_json_adversarial.py | 17 + tools/generate_env_vars_docs.py | 14 + 13 files changed, 787 insertions(+), 241 deletions(-) diff --git a/docs/benchmark-modes/trace-replay.md b/docs/benchmark-modes/trace-replay.md index 5e344ad24e..d0b3554842 100644 --- a/docs/benchmark-modes/trace-replay.md +++ b/docs/benchmark-modes/trace-replay.md @@ -109,6 +109,19 @@ aiperf profile \ AIPerf refuses parameter sweeps (e.g. `--concurrency 1,2,4`) against an auto-promoted trace; either pin a single value or pass `--no-fixed-schedule` to keep your sweep semantics. +### Large Baseten Parquet Traces + +The `baseten_trace` loader reads trace rows and sampled-session metadata in +bounded batches and projects only the columns required by the selected replay +options. It also skips `inputs.json` generation, which would otherwise +duplicate large recorded prompts. This is independent of the mmap dataset +cache; Baseten traces remain uncached. + +The same schema can be stored as one uncompressed Arrow IPC file with an +`.arrow` or `.ipc` suffix. AIPerf memory-maps that file and materializes only +rows selected for replay. Use `--custom-dataset-type baseten_trace`; no schema +or row-order conversion beyond the storage format is required. + ## Using Pre-formatted Messages Instead of synthetic prompts generated from `input_length` and `hash_ids`, you can provide an OpenAI-compatible `messages` array directly per trace entry. This is useful for replaying captured conversations (e.g., coding agent sessions) with exact prompt content. @@ -166,4 +179,4 @@ aiperf profile \ --custom-dataset-type mooncake_trace \ --fixed-schedule ``` - \ No newline at end of file + diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 8a1ae23f29..dcf5e2ff82 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -96,6 +96,7 @@ Dataset loading and configuration. Controls timeouts and behavior for dataset lo | Environment Variable | Default | Constraints | Description | |----------------------|---------|-------------|-------------| | `AIPERF_DATASET_CONFIGURATION_TIMEOUT` | `300.0` | ≥ 1.0, ≤ 100000.0 | Timeout in seconds for dataset configuration operations | +| `AIPERF_DATASET_BASETEN_SESSION_COLUMN` | `'provided_session_id'` | one of: 'provided_session_id' / 'poor_man_session_id' | Session column used by the Baseten trace loader when both supported columns exist. Set to poor_man_session_id for legacy traces. If the selected column is absent, the loader uses the available column. | | `AIPERF_DATASET_MMAP_BASE_PATH` | `None` | — | Base path for memory-mapped dataset files. If None, uses system temp directory. Set to a shared filesystem path for Kubernetes mounted volumes. Example: AIPERF_DATASET_MMAP_BASE_PATH=/mnt/shared-pvc creates files at /mnt/shared-pvc/aiperf_mmap_{benchmark_id}/ | | `AIPERF_DATASET_MMAP_CACHE_ENABLED` | `True` | — | If True, AIPerf reuses memory-mapped dataset files across runs whose input bytes, tokenizer identity, and prompt/input settings are byte-identical. Set to False to force every run to re-tokenize and re-write its mmap files. Cache misses still produce byte-identical mmap files to a non-cached run. | | `AIPERF_DATASET_MMAP_CACHE_DIR` | `None` | — | Directory holding the content-addressed mmap cache. If None, defaults to ~/.cache/aiperf/dataset_mmap. Each cache entry lives under a `dir/key` subpath and contains dataset.dat, index.dat, manifest.json, and (when produced) inputs.json. No automatic eviction is implemented yet -- delete the directory to reclaim disk. | @@ -159,7 +160,7 @@ HTTP client socket and connection configuration. Controls low-level socket optio | `AIPERF_HTTP_USE_DNS_CACHE` | `True` | — | Enable DNS cache | | `AIPERF_HTTP_SSL_VERIFY` | `True` | — | Enable SSL certificate verification. Set to False to disable verification. WARNING: Disabling this is insecure and should only be used for testing in a trusted environment. | | `AIPERF_HTTP_REQUEST_CANCELLATION_SEND_TIMEOUT` | `300.0` | ≥ 10.0, ≤ 3600.0 | Safety net timeout in seconds for waiting for HTTP request to be fully sent when request cancellation is enabled. Used as fallback when no explicit timeout is configured to prevent hanging indefinitely while waiting for the request to be written to the socket. | -| `AIPERF_HTTP_IP_VERSION` | `'4'` | — | IP version for HTTP socket connections. Options: '4' (AF_INET, default), '6' (AF_INET6), or 'auto' (AF_UNSPEC, system chooses). | +| `AIPERF_HTTP_IP_VERSION` | `'4'` | one of: '4' / '6' / 'auto' | IP version for HTTP socket connections. Options: '4' (AF_INET, default), '6' (AF_INET6), or 'auto' (AF_UNSPEC, system chooses). | | `AIPERF_HTTP_TRUST_ENV` | `False` | — | Trust environment variables for HTTP client configuration. When enabled, aiohttp will read proxy settings from HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables. | | `AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID` | `False` | — | Also send X-Session-ID with the stable X-Correlation-ID value. This transport setting is the supported way to enable generic HTTP session affinity. It is ADDITIVE (both headers are sent); --session-header only RENAMES the single correlation header. | | `AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID` | `False` | — | Also send X-SMG-Routing-Key with the stable X-Correlation-ID value. This transport setting is the supported affinity path for the SGLang Model Gateway manual routing policy. | @@ -186,7 +187,7 @@ Metrics collection and storage configuration. Controls metrics storage allocatio | `AIPERF_METRICS_OSL_MISMATCH_PCT_THRESHOLD` | `5.0` | ≥ 0.0, ≤ 100.0 | Percentage difference threshold for flagging discrepancies between requested and actual output sequence length (default: 5%) | | `AIPERF_METRICS_OSL_MISMATCH_MAX_TOKEN_THRESHOLD` | `50` | ≥ 1 | Maximum absolute token threshold for OSL mismatch. The effective threshold is min(requested_osl * pct_threshold, this value). Makes threshold tighter for large OSL values (default: 50 tokens) | | `AIPERF_METRICS_TDIGEST_COMPRESSION` | `500` | ≥ 20, ≤ 10000 | t-digest sketch compression for list-valued record metric aggregation. Higher = more centroids, tighter percentile accuracy, larger sketch. Default 500 measured to keep worst-case relative percentile error under 0.05% on 50M-sample workloads (40x under the 0.5% claimed accuracy band) at ~4 KB sketch size. | -| `AIPERF_METRICS_LIST_BACKEND` | `'ragged'` | — | Storage backend for list-valued RECORD metrics (today: only inter_chunk_latency). 'ragged' (default) keeps every value, enabling exact percentiles and ICL-aware throughput / tokens-in-flight sweep curves. 'tdigest' uses a bounded-memory crick.TDigest sketch (~4 KB regardless of sample count) — percentiles are approximate (at most 0.05% relative error at default compression), and ICL-aware sweep curves silently fall back to their non-ICL equivalents that use only request-level (start_ns, generation_start_ns, end_ns) timing. Choose tdigest when records-manager pod memory at 1M+ request scale is the binding constraint. | +| `AIPERF_METRICS_LIST_BACKEND` | `'ragged'` | one of: 'ragged' / 'tdigest' | Storage backend for list-valued RECORD metrics (today: only inter_chunk_latency). 'ragged' (default) keeps every value, enabling exact percentiles and ICL-aware throughput / tokens-in-flight sweep curves. 'tdigest' uses a bounded-memory crick.TDigest sketch (~4 KB regardless of sample count) — percentiles are approximate (at most 0.05% relative error at default compression), and ICL-aware sweep curves silently fall back to their non-ICL equivalents that use only request-level (start_ns, generation_start_ns, end_ns) timing. Choose tdigest when records-manager pod memory at 1M+ request scale is the binding constraint. | ## MLFLOW diff --git a/docs/tutorials/baseten-trace.md b/docs/tutorials/baseten-trace.md index 1e5a165671..33725687bf 100644 --- a/docs/tutorials/baseten-trace.md +++ b/docs/tutorials/baseten-trace.md @@ -30,8 +30,8 @@ Common optional columns: ## Replay Semantics -- Requests are grouped into sessions by whichever of `provided_session_id` / `poor_man_session_id` shows the stronger repeated-session signal; ties prefer `provided_session_id`. -- If neither column repeats (or the chosen column is null for a row), the loader generates a fresh per-row session ID. +- Requests are grouped by `provided_session_id` by default. For legacy traces that should use `poor_man_session_id`, set `AIPERF_DATASET_BASETEN_SESSION_COLUMN=poor_man_session_id`. If the selected column is absent, the loader uses the available session column. This explicit schema-based choice avoids scanning trace rows to infer a session column. +- The loader uses every non-null value from the selected column, including unique values. It generates a fresh per-row session ID only when neither supported column exists or the selected value is null. - All timestamps are normalized to `ms since first event in file`. - Rows inside each session are sorted by normalized timestamp before replay. - `prompt` is replayed as the literal completion prompt. diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 4484eb6f88..7318a88aad 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -306,6 +306,14 @@ class _DatasetSettings(BaseSettings): default=300.0, description="Timeout in seconds for dataset configuration operations", ) + BASETEN_SESSION_COLUMN: Literal["provided_session_id", "poor_man_session_id"] = ( + Field( + default="provided_session_id", + description="Session column used by the Baseten trace loader when both " + "supported columns exist. Set to poor_man_session_id for legacy traces. " + "If the selected column is absent, the loader uses the available column.", + ) + ) MMAP_BASE_PATH: Path | None = Field( default=None, description="Base path for memory-mapped dataset files. If None, uses system temp directory. " diff --git a/src/aiperf/config/flags/_converter_profiling.py b/src/aiperf/config/flags/_converter_profiling.py index ea8bc784f2..f407556674 100644 --- a/src/aiperf/config/flags/_converter_profiling.py +++ b/src/aiperf/config/flags/_converter_profiling.py @@ -384,16 +384,10 @@ def _apply_dataset_aware_autodefaults(prof: dict[str, Any], cli: CLIConfig) -> N _maybe_set_dag_root_sessions(prof, cli, file_path) -def _first_record_has_timestamp(file_path: object) -> bool: - """Return True when a trace file carries timestamp data.""" - from pathlib import Path - - from aiperf.common.utils import load_json_str - - path = Path(file_path) - if not path.is_file(): - return False - if path.suffix.lower() == ".parquet": +def _columnar_file_has_timestamp(path: Path) -> bool | None: + """Probe known columnar formats, or return None for another format.""" + suffix = path.suffix.lower() + if suffix == ".parquet": try: import pyarrow as pa import pyarrow.parquet as pq @@ -404,6 +398,24 @@ def _first_record_has_timestamp(file_path: object) -> bool: return "timestamp_start_unix_ms" in set(pq.read_schema(path).names) except (OSError, pa.ArrowException): return False + if suffix in {".arrow", ".ipc"}: + from aiperf.dataset.loader.baseten_trace import BasetenTraceDatasetLoader + + return BasetenTraceDatasetLoader.can_load(filename=path) + return None + + +def _first_record_has_timestamp(file_path: object) -> bool: + """Return True when a trace file carries timestamp data.""" + from pathlib import Path + + from aiperf.common.utils import load_json_str + + path = Path(file_path) + if not path.is_file(): + return False + if (columnar_result := _columnar_file_has_timestamp(path)) is not None: + return columnar_result try: with open(path, encoding="utf-8") as f: for line in f: @@ -422,7 +434,7 @@ def _first_record_has_timestamp(file_path: object) -> bool: def _count_dataset_records(file_path: object) -> int: - """Count records across a JSONL file/directory or Parquet trace file.""" + """Count records across JSONL, Parquet, or Arrow IPC input.""" from pathlib import Path path = Path(file_path) @@ -444,6 +456,12 @@ def _count_dataset_records(file_path: object) -> int: return pq.ParquetFile(path).metadata.num_rows except (OSError, pa.ArrowException): return 0 + if path.suffix.lower() in {".arrow", ".ipc"} and path.is_file(): + from aiperf.dataset.loader.baseten_trace import ( + count_baseten_records, + ) + + return count_baseten_records(str(path)) if path.is_file(): with open(path, encoding="utf-8") as f: return sum(1 for line in f if line.strip()) diff --git a/src/aiperf/dataset/dataset_manager.py b/src/aiperf/dataset/dataset_manager.py index bbc046d0f3..bfc095ae34 100644 --- a/src/aiperf/dataset/dataset_manager.py +++ b/src/aiperf/dataset/dataset_manager.py @@ -270,11 +270,14 @@ def _should_skip_inputs_json(self) -> bool: per-turn payloads are verbatim or synthesized at runtime and would be huge (and for weka, meaningless) in inputs.json. Uses the loader's DETECTED type so auto-detected traces (e.g. a bare BurstGPT CSV) are - skipped too. Same predicate the mmap cache uses - (``mmap_cache.is_trace_or_verbatim_dataset``), but the cache gate keys - off the pre-load CONFIG type: an explicitly-typed trace is both cached - and skipped, while an auto-detected trace is skipped-but-uncached (a - re-tokenize perf miss, never a correctness issue). + skipped too. Most formats share the mmap cache predicate + (``mmap_cache.is_trace_or_verbatim_dataset``), but Baseten is a + deliberate skip-only special case: its large recorded prompts make + ``inputs.json`` prohibitively expensive, while it remains outside the + mmap cache. The cache gate keys off the pre-load CONFIG type, so other + explicitly-typed traces are cached and skipped while auto-detected + traces are skipped-but-uncached (a re-tokenize perf miss, never a + correctness issue). """ default_dataset = self.run.cfg.get_default_dataset() public_dataset = ( @@ -282,8 +285,11 @@ def _should_skip_inputs_json(self) -> bool: if isinstance(default_dataset, PublicDataset) else None ) - return mmap_cache.is_trace_or_verbatim_dataset( - self._detected_dataset_type, public_dataset + return ( + self._detected_dataset_type == CustomDatasetType.BASETEN_TRACE + or mmap_cache.is_trace_or_verbatim_dataset( + self._detected_dataset_type, public_dataset + ) ) def _lookup_under_lock(self) -> mmap_cache.CacheHit | None: diff --git a/src/aiperf/dataset/loader/baseten_trace.py b/src/aiperf/dataset/loader/baseten_trace.py index ca7ee8f1f8..968bd7eec3 100644 --- a/src/aiperf/dataset/loader/baseten_trace.py +++ b/src/aiperf/dataset/loader/baseten_trace.py @@ -1,17 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Baseten.co, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Baseten Parquet trace replay loader.""" +"""Baseten Parquet and Arrow IPC trace replay loader.""" from __future__ import annotations -from collections import Counter, defaultdict +from collections import defaultdict +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from typing import Annotated, Any try: import pyarrow as pa import pyarrow.compute as pc + import pyarrow.ipc as ipc import pyarrow.parquet as pq except ImportError: # pragma: no cover - platform-dependent # pyarrow has no Windows-on-ARM wheel (apache/arrow#47195) and source-builds @@ -20,12 +23,14 @@ # pyarrow is unavailable. pa = None pc = None + ipc = None pq = None -from pydantic import ConfigDict, Field, field_validator +from pydantic import Field, field_validator from aiperf.common import random_generator as rng from aiperf.common.enums import ConversationContextMode +from aiperf.common.environment import Environment from aiperf.common.models import AIPerfBaseModel, Conversation from aiperf.dataset.loader._baseten_replay_timemodel import reflow_idle_gaps from aiperf.dataset.loader._delay_cap import DelayCapTracker @@ -37,11 +42,9 @@ METADATA_COLUMNS_TIME = "timestamp_start_unix_ms" METADATA_COLUMNS_SESSION = "provided_session_id" METADATA_COLUMNS_POOR_MAN_SESSION = "poor_man_session_id" -METADATA_COLUMNS = { - METADATA_COLUMNS_TIME, - METADATA_COLUMNS_SESSION, - METADATA_COLUMNS_POOR_MAN_SESSION, -} +_VALIDATION_SAMPLE_ROWS = 10 +_PARQUET_BATCH_SIZE = 128 +_ARROW_IPC_SUFFIXES = frozenset({".arrow", ".ipc"}) _REQUIRED_COLUMNS = { METADATA_COLUMNS_TIME, @@ -53,15 +56,10 @@ NonNegativeInt = Annotated[int, Field(ge=0)] PositiveInt = Annotated[int, Field(gt=0)] NonNegativeFloat = Annotated[float, Field(ge=0)] -RequestCanceledInt = Annotated[int, Field(ge=0, le=1)] class BasetenTrace(AIPerfBaseModel): - """Schema for Baseten completion traces exported as Parquet.""" - - # populate_by_name lets the model_dump()/model_validate round-trip used by - # synthesis reconstruction repopulate aliased fields (dataset_version). - model_config = ConfigDict(populate_by_name=True) + """Schema for Baseten completion traces exported as Parquet or Arrow IPC.""" timestamp_start_unix_ms: NonNegativeInt = Field( description="Recorded request start timestamp in Unix milliseconds." @@ -87,48 +85,10 @@ class BasetenTrace(AIPerfBaseModel): default=None, description="Recorded end-to-end request duration in milliseconds.", ) - duration_ttft_ms: NonNegativeInt | None = Field( - default=None, - description="Recorded time to first token in milliseconds.", - ) - request_canceled: RequestCanceledInt | None = Field( - default=None, - description="Whether the source request was canceled.", - ) - cached_tokens_reference: NonNegativeInt | None = Field( - default=None, - description="Recorded reference cached-token count.", - ) - model_name: str | None = Field( - default=None, - description="Model name recorded in the source trace.", - ) - org_id: str | NonNegativeInt | None = Field( - default=None, - description="Organization identifier recorded in the source trace.", - ) block_size: PositiveInt | None = Field( default=None, description="KV-cache block size associated with total_hashes.", ) - features: str | None = Field(default=None, description="Opaque feature metadata.") - speculation_ratio: NonNegativeFloat | None = Field( - default=None, - description="Average tokens per decode iteration.", - ) - output_text: str | None = Field( - default=None, - description="Recorded completion text retained for offline validation.", - ) - dataset_version: str | None = Field( - default=None, - alias="__version__", - description="Source dataset version.", - ) - total_hashes_len: NonNegativeInt | None = Field( - default=None, - description="Recorded total_hashes length, when exported separately.", - ) timestamp: NonNegativeInt | NonNegativeFloat | None = Field( default=None, @@ -168,71 +128,138 @@ def _coerce_null_hashes(cls, value: Any) -> Any: return value -def _score_session_groups( - session_ids: list[str | int | None], -) -> tuple[int, int]: - counts = Counter(session_id for session_id in session_ids if session_id is not None) - repeated_group_sizes = [count for count in counts.values() if count > 1] - return (sum(repeated_group_sizes), len(repeated_group_sizes)) - - -def choose_baseten_session_key( - provided_session_ids: list[str | int | None], - poor_man_session_ids: list[int | None], -) -> str | None: - """Return the session column with the strongest repeated-session signal.""" - provided_score = _score_session_groups(provided_session_ids) - poor_score = _score_session_groups(poor_man_session_ids) - - if provided_score > poor_score and provided_score[0] > 0: - return METADATA_COLUMNS_SESSION - if poor_score > provided_score and poor_score[0] > 0: - return METADATA_COLUMNS_POOR_MAN_SESSION - if provided_score == poor_score and provided_score[0] > 0: - return METADATA_COLUMNS_SESSION +def _baseten_session_key_from_schema(schema_names: set[str]) -> str | None: + """Choose the canonical session column without scanning trace rows.""" + configured = Environment.DATASET.BASETEN_SESSION_COLUMN + if configured in schema_names: + return configured + fallback = ( + METADATA_COLUMNS_POOR_MAN_SESSION + if configured == METADATA_COLUMNS_SESSION + else METADATA_COLUMNS_SESSION + ) + if fallback in schema_names: + return fallback return None +def _parquet_min_timestamp(parquet_file: pq.ParquetFile) -> int | None: + """Return the minimum timestamp only when every row group has statistics. + + Returning ``None`` for any missing statistic makes the caller scan the + timestamp column, preserving correctness when statistics are incomplete. + """ + timestamp_index = parquet_file.schema.names.index(METADATA_COLUMNS_TIME) + minimum: int | None = None + for row_group_index in range(parquet_file.metadata.num_row_groups): + statistics = ( + parquet_file.metadata.row_group(row_group_index) + .column(timestamp_index) + .statistics + ) + if statistics is None or not statistics.has_min_max: + return None + value = int(statistics.min) + minimum = value if minimum is None else min(minimum, value) + return minimum + + +def _is_arrow_ipc(file_path: str | Path) -> bool: + return Path(file_path).suffix.lower() in _ARROW_IPC_SUFFIXES + + +@contextmanager +def _open_arrow_ipc(file_path: str | Path) -> Iterator[Any]: + """Open a memory-mapped Arrow IPC file for the lifetime of the reader.""" + source = pa.memory_map(str(file_path), "r") + try: + yield ipc.open_file(source) + finally: + source.close() + + +def count_baseten_records(file_path: str) -> int: + """Return the row count for a Baseten Parquet or Arrow trace.""" + if pq is None: # pragma: no cover - platform-dependent + return 0 + try: + if not _is_arrow_ipc(file_path): + return pq.ParquetFile(file_path).metadata.num_rows + with _open_arrow_ipc(file_path) as reader: + return sum( + reader.get_batch(index).num_rows + for index in range(reader.num_record_batches) + ) + except (OSError, pa.ArrowException): + return 0 + + def count_baseten_parquet_records_and_sessions(file_path: str) -> tuple[int, int]: - """Return row and session counts for a Baseten Parquet trace file.""" + """Return row and session counts for a Baseten Parquet or Arrow trace.""" if pq is None: # pragma: no cover - platform-dependent return 0, 0 try: - parquet_file = pq.ParquetFile(file_path) - row_count = parquet_file.metadata.num_rows - schema_names = set(pq.read_schema(file_path).names) - session_columns = { - METADATA_COLUMNS_SESSION, - METADATA_COLUMNS_POOR_MAN_SESSION, - } & schema_names - if not session_columns: - return row_count, row_count - - table = pq.read_table(file_path, columns=sorted(session_columns)) + if _is_arrow_ipc(file_path): + with _open_arrow_ipc(file_path) as reader: + schema_names = set(reader.schema.names) + session_key = _baseten_session_key_from_schema(schema_names) + row_count = 0 + session_ids: set[str | int] = set() + null_count = 0 + for index in range(reader.num_record_batches): + batch = reader.get_batch(index) + row_count += batch.num_rows + if session_key is None: + continue + session_column = batch.column( + batch.schema.get_field_index(session_key) + ) + null_count += session_column.null_count + session_ids.update( + value + for value in pc.unique(session_column).to_pylist() + if value is not None + ) + session_count = len(session_ids) + null_count + else: + parquet_file = pq.ParquetFile(file_path) + row_count = parquet_file.metadata.num_rows + schema_names = set(parquet_file.schema_arrow.names) + session_key = _baseten_session_key_from_schema(schema_names) + if session_key is None: + return row_count, row_count + session_ids = set() + null_count = 0 + for batch in parquet_file.iter_batches( + columns=[session_key], + batch_size=_PARQUET_BATCH_SIZE, + use_threads=True, + ): + session_column = batch.column(0) + null_count += session_column.null_count + session_ids.update( + value + for value in pc.unique(session_column).to_pylist() + if value is not None + ) + session_count = len(session_ids) + null_count except (OSError, pa.ArrowException): return 0, 0 - rows = table.to_pylist() - session_key = choose_baseten_session_key( - [row.get(METADATA_COLUMNS_SESSION) for row in rows], - [row.get(METADATA_COLUMNS_POOR_MAN_SESSION) for row in rows], - ) if session_key is None: return row_count, row_count - - values = {str(row[session_key]) for row in rows if row.get(session_key) is not None} - return row_count, len(values) if values else row_count + return row_count, session_count or row_count class BasetenTraceDatasetLoader(BaseTraceDatasetLoader[BasetenTrace]): - """Loader for Baseten completion traces exported as Parquet.""" + """Loader for Baseten completion traces exported as Parquet or Arrow IPC.""" def __init__(self, *args, **kwargs) -> None: if pq is None: raise ValueError( "baseten_trace requires pyarrow, which is not installed (no " "Windows-on-ARM wheel is published; see apache/arrow#47195). " - "Install pyarrow to replay Parquet traces." + "Install pyarrow to replay Parquet or Arrow IPC traces." ) super().__init__(*args, **kwargs) dataset = self.run.cfg.get_default_dataset() @@ -288,6 +315,8 @@ def __init__(self, *args, **kwargs) -> None: # it because re-scoring the filtered subset can flip to the other # column and silently shred the sessions sampling kept whole. self._sampled_session_key: str | None = None + self._schema_names: set[str] | None = None + self._parquet_file: Any | None = None def _reject_extra_input_collisions(self) -> None: """Reject endpoint extra-inputs keys this loader injects per-turn. @@ -324,30 +353,41 @@ def can_load( ) -> bool: if pq is None: # pragma: no cover - platform-dependent return False - if filename is None or Path(filename).suffix.lower() != ".parquet": + if filename is None: + return False + + suffix = Path(filename).suffix.lower() + if suffix not in {".parquet", *_ARROW_IPC_SUFFIXES}: return False if data is not None: return _REQUIRED_COLUMNS.issubset(data.keys()) try: - schema = pq.read_schema(filename) + if suffix == ".parquet": + schema_names = set(pq.read_schema(filename).names) + else: + with _open_arrow_ipc(filename) as reader: + schema_names = set(reader.schema.names) except (FileNotFoundError, OSError, pa.ArrowException): return False - return _REQUIRED_COLUMNS.issubset(schema.names) + return _REQUIRED_COLUMNS.issubset(schema_names) def _parse_trace(self, record: dict) -> BasetenTrace: - raise NotImplementedError("BasetenTraceDatasetLoader reads Parquet, not JSONL.") + raise NotImplementedError( + "BasetenTraceDatasetLoader reads columnar files, not JSONL." + ) def _preprocess_trace(self, trace: BasetenTrace) -> None: + trace.total_hashes = list(trace.total_hashes or []) trace.timestamp = int(trace.timestamp_start_unix_ms) trace.input_length = int(trace.input_tokens) # Real traces contain canceled requests with output_tokens=0, but # Turn.max_tokens requires >= 1. trace.output_length = max(1, int(trace.output_tokens)) trace.text_input = trace.prompt - trace.hash_ids = list(trace.total_hashes or []) + trace.hash_ids = list(trace.total_hashes) def _set_request_body(self, trace: BasetenTrace) -> None: if trace.hash_ids is None: @@ -376,7 +416,7 @@ def _group_traces(self, items: list[BasetenTrace]) -> dict[str, list[BasetenTrac if not items: return {} - session_key = self._sampled_session_key or self._choose_session_key(items) + session_key = self._sampled_session_key if session_key is None: self.info( "No repeated Baseten trace session key found; generating session IDs." @@ -421,61 +461,89 @@ def _order_groups( session_entries.sort(key=lambda item: (item[0], item[1])) return {session_id: traces for _, session_id, traces in session_entries} - def _read_metadata_table(self) -> pa.Table: - schema = pq.read_schema(self.filename) - metadata_columns = [ - column for column in METADATA_COLUMNS if column in set(schema.names) - ] - return pq.read_table(self.filename, columns=metadata_columns) - - def _choose_session_key_from_metadata_rows( - self, rows: list[dict[str, Any]] - ) -> str | None: - return choose_baseten_session_key( - [row.get(METADATA_COLUMNS_SESSION) for row in rows], - [row.get(METADATA_COLUMNS_POOR_MAN_SESSION) for row in rows], + def _source_schema_names(self) -> set[str]: + if self._schema_names is None: + if _is_arrow_ipc(self.filename): + with _open_arrow_ipc(self.filename) as reader: + self._schema_names = set(reader.schema.names) + else: + assert self._parquet_file is not None + self._schema_names = set(self._parquet_file.schema_arrow.names) + return self._schema_names + + def _iter_source_batches(self, columns: list[str]) -> Iterator[pa.RecordBatch]: + """Yield projected batches from Parquet or memory-mapped Arrow IPC.""" + if _is_arrow_ipc(self.filename): + with _open_arrow_ipc(self.filename) as reader: + for index in range(reader.num_record_batches): + yield reader.get_batch(index).select(columns) + return + + assert self._parquet_file is not None + yield from self._parquet_file.iter_batches( + columns=columns, + batch_size=_PARQUET_BATCH_SIZE, + use_threads=True, ) + def _minimum_timestamp(self) -> int | None: + if not _is_arrow_ipc(self.filename): + assert self._parquet_file is not None + minimum = _parquet_min_timestamp(self._parquet_file) + if minimum is not None: + return minimum + minimum = None + for batch in self._iter_source_batches([METADATA_COLUMNS_TIME]): + batch_minimum = pc.min(batch.column(0)).as_py() + if batch_minimum is not None: + minimum = ( + int(batch_minimum) + if minimum is None + else min(minimum, int(batch_minimum)) + ) + return minimum + def _sample_session_ids( self, ) -> tuple[int | None, str | None, set[str | int] | None, set[int] | None]: - metadata_table = self._read_metadata_table() - - if metadata_table.num_rows == 0: - return None, None, None, None - - metadata_rows = metadata_table.to_pylist() - session_key = self._choose_session_key_from_metadata_rows(metadata_rows) + session_key = _baseten_session_key_from_schema(self._source_schema_names()) + self._sampled_session_key = session_key + if self._session_sample_ratio is None or self._session_sample_ratio >= 1.0: + return self._minimum_timestamp(), session_key, None, None min_timestamp: int | None = None session_first_ts: dict[str | int, int] = {} null_row_count = 0 - for row in metadata_rows: - timestamp = int(row[METADATA_COLUMNS_TIME]) - min_timestamp = ( - timestamp if min_timestamp is None else min(min_timestamp, timestamp) - ) - - if session_key is None: - continue - - session_id = row.get(session_key) - if session_id is None: - null_row_count += 1 - continue + columns = [METADATA_COLUMNS_TIME] + if session_key is not None: + columns.append(session_key) + for batch in self._iter_source_batches(columns): + for row in batch.to_pylist(): + timestamp = int(row[METADATA_COLUMNS_TIME]) + min_timestamp = ( + timestamp + if min_timestamp is None + else min(min_timestamp, timestamp) + ) - session_first_ts[session_id] = min( - timestamp, session_first_ts.get(session_id, timestamp) - ) + if session_key is None: + continue + session_id = row.get(session_key) + if session_id is None: + null_row_count += 1 + continue + session_first_ts[session_id] = min( + timestamp, session_first_ts.get(session_id, timestamp) + ) - if self._session_sample_ratio is None or self._session_sample_ratio >= 1.0: - return min_timestamp, session_key, None, None + if min_timestamp is None: + return None, session_key, None, None if session_key is None: self.warning( "trace_session_sample_ratio requested, but neither provided_session_id " - "nor poor_man_session_id forms multi-row sessions; skipping sampling." + "nor poor_man_session_id exists; skipping sampling." ) return min_timestamp, None, None, None @@ -508,7 +576,6 @@ def _sample_session_ids( f"using {session_key} with " f"trace_session_sample_ratio={self._session_sample_ratio}" ) - self._sampled_session_key = session_key return ( min_timestamp, session_key, @@ -522,7 +589,15 @@ def load_dataset(self) -> dict[str, list[BasetenTrace]]: self._capped_max_osl = 0 self._floored_zero_osl = 0 - items = self._read_traces(*self._sample_session_ids()) + if not _is_arrow_ipc(self.filename): + self._parquet_file = pq.ParquetFile(self.filename) + self._schema_names = set(self._parquet_file.schema_arrow.names) + try: + items = self._read_traces(*self._sample_session_ids()) + finally: + if self._parquet_file is not None: + self._parquet_file.close() + self._parquet_file = None # Closed-loop replay defers the gap cap to convert_to_conversations so # think-time delays derive from the recorded timestamps, not reflowed ones. @@ -560,25 +635,31 @@ def _read_traces( session_ids: set[str | int] | None, null_rows: set[int] | None, ) -> list[BasetenTrace]: - """Read, sample, normalize, and filter trace rows from the Parquet file.""" + """Read, sample, normalize, and filter trace rows from the source file.""" sampling = session_key is not None and session_ids is not None - table_kwargs: dict[str, Any] = {} - if sampling: - table_kwargs["filters"] = ( - pc.field(session_key).isin(sorted(session_ids)) - | pc.field(session_key).is_null() - ) + projected_columns = self._projected_trace_columns(session_key) items: list[BasetenTrace] = [] null_ordinal = 0 - for row in pq.read_table(self.filename, **table_kwargs).to_pylist(): + validated_rows = 0 + for row in self._iter_trace_rows( + projected_columns, + session_key if sampling else None, + session_ids, + ): if sampling and row.get(session_key) is None: kept = null_ordinal in null_rows null_ordinal += 1 if not kept: continue - trace = BasetenTrace.model_validate(row) + if validated_rows < _VALIDATION_SAMPLE_ROWS or self._row_needs_validation( + row + ): + trace = BasetenTrace.model_validate(row) + validated_rows += 1 + else: + trace = BasetenTrace.model_construct(**row) self._preprocess_trace(trace) normalized = min_timestamp is not None and trace.timestamp is not None if normalized: @@ -600,6 +681,66 @@ def _read_traces( items.append(trace) return items + @staticmethod + def _row_needs_validation(row: dict[str, Any]) -> bool: + """Route malformed required fields through Pydantic's clear errors.""" + if not isinstance(row.get("prompt"), str): + return True + return any( + not isinstance(value := row.get(field), int) or value < 0 + for field in ( + METADATA_COLUMNS_TIME, + "input_tokens", + "output_tokens", + ) + ) + + def _projected_trace_columns(self, session_key: str | None) -> list[str]: + """Return source columns needed by the configured replay behavior.""" + schema_names = self._source_schema_names() + columns = { + METADATA_COLUMNS_TIME, + "prompt", + "input_tokens", + "output_tokens", + } + if session_key is not None: + columns.add(session_key) + if not self._omit_kv_hints: + columns.update(("total_hashes", "block_size")) + if not self._open_loop: + columns.add("duration_e2e_ms") + return sorted(columns & schema_names) + + def _iter_trace_rows( + self, + projected_columns: list[str], + session_key: str | None, + session_ids: set[str | int] | None, + ) -> Iterator[dict[str, Any]]: + """Yield projected rows after applying sampling in bounded Arrow batches.""" + sampled_ids: pa.Array | None = None + for batch in self._iter_source_batches(projected_columns): + if session_key is not None and session_ids is not None: + if sampled_ids is None: + sampled_ids = pa.array( + sorted(session_ids), + type=batch.schema.field(session_key).type, + ) + session_column = batch.column(batch.schema.get_field_index(session_key)) + selected = pc.is_in(session_column, value_set=sampled_ids) + selected = pc.or_(selected, pc.is_null(session_column)) + if _is_arrow_ipc(self.filename): + columns = tuple(zip(batch.schema.names, batch.columns, strict=True)) + selected_rows = pc.indices_nonzero(selected).to_pylist() + for row_index in selected_rows: + yield { + name: column[row_index].as_py() for name, column in columns + } + continue + batch = batch.filter(selected) + yield from batch.to_pylist() + def _apply_idle_gap_cap(self, items: list[BasetenTrace]) -> None: """Collapse global idle gaps so a sparse (sampled) trace does not idle through long dead-air stretches under fixed-schedule replay. @@ -683,27 +824,11 @@ def _apply_back_pressure(self, data: dict[str, list[BasetenTrace]]) -> None: prev_ts = ts prev_e2e_ms = float(trace.duration_e2e_ms or 0) / self._speedup - def _choose_session_key(self, items: list[BasetenTrace]) -> str | None: - return choose_baseten_session_key( - [trace.provided_session_id for trace in items], - [trace.poor_man_session_id for trace in items], - ) - def _synthesis_exclude_fields(self) -> frozenset[str]: return frozenset( { "duration_e2e_ms", - "duration_ttft_ms", - "request_canceled", - "cached_tokens_reference", - "model_name", - "org_id", "block_size", - "features", - "speculation_ratio", - "output_text", - "dataset_version", - "total_hashes_len", "provided_session_id", "poor_man_session_id", "request_body", diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index afe28d621b..c0fe53bb26 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -543,10 +543,10 @@ custom_dataset_loader: baseten_trace: class: aiperf.dataset.loader.baseten_trace:BasetenTraceDatasetLoader description: | - Baseten completion trace dataset loader for Parquet exports that include - prompts, timestamps, session identifiers, output tokens, and optional - hash-based KV-cache metadata. Supports fixed_schedule timing replay for - /v1/completions workloads. + Baseten completion trace dataset loader for Parquet or Arrow IPC exports + that include prompts, timestamps, session identifiers, output tokens, and + optional hash-based KV-cache metadata. Arrow IPC files are memory-mapped. + Supports fixed_schedule timing replay for /v1/completions workloads. metadata: is_trace: true default_block_size: 64 diff --git a/tests/unit/config/test_baseten_replay_flags.py b/tests/unit/config/test_baseten_replay_flags.py index 2066183eb8..f9514b4996 100644 --- a/tests/unit/config/test_baseten_replay_flags.py +++ b/tests/unit/config/test_baseten_replay_flags.py @@ -25,11 +25,16 @@ pytest.importorskip("pyarrow") import pyarrow as pa +import pyarrow.ipc as ipc import pyarrow.parquet as pq from pydantic import ValidationError from pytest import param from aiperf.config.flags._converter_dataset import build_dataset +from aiperf.config.flags._converter_profiling import ( + _count_dataset_records, + _first_record_has_timestamp, +) from aiperf.config.flags.cli_config import CLIConfig from aiperf.config.flags.converter import convert_cli_to_aiperf @@ -78,6 +83,29 @@ def trace_parquet(tmp_path: Path) -> Path: return path +@pytest.mark.parametrize("suffix", [".arrow", ".ipc"]) +def test_arrow_ipc_profiling_metadata(tmp_path: Path, suffix: str) -> None: + path = tmp_path / f"trace{suffix}" + table = pa.Table.from_pylist( + [ + { + "timestamp_start_unix_ms": 100, + "prompt": "hello", + "input_tokens": 3, + "output_tokens": 4, + } + ] + ) + with ( + pa.OSFile(str(path), "wb") as sink, + ipc.new_file(sink, table.schema) as writer, + ): + writer.write_table(table) + + assert _first_record_has_timestamp(path) is True + assert _count_dataset_records(path) == 1 + + def _baseten_argv(trace_parquet: Path, *extra: str) -> list[str]: return [ "--url", diff --git a/tests/unit/dataset/loader/test_baseten_trace.py b/tests/unit/dataset/loader/test_baseten_trace.py index 62683b1097..d4209c9112 100644 --- a/tests/unit/dataset/loader/test_baseten_trace.py +++ b/tests/unit/dataset/loader/test_baseten_trace.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path +from typing import Any from unittest.mock import Mock import pytest @@ -9,11 +10,15 @@ pytest.importorskip("pyarrow") import pyarrow as pa +import pyarrow.ipc as ipc import pyarrow.parquet as pq +from pydantic import ValidationError from pytest import param from aiperf.common.enums import ConversationContextMode +from aiperf.common.environment import Environment from aiperf.config.flags.cli_config import CLIConfig +from aiperf.dataset.loader import baseten_trace as baseten_trace_module from aiperf.dataset.loader.baseten_trace import ( BasetenTrace, BasetenTraceDatasetLoader, @@ -29,6 +34,16 @@ def _write_parquet(path: Path, rows: list[dict]) -> Path: return path +def _write_arrow(path: Path, rows: list[dict], batch_size: int = 2) -> Path: + table = pa.Table.from_pylist(rows) + with ( + pa.OSFile(str(path), "wb") as sink, + ipc.new_file(sink, table.schema) as writer, + ): + writer.write_table(table, max_chunksize=batch_size) + return path + + def _mock_prompt_generator() -> Mock: generator = Mock() generator._decoded_cache = {} @@ -81,8 +96,13 @@ def test_can_load_parquet_schema_without_optional_columns(self, tmp_path: Path): assert BasetenTraceDatasetLoader.can_load(filename=path) is True def test_load_dataset_normalizes_timestamps_and_groups_sessions( - self, tmp_path: Path - ): + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + Environment.DATASET, + "BASETEN_SESSION_COLUMN", + "poor_man_session_id", + ) path = _write_parquet( tmp_path / "trace.parquet", [ @@ -132,7 +152,6 @@ def test_load_dataset_normalizes_timestamps_and_groups_sessions( assert list(dataset.keys()) == ["42", "99"] assert [trace.timestamp for trace in dataset["42"]] == [0, 150] assert dataset["42"][0].text_input == "first prompt" - assert dataset["42"][0].request_canceled == 1 # No "prompt" override: the completions endpoint emits single-prompt # payloads as bare strings itself. assert dataset["42"][0].request_body == { @@ -141,6 +160,130 @@ def test_load_dataset_normalizes_timestamps_and_groups_sessions( "block_size": 64, } + @pytest.mark.parametrize("suffix", [".arrow", ".ipc"]) + def test_arrow_ipc_load_matches_parquet(self, tmp_path: Path, suffix: str) -> None: + rows = [ + { + "timestamp_start_unix_ms": timestamp, + "prompt": f"prompt-{index}", + "input_tokens": index + 1, + "output_tokens": index, + "total_hashes": [index, index + 1], + "poor_man_session_id": index // 2, + "block_size": 64, + } + for index, timestamp in enumerate(range(100, 500, 10), start=1) + ] + parquet_path = _write_parquet(tmp_path / "trace.parquet", rows) + arrow_path = _write_arrow(tmp_path / f"trace{suffix}", rows, batch_size=3) + + def load(path: Path) -> dict[str, list[dict]]: + loader = BasetenTraceDatasetLoader( + filename=str(path), + run=_make_run( + path, + replay_speedup=2.0, + trace_session_sample_ratio=0.1, + random_seed=7, + ), + prompt_generator=_mock_prompt_generator(), + ) + return { + session_id: [trace.model_dump() for trace in traces] + for session_id, traces in loader.load_dataset().items() + } + + assert BasetenTraceDatasetLoader.can_load(filename=arrow_path) is True + assert load(arrow_path) == load(parquet_path) + + def test_count_arrow_ipc_records_and_sessions(self, tmp_path: Path) -> None: + path = _write_arrow( + tmp_path / "trace.arrow", + [ + { + "timestamp_start_unix_ms": index, + "prompt": str(index), + "input_tokens": 1, + "output_tokens": 1, + "poor_man_session_id": session_id, + } + for index, session_id in enumerate((1, 1, 2, None)) + ], + ) + + assert count_baseten_parquet_records_and_sessions(str(path)) == (4, 3) + + def test_arrow_ipc_opens_once_per_scan( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + path = _write_arrow( + tmp_path / "trace.arrow", + [ + { + "timestamp_start_unix_ms": index, + "prompt": str(index), + "input_tokens": 1, + "output_tokens": 1, + } + for index in range(12) + ], + batch_size=2, + ) + run = _make_run(path) + original_open = baseten_trace_module._open_arrow_ipc + open_calls = 0 + + def counting_open(file_path: str | Path) -> Any: + nonlocal open_calls + open_calls += 1 + return original_open(file_path) + + monkeypatch.setattr(baseten_trace_module, "_open_arrow_ipc", counting_open) + loader = BasetenTraceDatasetLoader( + filename=str(path), + run=run, + prompt_generator=_mock_prompt_generator(), + ) + + loader.load_dataset() + + assert open_calls == 3 + + def test_parquet_opens_once_per_load( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + path = _write_parquet( + tmp_path / "trace.parquet", + [ + { + "timestamp_start_unix_ms": index, + "prompt": str(index), + "input_tokens": 1, + "output_tokens": 1, + } + for index in range(12) + ], + ) + run = _make_run(path) + parquet_file_class = pq.ParquetFile + open_calls = 0 + + def counting_open(*args: Any, **kwargs: Any) -> Any: + nonlocal open_calls + open_calls += 1 + return parquet_file_class(*args, **kwargs) + + monkeypatch.setattr(baseten_trace_module.pq, "ParquetFile", counting_open) + loader = BasetenTraceDatasetLoader( + filename=str(path), + run=run, + prompt_generator=_mock_prompt_generator(), + ) + + loader.load_dataset() + + assert open_calls == 1 + def test_convert_to_conversations_uses_literal_prompts(self, tmp_path: Path): path = _write_parquet( tmp_path / "trace.parquet", @@ -428,9 +571,14 @@ def load() -> list[list[str]]: assert first == second assert first, "mid-ratio sampling should keep at least one session" - def test_trace_session_sampling_skips_when_no_effective_session_key( - self, tmp_path: Path - ): + def test_trace_session_sampling_uses_provided_session_id_fallback( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + Environment.DATASET, + "BASETEN_SESSION_COLUMN", + "poor_man_session_id", + ) path = _write_parquet( tmp_path / "trace.parquet", [ @@ -456,19 +604,34 @@ def test_trace_session_sampling_skips_when_no_effective_session_key( prompt_generator=_mock_prompt_generator(), ) + parquet_file_class = pq.ParquetFile + + class NoReadParquetFile: + def __init__(self, *args, **kwargs) -> None: + self._parquet_file = parquet_file_class(*args, **kwargs) + + def __getattr__(self, name: str): + if name == "read": + raise AssertionError("sampled metadata must use bounded batches") + return getattr(self._parquet_file, name) + + monkeypatch.setattr(baseten_trace_module.pq, "ParquetFile", NoReadParquetFile) + dataset = loader.load_dataset() - assert len(dataset) == 2 + assert list(dataset) == ["unique-1"] assert sorted( trace.text_input for traces in dataset.values() for trace in traces - ) == [ - "row-1", - "row-2", - ] + ) == ["row-1"] - def test_trace_session_sampling_falls_back_to_poor_man_session_id( - self, tmp_path: Path - ): + def test_trace_session_sampling_uses_configured_poor_man_session_id( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + Environment.DATASET, + "BASETEN_SESSION_COLUMN", + "poor_man_session_id", + ) path = _write_parquet( tmp_path / "trace.parquet", [ @@ -518,20 +681,21 @@ def test_trace_session_sampling_falls_back_to_poor_man_session_id( kept_session = next(iter(dataset.values())) assert len(kept_session) == 2 - def test_sampling_and_grouping_use_same_session_key(self, tmp_path: Path): - # Sampling filters rows by the session key scored on the FULL file; - # grouping must reuse that key. Re-scoring the filtered subset can flip - # to the other column, shredding the sessions sampling kept whole and - # grouping row-sampled null-key rows into multi-turn conversations with - # silently dropped middle turns. - # + def test_sampling_and_grouping_use_same_session_key( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + Environment.DATASET, + "BASETEN_SESSION_COLUMN", + "poor_man_session_id", + ) + + # Sampling and grouping must reuse the configured key; switching after + # filtering would shred sessions and regroup null-key rows. # Fixture shape is pinned to the sampling RNG (root seed 42 from the # autouse fixture): 7 poor_man sessions draw the first 7 uniforms (pairB # dropped, pairA kept, singletons eat the rest), then the 3 null-poor_man - # "s0" rows draw the next 3 (t0 kept, t1 dropped, t2 kept). Full-file - # scores: poor_man (4, 2) > provided (2·"s0"... 3 rows -> (3, 1)), so - # sampling keys on poor_man; the filtered subset scores (2, 1) vs (2, 1), - # which re-scoring would flip to provided_session_id. + # "s0" rows draw the next 3 (t0 kept, t1 dropped, t2 kept). def row(ts: int, prompt: str, poor: int | None, provided: str | None = None): return { "timestamp_start_unix_ms": ts, @@ -573,7 +737,25 @@ def row(ts: int, prompt: str, poor: int | None, provided: str | None = None): if any(prompt.startswith("s0-") for prompt in prompts): assert len(prompts) == 1 - def test_resolver_session_count_uses_same_key_as_loader(self, tmp_path: Path): + @pytest.mark.parametrize( + ("session_column", "expected_sessions"), + [ + param("provided_session_id", 4, id="provided"), + param("poor_man_session_id", 2, id="poor-man"), + ], + ) # fmt: skip + def test_resolver_session_count_uses_same_key_as_loader( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + session_column: str, + expected_sessions: int, + ) -> None: + monkeypatch.setattr( + Environment.DATASET, + "BASETEN_SESSION_COLUMN", + session_column, + ) path = _write_parquet( tmp_path / "trace.parquet", [ @@ -612,7 +794,10 @@ def test_resolver_session_count_uses_same_key_as_loader(self, tmp_path: Path): ], ) - assert count_baseten_parquet_records_and_sessions(str(path)) == (4, 2) + assert count_baseten_parquet_records_and_sessions(str(path)) == ( + 4, + expected_sessions, + ) def test_fixed_schedule_offsets_filter_relative_window_on_unix_timestamps( self, tmp_path: Path @@ -819,33 +1004,171 @@ def test_load_dataset_zero_output_tokens_floored_to_one(self, tmp_path: Path): assert turns["canceled row"].max_tokens == 1 assert turns["normal row"].max_tokens == 7 - def test_load_dataset_populates_dataset_version_from_version_column( - self, tmp_path: Path - ): - # The parquet column is named __version__; the pydantic alias must - # populate dataset_version instead of stranding it in model_extra. + @pytest.mark.parametrize( + ("run_kwargs", "expected_columns"), + [ + param( + {}, + { + "block_size", + "input_tokens", + "output_tokens", + "poor_man_session_id", + "prompt", + "timestamp_start_unix_ms", + "total_hashes", + }, + id="default", + ), + param( + {"omit_kv_hints": True}, + { + "input_tokens", + "output_tokens", + "poor_man_session_id", + "prompt", + "timestamp_start_unix_ms", + }, + id="omit-kv-hints", + ), + param( + {"open_loop_replay": False}, + { + "block_size", + "duration_e2e_ms", + "input_tokens", + "output_tokens", + "poor_man_session_id", + "prompt", + "timestamp_start_unix_ms", + "total_hashes", + }, + id="closed-loop", + ), + ], + ) # fmt: skip + def test_load_dataset_validates_sample_and_skips_unused_columns( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + run_kwargs: dict[str, bool], + expected_columns: set[str], + ) -> None: path = _write_parquet( tmp_path / "trace.parquet", [ { - "timestamp_start_unix_ms": 100, - "prompt": "versioned", + "timestamp_start_unix_ms": index, + "prompt": f"prompt-{index}", "input_tokens": 5, "output_tokens": 1, - "__version__": "0.0.11", + "output_text": "unused completion", + "model_name": "unused model", + "request_canceled": 1, + "__version__": "unused version", + "poor_man_session_id": 1, + "total_hashes": [1, 2], + "block_size": 64, + "duration_e2e_ms": 10, } + for index in range(12) ], ) + original_validate = BasetenTrace.model_validate + original_iter_batches = pq.ParquetFile.iter_batches + validation_calls = 0 + requested_columns: list[set[str]] = [] + + def count_validation_calls(row: dict) -> BasetenTrace: + nonlocal validation_calls + validation_calls += 1 + return original_validate(row) + + def capture_columns(parquet_file, *args, **kwargs): + requested_columns.append(set(kwargs["columns"])) + return original_iter_batches(parquet_file, *args, **kwargs) + + monkeypatch.setattr(BasetenTrace, "model_validate", count_validation_calls) + monkeypatch.setattr(pq.ParquetFile, "iter_batches", capture_columns) loader = BasetenTraceDatasetLoader( filename=str(path), - run=_make_run(), + run=_make_run(path, **run_kwargs), prompt_generator=_mock_prompt_generator(), ) dataset = loader.load_dataset() + traces = [trace for session in dataset.values() for trace in session] - trace = next(iter(dataset.values()))[0] - assert trace.dataset_version == "0.0.11" + assert validation_calls == 10 + assert len(traces) == 12 + assert requested_columns == [expected_columns] + assert all(not hasattr(trace, "output_text") for trace in traces) + assert all(not hasattr(trace, "model_name") for trace in traces) + assert all(not hasattr(trace, "request_canceled") for trace in traces) + assert all(not hasattr(trace, "dataset_version") for trace in traces) + + @pytest.mark.parametrize( + ("field", "invalid_value"), + [ + param("timestamp_start_unix_ms", -1, id="negative-timestamp"), + param("input_tokens", None, id="null-input-tokens"), + param("output_tokens", -1, id="negative-output-tokens"), + param("prompt", None, id="null-prompt"), + ], + ) # fmt: skip + def test_load_dataset_validates_malformed_required_field_after_sample( + self, + tmp_path: Path, + field: str, + invalid_value: object, + ) -> None: + rows = [ + { + "timestamp_start_unix_ms": index, + "prompt": f"prompt-{index}", + "input_tokens": 5, + "output_tokens": 1, + } + for index in range(11) + ] + rows[-1][field] = invalid_value + path = _write_parquet(tmp_path / "trace.parquet", rows) + loader = BasetenTraceDatasetLoader( + filename=str(path), + run=_make_run(path), + prompt_generator=_mock_prompt_generator(), + ) + + with pytest.raises(ValidationError, match=field): + loader.load_dataset() + + def test_load_dataset_normalizes_null_hashes_after_validation_sample( + self, tmp_path: Path + ) -> None: + path = _write_parquet( + tmp_path / "trace.parquet", + [ + { + "timestamp_start_unix_ms": index, + "prompt": f"prompt-{index}", + "input_tokens": 5, + "output_tokens": 1, + "total_hashes": [1] if index < 10 else None, + } + for index in range(11) + ], + ) + loader = BasetenTraceDatasetLoader( + filename=str(path), + run=_make_run(path), + prompt_generator=_mock_prompt_generator(), + ) + + traces = [ + trace for session in loader.load_dataset().values() for trace in session + ] + + assert traces[-1].total_hashes == [] def test_request_body_uses_capped_output_length(self, tmp_path: Path): path = _write_parquet( @@ -972,14 +1295,10 @@ def test_model_accepts_null_hashes_and_numeric_provided_session_id(self): output_tokens=20, total_hashes=None, provided_session_id=42, - request_canceled=1, - org_id=7, ) assert trace.total_hashes == [] assert trace.provided_session_id == 42 - assert trace.request_canceled == 1 - assert trace.org_id == 7 class TestSynthesisHooks: @@ -1017,7 +1336,6 @@ def test_reconstruct_traces_preserves_original_metadata( poor_man_session_id=7, total_hashes=[1, 2], block_size=64, - __version__="0.0.11", ) ] synth_dicts = [ @@ -1035,9 +1353,6 @@ def test_reconstruct_traces_preserves_original_metadata( assert result[0].prompt == "original" assert result[0].poor_man_session_id == 7 assert result[0].total_hashes == [1, 2] - # The model_dump()/model_validate round-trip must not strand aliased - # fields in model_extra. - assert result[0].dataset_version == "0.0.11" def test_reconstruct_traces_uses_last_original_for_extra_synth_rows( self, tmp_path: Path diff --git a/tests/unit/dataset/loader/test_baseten_trace_optional_pyarrow.py b/tests/unit/dataset/loader/test_baseten_trace_optional_pyarrow.py index 8a0c261084..24aae1ec5f 100644 --- a/tests/unit/dataset/loader/test_baseten_trace_optional_pyarrow.py +++ b/tests/unit/dataset/loader/test_baseten_trace_optional_pyarrow.py @@ -32,6 +32,7 @@ def find_spec(self, name, path=None, target=None): ) assert BasetenTraceDatasetLoader.can_load(filename="x.parquet") is False +assert BasetenTraceDatasetLoader.can_load(filename="x.arrow") is False assert count_baseten_parquet_records_and_sessions("x.parquet") == (0, 0) try: diff --git a/tests/unit/dataset/test_dataset_manager_inputs_json_adversarial.py b/tests/unit/dataset/test_dataset_manager_inputs_json_adversarial.py index 9d8c87fcea..9cbeb4eb06 100644 --- a/tests/unit/dataset/test_dataset_manager_inputs_json_adversarial.py +++ b/tests/unit/dataset/test_dataset_manager_inputs_json_adversarial.py @@ -330,6 +330,23 @@ def _gen(): class TestSkipInputsJsonAdversarial: + @pytest.mark.asyncio + async def test_skip_inputs_json_generation_for_baseten_trace( + self, tmp_path: Path + ) -> None: + mgr = _full_manager(tmp_path, CustomDatasetType.BASETEN_TRACE) + mgr._configure_dataset = AsyncMock() + mgr._configure_tokenizer = AsyncMock() + mgr._configure_dataset_client_and_free_memory = AsyncMock() + mgr._detected_dataset_type = CustomDatasetType.BASETEN_TRACE + mgr._try_cache_lookup = Mock(return_value=None) + + with patch.object( + mgr, "_generate_inputs_json_file", new_callable=AsyncMock + ) as mock_gen: + await mgr._profile_configure_command(Mock()) + mock_gen.assert_not_called() + @pytest.mark.asyncio async def test_skip_inputs_json_generation_for_raw_payload_dataset_type( self, tmp_path: Path diff --git a/tools/generate_env_vars_docs.py b/tools/generate_env_vars_docs.py index cb87eaf1a2..69bea0f261 100755 --- a/tools/generate_env_vars_docs.py +++ b/tools/generate_env_vars_docs.py @@ -81,6 +81,20 @@ def _parse_field(node: ast.AnnAssign) -> Field | None: constraints = [] description = "—" + if ( + isinstance(node.annotation, ast.Subscript) + and isinstance(node.annotation.value, ast.Name) + and node.annotation.value.id == "Literal" + ): + values = ( + node.annotation.slice.elts + if isinstance(node.annotation.slice, ast.Tuple) + else [node.annotation.slice] + ) + constraints.append( + f"one of: {' / '.join(ast.unparse(value) for value in values)}" + ) + if isinstance(node.value, ast.Call): func = node.value.func if isinstance(func, ast.Name) and func.id == "Field": From 0cd80441e7b219825c8848349980c2e479aee261 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:55:33 -0700 Subject: [PATCH 017/119] fix: always project recorded-outcome columns so fidelity survives load (#1269) Signed-off-by: lkomali (cherry picked from commit 215be05b6a534fb19b84bf83f711db2d20f5bea1) --- src/aiperf/dataset/loader/baseten_trace.py | 14 +++++++++++--- tests/unit/dataset/loader/test_baseten_trace.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/aiperf/dataset/loader/baseten_trace.py b/src/aiperf/dataset/loader/baseten_trace.py index 968bd7eec3..99ae30d75b 100644 --- a/src/aiperf/dataset/loader/baseten_trace.py +++ b/src/aiperf/dataset/loader/baseten_trace.py @@ -696,20 +696,28 @@ def _row_needs_validation(row: dict[str, Any]) -> bool: ) def _projected_trace_columns(self, session_key: str | None) -> list[str]: - """Return source columns needed by the configured replay behavior.""" + """Return source columns needed by the configured replay behavior. + + Recorded-outcome columns (durations, cached-token reference) are always + projected: they are ground truth that must survive load for later + fidelity comparison, independent of replay mode. They are small integer + columns, so keeping them costs negligibly against the projection's I/O + savings on the large prompt/hash columns. + """ schema_names = self._source_schema_names() columns = { METADATA_COLUMNS_TIME, "prompt", "input_tokens", "output_tokens", + "duration_e2e_ms", + "duration_ttft_ms", + "cached_tokens_reference", } if session_key is not None: columns.add(session_key) if not self._omit_kv_hints: columns.update(("total_hashes", "block_size")) - if not self._open_loop: - columns.add("duration_e2e_ms") return sorted(columns & schema_names) def _iter_trace_rows( diff --git a/tests/unit/dataset/loader/test_baseten_trace.py b/tests/unit/dataset/loader/test_baseten_trace.py index d4209c9112..45b51f222a 100644 --- a/tests/unit/dataset/loader/test_baseten_trace.py +++ b/tests/unit/dataset/loader/test_baseten_trace.py @@ -1011,6 +1011,9 @@ def test_load_dataset_zero_output_tokens_floored_to_one(self, tmp_path: Path): {}, { "block_size", + "cached_tokens_reference", + "duration_e2e_ms", + "duration_ttft_ms", "input_tokens", "output_tokens", "poor_man_session_id", @@ -1023,6 +1026,9 @@ def test_load_dataset_zero_output_tokens_floored_to_one(self, tmp_path: Path): param( {"omit_kv_hints": True}, { + "cached_tokens_reference", + "duration_e2e_ms", + "duration_ttft_ms", "input_tokens", "output_tokens", "poor_man_session_id", @@ -1035,7 +1041,9 @@ def test_load_dataset_zero_output_tokens_floored_to_one(self, tmp_path: Path): {"open_loop_replay": False}, { "block_size", + "cached_tokens_reference", "duration_e2e_ms", + "duration_ttft_ms", "input_tokens", "output_tokens", "poor_man_session_id", @@ -1070,6 +1078,8 @@ def test_load_dataset_validates_sample_and_skips_unused_columns( "total_hashes": [1, 2], "block_size": 64, "duration_e2e_ms": 10, + "duration_ttft_ms": 4, + "cached_tokens_reference": 2, } for index in range(12) ], @@ -1106,6 +1116,11 @@ def capture_columns(parquet_file, *args, **kwargs): assert all(not hasattr(trace, "model_name") for trace in traces) assert all(not hasattr(trace, "request_canceled") for trace in traces) assert all(not hasattr(trace, "dataset_version") for trace in traces) + # Recorded outcomes must survive load for later fidelity comparison, + # regardless of replay mode. + assert all(trace.duration_e2e_ms == 10 for trace in traces) + assert all(trace.duration_ttft_ms == 4 for trace in traces) + assert all(trace.cached_tokens_reference == 2 for trace in traces) @pytest.mark.parametrize( ("field", "invalid_value"), From 1cbd2f57cfe5caa556f2e32b8869e284c468c5d7 Mon Sep 17 00:00:00 2001 From: Harrison Saturley-Hall Date: Tue, 11 Aug 2026 19:15:55 -0400 Subject: [PATCH 018/119] chore: Update CODEOWNERS to aiperf-codeowners team (#1271) Signed-off-by: Harrison Saturley-Hall (cherry picked from commit 516faa12c84b742b8b40bfbdd4d540c2ce69c829) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3b3c56ed20..f01a06b13d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,4 +14,4 @@ # Docs: https://docs.github.com/articles/about-code-owners # Default owners for everything in the repository — the core NVIDIA AIPerf team. -* @ajcasagrande @debermudez @matthewkotila @ilana-n @lkomali @ganeshku1 @FrankD412 +* @ai-dynamo/aiperf-codeowners From ff97af649960018941ba5e0d2089ba0853001337 Mon Sep 17 00:00:00 2001 From: Harrison Saturley-Hall Date: Tue, 11 Aug 2026 19:41:26 -0400 Subject: [PATCH 019/119] docs: spell out Visual Studio Code (#1272) Signed-off-by: Harrison King Saturley-Hall (cherry picked from commit ce453582c7182b0e34205b65843ec5e70723828a) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5e08e0bf6..d5174b0bb9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -194,7 +194,7 @@ You can use the `act` tool to run GitHub Actions locally. See [act usage](https: act -j run-integration-tests ``` -You can also use the VSCode extension [GitHub Local Actions](https://marketplace.visualstudio.com/items?itemName=SanjulaGanepola.github-local-actions). +You can also use the Visual Studio Code extension [GitHub Local Actions](https://marketplace.visualstudio.com/items?itemName=SanjulaGanepola.github-local-actions). ## Developer Certificate of Origin From a25e9b3826e340becace90c7754d9c1ede19fc0e Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:10:55 -0700 Subject: [PATCH 020/119] feat(dataset): add tracelab dataset support (#1262) Signed-off-by: Elias Bermudez (cherry picked from commit 082a51827eb9f755aacb93122cafe0383cf99f6e) --- README.md | 1 + docs/benchmark-datasets.md | 5 + docs/benchmark-modes/trace-replay.md | 2 + docs/cli-options.md | 8 +- docs/environment-variables.md | 3 + docs/index.yml | 2 + docs/tutorials/tracelab-trace.md | 158 +++ src/aiperf/common/environment.py | 27 + src/aiperf/common/utils.py | 16 +- src/aiperf/config/dataset/config.py | 12 +- src/aiperf/config/dataset/resolver.py | 78 +- src/aiperf/config/flags/_converter_dataset.py | 14 +- .../config/flags/_converter_profiling.py | 36 +- src/aiperf/config/flags/cli_config.py | 1 + src/aiperf/dataset/composer/custom.py | 4 +- .../dataset/loader/_tracelab_convert.py | 702 +++++++++ src/aiperf/dataset/loader/base_loader.py | 7 +- src/aiperf/dataset/loader/tracelab_trace.py | 387 +++++ src/aiperf/dataset/mmap_cache.py | 18 +- src/aiperf/plugin/plugins.yaml | 17 + tests/unit/common/test_utils.py | 80 ++ .../unit/config/test_baseten_replay_flags.py | 81 ++ tests/unit/config/test_block_size_routing.py | 19 +- tests/unit/config/test_resolver_edge_cases.py | 63 + .../dataset/loader/test_tracelab_trace.py | 1251 +++++++++++++++++ tests/unit/dataset/test_mmap_cache.py | 78 + .../plugin/test_new_loader_plugin_entries.py | 1 + 27 files changed, 3021 insertions(+), 50 deletions(-) create mode 100644 docs/tutorials/tracelab-trace.md create mode 100644 src/aiperf/dataset/loader/_tracelab_convert.py create mode 100644 src/aiperf/dataset/loader/tracelab_trace.py create mode 100644 tests/unit/dataset/loader/test_tracelab_trace.py diff --git a/README.md b/README.md index cb3dc45555..2a4c8a1ca3 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,7 @@ Log File: /home/user/Code/aiperf/artifacts/granite4:350m-openai-chat-concurrency - [Synthetic Dataset Generation](docs/tutorials/synthetic-dataset.md) - Generate synthetic datasets - [Agentic Code Generator](docs/tutorials/agentic-code-generator.md) - Generate multi-turn coding-agent traces for KV cache benchmarking - [Weka Traces](docs/tutorials/weka-trace.md) - Replay real agentic coding sessions with KV-cache-aware Weka traces +- [TraceLab Traces](docs/tutorials/tracelab-trace.md) - Replay the public TraceLab corpus of real agentic coding sessions - [AgentX scenarios](docs/tutorials/agentx-mvp.md) - InferenceX defaults (`--scenario agentx`) and the original MVP benchmark (`--scenario inferencex-agentx-mvp`) - [Fixed Schedule](docs/tutorials/fixed-schedule.md) - Precise timestamp-based execution - [Time-based Benchmarking](docs/tutorials/time-based-benchmarking.md) - Duration-based testing diff --git a/docs/benchmark-datasets.md b/docs/benchmark-datasets.md index 36a48b7d9d..c2f5f50228 100644 --- a/docs/benchmark-datasets.md +++ b/docs/benchmark-datasets.md @@ -67,6 +67,11 @@ This document describes datasets that AIPerf can use to generate stimulus. Addit ✅ Synthetic multi-turn coding-agent traces with shared prompt layers, repository context, and cache-aware turn growth. Generated via aiperf synthesize agentic-code and replayed as a Mooncake trace. + + TraceLab + ✅ + Real agentic coding sessions from the TraceLab corpus, replayed with synthesized KV-cache block IDs and recovered subagent nesting --input-file syfi_coding_trace.jsonl.gz --custom-dataset-type tracelab + diff --git a/docs/benchmark-modes/trace-replay.md b/docs/benchmark-modes/trace-replay.md index d0b3554842..946cf33428 100644 --- a/docs/benchmark-modes/trace-replay.md +++ b/docs/benchmark-modes/trace-replay.md @@ -93,6 +93,8 @@ The `--fixed-schedule` flag tells AIPerf to send requests at the exact timestamp When you supply a trace dataset (`--custom-dataset-type mooncake_trace`, `bailian_trace`, `burst_gpt_trace`, ...) and the file's first record carries a `timestamp` field, AIPerf automatically switches the profiling phase to fixed-schedule mode and fills `--request-count` from the number of trace entries. You can pass `--fixed-schedule` explicitly for clarity, but it's no longer required. +A few formats carry timing somewhere other than a top-level `timestamp` and are recognized by type instead: `sagemaker_data_capture` nests it under `eventMetadata`, `burst_gpt_trace` enforces a `Timestamp` column, and `tracelab` derives a submission time per round from its `timing_events[]` array. + To override the auto-promotion — for example, to replay the same trace under a fresh `--concurrency` or `--request-rate` setting and ignore the captured timestamps — pass `--no-fixed-schedule`: ```bash diff --git a/docs/cli-options.md b/docs/cli-options.md index 6cf50bf877..1c55109faa 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -501,8 +501,8 @@ Dataset-specific filter in key=value form. Repeat for multiple filters. Only sup #### `--custom-dataset-type` `` -Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), `mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; when using `random_pool`, `--conversation-num` defaults to 100 if not specified; batch sizes > 1 sample each modality independently from a flat pool and do not preserve per-entry associations - use `single_turn` if paired modalities must stay together). Requires `--input-file`. Mutually exclusive with `--public-dataset`. -
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `weka_trace`]_ +Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), `tracelab` (TraceLab agentic-coding corpus, JSONL or gzipped JSONL), `mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; when using `random_pool`, `--conversation-num` defaults to 100 if not specified; batch sizes > 1 sample each modality independently from a flat pool and do not preserve per-entry associations - use `single_turn` if paired modalities must stay together). Requires `--input-file`. Mutually exclusive with `--public-dataset`. +
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `tracelab`, `weka_trace`]_ #### `--ignore-trace-delays` @@ -2036,8 +2036,8 @@ Dataset-specific filter in key=value form. Repeat for multiple filters. Only sup #### `--custom-dataset-type` `` -Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), `mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; when using `random_pool`, `--conversation-num` defaults to 100 if not specified; batch sizes > 1 sample each modality independently from a flat pool and do not preserve per-entry associations - use `single_turn` if paired modalities must stay together). Requires `--input-file`. Mutually exclusive with `--public-dataset`. -
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `weka_trace`]_ +Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), `tracelab` (TraceLab agentic-coding corpus, JSONL or gzipped JSONL), `mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; when using `random_pool`, `--conversation-num` defaults to 100 if not specified; batch sizes > 1 sample each modality independently from a flat pool and do not preserve per-entry associations - use `single_turn` if paired modalities must stay together). Requires `--input-file`. Mutually exclusive with `--public-dataset`. +
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `tracelab`, `weka_trace`]_ #### `--ignore-trace-delays` diff --git a/docs/environment-variables.md b/docs/environment-variables.md index dcf5e2ff82..49c5f8aab0 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -105,6 +105,9 @@ Dataset loading and configuration. Controls timeouts and behavior for dataset lo | `AIPERF_DATASET_MEDIA_DOWNLOAD_TIMEOUT` | `60.0` | ≥ 1.0, ≤ 100000.0 | Timeout in seconds per media URL download when inline encoding is required | | `AIPERF_DATASET_MEDIA_DOWNLOAD_MAX_CONCURRENCY` | `10` | ≥ 1, ≤ 100 | Maximum number of concurrent media URL downloads | | `AIPERF_DATASET_INLINE_RECORDS_WARN_THRESHOLD` | `500` | ≥ 1 | Soft warning threshold for the number of inline `records:` entries on a `FileDataset`. When total inline records exceed this value, the config loader logs a warning suggesting the user move the dataset to a JSONL file. No hard cap. | +| `AIPERF_DATASET_TRACELAB_SUBAGENT_JOIN` | `True` | — | When True (default), TraceLabTraceDatasetLoader recovers subagent parent/child links by timing containment and nests each recovered child as a subagent entry inside its parent trace. Set to False to emit every recorded session as an independent flat trace, which is the shape the corpus literally records. | +| `AIPERF_DATASET_TRACELAB_CODEX_SUBAGENT_JOIN` | `True` | — | When True (default), the TraceLab subagent join also runs over codex sessions. Codex uses an async spawn/wait/close agent lifecycle whose handles are stripped from the released corpus, so only a coarse session-level window is available there and a session fanning out several agents collapses them into one window. Set to False to keep only the precise blocking-tool-call join. | +| `AIPERF_DATASET_TRACELAB_MIN_SPAWN_MS` | `10000` | ≥ 0 | Minimum wall latency, in milliseconds, for a spawning tool call to be treated as a subagent round-trip by the TraceLab join. Short calls are overwhelmingly no-op or error returns, and admitting them widens the containment window enough to start capturing unrelated concurrent sessions. | | `AIPERF_DATASET_WEKA_PARALLEL_WORKERS` | `0` | ≥ 0, ≤ 256 | Number of worker processes for WekaTraceLoader parallel reconstruction. 0 = auto (min(cpu_count - 1, 16, num_traces)). Set to 1 to force serial reconstruction. | | `AIPERF_DATASET_WEKA_PARALLEL_THRESHOLD` | `8` | ≥ 1, ≤ 100000 | Minimum number of parent traces required before WekaTraceLoader switches to the multi-process parallel reconstruction path. Below this, the in-process serial path is used (Pool startup overhead exceeds the speedup for tiny corpora). | | `AIPERF_DATASET_WEKA_SPLIT_FLATTENED_AGENTS` | `True` | — | When True (default), WekaTraceLoader runs hash_id LCP chain detection at both layers: untagged agent fan-outs recorded as flat top-level requests split into per-agent child conversations (::fa:NNN), and each subagent entry's inner requests split into per-context-chain children (`::sa:agent_id` plus :fa:NNN siblings), all with SPAWN/SPAWN_JOIN linkage so replay reproduces the recorded concurrency. Set to False to disable detection at both layers: all top-level requests serialize into one root conversation and each subagent emits exactly one child with its inner requests in time order. Detected chains at both layers are further split into genuine agents and auxiliary one-shot sidecars (top-level ::fa: vs ::aux:; subagent overflow :fa: vs :aux:) per WEKA_AUX_MAX_REQUESTS / WEKA_AUX_ISL_RATIO / WEKA_AUX_ISL_FLOOR. | diff --git a/docs/index.yml b/docs/index.yml index d33b292228..a4fbda72d5 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -106,6 +106,8 @@ navigation: path: tutorials/agentic-code-generator.md - page: Replay Weka Agentic Coding Traces path: tutorials/weka-trace.md + - page: Replay TraceLab Agentic Coding Traces + path: tutorials/tracelab-trace.md - page: InferenceX AgentX MVP Benchmark path: tutorials/agentx-mvp.md - section: Load Patterns & Scheduling diff --git a/docs/tutorials/tracelab-trace.md b/docs/tutorials/tracelab-trace.md new file mode 100644 index 0000000000..ffbb900b26 --- /dev/null +++ b/docs/tutorials/tracelab-trace.md @@ -0,0 +1,158 @@ + + +# Profile with TraceLab Agentic Coding Traces + +AIPerf can replay [TraceLab](https://github.com/uw-syfi/TraceLab), a public corpus of real agentic coding sessions captured from 52 developers using Claude Code and Codex against production endpoints. It records 665,453 LLM rounds across 8,058 sessions and 743,819 tool calls, licensed CC BY 4.0. + +TraceLab is a good fit when you want a KV-cache workload whose shape was set by people rather than by a generator: long sessions, deep prefix reuse, agent fan-out, and multi-minute human pauses between turns. + +This guide covers fetching the corpus and replaying it, and is explicit about which parts of a session are reproduced and which are reconstructed. + +--- + +## Start a vLLM Server + +Launch a vLLM server with a chat model: + +```bash +docker pull vllm/vllm-openai:latest +docker run --gpus all -p 8000:8000 vllm/vllm-openai:latest \ + --model Qwen/Qwen3-0.6B +``` + +Verify the server is ready: +```bash +curl -s localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"test"}],"max_tokens":1}' +``` + +--- + +## Download the Corpus + +The corpus is published as a GitHub release asset. It is about 100 MB compressed and roughly 650 MB inflated, and AIPerf reads it gzipped, so there is no need to decompress it: + +```bash +curl -L -o syfi_coding_trace.jsonl.gz \ + https://github.com/uw-syfi/TraceLab/releases/download/v0.0.2/syfi_coding_trace.jsonl.gz +``` + +For a first run, take a slice rather than the whole corpus: + +```bash +zcat syfi_coding_trace.jsonl.gz | head -20000 | gzip > tracelab_slice.jsonl.gz +``` + +--- + +## Profile + +```bash +aiperf profile \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --url localhost:8000 \ + --endpoint-type chat \ + --custom-dataset-type tracelab \ + --input-file tracelab_slice.jsonl.gz \ + --num-dataset-entries 50 \ + --concurrency 4 \ + --no-fixed-schedule \ + --inter-turn-delay-cap-seconds 5 +``` + +`--no-fixed-schedule` keeps the run in concurrency mode so `--inter-turn-delay-cap-seconds` takes effect. Without it, AIPerf auto-promotes TraceLab (which carries per-round timestamps) to fixed-schedule replay, where the recorded timestamps are authoritative and the cap has no influence. + +`--inter-turn-delay-cap-seconds` is worth setting deliberately. The corpus records real human pauses, some of them tens of minutes long, and AIPerf will sleep for them faithfully. Capping makes a run finish in reasonable time; not capping is the more honest replay. Decide which you want rather than inheriting the default. + +--- + +## Fixed-Schedule Replay + +Every round carries an absolute timestamp, so the corpus can be replayed on its +recorded arrival schedule: + +```bash +aiperf profile ... --custom-dataset-type tracelab --fixed-schedule \ + --fixed-schedule-end-offset 300000 +``` + +Bound the window. A fixed-schedule run without one replays the sessions at their +real offsets, and a single TraceLab session can span hours of wall clock, so the +run takes that long by design. `--fixed-schedule-end-offset` is in milliseconds. + +--- + +## TraceLab Format + +Each JSONL line is one LLM round, not one session. AIPerf groups rounds into sessions by `session_id` and replays each session as a conversation. The fields it reads: + +- `session_id`: session this round belongs to, formatted `:` +- `round_index`: the corpus's own round ordering +- `input_tokens_total`, `prefix_tokens`, `newly_append_tokens`: engine-reported input decomposition, where `input_tokens_total = prefix_tokens + newly_append_tokens` +- `output_tokens`, `reasoning_output_tokens`: generated token counts +- `model`, `provider`, `user`, `project`: identity fields +- `timing_events[]`: per-event absolute ISO-8601 timestamps (`user_message`, `tool_result`, `text`, `reasoning`, `tool_call`) +- `tools[]`: tool calls with `tool_name`, `emitted_at`, `result_at`, `tool_wall_latency_ms` +- `first_input_event_type`: what fed this round + +The corpus contains no prompt or response text: the longest string in it is a 503-character command skeleton. AIPerf synthesizes prompt content to the recorded token counts, which is the same approach every hash-id trace format uses. + +--- + +## What Is Reproduced, and What Is Reconstructed + +Replay fidelity differs by property, and the difference matters when you interpret results. + +Reproduced from the record: + +- input and output token counts per round +- absolute submission times, so inter-turn think time is the real human gap +- which model served each round +- whether a round ended by requesting a tool + +Reconstructed, because the corpus does not record it: + +- **KV-cache block IDs.** TraceLab carries no content hashes at all. AIPerf mints per-session virtual block IDs from the `prefix_tokens` / `newly_append_tokens` split, which reproduces the recorded prefix reuse to block granularity and degrades correctly when an agent compacts its context. Cross-session sharing is not recoverable: the corpus has no cross-session content identity. +- **Subagent parent/child links.** A subagent round is filed under its own top-level `session_id` with no parent reference of any kind. AIPerf recovers the link by timing containment: a session sharing the same user and project whose entire span falls inside a spawning tool call's window is taken to be that call's subagent, and the tightest enclosing window wins. This is a containment rate, not an accuracy: the corpus has no ground truth for the join. Sessions that match no window are replayed as independent traces rather than attached to a guessed parent. +- **`api_time`.** Derived from the span between a round's last input event and its last model-emitted event. The corpus records no server-reported latency and no TTFT, so this is a proxy with nothing to check it against. + +Not recoverable at all: the subagent type requested at spawn, which lived in tool-call arguments the corpus strips. + +--- + +## Block Size + +Because the block IDs are synthesized rather than recorded, the block size is a real knob here. It defaults to 64 tokens; set `--isl-block-size` to match the engine under test: + +```bash +aiperf profile ... --custom-dataset-type tracelab --isl-block-size 128 +``` + +Changing it changes the synthesized ID chain, and therefore the prefix-cache hit rate the replay presents to the server. + +--- + +## Subagent Join Controls + +The join runs by default. Three environment variables control it: + +| Variable | Default | Effect | +| --- | --- | --- | +| `AIPERF_DATASET_TRACELAB_SUBAGENT_JOIN` | `true` | Set `false` to replay every recorded session as an independent flat trace, which is the shape the corpus literally records | +| `AIPERF_DATASET_TRACELAB_CODEX_SUBAGENT_JOIN` | `true` | Set `false` to keep only the precise blocking-tool-call join | +| `AIPERF_DATASET_TRACELAB_MIN_SPAWN_MS` | `10000` | Minimum spawning-call wall latency treated as a subagent round-trip | + +The Codex control exists because the two providers are not equally joinable. Claude Code spawns a subagent through a single blocking tool call, so that call's window is exactly the child's lifetime. Codex uses an async `spawn_agent` / `wait_agent` / `close_agent` lifecycle whose handles live in the stripped tool arguments, so a spawn cannot be paired to its own wait and only a coarse session-level window is available. A Codex session that fans out several agents collapses them into one window. + +--- + +## Related Tutorials + +- [Replay Weka Agentic Coding Traces](weka-trace.md) +- [Profile with Bailian Traces](bailian-trace.md) +- [Trace Benchmarking](../benchmark-modes/trace-replay.md) diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 7318a88aad..ec00def6bd 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -375,6 +375,33 @@ class _DatasetSettings(BaseSettings): "dataset to a JSONL file. No hard cap.", ) + TRACELAB_SUBAGENT_JOIN: bool = Field( + default=True, + description="When True (default), TraceLabTraceDatasetLoader recovers " + "subagent parent/child links by timing containment and nests each " + "recovered child as a subagent entry inside its parent trace. Set to " + "False to emit every recorded session as an independent flat trace, " + "which is the shape the corpus literally records.", + ) + TRACELAB_CODEX_SUBAGENT_JOIN: bool = Field( + default=True, + description="When True (default), the TraceLab subagent join also " + "runs over codex sessions. Codex uses an async spawn/wait/close agent " + "lifecycle whose handles are stripped from the released corpus, so " + "only a coarse session-level window is available there and a session " + "fanning out several agents collapses them into one window. Set to " + "False to keep only the precise blocking-tool-call join.", + ) + TRACELAB_MIN_SPAWN_MS: int = Field( + ge=0, + default=10000, + description="Minimum wall latency, in milliseconds, for a spawning " + "tool call to be treated as a subagent round-trip by the TraceLab " + "join. Short calls are overwhelmingly no-op or error returns, and " + "admitting them widens the containment window enough to start " + "capturing unrelated concurrent sessions.", + ) + WEKA_PARALLEL_WORKERS: int = Field( ge=0, le=256, diff --git a/src/aiperf/common/utils.py b/src/aiperf/common/utils.py index 15dc65162d..0da86e8e35 100644 --- a/src/aiperf/common/utils.py +++ b/src/aiperf/common/utils.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import gzip import inspect import multiprocessing as mp import os @@ -8,7 +9,7 @@ import types from collections.abc import Awaitable, Callable, Iterator from contextlib import contextmanager -from typing import Any +from typing import Any, TextIO import orjson @@ -84,6 +85,19 @@ async def call_all_functions(funcs: list[Callable], *args, **kwargs) -> None: raise AIPerfMultiError("Errors calling functions", exceptions) +def open_text_maybe_gzip(path: str | os.PathLike[str]) -> TextIO: + """Open a text file, transparently decompressing gzip data. + + Detection is by magic-byte sniff (``\\x1f\\x8b``) rather than filename + suffix, so a gzipped file without a ``.gz`` extension is handled correctly. + """ + with open(path, "rb") as _probe: + _magic = _probe.read(2) + if _magic == b"\x1f\x8b": + return gzip.open(path, "rt", encoding="utf-8") + return open(path, encoding="utf-8") + + def load_json_str( json_str: str | bytes, func: Callable = lambda x: x ) -> dict[str, Any]: diff --git a/src/aiperf/config/dataset/config.py b/src/aiperf/config/dataset/config.py index 7263fcb10a..4dba0cd2ef 100644 --- a/src/aiperf/config/dataset/config.py +++ b/src/aiperf/config/dataset/config.py @@ -305,8 +305,9 @@ class FileDataset(BaseConfig): description="Path to file or directory containing benchmark dataset. " "Can be absolute or relative. Mutually exclusive with `records:`. " "Supported formats depend on the format field: " - "JSONL for single_turn/multi_turn, JSONL trace files for mooncake_trace/" - "bailian_trace, Parquet for baseten_trace, directories for random_pool.", + "JSONL for single_turn/multi_turn, JSONL (optionally gzipped) for " + "tracelab, JSONL trace files for mooncake_trace/bailian_trace, " + "Parquet for baseten_trace, directories for random_pool.", ), ] @@ -330,6 +331,7 @@ class FileDataset(BaseConfig): description="Dataset file format determining parsing logic and expected file structure. " "single_turn: JSONL with single prompt-response exchanges. " "multi_turn: JSONL with conversation history. " + "tracelab: TraceLab agentic-coding corpus, JSONL or gzipped JSONL. " "mooncake_trace / bailian_trace / baseten_trace / burst_gpt_trace: " "timestamped trace files for replay. " "sagemaker_data_capture: JSONL captured by SageMaker DataCapture. " @@ -496,9 +498,9 @@ class FileDataset(BaseConfig): default=None, ge=1, description="Hash-id block granularity for trace replay (--isl-block-size). " - "hash-id trace loaders (mooncake_trace, bailian_trace, burst_gpt_trace, " - "sagemaker_data_capture) decode each hash_id into a cached block of this " - "many tokens; total ISL = (num_hash_ids - 1) * block_size + final_block_size. " + "hash-id trace loaders (mooncake_trace, bailian_trace, baseten_trace, " + "tracelab) decode each hash_id into a cached block of this many tokens; " + "total ISL = (num_hash_ids - 1) * block_size + final_block_size. " "When unset, the loader's plugin-metadata default applies (e.g. 512 for " "mooncake_trace, 16 for bailian_trace). Not used by weka, which carries its " "own inline per-block sizes.", diff --git a/src/aiperf/config/dataset/resolver.py b/src/aiperf/config/dataset/resolver.py index 1c37287721..b3a500e309 100644 --- a/src/aiperf/config/dataset/resolver.py +++ b/src/aiperf/config/dataset/resolver.py @@ -10,6 +10,9 @@ from __future__ import annotations +import functools +import gzip +import zlib from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -71,6 +74,31 @@ class _DatasetResolution: is_forking: dict[str, bool] = field(default_factory=dict) +@functools.cache +def _implicit_timing_types() -> frozenset[object]: + """Formats whose loader always produces timing the generic probe can't see. + + * ``sagemaker_data_capture`` nests it under ``eventMetadata.inferenceTime``. + * ``burst_gpt_trace`` is CSV; the loader enforces a ``Timestamp`` column at + load time (``BurstGPTTraceDatasetLoader._REQUIRED_COLUMNS``). + * ``baseten_trace`` is Parquet. + * ``tracelab`` nests it under ``timing_events[]`` and may be gzipped; the + loader derives an absolute submission time for every round it keeps. + + Built lazily: ``CustomDatasetType`` members are registered from plugins.yaml. + """ + from aiperf.plugin.enums import CustomDatasetType + + return frozenset( + { + CustomDatasetType.SAGEMAKER_DATA_CAPTURE, + CustomDatasetType.BURST_GPT_TRACE, + CustomDatasetType.BASETEN_TRACE, + CustomDatasetType.TRACELAB, + } + ) + + class DatasetResolver: """Resolve file-based dataset paths, detect types, timing, and sampling.""" @@ -282,8 +310,10 @@ def _read_first_jsonl_record(file_path: str) -> dict | None: """ import orjson + from aiperf.common.utils import open_text_maybe_gzip + try: - with open(file_path, encoding="utf-8") as f: + with open_text_maybe_gzip(file_path) as f: for line in f: if line := line.strip(): try: @@ -291,7 +321,7 @@ def _read_first_jsonl_record(file_path: str) -> dict | None: except orjson.JSONDecodeError: return None return parsed if isinstance(parsed, dict) else None - except UnicodeDecodeError: + except (UnicodeDecodeError, EOFError, gzip.BadGzipFile, zlib.error): return None return None @@ -347,37 +377,33 @@ def _check_timing_data( """Check whether the first record carries timing information. Most trace formats expose ``timestamp`` or ``delay`` at the top level, - but sagemaker_data_capture nests its timing under - ``eventMetadata.inferenceTime``. We branch per-loader so fixed_schedule - validation accepts every format whose loader actually produces timing. + and the generic probe below reads it off the first record. The formats + in :func:`_implicit_timing_types` carry it somewhere that probe cannot + see, so they are accepted by type instead. """ - from aiperf.plugin.enums import CustomDatasetType - - if dataset_type == CustomDatasetType.SAGEMAKER_DATA_CAPTURE: - return True - if dataset_type == CustomDatasetType.BURST_GPT_TRACE: - # BurstGPT is CSV; the loader enforces a ``Timestamp`` column at - # load time (see ``BurstGPTTraceDatasetLoader._REQUIRED_COLUMNS``), - # so the dataset cannot load without timing. - return True - if dataset_type == CustomDatasetType.BASETEN_TRACE: + if dataset_type in _implicit_timing_types(): return True record = first_record if record is None: from pathlib import Path - from aiperf.common.utils import load_json_str + from aiperf.common.utils import load_json_str, open_text_maybe_gzip if Path(file_path).is_dir(): return False try: - with open(file_path) as f: + with open_text_maybe_gzip(file_path) as f: for line in f: if line := line.strip(): record = load_json_str(line) break - except (OSError, ValueError): + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + _logger.warning( + f"Truncated or corrupt gzip in dataset file '{file_path}': {e}" + ) + return False + except (OSError, UnicodeDecodeError, ValueError): return False if record is None: @@ -405,19 +431,27 @@ def _count_records_and_sessions( is_multi_turn = dataset_type in ( CustomDatasetType.MULTI_TURN, CustomDatasetType.BAILIAN_TRACE, + CustomDatasetType.TRACELAB, ) + from aiperf.common.utils import open_text_maybe_gzip + record_count = 0 session_ids: set[str] = set() try: - with open(file_path) as f: + with open_text_maybe_gzip(file_path) as f: for line in f: if not (line := line.strip()): continue record_count += 1 if is_multi_turn: _add_session_id(line, session_ids) - except OSError: + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + _logger.warning( + f"Truncated or corrupt gzip in dataset file '{file_path}': {e}" + ) + return 0, 0 + except (OSError, UnicodeDecodeError): return 0, 0 if is_multi_turn and session_ids: @@ -512,11 +546,11 @@ def _collect_dag_session_and_fork_ids(file_path: str) -> tuple[set[str], set[str ``pre_session_spawns``. Anything in ``referenced_ids`` is NOT a root and must not be sampled standalone. """ - from aiperf.common.utils import load_json_str + from aiperf.common.utils import load_json_str, open_text_maybe_gzip all_ids: set[str] = set() referenced_ids: set[str] = set() - with open(file_path) as f: + with open_text_maybe_gzip(file_path) as f: for raw in f: if not (line := raw.strip()): continue diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index a201640e81..ca38de7194 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -910,8 +910,8 @@ def _apply_max_context_length(d: dict[str, Any], cli: CLIConfig) -> None: { "mooncake_trace", "bailian_trace", - "burst_gpt_trace", - "sagemaker_data_capture", + "baseten_trace", + "tracelab", } ) @@ -920,8 +920,8 @@ def _apply_block_size(d: dict[str, Any], cli: CLIConfig) -> None: """Route ``--isl-block-size`` onto ``FileDataset.block_size`` for hash-id trace datasets. - block_size is fundamentally a TRACE field: the mooncake/bailian/burst_gpt/ - sagemaker loaders decode each ``hash_id`` into a cached block of this many + block_size is fundamentally a TRACE field: the mooncake/bailian/baseten/ + tracelab loaders decode each ``hash_id`` into a cached block of this many tokens (default 512 / 16 from plugin metadata). Synthetic datasets carry it on ``prompts.block_size`` (written by ``_build_prompts``, then stripped for FILE/PUBLIC by ``_apply_dataset_type``), so for FILE traces it must be @@ -967,9 +967,9 @@ def _apply_block_size(d: dict[str, Any], cli: CLIConfig) -> None: return raise ValueError( "--isl-block-size only applies to synthetic generation or hash-id trace " - "replay (mooncake_trace, bailian_trace, burst_gpt_trace, " - "sagemaker_data_capture). The selected dataset does not decode hash-id " - "token blocks; drop --isl-block-size." + "replay (mooncake_trace, bailian_trace, baseten_trace, tracelab). " + "The selected dataset does not decode hash-id token blocks; " + "drop --isl-block-size." ) diff --git a/src/aiperf/config/flags/_converter_profiling.py b/src/aiperf/config/flags/_converter_profiling.py index f407556674..d9b8f17556 100644 --- a/src/aiperf/config/flags/_converter_profiling.py +++ b/src/aiperf/config/flags/_converter_profiling.py @@ -5,8 +5,14 @@ from __future__ import annotations +import gzip +import zlib from typing import TYPE_CHECKING, Any +from aiperf.common.aiperf_logger import AIPerfLogger + +_logger = AIPerfLogger(__name__) + if TYPE_CHECKING: from pathlib import Path @@ -405,11 +411,21 @@ def _columnar_file_has_timestamp(path: Path) -> bool | None: return None +def _has_timing_events_timestamp(data: dict) -> bool: + events = data.get("timing_events") + return bool( + events + and isinstance(events, list) + and isinstance(events[0], dict) + and events[0].get("timestamp") is not None + ) + + def _first_record_has_timestamp(file_path: object) -> bool: """Return True when a trace file carries timestamp data.""" from pathlib import Path - from aiperf.common.utils import load_json_str + from aiperf.common.utils import load_json_str, open_text_maybe_gzip path = Path(file_path) if not path.is_file(): @@ -417,7 +433,7 @@ def _first_record_has_timestamp(file_path: object) -> bool: if (columnar_result := _columnar_file_has_timestamp(path)) is not None: return columnar_result try: - with open(path, encoding="utf-8") as f: + with open_text_maybe_gzip(path) as f: for line in f: if not (stripped := line.strip()): continue @@ -427,8 +443,13 @@ def _first_record_has_timestamp(file_path: object) -> bool: return False if not isinstance(data, dict): return False - return data.get("timestamp") is not None - except OSError: + return data.get( + "timestamp" + ) is not None or _has_timing_events_timestamp(data) + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + _logger.warning(f"Truncated or corrupt gzip in '{file_path}': {e}") + return False + except (OSError, UnicodeDecodeError): return False return False @@ -437,6 +458,8 @@ def _count_dataset_records(file_path: object) -> int: """Count records across JSONL, Parquet, or Arrow IPC input.""" from pathlib import Path + from aiperf.common.utils import open_text_maybe_gzip + path = Path(file_path) try: if path.is_dir(): @@ -463,8 +486,11 @@ def _count_dataset_records(file_path: object) -> int: return count_baseten_records(str(path)) if path.is_file(): - with open(path, encoding="utf-8") as f: + with open_text_maybe_gzip(path) as f: return sum(1 for line in f if line.strip()) + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + _logger.warning(f"Truncated or corrupt gzip in '{file_path}': {e}") + return 0 except (OSError, UnicodeDecodeError): return 0 return 0 diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 706f02bb9a..3a2baab304 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -590,6 +590,7 @@ def url(self) -> str: Field( description="Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. " "Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), " + "`tracelab` (TraceLab agentic-coding corpus, JSONL or gzipped JSONL), " "`mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; " "when using `random_pool`, `--conversation-num` defaults to 100 if not specified; " "batch sizes > 1 sample each modality independently from a flat pool and do not preserve " diff --git a/src/aiperf/dataset/composer/custom.py b/src/aiperf/dataset/composer/custom.py index 39bf93ec8f..2887b57bb2 100644 --- a/src/aiperf/dataset/composer/custom.py +++ b/src/aiperf/dataset/composer/custom.py @@ -10,7 +10,7 @@ from aiperf.common.enums import ConversationContextMode, DatasetFormat from aiperf.common.models import Conversation from aiperf.common.tokenizer import Tokenizer -from aiperf.common.utils import load_json_str +from aiperf.common.utils import load_json_str, open_text_maybe_gzip from aiperf.config.dataset import FileDataset from aiperf.dataset.composer.base import BaseDatasetComposer from aiperf.dataset.loader.base_loader import BaseLoader @@ -138,7 +138,7 @@ def _infer_dataset_type(self, file_path: str) -> CustomDatasetType: return self._infer_type(data=None, filename=file_path) # For files, read first non-empty line and use both content and path detection - with open(file_path, encoding="utf-8") as f: + with open_text_maybe_gzip(file_path) as f: try: for line in f: if not (line := line.strip()): diff --git a/src/aiperf/dataset/loader/_tracelab_convert.py b/src/aiperf/dataset/loader/_tracelab_convert.py new file mode 100644 index 0000000000..52fa06621d --- /dev/null +++ b/src/aiperf/dataset/loader/_tracelab_convert.py @@ -0,0 +1,702 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pure-python TraceLab -> Weka-trace conversion. + +TraceLab publishes one JSONL row per LLM round. The Weka trace schema wants one +object per *session*, carrying a per-request KV-cache block id list. This module +groups rounds into sessions, synthesizes those block ids from TraceLab's +engine-reported prefix decomposition, and reconstructs subagent nesting. + +Apart from one file-opening helper, nothing here imports aiperf: the output is +a plain ``dict`` matching the Weka trace schema, which +:mod:`aiperf.dataset.loader.tracelab_trace` validates into a ``WekaTrace``. Keeping the conversion free of framework types makes it directly +unit-testable and keeps the loader itself a thin adapter. +""" + +from __future__ import annotations + +import re +from collections import defaultdict +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +__all__ = [ + "DEFAULT_MIN_SPAWN_MS", + "HashIdMinter", + "JoinStats", + "Spawn", + "build_join_index", + "build_trace", + "group_children_by_parent", + "order_rounds", + "safe_trace_id", + "session_span", + "synthesize_hash_ids", +] + +INPUT_EVENTS = frozenset({"user_message", "tool_result"}) +OUTPUT_EVENTS = frozenset({"text", "reasoning", "tool_call"}) + +# Claude Code spawns a subagent through a single blocking tool call. Codex +# instead uses an async spawn_agent/wait_agent/close_agent lifecycle; see +# :func:`build_join_index`. +CLAUDE_SPAWN_TOOLS = frozenset({"Agent", "Task"}) + +# A subagent round-trip is long. Short spawning-tool calls are overwhelmingly +# no-op or error returns, and admitting them widens the containment window +# enough to start capturing unrelated concurrent sessions. +DEFAULT_MIN_SPAWN_MS = 10000 + +_SAFE = re.compile(r"[^A-Za-z0-9._-]+") + +# Recorded in the trace ``totals`` so a reconstructed trace names its origin. +SOURCE_TAG = "tracelab" + + +def parse_ts(value: str) -> float: + """ISO-8601 -> POSIX seconds. Naive timestamps are assumed UTC.""" + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.timestamp() + + +def round_timing(row: dict[str, Any]) -> tuple[float | None, float | None]: + """``(submission time, api_time)`` in POSIX seconds / seconds. + + Submission time is the LATEST input event in the round: for a round fed by + several parallel tool results, the request cannot have been sent before the + last one landed. ``api_time`` is the span from submission to the last + model-emitted event. + + TraceLab records no server-reported latency and no TTFT, so ``api_time`` is + a derived proxy with no ground truth to check it against, not a recorded + value. + """ + events = row.get("timing_events") or [] + ins = [ + parse_ts(e["timestamp"]) + for e in events + if e.get("event_type") in INPUT_EVENTS and e.get("timestamp") + ] + outs = [ + parse_ts(e["timestamp"]) + for e in events + if e.get("event_type") in OUTPUT_EVENTS and e.get("timestamp") + ] + if not ins and not outs: + return None, None + submitted = max(ins) if ins else min(outs) + if not outs: + return submitted, None + return submitted, max(max(outs) - submitted, 0.0) + + +def session_span(rows: list[dict[str, Any]]) -> tuple[float, float] | None: + """Wall-clock extent of a whole session, across round events and tool calls.""" + stamps: list[float] = [] + for row in rows: + for event in row.get("timing_events") or []: + if event.get("timestamp"): + stamps.append(parse_ts(event["timestamp"])) + for tool in row.get("tools") or []: + for key in ("emitted_at", "result_at"): + if tool.get(key): + stamps.append(parse_ts(tool[key])) + if not stamps: + return None + return min(stamps), max(stamps) + + +class HashIdMinter: + """Block id allocator. Ids are unique positive ints within one trace. + + ``hash_id_scope: "local"`` is one namespace per trace, NOT per conversation: + a subagent shares its parent trace's scope, so a block id reused across + parent and child decodes to the same tokens and reproduces the real + cross-agent shared prefix. One minter therefore spans a parent and every + subagent nested inside it. + """ + + __slots__ = ("_next",) + + def __init__(self) -> None: + self._next = 1 + + def take(self, count: int) -> list[int]: + if count <= 0: + return [] + ids = list(range(self._next, self._next + count)) + self._next += count + return ids + + +def synthesize_hash_ids( + rows: list[dict[str, Any]], block_size: int, minter: HashIdMinter +) -> list[list[int]]: + """Rebuild a per-round KV-cache block chain from TraceLab's prefix split. + + TraceLab records ``input_tokens_total = prefix_tokens + newly_append_tokens``, + an engine-reported split of how much of this round's input was already + resident. That becomes block ids by keeping the first + ``prefix_tokens // block_size`` ids from the previous round and minting + fresh ids for the rest of the input. + + ``hash_ids`` describes the request's ENTIRE input, not just its cached part, + so the list is FLOOR(``input_tokens_total`` / ``block_size``) long: only + whole blocks are hashed and ``input_length % block_size`` is carried as an + unhashed partial tail. A ceil-length list is silently tolerated and wrong + twice over, since the loader drops the trailing id as content while the + cache-hit metric still counts it, and the extra id perturbs agent-chain + detection. The cache hit is recovered downstream by taking the longest + common prefix against the preceding request. + + Degrades correctly through context compaction: when the agent compacts, the + engine reports a smaller ``prefix_tokens``, the reused span shrinks, and + fresh ids are minted. + + The minter is passed in rather than created here so that a parent and its + nested subagents share one id namespace (see :class:`HashIdMinter`). + """ + prev: list[int] = [] + out: list[list[int]] = [] + for row in rows: + total = max(int(row.get("input_tokens_total") or 0), 0) + prefix = max(int(row.get("prefix_tokens") or 0), 0) + # Never claim more prefix than the input actually holds. + prefix = min(prefix, total) + n_blocks = total // block_size + n_reuse = min(prefix // block_size, len(prev), n_blocks) + ids = prev[:n_reuse] + minter.take(n_blocks - n_reuse) + out.append(ids) + prev = ids + return out + + +def order_rounds( + rows: list[dict[str, Any]], +) -> list[tuple[float, float | None, dict[str, Any]]]: + """Rows -> ``[(submitted, api_time, row)]`` in replay order; undated dropped. + + Ordered by SUBMISSION TIME first, with TraceLab's own ``round_index`` as the + tie-break. Sorting by ``round_index`` alone is wrong in two ways that both + bite: + + * A spawn turn must strictly precede its join turn, but spawn position is + read by ARRAY INDEX while the join is placed by TIMESTAMP. Where the + corpus's ``round_index`` disagrees with its timestamps the join lands + ahead of the spawn, orchestrator validation raises, and the ENTIRE trace + is discarded rather than degrading. + * The KV-cache prefix chain evolves in wall-clock order, not in + ``round_index`` order, so time ordering is also the more faithful basis + for the hash-id reuse chain. + + Remaining ties fall back to file order, which python's stable sort + preserves. Duplicate ``(session_id, round_index)`` pairs exist in the + corpus and must not crash it. + """ + timed: list[tuple[float, float | None, dict[str, Any]]] = [] + for row in rows: + submitted, api = round_timing(row) + if submitted is None: + continue + timed.append((submitted, api, row)) + timed.sort(key=lambda item: (item[0], item[2].get("round_index") or 0)) + return timed + + +@dataclass(frozen=True, kw_only=True, slots=True) +class Spawn: + """One recovered parent -> child link. + + Attributes: + parent_sid: Session id of the spawning (parent) session. + child_sid: Session id of the spawned (child) session. + start: POSIX seconds (from :func:`parse_ts`) at which the spawning + tool call was emitted — the left edge of the containment window. + end: POSIX seconds at which the spawning tool call returned — the + right edge of the containment window. + duration_ms: Wall-clock duration of the spawn window in milliseconds + (``int((end - start) * 1000)``). + kind: ``"claude"`` for a blocking Agent/Task call, ``"codex"`` for the + async spawn_agent/wait_agent lifecycle. + """ + + parent_sid: str + child_sid: str + start: float + end: float + duration_ms: int + kind: str + + +class JoinStats: + """Counters describing one subagent-join pass, for logging.""" + + __slots__ = ( + "ambiguous", + "grandchildren", + "matched", + "matched_claude", + "matched_codex", + "windows", + "windows_claude", + "windows_codex", + "windows_matched", + ) + + def __init__(self) -> None: + self.windows = 0 + self.windows_claude = 0 + self.windows_codex = 0 + self.windows_matched = 0 + self.matched = 0 + self.matched_claude = 0 + self.matched_codex = 0 + self.ambiguous = 0 + self.grandchildren = 0 + + def summary(self) -> str: + barren = self.windows - self.windows_matched + pct = f" ({barren / self.windows * 100:.1f}%)" if self.windows else "" + return ( + f"{self.windows} spawn windows " + f"({self.windows_claude} claude, {self.windows_codex} codex), " + f"{self.matched} children matched " + f"({self.matched_claude} claude, {self.matched_codex} codex), " + f"{self.ambiguous} ambiguous (tightest window kept), " + f"{barren} windows matched no child{pct}, " + f"{self.grandchildren} grandchildren kept as standalone traces" + ) + + +def build_join_index( + sessions: dict[str, list[dict[str, Any]]], + min_spawn_ms: int = DEFAULT_MIN_SPAWN_MS, + enable_codex: bool = True, +) -> tuple[dict[str, Spawn], JoinStats]: + """Recover parent -> child subagent links. Returns ``(child_sid -> Spawn, stats)``. + + TraceLab carries NO explicit parent link: a subagent round is its own + top-level ``session_id``, ``session_id`` is a flat ``:``, + ``trace_key`` is a row id, and no cross-reference to another session appears + anywhere in the row. + + The link is recovered by TIMING CONTAINMENT. For a spawning tool call + lasting at least ``min_spawn_ms``, a different session with the same + ``user`` and ``project`` whose ENTIRE span falls inside + ``[emitted_at, result_at]`` is taken to be that call's subagent. Where a + child fits inside several candidate windows the TIGHTEST window wins, and + the ambiguity is counted. + + This is a CONTAINMENT rate, not an accuracy: the corpus has no ground truth + for the join, so the counters describe how often the rule fired, never + whether an individual match is right. + + Two provider lifecycles, and they are not equally strong: + + * Claude (``Agent`` / ``Task``) blocks for the whole subagent run, so the + tool call's own window IS the subagent's extent. This join is precise and + attributes a child to a specific call. + * Codex uses an async ``spawn_agent`` -> ``wait_agent`` -> ``close_agent`` + lifecycle. ``spawn_agent`` returns in milliseconds with a handle, and that + handle lived in the tool ARGUMENTS, which the released corpus strips. A + spawn cannot therefore be paired to its own wait, and the only usable + window is the session-level ``[earliest spawn_agent, latest wait_agent + result]``. That is COARSER: it cannot attribute a child to a specific + spawn, and a session that fans out several agents collapses them all into + one window. Counted separately, and separately disableable, for that + reason. + """ + span: dict[str, tuple[float, float]] = {} + ident: dict[str, tuple[Any, Any]] = {} + for sid, rows in sessions.items(): + extent = session_span(rows) + if extent is None: + continue + span[sid] = extent + ident[sid] = (rows[0].get("user"), rows[0].get("project")) + + # (start, end, parent_sid, duration_ms, kind) + windows: list[tuple[float, float, str, int, str]] = [] + for sid, rows in sessions.items(): + if rows[0].get("provider") == "codex": + if not enable_codex: + continue + window = _codex_window(rows) + if window is not None: + start, end = window + windows.append((start, end, sid, int((end - start) * 1000), "codex")) + continue + windows.extend(_claude_windows(sid, rows, min_spawn_ms)) + + by_ident: dict[tuple[Any, Any], list[tuple[float, float, str, int, str]]] = ( + defaultdict(list) + ) + for win in windows: + by_ident[ident.get(win[2], (None, None))].append(win) + + stats = JoinStats() + stats.windows = len(windows) + stats.windows_claude = sum(1 for w in windows if w[4] == "claude") + stats.windows_codex = sum(1 for w in windows if w[4] == "codex") + return _match_children(span, ident, by_ident, stats), stats + + +def _match_children( + span: dict[str, tuple[float, float]], + ident: dict[str, tuple[Any, Any]], + by_ident: dict[tuple[Any, Any], list[tuple[float, float, str, int, str]]], + stats: JoinStats, +) -> dict[str, Spawn]: + """Assign each session to the tightest spawn window that contains it.""" + links: dict[str, Spawn] = {} + hit_windows: set[tuple[float, float, str]] = set() + for csid, (cstart, cend) in span.items(): + hits = [ + (wend - wstart, wstart, wend, psid, dur, kind) + for (wstart, wend, psid, dur, kind) in by_ident.get(ident[csid], ()) + if psid != csid and cstart >= wstart and cend <= wend + ] + if not hits: + continue + hits.sort(key=lambda h: (h[0], h[1], h[3])) + if len(hits) > 1: + stats.ambiguous += 1 + _, wstart, wend, psid, dur, kind = hits[0] + links[csid] = Spawn( + parent_sid=psid, + child_sid=csid, + start=wstart, + end=wend, + duration_ms=dur, + kind=kind, + ) + hit_windows.add((wstart, wend, psid)) + stats.matched += 1 + if kind == "claude": + stats.matched_claude += 1 + else: + stats.matched_codex += 1 + stats.windows_matched = len(hit_windows) + return links + + +def _codex_window(rows: list[dict[str, Any]]) -> tuple[float, float] | None: + """Session-level ``[earliest spawn_agent, latest wait_agent result]``.""" + spawns: list[float] = [] + waits: list[float] = [] + for row in rows: + for tool in row.get("tools") or []: + name = tool.get("tool_name") + if name == "spawn_agent" and tool.get("emitted_at"): + spawns.append(parse_ts(tool["emitted_at"])) + elif name == "wait_agent" and tool.get("result_at"): + waits.append(parse_ts(tool["result_at"])) + if not spawns or not waits: + return None + start, end = min(spawns), max(waits) + return (start, end) if end > start else None + + +def _claude_windows( + sid: str, rows: list[dict[str, Any]], min_spawn_ms: int +) -> Iterator[tuple[float, float, str, int, str]]: + """One window per blocking spawning tool call of sufficient duration.""" + for row in rows: + for tool in row.get("tools") or []: + if tool.get("tool_name") not in CLAUDE_SPAWN_TOOLS: + continue + latency = tool.get("tool_wall_latency_ms") + if latency is None or latency < min_spawn_ms: + continue + if not tool.get("emitted_at") or not tool.get("result_at"): + continue + yield ( + parse_ts(tool["emitted_at"]), + parse_ts(tool["result_at"]), + sid, + int(latency), + "claude", + ) + + +def subagent_type_for(models: list[str]) -> str: + """Best available stand-in for the child's role. + + The real subagent type is NOT RECOVERABLE: it lived in the spawning tool + call's arguments, and the released corpus strips tool arguments. Nothing in + the row records which agent type was requested. + + ``WekaSubagentEntry.subagent_type`` is required by the schema but is not + read during reconstruction, so emitting the child's model costs nothing at + replay and is at least true, where inventing a role name would read as + something that had been recovered. + """ + return models[0] if models else "unknown" + + +def build_requests( + timed: list[tuple[float, float | None, dict[str, Any]]], + hash_chains: list[list[int]], + t0: float, +) -> tuple[list[dict[str, Any]], list[str]]: + """Rounds -> Weka ``type: "n"`` requests, timestamped relative to ``t0``.""" + requests: list[dict[str, Any]] = [] + models: list[str] = [] + prev_end: float | None = None + for (submitted, api, row), hash_ids in zip(timed, hash_chains, strict=True): + model = row.get("model") or "unknown" + if model not in models: + models.append(model) + + t = submitted - t0 + # think_time is the client-side idle gap before this request: the span + # from the previous request COMPLETING to this one being sent. For this + # corpus that gap is the human reading and typing, plus the local tool + # execution. + think = None if prev_end is None else max(t - prev_end, 0.0) + + out_tokens = int(row.get("output_tokens") or 0) + reasoning = row.get("reasoning_output_tokens") + if reasoning: + # There is no separate reasoning-token field in the trace schema; + # reasoning is part of what the server has to generate, so it folds + # into output_length. + out_tokens += int(reasoning) + + input_types = ( + ["tool_result"] + if row.get("first_input_event_type") == "tool_result" + else ["text"] + ) + # A round that emitted a tool_call ended because the model wanted a tool. + emitted_tool = any( + e.get("event_type") == "tool_call" for e in (row.get("timing_events") or []) + ) + + requests.append( + { + "t": round(t, 6), + "type": "n", + "model": model, + "in": max(int(row.get("input_tokens_total") or 0), 1), + "out": max(out_tokens, 1), + "hash_ids": hash_ids, + "input_types": input_types, + "output_types": ["text"], + "stop": "tool_use" if emitted_tool else "end_turn", + "api_time": None if api is None else round(api, 6), + "think_time": None if think is None else round(think, 6), + } + ) + prev_end = t + (api or 0.0) + return requests, models + + +def build_subagent_entry( + spawn: Spawn, + *, + child_rows: list[dict[str, Any]], + block_size: int, + minter: HashIdMinter, + t0: float, +) -> dict[str, Any] | None: + """One matched child session -> a ``type: "subagent"`` entry, or None.""" + timed = order_rounds(child_rows) + if not timed: + return None + + hash_chains = synthesize_hash_ids([item[2] for item in timed], block_size, minter) + inner, models = build_requests(timed, hash_chains, t0) + + entry_t = spawn.start - t0 + # An inner timestamp EARLIER than the spawn marker is read downstream as + # subagent-RELATIVE and rewritten to entry.t + req.t. That heuristic is + # per-request and silent, so one inner request slipping below entry_t would + # be flung far into the future while its siblings stayed put. + # + # The containment join makes that unreachable: a child only matches when + # its whole span falls inside the spawn window, and the span mins over + # timing events AND tool stamps while a request time is drawn from timing + # events alone, so every request is at or after entry_t already. This is a + # guard on that precondition, not a live transform. It stays because the + # failure it prevents is silent and whole-trace, while the check is one + # comparison per request, and any future join rule that is not + # containment-based would need it. + for req in inner: + if req["t"] < entry_t: + req["t"] = round(entry_t, 6) + # think_time on the child's first request would be a gap measured from + # nothing. + inner[0]["think_time"] = None + + child_end = max(item[0] + (item[1] or 0.0) for item in timed) + return { + "t": round(entry_t, 6), + "type": "subagent", + # Unique within the trace: duplicate agent_ids among retained subagents + # are rejected at load, and each child maps to exactly one parent. + "agent_id": safe_trace_id(spawn.child_sid), + "subagent_type": subagent_type_for(models), + "duration_ms": spawn.duration_ms, + "total_tokens": sum(r["in"] + r["out"] for r in inner), + "tool_use_count": sum(len(row.get("tools") or []) for row in child_rows), + "status": "completed" if child_end <= spawn.end else "incomplete", + "requests": inner, + "models": models, + "tool_tokens": 0, + "system_tokens": 0, + } + + +def build_trace( + session_id: str, + rows: list[dict[str, Any]], + block_size: int, + children: dict[str, tuple[Spawn, list[dict[str, Any]]]] | None = None, + *, + placed_sids: set[str] | None = None, +) -> dict[str, Any] | None: + """One TraceLab session (plus any recovered subagents) -> one trace dict.""" + timed = order_rounds(rows) + if not timed: + return None + + t0 = timed[0][0] + # One namespace for the parent and every nested child (hash_id_scope local). + minter = HashIdMinter() + hash_chains = synthesize_hash_ids([item[2] for item in timed], block_size, minter) + requests, models = build_requests(timed, hash_chains, t0) + + n_subagents = 0 + if children: + requests, n_subagents = _merge_subagent_entries( + requests=requests, + models=models, + children=children, + block_size=block_size, + minter=minter, + t0=t0, + placed_sids=placed_sids, + ) + + normals = [r for r in requests if r["type"] == "n"] + return { + "id": session_id, + "models": models, + "block_size": block_size, + "hash_id_scope": "local", + "tool_tokens": 0, + "system_tokens": 0, + "requests": requests, + "totals": { + "rounds": len(normals), + "subagents": n_subagents, + "input_tokens": sum(r["in"] for r in normals), + "output_tokens": sum(r["out"] for r in normals), + "source": SOURCE_TAG, + }, + } + + +def _merge_subagent_entries( + *, + requests: list[dict[str, Any]], + models: list[str], + children: dict[str, tuple[Spawn, list[dict[str, Any]]]], + block_size: int, + minter: HashIdMinter, + t0: float, + placed_sids: set[str] | None = None, +) -> tuple[list[dict[str, Any]], int]: + """Interleave subagent markers into a parent's request array. + + Each marker goes after the LAST parent request whose t is at or before the + spawn. Anchoring to the emitting round's array position instead looks more + direct but is wrong: spawn position is read by ARRAY INDEX while the join is + placed by TIMESTAMP, so any disagreement between the two makes the join land + ahead of the spawn, orchestrator validation raises, and the entire trace is + discarded rather than degrading. Ordering by time keeps the request array + monotonic in t, which is the condition that reconciles them. + + This satisfies the other placement rule for free: a subagent needs some + top-level request at a lower array index or it is silently dropped. There is + always at least one, because the spawning round was submitted before its own + tool call was emitted. + + ``models`` is extended in place with any model only the children used. + """ + req_times = [r["t"] for r in requests] + pending: dict[int, list[dict[str, Any]]] = defaultdict(list) + n_subagents = 0 + for csid, (spawn, child_rows) in children.items(): + entry = build_subagent_entry( + spawn, + child_rows=child_rows, + block_size=block_size, + minter=minter, + t0=t0, + ) + if entry is None: + continue + anchor = _anchor_index(req_times, entry["t"]) + if anchor is None: + continue + pending[anchor].append(entry) + if placed_sids is not None: + placed_sids.add(csid) + for model in entry["models"]: + if model not in models: + models.append(model) + n_subagents += 1 + + if not pending: + return requests, n_subagents + + merged: list[dict[str, Any]] = [] + for idx, req in enumerate(requests): + merged.append(req) + merged.extend(sorted(pending.get(idx, ()), key=lambda e: e["t"])) + return merged, n_subagents + + +def _anchor_index(req_times: list[float], entry_t: float) -> int | None: + """Index of the last request at or before ``entry_t``, or None.""" + anchor = None + for i, rt in enumerate(req_times): + if rt <= entry_t: + anchor = i + return anchor + + +def group_children_by_parent( + sessions: dict[str, list[dict[str, Any]]], + links: dict[str, Spawn], + stats: JoinStats, +) -> dict[str, dict[str, tuple[Spawn, list[dict[str, Any]]]]]: + """Bucket recovered children under their parents. + + The schema nests exactly ONE level: a subagent entry's inner requests are + plain calls and cannot themselves carry a subagent marker. So a child is + only nested when its parent is itself a root. A GRANDCHILD (whose parent is + someone else's child) is kept as its own independent trace rather than being + dropped, since losing it silently would discard exactly the deepest agentic + structure this pass exists to recover. + """ + out: dict[str, dict[str, tuple[Spawn, list[dict[str, Any]]]]] = defaultdict(dict) + for csid, spawn in links.items(): + if spawn.parent_sid not in sessions or csid not in sessions: + continue + if spawn.parent_sid in links: + stats.grandchildren += 1 + continue + out[spawn.parent_sid][csid] = (spawn, sessions[csid]) + return out + + +def safe_trace_id(session_id: str) -> str: + """Session id -> an identifier safe to use as a trace or agent id.""" + return _SAFE.sub("_", session_id)[:150] diff --git a/src/aiperf/dataset/loader/base_loader.py b/src/aiperf/dataset/loader/base_loader.py index fb58d03d8c..66c45648fc 100644 --- a/src/aiperf/dataset/loader/base_loader.py +++ b/src/aiperf/dataset/loader/base_loader.py @@ -14,6 +14,7 @@ from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.common.models import Conversation from aiperf.common.session_id_generator import SessionIDGenerator +from aiperf.common.utils import open_text_maybe_gzip from aiperf.dataset.loader.models import CustomDatasetT from aiperf.plugin.enums import DatasetSamplingStrategy @@ -163,7 +164,9 @@ def _iter_record_dicts( raises ``ValueError`` if absent). Unknown pool names raise ``KeyError``. For file mode: - - Yields ``orjson.loads`` of each non-empty stripped line. + - Yields ``orjson.loads`` of each non-empty stripped line. A ``.gz`` + suffix is transparently decompressed, so a gzipped JSONL corpus + can be replayed without inflating it first. - ``source`` (if provided) is treated as an alternate file path (used by directory-walking loaders such as ``random_pool``). """ @@ -180,7 +183,7 @@ def _iter_record_dicts( return target = source if source is not None else self.filename - with open(target, encoding="utf-8") as f: + with open_text_maybe_gzip(target) as f: for lineno, line in enumerate(f, start=1): if line := line.strip(): try: diff --git a/src/aiperf/dataset/loader/tracelab_trace.py b/src/aiperf/dataset/loader/tracelab_trace.py new file mode 100644 index 0000000000..332e69bde6 --- /dev/null +++ b/src/aiperf/dataset/loader/tracelab_trace.py @@ -0,0 +1,387 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""TraceLab agentic-coding trace loader. + +Reads the TraceLab (uw-syfi) corpus, which publishes one JSONL row per LLM +round, groups those rounds into sessions, and hands the result to +:class:`WekaTraceLoader` for reconstruction. File-based Weka replay and +TraceLab replay therefore share one reconstruction body: the same hash_id +replay, model mapping, branch / spawn-join linkage and delay capping. + +Two things this loader does that a plain format adapter does not, both because +the corpus does not record them: + +* **Block ids.** TraceLab has no content hashes of any kind. It does record an + engine-reported ``prefix_tokens`` / ``newly_append_tokens`` split of every + round's input, which is enough to mint per-session virtual block ids that + reproduce the recorded prefix-reuse to block granularity. See + :func:`~aiperf.dataset.loader._tracelab_convert.synthesize_hash_ids`. +* **Subagent nesting.** A subagent round appears in the corpus as its own + top-level ``session_id`` with no parent link of any kind. The link is + recovered by timing containment against the spawning tool call's window. See + :func:`~aiperf.dataset.loader._tracelab_convert.build_join_index`. + +Usage:: + + aiperf profile --custom-dataset-type tracelab \\ + --input-file syfi_coding_trace.jsonl.gz ... + +Conversion runs in memory; no intermediate files are written. +""" + +from __future__ import annotations + +import time +import zlib +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +from pydantic import ValidationError + +from aiperf.common.environment import Environment +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.common.models import Conversation +from aiperf.common.utils import open_text_maybe_gzip +from aiperf.dataset.generator.prompt import PromptGenerator +from aiperf.dataset.loader._tracelab_convert import ( + build_join_index, + build_trace, + group_children_by_parent, + safe_trace_id, +) +from aiperf.dataset.loader.base_loader import BaseFileLoader +from aiperf.dataset.loader.weka_trace import WekaTraceLoader +from aiperf.dataset.loader.weka_trace_models import WekaTrace +from aiperf.plugin.enums import DatasetSamplingStrategy + +if TYPE_CHECKING: + from aiperf.config.resolution.plan import BenchmarkRun + +# Historical default, and the value this loader declares in its plugin +# metadata. Kept as a named constant so the fallback and the metadata cannot +# drift apart silently. +DEFAULT_BLOCK_SIZE = 64 + +# Fields every TraceLab row carries. Used only for content auto-detection, so +# it is deliberately a small distinctive subset rather than the full schema: +# ``prefix_tokens`` + ``newly_append_tokens`` + ``timing_events`` together do +# not co-occur in any other format this repo reads. +_DETECT_KEYS = frozenset( + { + "session_id", + "round_index", + "input_tokens_total", + "prefix_tokens", + "newly_append_tokens", + "timing_events", + } +) + +_JSONL_SUFFIXES = (".jsonl", ".jsonl.gz", ".json.gz", ".gz") + + +class TraceLabTraceDatasetLoader(BaseFileLoader): + """Dataset loader for the TraceLab agentic-coding corpus. + + Parses TraceLab JSONL (optionally gzipped), builds :class:`WekaTrace` + objects in memory, and delegates conversation reconstruction to + :class:`WekaTraceLoader` so this path and file-based Weka replay cannot + diverge. + + Each recovered session produces: + + - one root :class:`Conversation` from its rounds, and + - one child :class:`Conversation` per recovered subagent, linked by + SPAWN + SPAWN_JOIN prerequisites on the parent's turns. + + Behaviour knobs live on the ``AIPERF_DATASET_TRACELAB_*`` environment + variables rather than the CLI, since they select between reconstruction + hypotheses rather than benchmark parameters. + + Raises: + DatasetLoaderError: unreadable input, or a converted session that does + not satisfy the trace schema. + """ + + tag: ClassVar[str] = "TraceLabTrace" + + def __init__( + self, + *, + filename: str | Path | None = None, + run: BenchmarkRun | None = None, + prompt_generator: PromptGenerator | None = None, + default_block_size: int | None = None, + **kwargs: Any, + ) -> None: + super().__init__(filename=filename, run=run, **kwargs) + self.prompt_generator = prompt_generator + + dataset = self.run.cfg.get_default_dataset() + # --isl-block-size lands on the flat FileDataset.block_size (routed by + # the config converter for hash-id trace formats); a synthetic-via-file + # dataset carries it on prompts.block_size. Plugin metadata is the + # fallback, and DEFAULT_BLOCK_SIZE the floor. Unlike Weka -- whose + # traces declare their own per-file block size, so an override would be + # meaningless -- TraceLab has no recorded block size at all: the value + # chosen here is what the block ids are synthesized AT, so it is a real + # knob and the user's value must win. + prompts = getattr(dataset, "prompts", None) + block_size = getattr(dataset, "block_size", None) + if block_size is None and prompts is not None: + block_size = getattr(prompts, "block_size", None) + if block_size is None: + block_size = default_block_size + self._block_size = ( + int(block_size) if block_size is not None else DEFAULT_BLOCK_SIZE + ) + if self._block_size <= 0: + raise DatasetLoaderError( + f"TraceLab block size must be positive, got {self._block_size}." + ) + + self._join_subagents = Environment.DATASET.TRACELAB_SUBAGENT_JOIN + self._join_codex = Environment.DATASET.TRACELAB_CODEX_SUBAGENT_JOIN + self._min_spawn_ms = Environment.DATASET.TRACELAB_MIN_SPAWN_MS + + # The delegate is constructed in HF-delegation mode (filename=None): it + # never touches a file, it only reconstructs the traces handed to it. + # Selection (--num-dataset-entries / --max-context-length) is therefore + # this loader's job, exactly as it is the HF loader's. + self._weka = WekaTraceLoader( + filename=None, + run=self.run, + prompt_generator=prompt_generator, + default_block_size=self._block_size, + ) + + @classmethod + def can_load( + cls, + data: dict[str, Any] | None = None, + filename: str | Path | None = None, + ) -> bool: + """Return True when the source looks like TraceLab JSONL. + + Auto-detection is content-based off the first record when the caller + already parsed one, and single-read otherwise. Directories are rejected: + TraceLab is distributed as a single JSONL file. + """ + if data is not None: + return _DETECT_KEYS.issubset(data.keys()) + if filename is None: + return False + path = Path(filename) + try: + if path.is_dir() or not path.is_file(): + return False + if not str(path).endswith(_JSONL_SUFFIXES): + return False + first = cls._first_record(path) + except Exception: + return False + return first is not None and _DETECT_KEYS.issubset(first.keys()) + + @staticmethod + def _first_record(path: Path) -> dict[str, Any] | None: + from aiperf.common.utils import load_json_str + + with open_text_maybe_gzip(path) as handle: + for line in handle: + if line := line.strip(): + record = load_json_str(line) + return record if isinstance(record, dict) else None + return None + + def load_dataset(self) -> dict[str, list[WekaTrace]]: + """Read the corpus, group rounds into sessions, return ``{id: [trace]}``. + + Overrides :meth:`BaseFileLoader._iter_record_dicts` indirectly by not + using it: that helper opens the target in text mode, and the released + corpus ships gzipped. + """ + if self.filename is None: + raise DatasetLoaderError( + "TraceLab replay requires --input-file pointing at a TraceLab " + "JSONL file (optionally gzipped)." + ) + t0 = time.monotonic() + sessions = self._read_sessions(self.filename) + if not sessions: + raise DatasetLoaderError( + f"No TraceLab sessions found in '{self.filename}'. Every row " + "must carry a 'session_id'." + ) + + children_by_parent: dict[str, dict[str, tuple[Any, list[dict[str, Any]]]]] = ( + defaultdict(dict) + ) + if self._join_subagents: + links, stats = build_join_index( + sessions, + min_spawn_ms=self._min_spawn_ms, + enable_codex=self._join_codex, + ) + children_by_parent = group_children_by_parent(sessions, links, stats) + self.info(f"TraceLab subagent join: {stats.summary()}") + + nested_ids = {csid for kids in children_by_parent.values() for csid in kids} + traces = self._build_traces(sessions, children_by_parent, nested_ids) + data = self._select(traces) + self.info( + f"TraceLab: {len(sessions)} sessions -> {len(data)} traces " + f"(block_size={self._block_size}) in {time.monotonic() - t0:.1f}s" + ) + return data + + def _read_sessions(self, path: Path) -> dict[str, list[dict[str, Any]]]: + """Group every row of the corpus by ``session_id``, preserving file order.""" + from aiperf.common.utils import load_json_str + + sessions: dict[str, list[dict[str, Any]]] = defaultdict(list) + try: + with open_text_maybe_gzip(path) as handle: + for lineno, line in enumerate(handle, start=1): + if not (line := line.strip()): + continue + try: + row = load_json_str(line) + except ValueError as e: + raise DatasetLoaderError( + f"Invalid JSON in TraceLab file {path} at line {lineno}: {e}" + ) from None + if isinstance(row, dict) and (sid := row.get("session_id")): + sessions[sid].append(row) + except (OSError, UnicodeDecodeError, EOFError, zlib.error) as e: + raise DatasetLoaderError(f"Cannot read TraceLab file {path}: {e}") from e + return dict(sessions) + + def _build_traces( + self, + sessions: dict[str, list[dict[str, Any]]], + children_by_parent: dict[str, dict[str, tuple[Any, list[dict[str, Any]]]]], + nested_ids: set[str], + ) -> dict[str, WekaTrace]: + """Convert each root session into a validated :class:`WekaTrace`.""" + traces: dict[str, WekaTrace] = {} + actually_nested: set[str] = set() + + for sid, rows in sessions.items(): + if sid in nested_ids: + continue + try: + blob = build_trace( + sid, + rows, + self._block_size, + children_by_parent.get(sid), + placed_sids=actually_nested, + ) + except (ValueError, TypeError, AttributeError) as e: + raise DatasetLoaderError( + f"Failed to convert TraceLab session '{sid}': {e}" + ) from e + if blob is None or not blob["requests"]: + continue + self._add_trace(traces, blob, sid) + + fallback = nested_ids - actually_nested + if fallback: + self.warning( + f"TraceLab: {len(fallback)} subagent session(s) failed to nest " + "and will be emitted as standalone traces." + ) + for csid in sessions: + if csid not in fallback: + continue + try: + blob = build_trace(csid, sessions[csid], self._block_size) + except (ValueError, TypeError, AttributeError) as e: + raise DatasetLoaderError( + f"Failed to convert TraceLab session '{csid}': {e}" + ) from e + if blob is None or not blob["requests"]: + continue + self._add_trace(traces, blob, csid) + + return traces + + def _add_trace( + self, traces: dict[str, WekaTrace], blob: dict[str, Any], sid: str + ) -> None: + """Validate and register one converted trace blob.""" + trace_id = safe_trace_id(sid) + blob["id"] = trace_id + if trace_id in traces: + raise DatasetLoaderError( + f"Duplicate TraceLab trace id '{trace_id}': session " + f"'{sid}' collides with an earlier session." + ) + try: + traces[trace_id] = WekaTrace.model_validate(blob) + except ValidationError as e: + raise DatasetLoaderError( + f"Converted TraceLab session '{sid}' does not satisfy the " + f"trace schema: {e}" + ) from e + + def _select(self, traces: dict[str, WekaTrace]) -> dict[str, list[WekaTrace]]: + """Apply --max-context-length filtering then --num-dataset-entries cap. + + The delegate skips its own selection in delegation mode, so it happens + here. Filtering strictly precedes the cap: capping the raw prefix first + would silently shrink the eligible pool. + """ + from aiperf.dataset.loader.selection import ( + filter_then_cap, + log_selection_summary, + ) + from aiperf.dataset.loader.weka_trace import _trace_peak_context_length + + dataset = self.run.cfg.get_default_dataset() + entries = getattr(dataset, "entries", None) + explicit = bool(getattr(dataset, "entries_explicit", False)) or ( + "entries" in dataset.model_fields_set and entries is not None + ) + num_entries = entries if explicit else None + max_ctx = getattr(dataset, "max_context_length", None) + synthesis = getattr(dataset, "synthesis", None) + max_osl = getattr(synthesis, "max_osl", None) if synthesis else None + + if num_entries is None and max_ctx is None: + return {tid: [trace] for tid, trace in traces.items()} + + candidates = ( + ((tid, trace), _trace_peak_context_length(trace, max_osl=max_osl)) + for tid, trace in traces.items() + ) + kept, stats = filter_then_cap( + candidates, + num_dataset_entries=num_entries, + max_context_length=max_ctx, + ) + log_selection_summary( + stats, + source=str(self.filename), + num_dataset_entries=num_entries, + max_context_length=max_ctx, + ) + if not kept: + raise DatasetLoaderError( + f"No eligible TraceLab traces in '{self.filename}' after " + f"filter-then-cap (scanned {stats.scanned}, " + f"--max-context-length={max_ctx}, " + f"--num-dataset-entries={num_entries})." + ) + return {tid: [trace] for tid, trace in kept} + + def convert_to_conversations( + self, data: dict[str, list[WekaTrace]] + ) -> list[Conversation]: + """Delegate reconstruction to the Weka loader (same code path).""" + return self._weka.convert_to_conversations(data) + + @classmethod + def get_preferred_sampling_strategy(cls) -> DatasetSamplingStrategy: + return DatasetSamplingStrategy.SEQUENTIAL diff --git a/src/aiperf/dataset/mmap_cache.py b/src/aiperf/dataset/mmap_cache.py index 7e7b992487..0ee0286b74 100644 --- a/src/aiperf/dataset/mmap_cache.py +++ b/src/aiperf/dataset/mmap_cache.py @@ -200,6 +200,7 @@ "raw_payload", "inputs_json", "weka_trace", + "tracelab", } ) @@ -792,6 +793,7 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: "use_legacy_max_tokens": cfg.endpoint.use_legacy_max_tokens, "use_server_token_count": cfg.endpoint.use_server_token_count, "extra": cfg.endpoint.extra, + "force_content_parts": Environment.ENDPOINT.FORCE_CONTENT_PARTS, } if Environment.DATASET.PREFORMAT_PAYLOADS else None @@ -799,7 +801,7 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: "inline_records_sha256": records_hash, "prompt": prompt_dump, "endpoint_type": str(cfg.endpoint.type), - "model_name": model_names[0] if model_names else "", + "model_names": model_names, "fixed_schedule_start_offset": start_offset, "fixed_schedule_end_offset": end_offset, # Load-time timing knobs bake into the cached Turn timestamps/delays @@ -832,10 +834,24 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: "weka_seam_min_overlap_ratio": ( Environment.DATASET.WEKA_SEAM_MIN_OVERLAP_RATIO ), + # --isl-block-size lands here for the hash-id trace formats. It sets + # the token width every hash_id decodes to, so it rewrites the cached + # prompt bytes outright; without it in the key, changing the flag + # serves the previous run's mmap unchanged. + "block_size": getattr(dataset, "block_size", None), + # TraceLab synthesizes its block ids and recovers its subagent nesting + # at load time, so these select between reconstructions that are baked + # into the cached conversations. + "tracelab_subagent_join": Environment.DATASET.TRACELAB_SUBAGENT_JOIN, + "tracelab_codex_subagent_join": ( + Environment.DATASET.TRACELAB_CODEX_SUBAGENT_JOIN + ), + "tracelab_min_spawn_ms": Environment.DATASET.TRACELAB_MIN_SPAWN_MS, # Full synthesis dump: max_isl/max_osl caps AND the speedup_ratio / # *_multiplier transforms, all of which rewrite the decoded trace bytes. "synthesis": synthesis_dump, "max_context_length": getattr(dataset, "max_context_length", None), + "entries_explicit": getattr(dataset, "entries_explicit", False), } diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index c0fe53bb26..bf07c81a16 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -846,6 +846,23 @@ custom_dataset_loader: metadata: category: high_entropy + tracelab: + class: aiperf.dataset.loader.tracelab_trace:TraceLabTraceDatasetLoader + description: | + TraceLab agentic coding trace loader for the uw-syfi corpus published at + https://github.com/uw-syfi/TraceLab. Input is a single JSONL file, gzipped + or plain, holding one row per LLM round. Rounds are grouped into sessions, + per-request hash_ids are synthesized from the engine-reported prefix / + append token split, and subagent parent-child links are recovered by + timing containment before reconstruction is delegated to the Weka trace + loader. Subagent entries spawn concurrent child sessions via SPAWN + + SPAWN_JOIN prerequisites. + Usage: --custom-dataset-type tracelab --input-file syfi_coding_trace.jsonl.gz + metadata: + is_trace: true + default_block_size: 64 + default_prompt_corpus: coding + weka_trace: class: aiperf.dataset.loader.weka_trace:WekaTraceLoader description: | diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index f4fd8b1121..58a71c4ab4 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -1,8 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import gzip import multiprocessing as mp import threading +from pathlib import Path import orjson import pytest @@ -17,6 +19,84 @@ ) +def _write_gz_jsonl(path: Path, rows: list[dict]) -> Path: + with gzip.open(path, "wb") as f: + for r in rows: + f.write(orjson.dumps(r) + b"\n") + return path + + +def _write_plain_jsonl(path: Path, rows: list[dict]) -> Path: + path.write_bytes(b"".join(orjson.dumps(r) + b"\n" for r in rows)) + return path + + +class TestOpenTextMaybeGzip: + """``open_text_maybe_gzip`` detects gzip by magic bytes, not suffix.""" + + def test_plain_and_gz_produce_same_content(self, tmp_path: Path) -> None: + from aiperf.common.utils import open_text_maybe_gzip + + row = {"session_id": "s1", "data": "hello"} + plain = _write_plain_jsonl(tmp_path / "a.jsonl", [row]) + gz = _write_gz_jsonl(tmp_path / "a.jsonl.gz", [row]) + with open_text_maybe_gzip(plain) as f: + plain_text = f.read() + with open_text_maybe_gzip(gz) as f: + gz_text = f.read() + assert plain_text == gz_text + + def test_magic_byte_sniff_detects_gz_without_extension( + self, tmp_path: Path + ) -> None: + from aiperf.common.utils import open_text_maybe_gzip + + row = {"k": "v"} + no_ext = tmp_path / "data" + with gzip.open(no_ext, "wb") as f: + f.write(orjson.dumps(row) + b"\n") + with open_text_maybe_gzip(no_ext) as f: + assert orjson.loads(f.readline()) == row + + def test_record_counting_handles_gzipped_tracelab(self, tmp_path: Path) -> None: + """Two rows with the same session_id count as 1 session.""" + from aiperf.config.dataset.resolver import DatasetResolver + from aiperf.plugin.enums import CustomDatasetType + + rows = [ + {"session_id": "s1", "round_index": 0}, + {"session_id": "s1", "round_index": 1}, + ] + gz = _write_gz_jsonl(tmp_path / "a.jsonl.gz", rows) + count, sessions = DatasetResolver._count_records_and_sessions( + str(gz), CustomDatasetType.TRACELAB + ) + assert count == 2 + assert sessions == 1 + + def test_type_inference_reads_first_gzipped_record(self, tmp_path: Path) -> None: + from aiperf.config.dataset.resolver import DatasetResolver + + rows = [{"session_id": "s1", "prefix_tokens": 0}] + gz = _write_gz_jsonl(tmp_path / "a.jsonl.gz", rows) + assert DatasetResolver._read_first_jsonl_record(str(gz)) is not None + + def test_count_records_returns_zero_on_truncated_gzip(self, tmp_path: Path) -> None: + """A truncated gzip triggers the EOFError/BadGzipFile handler and returns (0, 0).""" + + from aiperf.config.dataset.resolver import DatasetResolver + + gz = tmp_path / "trunc.jsonl.gz" + rows = [{"session_id": "s1"}] * 50 + _write_gz_jsonl(gz, rows) + # Chop the last 30 bytes to corrupt the gzip stream. + data = gz.read_bytes() + gz.write_bytes(data[: max(len(data) - 30, 2)]) + count, sessions = DatasetResolver._count_records_and_sessions(str(gz), None) + assert count == 0 + assert sessions == 0 + + class TestAllowDaemonChildren: """``allow_daemon_children`` clears the current process's daemon flag so daemon services (every AIPerf service is spawned ``daemon=True``) can fan diff --git a/tests/unit/config/test_baseten_replay_flags.py b/tests/unit/config/test_baseten_replay_flags.py index f9514b4996..da13d2cb47 100644 --- a/tests/unit/config/test_baseten_replay_flags.py +++ b/tests/unit/config/test_baseten_replay_flags.py @@ -106,6 +106,87 @@ def test_arrow_ipc_profiling_metadata(tmp_path: Path, suffix: str) -> None: assert _count_dataset_records(path) == 1 +def _truncated_gz(tmp_path: Path) -> Path: + import gzip + import json + + gz = tmp_path / "trunc.jsonl.gz" + with gzip.open(gz, "wt", encoding="utf-8") as f: + for _ in range(50): + f.write(json.dumps({"timestamp": 0, "input_length": 4}) + "\n") + # Keep only the gzip magic bytes — the first record is guaranteed cut off. + gz.write_bytes(gz.read_bytes()[:20]) + return gz + + +def test_first_record_has_timestamp_returns_false_on_truncated_gz( + tmp_path: Path, +) -> None: + """A truncated gzip must degrade to False, not raise EOFError.""" + assert _first_record_has_timestamp(_truncated_gz(tmp_path)) is False + + +def test_count_dataset_records_returns_zero_on_truncated_gz(tmp_path: Path) -> None: + """A truncated gzip must degrade to 0, not raise EOFError.""" + assert _count_dataset_records(_truncated_gz(tmp_path)) == 0 + + +def _crc_corrupt_gz(tmp_path: Path) -> Path: + # Valid 10-byte gzip header + garbage deflate payload → zlib.error on any read. + gz = tmp_path / "crc_corrupt.jsonl.gz" + gz.write_bytes( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03" + b"\xff" * 20 + b"\x00" * 8 + ) + return gz + + +def _corrupt_gz_header(tmp_path: Path) -> Path: + # gzip magic + method byte 0x09 (not 8=deflate) → gzip.BadGzipFile. + # BadGzipFile is an OSError subclass; if the except clauses are in the + # wrong order the OSError arm wins silently with no log line. + gz = tmp_path / "corrupt.jsonl.gz" + gz.write_bytes(b"\x1f\x8b\x09\x00\x00\x00\x00\x00\x00\x00") + return gz + + +def test_first_record_has_timestamp_logs_on_corrupt_gz_header( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """BadGzipFile (corrupt header) must be caught by the logged branch.""" + import logging + + with caplog.at_level(logging.WARNING): + result = _first_record_has_timestamp(_corrupt_gz_header(tmp_path)) + assert result is False + assert any("corrupt" in r.message.lower() for r in caplog.records) + + +def test_count_dataset_records_logs_on_corrupt_gz_header( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """BadGzipFile (corrupt header) must be caught by the logged branch.""" + import logging + + with caplog.at_level(logging.WARNING): + result = _count_dataset_records(_corrupt_gz_header(tmp_path)) + assert result == 0 + assert any("corrupt" in r.message.lower() for r in caplog.records) + + +def test_first_record_has_timestamp_returns_false_on_crc_corrupt_gz( + tmp_path: Path, +) -> None: + """zlib.error from CRC corruption must degrade to False, not propagate.""" + assert _first_record_has_timestamp(_crc_corrupt_gz(tmp_path)) is False + + +def test_count_dataset_records_returns_zero_on_crc_corrupt_gz( + tmp_path: Path, +) -> None: + """zlib.error from CRC corruption must degrade to 0, not propagate.""" + assert _count_dataset_records(_crc_corrupt_gz(tmp_path)) == 0 + + def _baseten_argv(trace_parquet: Path, *extra: str) -> list[str]: return [ "--url", diff --git a/tests/unit/config/test_block_size_routing.py b/tests/unit/config/test_block_size_routing.py index 1b841f97cd..199f0347a7 100644 --- a/tests/unit/config/test_block_size_routing.py +++ b/tests/unit/config/test_block_size_routing.py @@ -26,7 +26,7 @@ def trace_file(tmp_path: Path) -> Path: @pytest.mark.parametrize( "fmt", - ["mooncake_trace", "bailian_trace", "burst_gpt_trace", "sagemaker_data_capture"], + ["mooncake_trace", "bailian_trace", "baseten_trace", "tracelab"], ) def test_block_size_routed_onto_filedataset_for_hash_id_traces( trace_file: Path, fmt: str @@ -45,6 +45,23 @@ def test_block_size_routed_onto_filedataset_for_hash_id_traces( assert "prompts" not in out +@pytest.mark.parametrize( + "fmt", + ["burst_gpt_trace", "sagemaker_data_capture"], +) +def test_block_size_rejected_for_non_hash_id_traces(trace_file: Path, fmt: str) -> None: + with pytest.raises(ValueError, match="--isl-block-size"): + build_dataset( + CLIConfig( + model_names=["m"], + input_file=str(trace_file), + custom_dataset_type=fmt, + prompt_input_tokens_block_size=256, + prompt_output_tokens_mean=64, + ) + ) + + def test_block_size_routed_onto_autodetected_file_trace(trace_file: Path) -> None: # No --custom-dataset-type: the loader is content-auto-detected, so format # is None at convert time. --isl-block-size must still route onto diff --git a/tests/unit/config/test_resolver_edge_cases.py b/tests/unit/config/test_resolver_edge_cases.py index b74c5bf88e..c04c571a46 100644 --- a/tests/unit/config/test_resolver_edge_cases.py +++ b/tests/unit/config/test_resolver_edge_cases.py @@ -267,6 +267,14 @@ def test_result_stored_in_resolved(self, tmp_path) -> None: assert run.resolved.tokenizer_names == expected +def _make_crc_corrupt_gz(path: Path, row: dict) -> Path: + # Valid 10-byte gzip header + garbage deflate payload → zlib.error on any read. + path.write_bytes( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03" + b"\xff" * 20 + b"\x00" * 8 + ) + return path + + # ============================================================ # DatasetResolver Edge Cases # ============================================================ @@ -456,6 +464,61 @@ def test_read_first_jsonl_record_returns_none_for_empty_file( assert DatasetResolver._read_first_jsonl_record(str(empty_file)) is None + def test_read_first_jsonl_record_returns_none_on_truncated_gz( + self, tmp_path + ) -> None: + """A truncated gzip must degrade to None, not raise EOFError.""" + import gzip + import json + + gz = tmp_path / "trunc.jsonl.gz" + with gzip.open(gz, "wt", encoding="utf-8") as f: + for _ in range(50): + f.write(json.dumps({"session_id": "s1"}) + "\n") + gz.write_bytes(gz.read_bytes()[:20]) + assert DatasetResolver._read_first_jsonl_record(str(gz)) is None + + def test_check_timing_data_returns_false_on_truncated_gz(self, tmp_path) -> None: + """A truncated gzip must degrade to False, not raise EOFError.""" + import gzip + import json + + gz = tmp_path / "trunc.jsonl.gz" + with gzip.open(gz, "wt", encoding="utf-8") as f: + for _ in range(50): + f.write(json.dumps({"timestamp": 0, "input_length": 4}) + "\n") + gz.write_bytes(gz.read_bytes()[:20]) + assert DatasetResolver._check_timing_data(str(gz), None) is False + + def test_read_first_jsonl_record_returns_none_on_crc_corrupt_gz( + self, tmp_path + ) -> None: + """zlib.error from CRC payload corruption must degrade to None.""" + gz = _make_crc_corrupt_gz(tmp_path / "crc.jsonl.gz", {"session_id": "s1"}) + assert DatasetResolver._read_first_jsonl_record(str(gz)) is None + + def test_check_timing_data_returns_false_on_crc_corrupt_gz(self, tmp_path) -> None: + """zlib.error from CRC payload corruption must degrade to False.""" + gz = _make_crc_corrupt_gz(tmp_path / "crc.jsonl.gz", {"timestamp": 0}) + assert DatasetResolver._check_timing_data(str(gz), None) is False + + def test_check_timing_data_returns_false_on_invalid_json_first_line( + self, tmp_path + ) -> None: + """orjson.JSONDecodeError (ValueError subclass) from a malformed first line + must degrade to False — the original handler caught ValueError and the + narrowed refactor must not silently drop that protection.""" + bad = tmp_path / "bad.jsonl" + bad.write_text("{not json}\n", encoding="utf-8") + assert DatasetResolver._check_timing_data(str(bad), None) is False + + def test_count_records_returns_zero_on_crc_corrupt_gz(self, tmp_path) -> None: + """zlib.error from CRC payload corruption must degrade to (0, 0).""" + gz = _make_crc_corrupt_gz(tmp_path / "crc.jsonl.gz", {"session_id": "s1"}) + count, sessions = DatasetResolver._count_records_and_sessions(str(gz), None) + assert count == 0 + assert sessions == 0 + def test_detect_type_returns_none_none_when_file_unreadable(self, tmp_path) -> None: """OSError on file open bails the whole detection with (None, None). diff --git a/tests/unit/dataset/loader/test_tracelab_trace.py b/tests/unit/dataset/loader/test_tracelab_trace.py new file mode 100644 index 0000000000..6a23e86b7e --- /dev/null +++ b/tests/unit/dataset/loader/test_tracelab_trace.py @@ -0,0 +1,1251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the TraceLab trace loader and its conversion layer.""" + +import gzip +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from pytest import param + +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.dataset.loader._tracelab_convert import ( + HashIdMinter, + build_join_index, + build_trace, + group_children_by_parent, + order_rounds, + round_timing, + safe_trace_id, + session_span, + synthesize_hash_ids, +) +from aiperf.dataset.loader.tracelab_trace import ( + DEFAULT_BLOCK_SIZE, + TraceLabTraceDatasetLoader, +) +from aiperf.dataset.loader.weka_trace_models import WekaTrace + +T0 = datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC) + + +def iso(offset_s: float) -> str: + """Seconds after the fixture epoch as a TraceLab-style ISO-8601 stamp.""" + stamp = (T0 + timedelta(seconds=offset_s)).isoformat() + return stamp.replace("+00:00", "Z") + + +def make_row( + *, + session_id: str = "claude:s1", + round_index: int = 0, + submitted: float = 0.0, + responded: float | None = 1.0, + input_tokens: int = 128, + prefix_tokens: int = 0, + output_tokens: int = 10, + reasoning_tokens: int | None = None, + model: str = "claude-opus-4-7", + provider: str = "claude", + user: str = "user_a", + project: str = "project_a", + first_input_event_type: str = "user_message", + emits_tool_call: bool = False, + tools: list[dict] | None = None, + timing_events: list[dict] | None = None, +) -> dict: + """One TraceLab JSONL row, with only the fields the loader reads.""" + if timing_events is None: + timing_events = [ + {"event_type": first_input_event_type, "timestamp": iso(submitted)} + ] + if responded is not None: + timing_events.append( + { + "event_type": "tool_call" if emits_tool_call else "text", + "timestamp": iso(responded), + } + ) + return { + "provider": provider, + "project": project, + "user": user, + "session_id": session_id, + "round_index": round_index, + "model": model, + "input_tokens_total": input_tokens, + "prefix_tokens": prefix_tokens, + "newly_append_tokens": max(input_tokens - prefix_tokens, 0), + "output_tokens": output_tokens, + "reasoning_output_tokens": reasoning_tokens, + "first_input_event_type": first_input_event_type, + "timing_events": timing_events, + "tools": tools or [], + } + + +def spawn_tool(*, emitted: float, result: float, name: str = "Agent") -> dict: + return { + "tool_name": name, + "emitted_at": iso(emitted), + "result_at": iso(result), + "tool_wall_latency_ms": int((result - emitted) * 1000), + } + + +def make_tracelab_run( + *, + block_size: int | None = None, + entries: int | None = None, + max_context_length: int | None = None, +): + """A real ``BenchmarkRun`` whose default dataset is a TraceLab file dataset.""" + from aiperf.config import BenchmarkConfig, BenchmarkRun + + dataset: dict = { + "name": "default", + "type": "file", + "records": [{"text": "placeholder"}], + "format": "tracelab", + } + if block_size is not None: + dataset["block_size"] = block_size + if entries is not None: + dataset["entries"] = entries + if max_context_length is not None: + dataset["max_context_length"] = max_context_length + + cfg = BenchmarkConfig.model_validate( + { + "models": ["test-model"], + "endpoint": { + "urls": ["http://localhost:8000/v1/chat/completions"], + "wait_for_model_timeout": 0, + }, + "datasets": [dataset], + "phases": [ + { + "name": "profiling", + "type": "concurrency", + "requests": 100, + "concurrency": 1, + } + ], + "tokenizer": {"name": "test-tok", "trust_remote_code": False}, + "runtime": {"ui": "simple"}, + } + ) + return BenchmarkRun( + benchmark_id="test-tracelab-run", + cfg=cfg, + artifact_dir=cfg.artifacts.dir, + random_seed=0, + ) + + +def write_jsonl(path: Path, rows: list[dict], *, gz: bool = False) -> Path: + payload = "".join(json.dumps(r) + "\n" for r in rows) + if gz: + with gzip.open(path, "wt", encoding="utf-8") as f: + f.write(payload) + else: + path.write_text(payload, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# Hash-id synthesis +# --------------------------------------------------------------------------- + + +class TestSynthesizeHashIds: + def test_length_is_floor_not_ceil(self): + """A partial trailing block is carried unhashed, so the list floors. + + A ceil-length list is silently accepted downstream and wrong twice: the + trailing id is dropped as content while the cache metric still counts + it, and the extra id perturbs agent-chain detection. + """ + rows = [make_row(input_tokens=130, prefix_tokens=0)] + (ids,) = synthesize_hash_ids(rows, 64, HashIdMinter()) + assert len(ids) == 130 // 64 == 2 + + @pytest.mark.parametrize( + "total,block,expected", + [ + param(0, 64, 0, id="empty"), + param(63, 64, 0, id="below_one_block"), + param(64, 64, 1, id="exactly_one_block"), + param(127, 64, 1, id="partial_second_block"), + param(256, 64, 4, id="four_blocks"), + param(100, 16, 6, id="small_block_size"), + ], + ) # fmt: skip + def test_length_matches_floor_division(self, total, block, expected): + rows = [make_row(input_tokens=total)] + (ids,) = synthesize_hash_ids(rows, block, HashIdMinter()) + assert len(ids) == expected + + def test_prefix_is_reused_from_the_previous_round(self): + rows = [ + make_row(round_index=0, input_tokens=256, prefix_tokens=0), + make_row(round_index=1, input_tokens=384, prefix_tokens=256), + ] + first, second = synthesize_hash_ids(rows, 64, HashIdMinter()) + assert first == [1, 2, 3, 4] + # Four reused blocks, then two freshly minted ones. + assert second[:4] == first + assert len(second) == 6 + assert second[4:] == [5, 6] + + def test_ids_are_unique_when_nothing_is_reused(self): + rows = [ + make_row(round_index=0, input_tokens=128, prefix_tokens=0), + make_row(round_index=1, input_tokens=128, prefix_tokens=0), + ] + first, second = synthesize_hash_ids(rows, 64, HashIdMinter()) + assert set(first).isdisjoint(second) + + def test_compaction_shrinks_the_reused_span(self): + """A smaller reported prefix means the agent compacted its context.""" + rows = [ + make_row(round_index=0, input_tokens=640, prefix_tokens=0), + make_row(round_index=1, input_tokens=704, prefix_tokens=640), + make_row(round_index=2, input_tokens=256, prefix_tokens=128), + ] + chains = synthesize_hash_ids(rows, 64, HashIdMinter()) + assert len(chains[1]) == 11 and chains[1][:10] == chains[0] + # Only two blocks survive the compaction; the rest are fresh. + assert chains[2][:2] == chains[1][:2] + assert len(chains[2]) == 4 + assert set(chains[2][2:]).isdisjoint(chains[1]) + + def test_prefix_larger_than_input_is_clamped(self): + """Never claim more prefix than the round's input actually holds.""" + rows = [ + make_row(round_index=0, input_tokens=640, prefix_tokens=0), + make_row(round_index=1, input_tokens=128, prefix_tokens=999999), + ] + chains = synthesize_hash_ids(rows, 64, HashIdMinter()) + assert len(chains[1]) == 2 + assert chains[1] == chains[0][:2] + + def test_missing_token_fields_do_not_raise(self): + rows = [{"input_tokens_total": None, "prefix_tokens": None}] + assert synthesize_hash_ids(rows, 64, HashIdMinter()) == [[]] + + def test_parent_and_child_share_one_namespace(self): + minter = HashIdMinter() + parent = synthesize_hash_ids([make_row(input_tokens=128)], 64, minter) + child = synthesize_hash_ids([make_row(input_tokens=128)], 64, minter) + assert set(parent[0]).isdisjoint(child[0]) + + +class TestHashIdMinter: + def test_take_is_monotonic_and_non_overlapping(self): + m = HashIdMinter() + assert m.take(3) == [1, 2, 3] + assert m.take(2) == [4, 5] + + def test_take_zero_or_negative_is_empty(self): + m = HashIdMinter() + assert m.take(0) == [] + assert m.take(-1) == [] + assert m.take(1) == [1] + + +# --------------------------------------------------------------------------- +# parse_ts +# --------------------------------------------------------------------------- + + +class TestParseTs: + def test_utc_suffix_z(self): + from aiperf.dataset.loader._tracelab_convert import parse_ts + + assert parse_ts("2026-01-01T00:00:00Z") == parse_ts("2026-01-01T00:00:00+00:00") + + def test_naive_timestamp_assumed_utc(self): + """A timestamp without tzinfo must be treated as UTC, not local time.""" + from aiperf.dataset.loader._tracelab_convert import parse_ts + + naive = parse_ts("2026-06-15T12:00:00") + utc = parse_ts("2026-06-15T12:00:00Z") + assert naive == utc + + +# --------------------------------------------------------------------------- +# Round timing and ordering +# --------------------------------------------------------------------------- + + +class TestRoundTiming: + def test_api_time_spans_submission_to_last_output(self): + _, api = round_timing(make_row(submitted=0.0, responded=2.5)) + assert api == pytest.approx(2.5) + + def test_submission_is_the_latest_input_event(self): + """Parallel tool results: the request cannot predate the last one.""" + row = make_row( + timing_events=[ + {"event_type": "tool_result", "timestamp": iso(1.0)}, + {"event_type": "tool_result", "timestamp": iso(3.0)}, + {"event_type": "text", "timestamp": iso(5.0)}, + ] + ) + submitted, api = round_timing(row) + assert submitted == pytest.approx((T0 + timedelta(seconds=3)).timestamp()) + assert api == pytest.approx(2.0) + + def test_output_only_round_has_no_api_time_offset(self): + row = make_row(timing_events=[{"event_type": "text", "timestamp": iso(4.0)}]) + submitted, api = round_timing(row) + assert submitted == pytest.approx((T0 + timedelta(seconds=4)).timestamp()) + assert api == pytest.approx(0.0) + + def test_input_only_round_has_no_api_time(self): + row = make_row( + timing_events=[{"event_type": "user_message", "timestamp": iso(1.0)}] + ) + _, api = round_timing(row) + assert api is None + + def test_no_timing_events_is_undated(self): + assert round_timing(make_row(timing_events=[])) == (None, None) + + def test_api_time_is_never_negative(self): + row = make_row( + timing_events=[ + {"event_type": "user_message", "timestamp": iso(9.0)}, + {"event_type": "text", "timestamp": iso(1.0)}, + ] + ) + _, api = round_timing(row) + assert api == 0.0 + + +class TestOrderRounds: + def test_orders_by_submission_time_not_round_index(self): + """round_index genuinely disagrees with recorded timestamps here. + + Spawn position is read by array index while the join is placed by + timestamp, so a request array that is not monotonic in ``t`` puts a + join ahead of its spawn and the whole trace is discarded. + """ + rows = [ + make_row(round_index=5, submitted=1.0), + make_row(round_index=1, submitted=9.0), + make_row(round_index=3, submitted=4.0), + ] + ordered = order_rounds(rows) + assert [r[2]["round_index"] for r in ordered] == [5, 3, 1] + assert [r[0] for r in ordered] == sorted(r[0] for r in ordered) + + def test_round_index_breaks_timestamp_ties(self): + rows = [ + make_row(round_index=7, submitted=2.0), + make_row(round_index=2, submitted=2.0), + ] + assert [r[2]["round_index"] for r in order_rounds(rows)] == [2, 7] + + def test_undated_rows_are_dropped(self): + rows = [make_row(round_index=0), make_row(round_index=1, timing_events=[])] + assert len(order_rounds(rows)) == 1 + + def test_duplicate_session_and_round_index_do_not_crash(self): + rows = [ + make_row(round_index=1, submitted=1.0), + make_row(round_index=1, submitted=2.0), + ] + assert len(order_rounds(rows)) == 2 + + +class TestSessionSpan: + def test_span_covers_events_and_tool_calls(self): + rows = [ + make_row( + submitted=1.0, + responded=2.0, + tools=[spawn_tool(emitted=3.0, result=8.0)], + ) + ] + start, end = session_span(rows) + assert start == pytest.approx((T0 + timedelta(seconds=1)).timestamp()) + assert end == pytest.approx((T0 + timedelta(seconds=8)).timestamp()) + + def test_no_stamps_is_none(self): + assert session_span([make_row(timing_events=[], tools=[])]) is None + + +# --------------------------------------------------------------------------- +# Request construction +# --------------------------------------------------------------------------- + + +class TestBuildTraceRequests: + def test_basic_shape_validates_against_the_schema(self): + rows = [ + make_row(round_index=i, submitted=i * 10.0, responded=i * 10.0 + 1.0) + for i in range(3) + ] + blob = build_trace("claude:s1", rows, 64) + trace = WekaTrace.model_validate(blob) + assert trace.hash_id_scope == "local" + assert trace.block_size == 64 + assert len(trace.requests) == 3 + assert blob["totals"]["rounds"] == 3 + assert blob["totals"]["source"] == "tracelab" + + def test_reasoning_tokens_fold_into_output_length(self): + rows = [make_row(output_tokens=10, reasoning_tokens=90)] + blob = build_trace("s", rows, 64) + assert blob["requests"][0]["out"] == 100 + + def test_think_time_is_the_gap_after_the_previous_response(self): + rows = [ + make_row(round_index=0, submitted=0.0, responded=2.0), + make_row(round_index=1, submitted=10.0, responded=11.0), + ] + blob = build_trace("s", rows, 64) + assert blob["requests"][0]["think_time"] is None + assert blob["requests"][1]["think_time"] == pytest.approx(8.0) + + def test_stop_reason_reflects_whether_a_tool_was_requested(self): + tool_round = build_trace("s", [make_row(emits_tool_call=True)], 64) + plain_round = build_trace("s", [make_row(emits_tool_call=False)], 64) + assert tool_round["requests"][0]["stop"] == "tool_use" + assert plain_round["requests"][0]["stop"] == "end_turn" + + def test_input_types_follow_the_first_input_event(self): + cont = build_trace("s", [make_row(first_input_event_type="tool_result")], 64) + fresh = build_trace("s", [make_row(first_input_event_type="user_message")], 64) + assert cont["requests"][0]["input_types"] == ["tool_result"] + assert fresh["requests"][0]["input_types"] == ["text"] + + def test_token_counts_floor_at_one(self): + blob = build_trace("s", [make_row(input_tokens=0, output_tokens=0)], 64) + assert blob["requests"][0]["in"] == 1 + assert blob["requests"][0]["out"] == 1 + + def test_models_are_collected_in_first_use_order(self): + rows = [ + make_row(round_index=0, submitted=0.0, model="model-b"), + make_row(round_index=1, submitted=1.0, model="model-a"), + make_row(round_index=2, submitted=2.0, model="model-b"), + ] + assert build_trace("s", rows, 64)["models"] == ["model-b", "model-a"] + + def test_all_undated_session_returns_none(self): + assert build_trace("s", [make_row(timing_events=[])], 64) is None + + +# --------------------------------------------------------------------------- +# Subagent join +# --------------------------------------------------------------------------- + + +def parent_with_spawn(*, sid="claude:parent", emitted=5.0, result=60.0, **kw): + return [ + make_row(session_id=sid, round_index=0, submitted=0.0, responded=1.0, **kw), + make_row( + session_id=sid, + round_index=1, + submitted=4.0, + responded=4.5, + tools=[spawn_tool(emitted=emitted, result=result)], + **kw, + ), + make_row(session_id=sid, round_index=2, submitted=70.0, responded=71.0, **kw), + ] + + +def child_session(*, sid="claude:child", start=10.0, n=3, **kw): + return [ + make_row( + session_id=sid, + round_index=i, + submitted=start + i * 2.0, + responded=start + i * 2.0 + 1.0, + **kw, + ) + for i in range(n) + ] + + +class TestSubagentJoin: + def test_contained_child_is_matched_to_its_spawn(self): + sessions = { + "claude:parent": parent_with_spawn(), + "claude:child": child_session(), + } + links, stats = build_join_index(sessions) + assert links["claude:child"].parent_sid == "claude:parent" + assert stats.matched == 1 and stats.matched_claude == 1 + assert stats.windows == 1 and stats.windows_matched == 1 + + def test_child_outside_the_window_is_not_matched(self): + sessions = { + "claude:parent": parent_with_spawn(emitted=5.0, result=15.0), + "claude:child": child_session(start=100.0), + } + links, stats = build_join_index(sessions) + assert links == {} and stats.matched == 0 + + def test_child_only_partly_contained_is_not_matched(self): + """Containment is total: a child overrunning its window is not one.""" + sessions = { + "claude:parent": parent_with_spawn(emitted=5.0, result=15.0), + "claude:child": child_session(start=10.0, n=6), + } + links, _ = build_join_index(sessions) + assert links == {} + + def test_a_different_user_or_project_never_matches(self): + sessions = { + "claude:parent": parent_with_spawn(), + "claude:child": child_session(user="someone_else"), + } + assert build_join_index(sessions)[0] == {} + + sessions = { + "claude:parent": parent_with_spawn(), + "claude:child": child_session(project="other_project"), + } + assert build_join_index(sessions)[0] == {} + + def test_short_spawning_calls_are_ignored(self): + """Short calls are overwhelmingly no-ops; admitting them over-captures.""" + child = [ + make_row( + session_id="claude:child", round_index=0, submitted=5.1, responded=5.3 + ) + ] + sessions = { + "claude:parent": parent_with_spawn(emitted=5.0, result=5.5), + "claude:child": child, + } + links, stats = build_join_index(sessions) + assert stats.windows == 0 and links == {} + # Same data, floor lowered: the window appears and matches. + links, stats = build_join_index(sessions, min_spawn_ms=100) + assert stats.windows == 1 and "claude:child" in links + + def test_tightest_window_wins_and_ambiguity_is_counted(self): + parent = parent_with_spawn(sid="claude:p", emitted=5.0, result=90.0) + parent[1]["tools"].append(spawn_tool(emitted=8.0, result=40.0)) + sessions = {"claude:p": parent, "claude:child": child_session(start=10.0)} + links, stats = build_join_index(sessions) + assert stats.ambiguous == 1 + # The 32s window, not the 85s one. + link = links["claude:child"] + assert link.duration_ms == 32000 + + def test_a_session_never_matches_its_own_window(self): + rows = parent_with_spawn(sid="claude:solo") + links, _ = build_join_index({"claude:solo": rows}) + assert "claude:solo" not in links + + def test_task_tool_also_spawns(self): + parent = parent_with_spawn() + parent[1]["tools"] = [spawn_tool(emitted=5.0, result=60.0, name="Task")] + sessions = {"claude:parent": parent, "claude:child": child_session()} + assert "claude:child" in build_join_index(sessions)[0] + + def test_unrelated_tool_names_do_not_spawn(self): + parent = parent_with_spawn() + parent[1]["tools"] = [spawn_tool(emitted=5.0, result=60.0, name="Bash")] + sessions = {"claude:parent": parent, "claude:child": child_session()} + assert build_join_index(sessions)[0] == {} + + +class TestCodexJoin: + @staticmethod + def _codex_parent(sid="codex:parent"): + return [ + make_row( + session_id=sid, + provider="codex", + round_index=0, + submitted=0.0, + responded=1.0, + tools=[{"tool_name": "spawn_agent", "emitted_at": iso(5.0)}], + ), + make_row( + session_id=sid, + provider="codex", + round_index=1, + submitted=70.0, + responded=71.0, + tools=[{"tool_name": "wait_agent", "result_at": iso(60.0)}], + ), + ] + + def test_codex_session_window_matches_a_child(self): + sessions = { + "codex:parent": self._codex_parent(), + "codex:child": child_session(sid="codex:child", provider="codex"), + } + links, stats = build_join_index(sessions) + assert links["codex:child"].kind == "codex" + assert stats.matched_codex == 1 and stats.windows_codex == 1 + + def test_codex_join_can_be_disabled_independently(self): + """Codex handles are stripped, so its window cannot attribute a spawn.""" + sessions = { + "codex:parent": self._codex_parent(), + "codex:child": child_session(sid="codex:child", provider="codex"), + } + links, stats = build_join_index(sessions, enable_codex=False) + assert links == {} and stats.windows_codex == 0 + + def test_spawn_without_a_wait_yields_no_window(self): + rows = self._codex_parent() + rows[1]["tools"] = [] + _, stats = build_join_index({"codex:parent": rows}) + assert stats.windows == 0 + + +class TestGrouping: + def test_grandchild_is_kept_as_a_standalone_trace(self): + """The schema nests one level; dropping the grandchild would lose the + deepest structure the join exists to recover.""" + sessions = { + "claude:root": parent_with_spawn( + sid="claude:root", emitted=5.0, result=200.0 + ), + "claude:mid": [ + make_row( + session_id="claude:mid", + round_index=0, + submitted=10.0, + responded=11.0, + ), + make_row( + session_id="claude:mid", + round_index=1, + submitted=12.0, + responded=13.0, + tools=[spawn_tool(emitted=20.0, result=80.0)], + ), + make_row( + session_id="claude:mid", + round_index=2, + submitted=100.0, + responded=101.0, + ), + ], + "claude:leaf": child_session(sid="claude:leaf", start=30.0, n=2), + } + links, stats = build_join_index(sessions) + grouped = group_children_by_parent(sessions, links, stats) + assert "claude:mid" in grouped["claude:root"] + assert stats.grandchildren == 1 + # The leaf is not nested anywhere, so it survives on its own. + assert all("claude:leaf" not in kids for kids in grouped.values()) + + +class TestNestedTrace: + @staticmethod + def _nested(): + sessions = { + "claude:parent": parent_with_spawn(), + "claude:child": child_session(), + } + links, stats = build_join_index(sessions) + grouped = group_children_by_parent(sessions, links, stats) + return build_trace( + "claude:parent", sessions["claude:parent"], 64, grouped.get("claude:parent") + ) + + def test_subagent_entry_is_emitted_and_validates(self): + blob = self._nested() + trace = WekaTrace.model_validate(blob) + subs = [r for r in trace.requests if r.type == "subagent"] + assert len(subs) == 1 + assert subs[0].requests and subs[0].status == "completed" + assert blob["totals"]["subagents"] == 1 + + def test_request_array_is_monotonic_in_t(self): + """The invariant that reconciles array-index spawns with timestamp joins.""" + times = [r["t"] for r in self._nested()["requests"]] + assert times == sorted(times) + + def test_marker_lands_after_a_parent_request_not_first(self): + reqs = self._nested()["requests"] + marker = next(i for i, r in enumerate(reqs) if r["type"] == "subagent") + assert marker > 0 + assert reqs[marker - 1]["type"] == "n" + + def test_inner_requests_never_precede_the_marker(self): + """An inner t below the marker is re-read as relative and flung forward.""" + blob = self._nested() + entry = next(r for r in blob["requests"] if r["type"] == "subagent") + assert all(r["t"] >= entry["t"] for r in entry["requests"]) + + def test_containment_makes_early_inner_requests_unreachable(self): + """The join's own precondition supplies the invariant. + + A child only matches when its whole span is inside the spawn window, + and the span mins over tool stamps as well as timing events, so a + matched child can never carry a request before its own marker. This + asserts the invariant on a child that reaches right to both edges of + its window, where a violation would be likeliest. + """ + child = [ + make_row( + session_id="claude:child", + round_index=0, + submitted=5.0, + responded=6.0, + tools=[spawn_tool(emitted=6.0, result=7.0, name="Bash")], + ), + make_row( + session_id="claude:child", round_index=1, submitted=8.0, responded=60.0 + ), + ] + sessions = { + "claude:parent": parent_with_spawn(emitted=5.0, result=60.0), + "claude:child": child, + } + links, stats = build_join_index(sessions) + assert "claude:child" in links, "fixture must actually match" + grouped = group_children_by_parent(sessions, links, stats) + blob = build_trace( + "claude:parent", sessions["claude:parent"], 64, grouped["claude:parent"] + ) + entry = next(r for r in blob["requests"] if r["type"] == "subagent") + assert min(r["t"] for r in entry["requests"]) >= entry["t"] + + def test_guard_clamps_an_inner_request_that_precedes_its_marker(self): + """Drive the guard directly: containment cannot produce this input. + + Downstream reads an inner t below the marker as subagent-relative and + rewrites it, which would scatter one request far into the future. The + guard is only reachable by handing ``build_subagent_entry`` a spawn that + starts after its child, which the join never does. + """ + from aiperf.dataset.loader._tracelab_convert import Spawn, build_subagent_entry + + child = child_session(start=0.0, n=3) + t0 = order_rounds(child)[0][0] + late = Spawn( + parent_sid="claude:parent", + child_sid="claude:child", + start=t0 + 100.0, + end=t0 + 200.0, + duration_ms=100000, + kind="claude", + ) + entry = build_subagent_entry( + late, child_rows=child, block_size=64, minter=HashIdMinter(), t0=t0 + ) + assert entry["t"] == pytest.approx(100.0) + assert all(r["t"] >= entry["t"] for r in entry["requests"]) + assert entry["requests"][0]["t"] == pytest.approx(100.0) + + def test_child_first_request_has_no_think_time(self): + entry = next(r for r in self._nested()["requests"] if r["type"] == "subagent") + assert entry["requests"][0]["think_time"] is None + + def test_agent_id_is_filesystem_and_schema_safe(self): + entry = next(r for r in self._nested()["requests"] if r["type"] == "subagent") + assert entry["agent_id"] == "claude_child" + + def test_child_hash_ids_do_not_collide_with_the_parent(self): + blob = self._nested() + parent_ids = { + i for r in blob["requests"] if r["type"] == "n" for i in r["hash_ids"] + } + entry = next(r for r in blob["requests"] if r["type"] == "subagent") + child_ids = {i for r in entry["requests"] for i in r["hash_ids"]} + assert parent_ids and child_ids + assert parent_ids.isdisjoint(child_ids) + + def test_totals_count_only_top_level_rounds(self): + blob = self._nested() + assert blob["totals"]["rounds"] == 3 + + +class TestSafeTraceId: + @pytest.mark.parametrize( + "raw,expected", + [ + param("claude:abc-123", "claude_abc-123", id="colon_and_hyphen"), + param("a/b\\c", "a_b_c", id="slash_and_backslash"), + param("plain.id_1", "plain.id_1", id="already_safe"), + ], + ) # fmt: skip + def test_unsafe_characters_are_replaced(self, raw, expected): + assert safe_trace_id(raw) == expected + + def test_long_ids_are_truncated(self): + assert len(safe_trace_id("x" * 500)) == 150 + + +# --------------------------------------------------------------------------- +# Loader +# --------------------------------------------------------------------------- + + +def build_loader(path, **run_kw): + return TraceLabTraceDatasetLoader( + filename=str(path), + run=make_tracelab_run(**run_kw), + default_block_size=DEFAULT_BLOCK_SIZE, + ) + + +class TestLoaderCanLoad: + def test_recognizes_a_tracelab_row(self): + assert TraceLabTraceDatasetLoader.can_load(data=make_row()) + + @pytest.mark.parametrize( + "row", + [ + param({"timestamp": 1, "input_length": 5}, id="mooncake_shape"), + param({"session_id": "s", "round_index": 0}, id="partial_tracelab"), + param({}, id="empty"), + ], + ) # fmt: skip + def test_rejects_other_shapes(self, row): + assert not TraceLabTraceDatasetLoader.can_load(data=row) + + def test_recognizes_a_file_and_its_gzipped_twin(self, tmp_path): + plain = write_jsonl(tmp_path / "t.jsonl", [make_row()]) + gz = write_jsonl(tmp_path / "t.jsonl.gz", [make_row()], gz=True) + assert TraceLabTraceDatasetLoader.can_load(filename=str(plain)) + assert TraceLabTraceDatasetLoader.can_load(filename=str(gz)) + + def test_rejects_directories_and_missing_files(self, tmp_path): + assert not TraceLabTraceDatasetLoader.can_load(filename=str(tmp_path)) + assert not TraceLabTraceDatasetLoader.can_load( + filename=str(tmp_path / "nope.jsonl") + ) + + def test_rejects_a_non_tracelab_jsonl(self, tmp_path): + path = write_jsonl( + tmp_path / "other.jsonl", [{"timestamp": 0, "input_length": 4}] + ) + assert not TraceLabTraceDatasetLoader.can_load(filename=str(path)) + + def test_none_arguments_are_safe(self): + assert not TraceLabTraceDatasetLoader.can_load() + + def test_malformed_json_does_not_raise(self, tmp_path): + path = tmp_path / "bad.jsonl" + path.write_text("{not json\n", encoding="utf-8") + assert TraceLabTraceDatasetLoader.can_load(filename=str(path)) is False + + +class TestLoaderLoadDataset: + @staticmethod + def _corpus(): + return ( + parent_with_spawn() + + child_session() + + [ + make_row( + session_id="claude:solo", + round_index=i, + submitted=i * 3.0, + responded=i * 3.0 + 1.0, + ) + for i in range(2) + ] + ) + + def test_loads_sessions_and_nests_the_recovered_child(self, tmp_path): + path = write_jsonl(tmp_path / "c.jsonl", self._corpus()) + data = build_loader(path).load_dataset() + assert set(data) == {"claude_parent", "claude_solo"} + parent = data["claude_parent"][0] + assert any(r.type == "subagent" for r in parent.requests) + + def test_gzip_and_plain_produce_identical_traces(self, tmp_path): + rows = self._corpus() + plain = build_loader(write_jsonl(tmp_path / "c.jsonl", rows)).load_dataset() + gz = build_loader( + write_jsonl(tmp_path / "c.jsonl.gz", rows, gz=True) + ).load_dataset() + assert plain.keys() == gz.keys() + for key in plain: + assert plain[key][0].model_dump() == gz[key][0].model_dump() + + def test_block_size_defaults_to_the_plugin_metadata_value(self, tmp_path): + path = write_jsonl(tmp_path / "c.jsonl", self._corpus()) + data = build_loader(path).load_dataset() + assert all(t[0].block_size == DEFAULT_BLOCK_SIZE for t in data.values()) + + def test_isl_block_size_overrides_the_plugin_metadata_value(self, tmp_path): + """Unlike Weka, TraceLab has no recorded block size: the user's wins.""" + path = write_jsonl(tmp_path / "c.jsonl", self._corpus()) + data = build_loader(path, block_size=128).load_dataset() + assert all(t[0].block_size == 128 for t in data.values()) + # And it actually changes synthesis, not just the declared field. + wide = data["claude_solo"][0].requests[0] + narrow = build_loader(path).load_dataset()["claude_solo"][0].requests[0] + assert len(wide.hash_ids) < len(narrow.hash_ids) + + def test_num_dataset_entries_caps_the_trace_count(self, tmp_path): + path = write_jsonl(tmp_path / "c.jsonl", self._corpus()) + assert len(build_loader(path, entries=1).load_dataset()) == 1 + + def test_subagent_join_can_be_disabled(self, tmp_path, monkeypatch): + from aiperf.common import environment as env_mod + + monkeypatch.setattr( + env_mod.Environment.DATASET, "TRACELAB_SUBAGENT_JOIN", False + ) + path = write_jsonl(tmp_path / "c.jsonl", self._corpus()) + data = build_loader(path).load_dataset() + assert "claude_child" in data + assert not any( + r.type == "subagent" for t in data.values() for r in t[0].requests + ) + + def test_child_that_fails_to_nest_is_emitted_standalone(self, tmp_path): + """A child linked by join but that fails to nest (e.g. anchor lookup + returns None) must appear as a standalone trace, not be silently dropped.""" + from unittest.mock import patch + + from aiperf.dataset.loader._tracelab_convert import Spawn + + parent_rows = parent_with_spawn() + child_rows = child_session() + loader = build_loader( + write_jsonl(tmp_path / "c.jsonl", parent_rows + child_rows) + ) + sessions = {"claude:parent": parent_rows, "claude:child": child_rows} + spawn = Spawn( + parent_sid="claude:parent", + child_sid="claude:child", + start=5.0, + end=60.0, + duration_ms=55000, + kind="claude", + ) + children_by_parent = {"claude:parent": {"claude:child": (spawn, child_rows)}} + nested_ids = {"claude:child"} + + with patch( + "aiperf.dataset.loader._tracelab_convert._anchor_index", + return_value=None, + ): + data = loader._build_traces(sessions, children_by_parent, nested_ids) + + assert "claude_child" in data + assert "claude_parent" in data + assert not any(r.type == "subagent" for r in data["claude_parent"].requests) + + def test_fallback_sessions_emitted_in_file_order(self, tmp_path): + """Fallback traces must follow sessions-dict insertion order, not set + iteration order, so --num-dataset-entries is deterministic.""" + from unittest.mock import patch + + rows_a = child_session(sid="claude:a", start=10.0) + rows_b = child_session(sid="claude:b", start=12.0) + rows_c = child_session(sid="claude:c", start=14.0) + # Build a loader with a file that provides the sessions dict ordering. + loader = build_loader( + write_jsonl( + tmp_path / "c.jsonl", + parent_with_spawn() + rows_a + rows_b + rows_c, + ) + ) + sessions = { + "claude:parent": parent_with_spawn(), + "claude:a": rows_a, + "claude:b": rows_b, + "claude:c": rows_c, + } + nested_ids = {"claude:a", "claude:b", "claude:c"} + + with patch( + "aiperf.dataset.loader._tracelab_convert._anchor_index", + return_value=None, + ): + traces = loader._build_traces(sessions, {}, nested_ids) + + keys = list(traces.keys()) + # All three fallback sessions present (parent also emitted as a root). + assert {"claude_a", "claude_b", "claude_c"}.issubset(set(keys)) + # Fallback entries must follow sessions-dict order (a, b, c), not set order. + fallback_keys = [k for k in keys if k in {"claude_a", "claude_b", "claude_c"}] + assert fallback_keys == ["claude_a", "claude_b", "claude_c"] + + def test_build_traces_wraps_conversion_error_with_session_id(self, tmp_path): + """ValueError/TypeError from build_trace is re-raised as DatasetLoaderError + with the session id attached.""" + from unittest.mock import patch + + loader = build_loader(write_jsonl(tmp_path / "c.jsonl", parent_with_spawn())) + sessions = {"claude:parent": parent_with_spawn()} + + with ( + patch( + "aiperf.dataset.loader.tracelab_trace.build_trace", + side_effect=ValueError("bad timestamp"), + ), + pytest.raises(DatasetLoaderError, match="claude:parent"), + ): + loader._build_traces(sessions, {}, set()) + + def test_merge_skips_child_with_no_timed_rounds(self, tmp_path): + """When build_subagent_entry returns None (child has no timed rounds), + _merge_subagent_entries skips it without crashing.""" + from aiperf.dataset.loader._tracelab_convert import ( + HashIdMinter, + Spawn, + _merge_subagent_entries, + build_requests, + synthesize_hash_ids, + ) + + parent_rows = parent_with_spawn() + timed_parent = [ + (r[0], r[1], r[2]) + for r in [(0.0, 1.0, parent_rows[0]), (4.0, 0.5, parent_rows[1])] + ] + t0 = 0.0 + minter = HashIdMinter() + hashes = synthesize_hash_ids([r[2] for r in timed_parent], 64, minter) + requests, models = build_requests(timed_parent, hashes, t0) + + child_no_timing = [make_row(session_id="claude:child", timing_events=[])] + spawn = Spawn( + parent_sid="claude:parent", + child_sid="claude:child", + start=5.0, + end=60.0, + duration_ms=55000, + kind="claude", + ) + placed: set[str] = set() + merged, n = _merge_subagent_entries( + requests=requests, + models=models, + children={"claude:child": (spawn, child_no_timing)}, + block_size=64, + minter=minter, + t0=t0, + placed_sids=placed, + ) + assert n == 0 + assert placed == set() + assert not any(r.get("type") == "subagent" for r in merged) + + def test_merge_adds_child_model_to_parent_models(self, tmp_path): + """When the child uses a model not yet in the parent's list, it is appended.""" + from aiperf.dataset.loader._tracelab_convert import ( + HashIdMinter, + Spawn, + _merge_subagent_entries, + build_requests, + synthesize_hash_ids, + ) + + parent_rows = parent_with_spawn() + child_rows = [ + make_row(session_id="claude:child", model="different-model", submitted=10.0) + ] + t0 = 0.0 + minter = HashIdMinter() + timed_parent = [(0.0, 1.0, parent_rows[0]), (4.0, 0.5, parent_rows[1])] + hashes = synthesize_hash_ids([r[2] for r in timed_parent], 64, minter) + requests, models = build_requests(timed_parent, hashes, t0) + + spawn = Spawn( + parent_sid="claude:parent", + child_sid="claude:child", + start=5.0, + end=60.0, + duration_ms=55000, + kind="claude", + ) + placed: set[str] = set() + _merge_subagent_entries( + requests=requests, + models=models, + children={"claude:child": (spawn, child_rows)}, + block_size=64, + minter=minter, + t0=t0, + placed_sids=placed, + ) + assert "different-model" in models + assert "claude:child" in placed + + def test_rows_without_a_session_id_are_skipped(self, tmp_path): + rows = self._corpus() + [{"round_index": 0, "timing_events": []}] + path = write_jsonl(tmp_path / "c.jsonl", rows) + assert set(build_loader(path).load_dataset()) == { + "claude_parent", + "claude_solo", + } + + def test_missing_filename_is_a_clear_error(self): + loader = TraceLabTraceDatasetLoader(inline_records=[], run=make_tracelab_run()) + with pytest.raises(DatasetLoaderError, match="requires --input-file"): + loader.load_dataset() + + def test_empty_file_is_a_clear_error(self, tmp_path): + path = write_jsonl(tmp_path / "empty.jsonl", []) + with pytest.raises(DatasetLoaderError, match="No TraceLab sessions"): + build_loader(path).load_dataset() + + def test_invalid_json_names_the_line(self, tmp_path): + path = tmp_path / "bad.jsonl" + path.write_text(json.dumps(make_row()) + "\n{oops\n", encoding="utf-8") + with pytest.raises(DatasetLoaderError, match="line 2"): + build_loader(path).load_dataset() + + def test_truncated_gzip_raises_dataset_loader_error(self, tmp_path): + """A partially-downloaded or corrupt .gz raises DatasetLoaderError, not + a raw zlib.error traceback.""" + import gzip as _gzip + + gz = tmp_path / "bad.jsonl.gz" + with _gzip.open(gz, "wt", encoding="utf-8") as f: + for _ in range(50): + f.write(json.dumps(make_row()) + "\n") + data = gz.read_bytes() + gz.write_bytes(data[: max(len(data) - 30, 2)]) + with pytest.raises(DatasetLoaderError, match="Cannot read TraceLab file"): + build_loader(gz).load_dataset() + + def test_non_utf8_file_raises_dataset_loader_error(self, tmp_path): + """A file with invalid UTF-8 bytes raises DatasetLoaderError, not a + raw UnicodeDecodeError traceback.""" + bad = tmp_path / "bad.jsonl" + bad.write_bytes(b'{"session_id": "s1"}\n\xff\xfe\n') + with pytest.raises(DatasetLoaderError, match="Cannot read TraceLab file"): + build_loader(bad).load_dataset() + + def test_undated_sessions_are_dropped_not_fatal(self, tmp_path): + rows = self._corpus() + [ + make_row(session_id="claude:ghost", timing_events=[], tools=[]) + ] + path = write_jsonl(tmp_path / "c.jsonl", rows) + assert "claude_ghost" not in build_loader(path).load_dataset() + + def test_sampling_strategy_is_sequential(self): + from aiperf.plugin.enums import DatasetSamplingStrategy + + assert ( + TraceLabTraceDatasetLoader.get_preferred_sampling_strategy() + == DatasetSamplingStrategy.SEQUENTIAL + ) + + +class TestLoaderReconstruction: + def test_conversations_include_the_subagent_child(self, tmp_path): + from tests.unit.dataset.loader._shared_helpers import _stub_loader + + rows = parent_with_spawn() + child_session() + loader = build_loader(write_jsonl(tmp_path / "c.jsonl", rows)) + data = loader.load_dataset() + _stub_loader(loader._weka) + conversations = loader.convert_to_conversations(data) + ids = {c.session_id for c in conversations} + assert "claude_parent" in ids + # One root plus one child carrying the recovered agent id. + assert any("::sa:" in sid for sid in ids) + assert len(conversations) >= 2 + + +class TestRegistration: + def test_loader_resolves_from_the_plugin_registry(self): + from aiperf.plugin import plugins + from aiperf.plugin.enums import CustomDatasetType, PluginType + + cls = plugins.get_class( + PluginType.CUSTOM_DATASET_LOADER, CustomDatasetType.TRACELAB + ) + assert cls is TraceLabTraceDatasetLoader + + def test_plugin_metadata_declares_our_own_block_size(self): + """Declared explicitly so it becomes ours to change independently.""" + from aiperf.plugin import plugins + from aiperf.plugin.enums import CustomDatasetType + + meta = plugins.get_dataset_loader_metadata(CustomDatasetType.TRACELAB) + assert meta.is_trace is True + assert meta.default_block_size == DEFAULT_BLOCK_SIZE + + def test_isl_block_size_is_accepted_for_tracelab(self): + """Without the format in the block-size set this raises.""" + from aiperf.config.flags._converter_dataset import _BLOCK_SIZE_TRACE_FORMATS + + assert "tracelab" in _BLOCK_SIZE_TRACE_FORMATS + + def test_mmap_cache_treats_tracelab_as_a_trace(self): + from aiperf.dataset.mmap_cache import is_trace_or_verbatim_dataset + + assert is_trace_or_verbatim_dataset("tracelab", None) + + def test_isl_block_size_survives_the_config_converter(self): + """The flag must reach FileDataset.block_size, not be rejected or dropped. + + Weka rejects it outright because its traces declare their own per-block + sizes; TraceLab synthesizes its block ids, so the value is load-bearing + and has to arrive. + """ + from aiperf.common.enums import DatasetType + from aiperf.config.flags._converter_dataset import _apply_block_size + + cli = type( + "StubCLI", + (), + { + "model_fields_set": {"prompt_input_tokens_block_size"}, + "prompt_input_tokens_block_size": 128, + }, + )() + d: dict = {"format": "tracelab", "type": DatasetType.FILE, "dataset": None} + _apply_block_size(d, cli) + assert d["block_size"] == 128 + + def test_block_size_zero_raises_not_silently_defaults(self): + """block_size=0 passed via default_block_size must raise, not silently + fall back to DEFAULT_BLOCK_SIZE (regression: 'if block_size' coerces 0 + to False and skips the validation that follows).""" + with pytest.raises(DatasetLoaderError, match="must be positive"): + TraceLabTraceDatasetLoader( + inline_records=[], + run=make_tracelab_run(), + default_block_size=0, + ) + + def test_first_record_has_timestamp_detects_tracelab(self, tmp_path): + """_first_record_has_timestamp must return True for TraceLab rows so + _maybe_auto_promote_trace can promote them to fixed-schedule.""" + from aiperf.config.flags._converter_profiling import _first_record_has_timestamp + + path = write_jsonl(tmp_path / "t.jsonl", [make_row()]) + assert _first_record_has_timestamp(path) + + def test_first_record_has_timestamp_detects_gzipped_tracelab(self, tmp_path): + from aiperf.config.flags._converter_profiling import _first_record_has_timestamp + + gz = write_jsonl(tmp_path / "t.jsonl.gz", [make_row()], gz=True) + assert _first_record_has_timestamp(gz) + + def test_fixed_schedule_accepts_tracelab_timing(self): + """Timing is nested under timing_events, so the generic probe misses it.""" + from aiperf.config.dataset.resolver import DatasetResolver + from aiperf.plugin.enums import CustomDatasetType + + assert DatasetResolver._check_timing_data( + "ignored.jsonl", None, CustomDatasetType.TRACELAB + ) + + +class TestGzipPlumbing: + """BaseFileLoader._iter_record_dicts transparently handles gzipped sources.""" + + def test_base_file_loader_iterates_a_gzipped_source(self, tmp_path): + gz = write_jsonl(tmp_path / "a.jsonl.gz", [make_row()], gz=True) + records = list(build_loader(gz)._iter_record_dicts()) + assert len(records) == 1 and records[0]["session_id"] == "claude:s1" diff --git a/tests/unit/dataset/test_mmap_cache.py b/tests/unit/dataset/test_mmap_cache.py index 90eeb801ab..c4ec89ae7e 100644 --- a/tests/unit/dataset/test_mmap_cache.py +++ b/tests/unit/dataset/test_mmap_cache.py @@ -248,6 +248,84 @@ def _key(seed: int) -> str | None: assert k1 is not None and k2 is not None assert k1 != k2, "random_seed must distinguish the cache key" + def test_settings_payload_key_set_is_frozen( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Any new cache-affecting field must be added here intentionally. + + This test is the gate for the class of cache-key holes where a setting + that rewrites cached bytes is accidentally omitted from the payload. + Covers both the top-level key set and the nested preformat_endpoint + sub-dict (the latter only populated when PREFORMAT_PAYLOADS is on). + """ + from aiperf.common.environment import Environment + from aiperf.plugin.enums import CustomDatasetType + + trace = _write_input_file(tmp_path, b'{"session_id": "s1"}\n') + run = make_run_from_cli( + CLIConfig( + model_names=["test-model"], + input_file=str(trace), + custom_dataset_type=CustomDatasetType.MOONCAKE_TRACE, + ) + ) + payload = mmap_cache._settings_payload_from_run(run) + assert set(payload.keys()) == { + "num_dataset_entries", + "dataset_sampling_strategy", + "custom_dataset_type", + "public_dataset_source", + "random_seed", + "dataset_random_seed", + "corpus", + "osl_fallback", + "preformat_payloads", + "preformat_endpoint", + "inline_records_sha256", + "prompt", + "endpoint_type", + "model_names", + "fixed_schedule_start_offset", + "fixed_schedule_end_offset", + "ignore_trace_delays", + "use_think_time_only", + "inter_turn_delay_cap_seconds", + "trace_idle_gap_cap_seconds", + "weka_split_flattened_agents", + "weka_aux_max_requests", + "weka_aux_isl_ratio", + "weka_aux_isl_floor", + "weka_aux_cross_model", + "weka_aux_reduction_osl_max", + "weka_aux_reduction_ratio", + "weka_worker_group_min", + "weka_tool_shaped_messages", + "weka_seam_max_gap_seconds", + "weka_seam_min_overlap_ratio", + "block_size", + "tracelab_subagent_join", + "tracelab_codex_subagent_join", + "tracelab_min_spawn_ms", + "synthesis", + "max_context_length", + "entries_explicit", + } + + # Also gate the nested preformat_endpoint sub-dict — force_content_parts + # lives there, not at the top level, so the key-set check above doesn't + # catch a future omission inside this dict. + monkeypatch.setattr(Environment.DATASET, "PREFORMAT_PAYLOADS", True) + payload_with_preformat = mmap_cache._settings_payload_from_run(run) + preformat = payload_with_preformat["preformat_endpoint"] + assert preformat is not None + assert set(preformat.keys()) == { + "streaming", + "use_legacy_max_tokens", + "use_server_token_count", + "extra", + "force_content_parts", + } + def test_settings_payload_includes_seed_corpus_osl(self, tmp_path: Path) -> None: # Regression guard for the wrong-cache-hit findings: random_seed, # prompts.corpus, and the per-record OSL fallback must enter the key. diff --git a/tests/unit/plugin/test_new_loader_plugin_entries.py b/tests/unit/plugin/test_new_loader_plugin_entries.py index 17845db882..7645fab2c8 100644 --- a/tests/unit/plugin/test_new_loader_plugin_entries.py +++ b/tests/unit/plugin/test_new_loader_plugin_entries.py @@ -20,6 +20,7 @@ class TestLoaderEntriesRegistered: [ ("raw_payload", "RawPayloadDatasetLoader"), ("inputs_json", "InputsJsonPayloadLoader"), + ("tracelab", "TraceLabTraceDatasetLoader"), ], ) def test_loader_resolves(self, loader_name: str, expected_cls: str): From 31bf5703af77fe5d09286904cf93c75a2bf69251 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 12 Aug 2026 18:08:20 -0700 Subject: [PATCH 021/119] perf(timing): high-resolution rate-loop pacing for exact rate delivery (#1185) Signed-off-by: Anthony Casagrande Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 21f8ad7b3e621285a1682b336df16607e7d3bb9f) --- .../benchmark-modes/timing-modes-reference.md | 30 ++ docs/environment-variables.md | 2 + src/aiperf/common/environment.py | 18 ++ src/aiperf/timing/high_res_timer.py | 234 +++++++++++++++ src/aiperf/timing/strategies/request_rate.py | 264 +++++++++++------ .../timing/strategies/test_request_rate.py | 268 ++++++++++++++++++ tests/unit/timing/test_high_res_timer.py | 122 ++++++++ 7 files changed, 856 insertions(+), 82 deletions(-) create mode 100644 src/aiperf/timing/high_res_timer.py create mode 100644 tests/unit/timing/test_high_res_timer.py diff --git a/docs/benchmark-modes/timing-modes-reference.md b/docs/benchmark-modes/timing-modes-reference.md index 8169af8aa0..0e90eceec6 100644 --- a/docs/benchmark-modes/timing-modes-reference.md +++ b/docs/benchmark-modes/timing-modes-reference.md @@ -27,6 +27,36 @@ When multiple options are specified, AIPerf uses this priority: 3. `--request-rate` → Rate-based scheduling with arrival patterns 4. `--concurrency` only → Burst mode (as fast as possible within limits) +--- + +## High-Resolution Rate Pacing + +At high request rates (≥ 500 req/s), event-loop timers quantize sub-millisecond +sleeps to ~1ms intervals, causing AIPerf to silently under-deliver the target QPS. +High-resolution pacing bypasses the timer wheel with a platform-specific mechanism: + +| Platform | Pacer | Precision | +|----------|-------|-----------| +| Linux | `TimerFdPacer` — `timerfd_create(CLOCK_MONOTONIC)` kernel hrtimer | ~50µs | +| macOS / Windows / other | `ThreadPacer` — dedicated sleep thread with `clock_nanosleep` / waitable timer | ~100µs (POSIX), ~500µs (Windows) | + +High-res pacing is **enabled by default** when `--request-rate` is used. The +selection is automatic: timerfd on Linux, thread-based elsewhere. + +### Pacing env vars + +| Variable | Default | Description | +|----------|---------|-------------| +| `AIPERF_TIMING_HIGH_RES_TIMER` | `true` | Set to `false` to force event-loop timer pacing (useful for isolating scheduling behaviour). | +| `AIPERF_TIMING_MAX_CATCHUP_SECONDS` | `0.01` | Maximum schedule backlog (seconds) the rate loop may catch up on before re-anchoring to the current time. Without a catch-up window, every oversleep permanently forfeits one schedule slot — at 5,000 req/s the loop falls behind on each tick. Increase if you see a schedule-backlog warning; decrease toward `0` for strict no-burst behaviour (at the cost of some under-delivery). | + +### Diagnosing under-delivery at high QPS + +If measured QPS is consistently lower than `--request-rate`: + +1. **Confirm the pacer is active.** Check the startup log for `"Using TimerFdPacer"` or `"Using ThreadPacer"`. If absent, `AIPERF_TIMING_HIGH_RES_TIMER` may be `false`. +2. **Check for schedule-backlog warnings.** If present, the pacer is waking on time but the event loop is stalling between wake and dispatch. Try `uvloop` (`pip install uvloop`) or reduce `--concurrency`. +3. **Widen the catch-up window.** At very high rates (≥ 5,000 req/s), increase `AIPERF_TIMING_MAX_CATCHUP_SECONDS` (e.g. `0.05`) to let the loop absorb brief stalls without re-anchoring too eagerly. --- diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 49c5f8aab0..2ccf049469 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -300,6 +300,8 @@ Timing manager configuration. Controls timing-related settings for credit phase |----------------------|---------|-------------|-------------| | `AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT` | `10.0` | ≥ 1.0, ≤ 300.0 | Timeout in seconds for waiting for cancelled credits to drain after phase timeout | | `AIPERF_TIMING_RATE_RAMP_UPDATE_INTERVAL` | `0.1` | ≥ 0.01, ≤ 10.0 | Update interval in seconds for continuous rate ramping (default 0.1s = 100ms) | +| `AIPERF_TIMING_HIGH_RES_TIMER` | `True` | — | Use high-resolution rate-loop pacing instead of event-loop timers, which quantize sub-millisecond sleeps to ~1ms granularity. Restores exact rate delivery and arrival-distribution fidelity at high request rates. Uses a Linux timerfd (kernel hrtimer, ~50us wakeup precision) when available, and a dedicated sleep thread on other platforms (~100us POSIX, ~0.5ms Windows). Set to false to force event-loop timer pacing. | +| `AIPERF_TIMING_MAX_CATCHUP_SECONDS` | `0.01` | ≥ 0.0, ≤ 10.0 | Maximum schedule backlog in seconds the rate loop is allowed to catch up on before re-anchoring to the current time. Event-loop timers oversleep sub-millisecond waits (~1ms granularity under uvloop/libuv); without a catch-up window every oversleep permanently forfeits schedule and high request rates silently under-deliver. Bounded so a genuine multi-second stall still re-anchors instead of firing a burst storm. | ## TOKENIZER diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index ec00def6bd..ed1b4e50e8 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -1227,6 +1227,24 @@ class _TimingSettings(BaseSettings): default=0.1, description="Update interval in seconds for continuous rate ramping (default 0.1s = 100ms)", ) + HIGH_RES_TIMER: bool = Field( + default=True, + description="Use high-resolution rate-loop pacing instead of event-loop timers, which " + "quantize sub-millisecond sleeps to ~1ms granularity. Restores exact rate delivery and " + "arrival-distribution fidelity at high request rates. Uses a Linux timerfd (kernel " + "hrtimer, ~50us wakeup precision) when available, and a dedicated sleep thread on other " + "platforms (~100us POSIX, ~0.5ms Windows). Set to false to force event-loop timer pacing.", + ) + MAX_CATCHUP_SECONDS: float = Field( + ge=0.0, + le=10.0, + default=0.01, + description="Maximum schedule backlog in seconds the rate loop is allowed to catch up on " + "before re-anchoring to the current time. Event-loop timers oversleep sub-millisecond " + "waits (~1ms granularity under uvloop/libuv); without a catch-up window every oversleep " + "permanently forfeits schedule and high request rates silently under-deliver. Bounded so " + "a genuine multi-second stall still re-anchors instead of firing a burst storm.", + ) class _ServiceSettings(BaseSettings): diff --git a/src/aiperf/timing/high_res_timer.py b/src/aiperf/timing/high_res_timer.py new file mode 100644 index 0000000000..20222c5891 --- /dev/null +++ b/src/aiperf/timing/high_res_timer.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""High-resolution absolute-deadline sleeps via Linux timerfd. + +Event-loop timers (libuv under uvloop, epoll_wait under stock asyncio) have +~1ms granularity, so the sub-millisecond sleeps needed by high request rates +(e.g. 200us intervals at 5,000 req/s) reliably oversleep to the next timer +tick. ``timerfd_create(CLOCK_MONOTONIC)`` timers are backed by kernel hrtimers +instead: expirations are delivered as fd readability with ~50us wakeup +precision, and the event loop observes them through ``loop.add_reader`` — the +same fd path the ZMQ clients use — bypassing the loop's timer wheel entirely. + +``time.perf_counter()`` is ``clock_gettime(CLOCK_MONOTONIC)`` on Linux +(asserted at construction), so perf_counter deadlines are passed directly to +``timerfd_settime(TFD_TIMER_ABSTIME)`` without clock conversion. + +Example: + pacer = TimerFdPacer() + deadline = time.perf_counter() + 0.0002 + await pacer.sleep_until(deadline) # wakes ~50us after deadline, not ~1ms + pacer.close() + +Linux-only: callers must gate on ``IS_LINUX`` and fall back to +``asyncio.sleep`` elsewhere. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import ctypes +import os +import threading +import time + +_CLOCK_MONOTONIC = 1 +_TFD_NONBLOCK = 0o4000 +_TFD_CLOEXEC = 0o2000000 +_TFD_TIMER_ABSTIME = 1 +_NANOS_PER_SECOND = 1_000_000_000 + + +class _Timespec(ctypes.Structure): + _fields_ = [("tv_sec", ctypes.c_long), ("tv_nsec", ctypes.c_long)] + + +class _Itimerspec(ctypes.Structure): + _fields_ = [("it_interval", _Timespec), ("it_value", _Timespec)] + + +class TimerFdPacer: + """Await absolute perf_counter deadlines with hrtimer (~50us) precision. + + Must be constructed inside a running event loop. Not thread-safe and + supports one waiter at a time (the rate loop is a single coroutine). + """ + + def __init__(self) -> None: + clock_impl = time.get_clock_info("perf_counter").implementation + if "CLOCK_MONOTONIC" not in clock_impl: + raise OSError( + f"TimerFdPacer requires perf_counter to be CLOCK_MONOTONIC-based, " + f"got {clock_impl!r}: timerfd deadlines would be on a different clock" + ) + self._libc = ctypes.CDLL("libc.so.6", use_errno=True) + self._libc.timerfd_create.argtypes = [ctypes.c_int, ctypes.c_int] + self._libc.timerfd_create.restype = ctypes.c_int + self._libc.timerfd_settime.argtypes = [ + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(_Itimerspec), + ctypes.POINTER(_Itimerspec), + ] + self._libc.timerfd_settime.restype = ctypes.c_int + fd = self._libc.timerfd_create(_CLOCK_MONOTONIC, _TFD_NONBLOCK | _TFD_CLOEXEC) + if fd < 0: + errno = ctypes.get_errno() + raise OSError(errno, f"timerfd_create failed: {os.strerror(errno)}") + self._fd = fd + self._loop = asyncio.get_running_loop() + self._tick = asyncio.Event() + self._loop.add_reader(self._fd, self._on_readable) + self._closed = False + self._active_waiter = False + + def _on_readable(self) -> None: + with contextlib.suppress(BlockingIOError): + os.read(self._fd, 8) # drain the expiration count + self._tick.set() + + async def sleep_until(self, deadline_perf_s: float) -> None: + """Sleep until an absolute ``time.perf_counter()`` deadline. + + Returns immediately if the deadline is already in the past (the kernel + fires expired absolute timers right away). + """ + assert not self._active_waiter, ( + "TimerFdPacer supports only one concurrent waiter" + ) + self._active_waiter = True + try: + deadline_ns = int(deadline_perf_s * _NANOS_PER_SECOND) + spec = _Itimerspec() + spec.it_value.tv_sec = deadline_ns // _NANOS_PER_SECOND + spec.it_value.tv_nsec = deadline_ns % _NANOS_PER_SECOND or 1 + self._tick.clear() + rc = self._libc.timerfd_settime( + self._fd, _TFD_TIMER_ABSTIME, ctypes.byref(spec), None + ) + if rc != 0: + errno = ctypes.get_errno() + raise OSError(errno, f"timerfd_settime failed: {os.strerror(errno)}") + await self._tick.wait() + finally: + self._active_waiter = False + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._loop.remove_reader(self._fd) + os.close(self._fd) + + +class ThreadPacer: + """Cross-platform fallback: absolute-deadline sleeps in a dedicated thread. + + ``time.sleep`` in a plain thread bypasses the event loop's ~1ms timer + wheel entirely: CPython backs it with ``clock_nanosleep`` on POSIX + (~50-100us wakeup precision, ``mach``-timer based on macOS) and + high-resolution waitable timers on Windows (~0.5ms, Python 3.11+). The + thread wakes the event loop via ``call_soon_threadsafe``, which every + loop implementation supports — including the Windows proactor loop, + where fd-reader integration (the timerfd approach) is unavailable. + + Same contract as :class:`TimerFdPacer`: construct inside a running event + loop, one waiter at a time, ``close()`` when done. + """ + + def __init__(self) -> None: + self._loop = asyncio.get_running_loop() + self._tick = asyncio.Event() + self._condition = threading.Condition() + self._deadline: float | None = None + self._generation = 0 + self._waiting_generation: int | None = None + self._closed = False + self._active_waiter = False + self._thread = threading.Thread( + target=self._run, name="aiperf-rate-pacer", daemon=True + ) + self._thread.start() + + def _set_tick_if_current(self, generation: int) -> None: + """Wake the waiter only if ``generation`` is still the live deadline. + + Runs on the loop thread. A cancelled or superseded ``sleep_until`` + bumps ``_generation``, so its in-flight wakeup is dropped here rather + than releasing the next waiter ahead of its own deadline. + """ + with self._condition: + if not self._closed and self._generation == generation: + self._tick.set() + + def _run(self) -> None: + """Sleep to each deadline, waking the loop when it expires. + + Waits on the condition rather than ``time.sleep`` so a cancellation or + a replacement deadline interrupts the current wait immediately. + """ + while True: + with self._condition: + while not self._closed: + deadline = self._deadline + if deadline is None: + self._condition.wait() + continue + + generation = self._generation + remaining = deadline - time.perf_counter() + if remaining <= 0: + self._deadline = None + self._waiting_generation = None + break + + self._waiting_generation = generation + self._condition.wait(timeout=remaining) + if self._waiting_generation == generation: + self._waiting_generation = None + if self._closed: + return + + # The loop may already be closing during teardown. Check the + # generation on the loop thread so a stale callback cannot wake a + # later sleep_until call after cancellation and reuse. + with contextlib.suppress(RuntimeError): + self._loop.call_soon_threadsafe(self._set_tick_if_current, generation) + + async def sleep_until(self, deadline_perf_s: float) -> None: + """Sleep until an absolute ``time.perf_counter()`` deadline. + + Returns immediately if the deadline is already in the past. + """ + assert not self._active_waiter, ( + "ThreadPacer supports only one concurrent waiter" + ) + self._active_waiter = True + try: + with self._condition: + self._generation += 1 + generation = self._generation + self._deadline = deadline_perf_s + self._tick.clear() + self._condition.notify() + try: + await self._tick.wait() + except asyncio.CancelledError: + with self._condition: + if self._generation == generation: + self._generation += 1 + self._deadline = None + self._condition.notify() + raise + finally: + self._active_waiter = False + + def close(self) -> None: + with self._condition: + if self._closed: + return + self._closed = True + self._generation += 1 + self._deadline = None + self._condition.notify_all() diff --git a/src/aiperf/timing/strategies/request_rate.py b/src/aiperf/timing/strategies/request_rate.py index 1237a42288..b948a4ed20 100644 --- a/src/aiperf/timing/strategies/request_rate.py +++ b/src/aiperf/timing/strategies/request_rate.py @@ -8,7 +8,8 @@ import time from typing import TYPE_CHECKING -from aiperf.common.constants import MILLIS_PER_SECOND, NANOS_PER_SECOND +from aiperf.common.constants import IS_LINUX, MILLIS_PER_SECOND, NANOS_PER_SECOND +from aiperf.common.environment import Environment from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.common.utils import yield_to_event_loop from aiperf.credit.dispatch import ChildDispatchResult @@ -23,6 +24,7 @@ from aiperf.timing.branch_orchestrator import BranchOrchestrator from aiperf.timing.config import CreditPhaseConfig from aiperf.timing.conversation_source import ConversationSource + from aiperf.timing.high_res_timer import ThreadPacer, TimerFdPacer from aiperf.timing.phase.lifecycle import PhaseLifecycle from aiperf.timing.phase.stop_conditions import StopConditionChecker @@ -121,11 +123,114 @@ def __init__( ) self._rate_generator = GeneratorClass(interval_config) self._rate_update_event = asyncio.Event() + # High-resolution pacer for sub-millisecond sleep precision; created + # lazily in execute_phase (requires a running event loop), closed on exit. + self._pacer: TimerFdPacer | ThreadPacer | None = None async def setup_phase(self) -> None: """Setup the phase.""" pass # Already setup in __init__ + def _create_high_res_pacer(self) -> TimerFdPacer | ThreadPacer | None: + """Create the best available high-resolution pacer, else None. + + Selection order: Linux timerfd (kernel hrtimer, in-loop fd) -> + dedicated sleep thread (all platforms) -> event-loop timers (None). + """ + if not Environment.TIMING.HIGH_RES_TIMER: + return None + if IS_LINUX: + try: + from aiperf.timing.high_res_timer import TimerFdPacer + + pacer = TimerFdPacer() + self.info("Rate loop pacing via timerfd (kernel hrtimer)") + return pacer + except OSError as e: + self.warning(f"timerfd unavailable ({e}); trying pacing thread") + try: + from aiperf.timing.high_res_timer import ThreadPacer + + pacer = ThreadPacer() + self.info("Rate loop pacing via dedicated sleep thread") + return pacer + except (OSError, RuntimeError) as e: + self.warning( + f"high-res pacing unavailable ({e}); falling back to event-loop timers" + ) + return None + + def _reschedule_for_rate_update(self, next_target_perf: float) -> float: + """Clear a rate update and calculate its earliest eligible target.""" + self._rate_update_event.clear() + return min( + next_target_perf, + time.perf_counter() + self._rate_generator.next_interval(), + ) + + async def _wait_for_pacer_or_rate_update(self, deadline_perf_s: float) -> bool: + """Wait for a high-resolution deadline or a request-rate update.""" + if self._pacer is None: + return False + if self._rate_update_event.is_set(): + return True + + pacer_task = asyncio.create_task(self._pacer.sleep_until(deadline_perf_s)) + rate_update_task = asyncio.create_task(self._rate_update_event.wait()) + try: + done, _ = await asyncio.wait( + {pacer_task, rate_update_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + for task in (pacer_task, rate_update_task): + if not task.done(): + task.cancel() + # Shielded so a second cancellation of the enclosing task (eager + # cancellation, or an external shutdown racing a first cancel) + # cannot interrupt the gather and leave the pacer's tick wait + # pending. The outer CancelledError is re-raised after cleanup. + await asyncio.shield( + asyncio.gather(pacer_task, rate_update_task, return_exceptions=True) + ) + + if pacer_task in done: + pacer_task.result() + return rate_update_task in done + + async def _wait_until_next_target( + self, next_target_perf: float, now: float + ) -> bool: + """Wait until the target and return whether a rate update won.""" + sleep_duration = next_target_perf - now + if sleep_duration <= 0: + await yield_to_event_loop() + return False + + if ( + self._pacer is not None + and sleep_duration <= Environment.TIMING.RATE_RAMP_UPDATE_INTERVAL + ): + try: + return await self._wait_for_pacer_or_rate_update(next_target_perf) + except OSError as e: + self.warning( + f"high-res pacing failed ({e}); falling back to event-loop timers" + ) + self._pacer.close() + self._pacer = None + return await self._wait_until_next_target( + next_target_perf, time.perf_counter() + ) + + try: + await asyncio.wait_for( + self._rate_update_event.wait(), timeout=sleep_duration + ) + except asyncio.TimeoutError: # noqa: UP041 - distinct on Python 3.10 + return False + return True + async def execute_phase(self) -> None: """Execute request rate main loop until stop condition reached. @@ -133,10 +238,16 @@ async def execute_phase(self) -> None: sleeping for relative intervals. This prevents drift accumulation over many iterations (relative sleeps compound small timing errors). - When falling behind (credit issuance took longer than the interval), - we reset to "now" rather than trying to catch up. This prioritizes - maintaining throughput (preventing bursts) over preserving the exact - arrival distribution. + When falling behind, we re-anchor to "now" only once the backlog + exceeds a bounded catch-up window (Environment.TIMING.MAX_CATCHUP_SECONDS) + rather than on every oversleep. Event-loop timers oversleep sub-ms waits + (~1ms under uvloop/libuv); unconditionally re-anchoring forfeits schedule + on every tick and silently under-delivers high request rates. Genuine + stalls beyond the window still re-anchor to avoid a burst storm. + + A high-resolution pacer (Environment.TIMING.HIGH_RES_TIMER) sleeps to + absolute deadlines with ~50us precision (Linux timerfd, else a dedicated + sleep thread), bypassing the event-loop timer wheel entirely. """ if self._lifecycle.started_at_perf_ns is None: raise RuntimeError("started_at_perf_ns is not set in the lifecycle") @@ -147,90 +258,79 @@ async def execute_phase(self) -> None: # The first turn of the next new session. Cached to avoid wasting samples from shuffle/sequential samplers. next_new_session_turn = self._conversation_source.next().build_first_turn() - while True: - now = time.perf_counter() + self._pacer = self._create_high_res_pacer() + try: + while True: + now = time.perf_counter() - # Behind schedule: reset to now instead of sending a burst to catch up. - # This sacrifices inter-arrival distribution accuracy for stable throughput. - if next_target_perf < now: - next_target_perf = now + # Behind schedule: re-anchor to now only past the bounded catch-up + # window, so sub-ms oversleeps stay on the original schedule. + if next_target_perf < now - Environment.TIMING.MAX_CATCHUP_SECONDS: + next_target_perf = now - if self._rate_update_event.is_set(): - self._rate_update_event.clear() - next_target_perf = min( - next_target_perf, - time.perf_counter() + self._rate_generator.next_interval(), - ) - continue - - sleep_duration = next_target_perf - now - if sleep_duration > 0: - try: - await asyncio.wait_for( - self._rate_update_event.wait(), timeout=sleep_duration + if self._rate_update_event.is_set(): + next_target_perf = self._reschedule_for_rate_update( + next_target_perf ) - except asyncio.TimeoutError: # noqa: UP041 - distinct on Python 3.10 - pass - else: - self._rate_update_event.clear() - next_target_perf = min( - next_target_perf, - time.perf_counter() + self._rate_generator.next_interval(), + continue + + if await self._wait_until_next_target(next_target_perf, now): + next_target_perf = self._reschedule_for_rate_update( + next_target_perf ) continue - else: - # CRITICAL: Always yield to event loop to allow callbacks to run. - # Without this, CONCURRENCY_BURST mode (0 interval) busy-loops and - # starves credit return callbacks, causing deadlock. - await yield_to_event_loop() - - # Schedule next interval BEFORE issuing credit. This way, variable - # credit issuance latency doesn't affect the timing of the next interval. - next_target_perf += self._rate_generator.next_interval() - - # Priority 1: Queued continuation turns from completed previous turns. - # These already hold session slots, so we just need prefill slots. - if not self._continuation_turns.empty(): - should_continue = await self._credit_issuer.issue_credit( - self._continuation_turns.get_nowait() - ) - if not should_continue: - return - # Priority 2: Start new session if allowed and slots available. - # try_issue_credit returns None if no slot (skip interval), False if - # stop condition reached (exit loop), True if issued successfully. - elif self._stop_checker.can_start_new_session(): - result = await self._credit_issuer.try_issue_credit( - next_new_session_turn - ) - match result: - case True: # Successfully issued credit - # Re-sample the next new turn for the next interval. - next_new_session_turn = ( - self._conversation_source.next().build_first_turn() - ) - case False: # Stop condition reached - self.debug( - "Exiting: stop condition reached after try_issue_credit" - ) + # Schedule next interval BEFORE issuing credit. This way, variable + # credit issuance latency doesn't affect the timing of the next interval. + next_target_perf += self._rate_generator.next_interval() + + # Priority 1: Queued continuation turns from completed previous turns. + # These already hold session slots, so we just need prefill slots. + if not self._continuation_turns.empty(): + should_continue = await self._credit_issuer.issue_credit( + self._continuation_turns.get_nowait() + ) + if not should_continue: return - case None: # No slot available, retry later - # Always yield to event loop to allow callbacks to run. - # This is especially critical to prevent deadlock in CONCURRENCY_BURST mode (0 interval). - await yield_to_event_loop() - - # Priority 3: No more sessions to start and queue is empty. - # Check if we're done sending entirely. - elif not self._stop_checker.can_send_any_turn(): - return - else: - # Can still send turns but queue is empty and can't start new - # sessions (session limit reached). Skip this interval and wait for - # continuation turns to arrive from callbacks. - # Always yield to event loop to allow callbacks to run. - # This is especially critical to prevent deadlock in CONCURRENCY_BURST mode (0 interval). - await yield_to_event_loop() + + # Priority 2: Start new session if allowed and slots available. + # try_issue_credit returns None if no slot (skip interval), False if + # stop condition reached (exit loop), True if issued successfully. + elif self._stop_checker.can_start_new_session(): + result = await self._credit_issuer.try_issue_credit( + next_new_session_turn + ) + match result: + case True: # Successfully issued credit + # Re-sample the next new turn for the next interval. + next_new_session_turn = ( + self._conversation_source.next().build_first_turn() + ) + case False: # Stop condition reached + self.debug( + "Exiting: stop condition reached after try_issue_credit" + ) + return + case None: # No slot available, retry later + # Always yield to event loop to allow callbacks to run. + # This is especially critical to prevent deadlock in CONCURRENCY_BURST mode (0 interval). + await yield_to_event_loop() + + # Priority 3: No more sessions to start and queue is empty. + # Check if we're done sending entirely. + elif not self._stop_checker.can_send_any_turn(): + return + else: + # Can still send turns but queue is empty and can't start new + # sessions (session limit reached). Skip this interval and wait for + # continuation turns to arrive from callbacks. + # Always yield to event loop to allow callbacks to run. + # This is especially critical to prevent deadlock in CONCURRENCY_BURST mode (0 interval). + await yield_to_event_loop() + finally: + if self._pacer is not None: + self._pacer.close() + self._pacer = None async def handle_credit_return( self, credit: Credit, *, error: str | None = None diff --git a/tests/unit/timing/strategies/test_request_rate.py b/tests/unit/timing/strategies/test_request_rate.py index 354d650eb6..5ef13287c5 100644 --- a/tests/unit/timing/strategies/test_request_rate.py +++ b/tests/unit/timing/strategies/test_request_rate.py @@ -241,6 +241,274 @@ def set_rate(self, new_rate: float) -> None: credit_issuer.try_issue_credit.assert_awaited_once() +@pytest.mark.asyncio +async def test_high_res_pacer_rate_update_reschedules_before_issuing_credit() -> None: + class FakeIntervalGenerator: + def __init__(self, config) -> None: + self.rate = config.request_rate + + def next_interval(self) -> float: + return 0.05 if self.rate == 0.1 else 0.001 + + def set_rate(self, new_rate: float) -> None: + self.rate = new_rate + + class FakePacer: + def __init__(self) -> None: + self.started = asyncio.Event() + self.cancelled = False + self.deadlines: list[float] = [] + + async def sleep_until(self, deadline_perf_s: float) -> None: + self.deadlines.append(deadline_perf_s) + if len(self.deadlines) == 1: + self.started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + self.cancelled = True + raise + + def close(self) -> None: + pass + + lifecycle = MagicMock() + lifecycle.started_at_perf_ns = int(time.perf_counter() * NANOS_PER_SECOND) + conversation_source = MagicMock() + conversation_source.next.return_value.build_first_turn.return_value = object() + credit_issuer = MagicMock() + credit_issuer.try_issue_credit = AsyncMock(return_value=False) + stop_checker = MagicMock() + stop_checker.can_start_new_session.return_value = True + config = CreditPhaseConfig( + phase=CreditPhase.PROFILING, + timing_mode=TimingMode.REQUEST_RATE, + request_rate=0.1, + total_expected_requests=1, + ) + with patch( + "aiperf.timing.strategies.request_rate.plugins.get_class", + return_value=FakeIntervalGenerator, + ): + strategy = RequestRateStrategy( + config=config, + conversation_source=conversation_source, + scheduler=MagicMock(), + stop_checker=stop_checker, + credit_issuer=credit_issuer, + lifecycle=lifecycle, + ) + + pacer = FakePacer() + with patch.object(strategy, "_create_high_res_pacer", return_value=pacer): + task = asyncio.create_task(strategy.execute_phase()) + await pacer.started.wait() + strategy.set_request_rate(1000.0) + await asyncio.wait_for(task, timeout=1.0) + + assert pacer.cancelled + assert len(pacer.deadlines) == 2 + assert pacer.deadlines[1] < pacer.deadlines[0] + credit_issuer.try_issue_credit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_high_res_pacer_os_error_falls_back_to_event_loop_timers() -> None: + class FakeIntervalGenerator: + def __init__(self, config) -> None: + self.rate = config.request_rate + + def next_interval(self) -> float: + return 0.001 + + class FailingPacer: + def __init__(self) -> None: + self.closed = False + + async def sleep_until(self, _deadline_perf_s: float) -> None: + raise OSError("timerfd failed") + + def close(self) -> None: + self.closed = True + + lifecycle = MagicMock() + lifecycle.started_at_perf_ns = int(time.perf_counter() * NANOS_PER_SECOND) + conversation_source = MagicMock() + conversation_source.next.return_value.build_first_turn.return_value = object() + credit_issuer = MagicMock() + credit_issuer.try_issue_credit = AsyncMock(return_value=False) + stop_checker = MagicMock() + stop_checker.can_start_new_session.return_value = True + config = CreditPhaseConfig( + phase=CreditPhase.PROFILING, + timing_mode=TimingMode.REQUEST_RATE, + request_rate=1000.0, + total_expected_requests=1, + ) + with patch( + "aiperf.timing.strategies.request_rate.plugins.get_class", + return_value=FakeIntervalGenerator, + ): + strategy = RequestRateStrategy( + config=config, + conversation_source=conversation_source, + scheduler=MagicMock(), + stop_checker=stop_checker, + credit_issuer=credit_issuer, + lifecycle=lifecycle, + ) + + pacer = FailingPacer() + with patch.object(strategy, "_create_high_res_pacer", return_value=pacer): + await asyncio.wait_for(strategy.execute_phase(), timeout=1.0) + + assert pacer.closed + credit_issuer.try_issue_credit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_execute_phase_cancelled_mid_pace_drains_pacer_and_closes() -> None: + """A cancel during active pacing drains the pacer instead of orphaning it. + + ``_wait_for_pacer_or_rate_update`` cancels its two child tasks in a + ``finally`` block and then awaits them through a shielded gather, so an + outer cancellation cannot abandon the pacer's in-flight sleep as a pending + task ("Task was destroyed but it is pending!") and leave its tick event + stale for the next ``sleep_until`` caller. Cancels twice to also cover a + repeat cancel (external shutdown racing a first cancel). + """ + + class FakeIntervalGenerator: + def __init__(self, config) -> None: + self.rate = config.request_rate + + def next_interval(self) -> float: + # Comfortably inside RATE_RAMP_UPDATE_INTERVAL (0.1s) so the + # high-res branch is taken, but long enough that setup overhead + # cannot push the first deadline into the past. + return 0.05 + + class FakePacer: + def __init__(self) -> None: + self.started = asyncio.Event() + self.sleep_cancelled = False + self.closed = False + + async def sleep_until(self, _deadline_perf_s: float) -> None: + self.started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + self.sleep_cancelled = True + raise + + def close(self) -> None: + self.closed = True + + lifecycle = MagicMock() + lifecycle.started_at_perf_ns = int(time.perf_counter() * NANOS_PER_SECOND) + conversation_source = MagicMock() + conversation_source.next.return_value.build_first_turn.return_value = object() + credit_issuer = MagicMock() + credit_issuer.try_issue_credit = AsyncMock(return_value=False) + stop_checker = MagicMock() + stop_checker.can_start_new_session.return_value = True + config = CreditPhaseConfig( + phase=CreditPhase.PROFILING, + timing_mode=TimingMode.REQUEST_RATE, + request_rate=20.0, + total_expected_requests=1, + ) + with patch( + "aiperf.timing.strategies.request_rate.plugins.get_class", + return_value=FakeIntervalGenerator, + ): + strategy = RequestRateStrategy( + config=config, + conversation_source=conversation_source, + scheduler=MagicMock(), + stop_checker=stop_checker, + credit_issuer=credit_issuer, + lifecycle=lifecycle, + ) + + pacer = FakePacer() + with patch.object(strategy, "_create_high_res_pacer", return_value=pacer): + task = asyncio.create_task(strategy.execute_phase()) + await asyncio.wait_for(pacer.started.wait(), timeout=5.0) + # A repeat cancel models an external shutdown racing the first one. + task.cancel() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + # The shielded gather keeps running after the outer cancel lands; give + # it the loop iterations it needs to finish draining. + for _ in range(10): + if pacer.sleep_cancelled: + break + await asyncio.sleep(0) + + assert pacer.sleep_cancelled, "pacer sleep was orphaned instead of drained" + assert pacer.closed, "pacer was not closed on the cancellation path" + credit_issuer.try_issue_credit.assert_not_awaited() + + +def _build_minimal_strategy() -> RequestRateStrategy: + class FakeIntervalGenerator: + def __init__(self, config): + self.rate = config.request_rate + + def next_interval(self) -> float: + return 0.01 + + config = CreditPhaseConfig( + phase=CreditPhase.PROFILING, + timing_mode=TimingMode.REQUEST_RATE, + request_rate=100.0, + total_expected_requests=1, + ) + with patch( + "aiperf.timing.strategies.request_rate.plugins.get_class", + return_value=FakeIntervalGenerator, + ): + return RequestRateStrategy( + config=config, + conversation_source=MagicMock(), + scheduler=MagicMock(), + stop_checker=MagicMock(), + credit_issuer=MagicMock(), + lifecycle=MagicMock(), + ) + + +class TestHighResPacerFactory: + """Tests for the high-resolution pacer selection factory.""" + + def test_returns_none_when_disabled(self) -> None: + """No pacer is created when HIGH_RES_TIMER is off.""" + strategy = _build_minimal_strategy() + with patch( + "aiperf.timing.strategies.request_rate.Environment.TIMING.HIGH_RES_TIMER", + False, + ): + assert strategy._create_high_res_pacer() is None + + async def test_returns_pacer_when_enabled(self) -> None: + """A pacer is created (and closes cleanly) when HIGH_RES_TIMER is on. + + Must run inside an event loop: both pacers capture the running loop. + """ + strategy = _build_minimal_strategy() + with patch( + "aiperf.timing.strategies.request_rate.Environment.TIMING.HIGH_RES_TIMER", + True, + ): + pacer = strategy._create_high_res_pacer() + assert pacer is not None + pacer.close() + + class TestConcurrencyBurstGenerator: def test_returns_zero(self) -> None: cfg = IntervalGeneratorConfig( diff --git a/tests/unit/timing/test_high_res_timer.py b/tests/unit/timing/test_high_res_timer.py new file mode 100644 index 0000000000..898bf96424 --- /dev/null +++ b/tests/unit/timing/test_high_res_timer.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import asyncio +import time + +import pytest + +from aiperf.common.constants import IS_LINUX +from aiperf.timing.high_res_timer import ThreadPacer, TimerFdPacer + +# The autouse no_sleep fixture rewrites asyncio.sleep to a bare event-loop yield, +# so spinning on it never releases the GIL and the pacer worker thread may never +# be scheduled. These helpers block in a worker thread (via asyncio.to_thread) so +# the real time.sleep hands the interpreter over without stalling the loop. + + +def _worker_is_waiting(pacer: ThreadPacer, timeout: float = 5.0) -> bool: + """Block until the pacer worker has parked on its deadline.""" + deadline = time.perf_counter() + timeout + while time.perf_counter() < deadline: + with pacer._condition: + if pacer._waiting_generation is not None: + return True + time.sleep(0.001) + return False + + +def _block_until(deadline_perf_s: float) -> None: + """Block in real time until an absolute perf_counter deadline.""" + remaining = deadline_perf_s - time.perf_counter() + if remaining > 0: + time.sleep(remaining) + + +@pytest.mark.asyncio +class TestThreadPacer: + async def test_sleep_until_wakes_at_deadline_not_before(self): + pacer = ThreadPacer() + try: + errors_us = [] + for _ in range(50): + deadline = time.perf_counter() + 0.0002 + await pacer.sleep_until(deadline) + errors_us.append((time.perf_counter() - deadline) * 1e6) + assert min(errors_us) >= 0.0 + finally: + pacer.close() + + async def test_sleep_until_past_deadline_returns_immediately(self): + pacer = ThreadPacer() + try: + start = time.perf_counter() + await pacer.sleep_until(start - 1.0) + assert time.perf_counter() - start < 0.1 + finally: + pacer.close() + + async def test_cancelled_sleep_does_not_wake_replacement_early(self) -> None: + pacer = ThreadPacer() + try: + first_deadline = time.perf_counter() + 0.3 + first_sleep = asyncio.create_task(pacer.sleep_until(first_deadline)) + assert await asyncio.to_thread(_worker_is_waiting, pacer), ( + "pacer worker did not start waiting" + ) + + first_sleep.cancel() + with pytest.raises(asyncio.CancelledError): + await first_sleep + + replacement_deadline = time.perf_counter() + 0.5 + replacement_sleep = asyncio.create_task( + pacer.sleep_until(replacement_deadline) + ) + # Past the cancelled sleep's deadline: a leaked stale tick would have + # completed the replacement early. + await asyncio.to_thread(_block_until, first_deadline + 0.05) + assert not replacement_sleep.done() + + await replacement_sleep + assert time.perf_counter() >= replacement_deadline + finally: + pacer.close() + await asyncio.to_thread(pacer._thread.join, 2) + assert not pacer._thread.is_alive() + + async def test_close_is_idempotent_and_stops_thread(self) -> None: + pacer = ThreadPacer() + pacer.close() + pacer.close() + await asyncio.to_thread(pacer._thread.join, 2) + assert not pacer._thread.is_alive() + + +@pytest.mark.asyncio +@pytest.mark.skipif(not IS_LINUX, reason="timerfd is Linux-only") +class TestTimerFdPacer: + async def test_sleep_until_wakes_at_deadline_not_before(self): + pacer = TimerFdPacer() + try: + errors_us = [] + for _ in range(50): + deadline = time.perf_counter() + 0.0002 + await pacer.sleep_until(deadline) + errors_us.append((time.perf_counter() - deadline) * 1e6) + assert min(errors_us) >= 0.0 + finally: + pacer.close() + + async def test_sleep_until_past_deadline_returns_immediately(self): + pacer = TimerFdPacer() + try: + start = time.perf_counter() + await pacer.sleep_until(start - 1.0) + assert time.perf_counter() - start < 0.05 + finally: + pacer.close() + + async def test_close_is_idempotent(self): + pacer = TimerFdPacer() + pacer.close() + pacer.close() From 6f5bd6989dfa904d7ce6a5c69703ea999c0d69f9 Mon Sep 17 00:00:00 2001 From: Anant Sharma Date: Fri, 14 Aug 2026 13:08:41 -0400 Subject: [PATCH 022/119] feat: build ffmpeg with a minimal codec allowlist (#1232) Signed-off-by: Anant Sharma (cherry picked from commit e659d2a95a4fd3f90defc45636f4e75ab27b6ae9) --- ATTRIBUTIONS-container.md | 72 ++++++++++++++++++- Dockerfile | 43 ++++++++--- docs/cli-options.md | 20 +++--- docs/genai-perf-feature-comparison.md | 2 +- docs/tutorials/synthetic-dataset.md | 2 +- docs/tutorials/synthetic-video.md | 56 ++++++++++----- pyproject.toml | 2 +- src/aiperf/common/enums/enums.py | 6 +- src/aiperf/config/dataset/video.py | 22 +++--- src/aiperf/config/flags/cli_config.py | 25 ++++--- .../component_integration/media/test_video.py | 14 ++-- tests/integration/test_video.py | 16 +++-- tests/scripts/chaos/local_cases.py | 4 +- .../dataset/generator/test_video_generator.py | 9 +-- tests/unit/dataset/loader/conftest.py | 9 +-- 15 files changed, 218 insertions(+), 84 deletions(-) diff --git a/ATTRIBUTIONS-container.md b/ATTRIBUTIONS-container.md index 75a9d079dd..25a8665fc5 100644 --- a/ATTRIBUTIONS-container.md +++ b/ATTRIBUTIONS-container.md @@ -17,7 +17,7 @@ This document provides attribution information for third-party software componen - **Website**: https://ffmpeg.org/ - **License**: LGPL v2.1+ - **Usage**: Video and audio processing library (included in runtime container) -- **Build Configuration**: Built without GPL components (`--disable-gpl --disable-nonfree --enable-libvorbis --enable-libvpx`) +- **Build Configuration**: Built without GPL or non-free components, with an explicit codec allowlist (`--disable-gpl --disable-nonfree --disable-everything --enable-libvorbis --enable-libvpx --enable-libopus`) **License Text:** @@ -46,6 +46,7 @@ The FFmpeg source code used to build this container is available at: - FFmpeg is dynamically linked and can be replaced by users - No FFmpeg source code modifications were made - Build configuration excludes GPL-licensed components +- The codec set is restricted to VP8/VP9 video and Vorbis/Opus audio; other codecs are not built into the distributed binary - Apache 2.0 licensed code in this project remains separate from LGPL components ### libvpx @@ -221,6 +222,67 @@ The libogg source code is available at: - libogg is dynamically linked with FFmpeg (via libvorbis) - BSD license is compatible with Apache 2.0 +### libopus + +**Component Information:** +- **Software**: libopus (Opus Interactive Audio Codec) +- **Version**: 1.3.1 (from Debian Bookworm) +- **Source**: Debian Bookworm +- **Website**: https://opus-codec.org/ +- **License**: BSD 3-Clause +- **Usage**: Opus audio codec library (included in runtime container, used by FFmpeg for MP4 audio encoding) + +**License Text:** + +> Copyright 2001-2011 Xiph.Org, Skype Limited, Octasic, +> Jean-Marc Valin, Timothy B. Terriberry, +> CSIRO, Gregory Maxwell, Mark Borgerding, +> Erik de Castro Lopo +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions +> are met: +> +> - Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> +> - Redistributions in binary form must reproduce the above copyright +> notice, this list of conditions and the following disclaimer in the +> documentation and/or other materials provided with the distribution. +> +> - Neither the name of Internet Society, IETF or IETF Trust, nor the +> names of specific contributors, may be used to endorse or promote +> products derived from this software without specific prior written +> permission. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +> ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +> A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +> OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +> SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +> LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +> DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +> THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +> (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +> OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +> +> Full license text: https://gitlab.xiph.org/xiph/opus/-/blob/v1.3.1/COPYING + +**Source Code Availability:** + +The libopus source code is available at: +- Debian package: https://packages.debian.org/bookworm/libopus-dev +- Debian source package: `apt-get source libopus` from Debian Bookworm repositories +- Upstream Xiph.org source: https://gitlab.xiph.org/xiph/opus + +**Compliance Notes:** + +- libopus binary is copied from Debian Bookworm base image +- No modifications were made to libopus source code +- libopus is dynamically linked with FFmpeg +- BSD license is compatible with Apache 2.0 + ### Bash **Component Information:** @@ -288,6 +350,12 @@ BSD 3-Clause is compatible with Apache 2.0: - Attribution requirements are satisfied through this document - No conflict with Apache 2.0 terms +### libopus (BSD 3-Clause) +BSD 3-Clause is compatible with Apache 2.0: +- BSD is a permissive license that allows redistribution with minimal restrictions +- Attribution requirements are satisfied through this document +- No conflict with Apache 2.0 terms + ### Bash (GPL v3+) GPL is compatible with Apache 2.0 when: - Bash runs as a separate executable and is not linked with Apache 2.0 code @@ -296,4 +364,4 @@ GPL is compatible with Apache 2.0 when: - Proper attribution is provided (as above) --- -*Last updated: March 30, 2026* +*Last updated: July 30, 2026* diff --git a/Dockerfile b/Dockerfile index b00c5a69c4..eee6984c9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -97,6 +97,7 @@ RUN mkdir -p /opt/licenses/dpkg \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ libogg-dev \ + libopus-dev \ libvorbis-dev \ libvpx-dev \ nasm \ @@ -106,7 +107,12 @@ RUN mkdir -p /opt/licenses/dpkg \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -# Download and build ffmpeg with libvpx (VP9 codec) +# Build ffmpeg with libvpx (VP9), libvorbis and libopus. A default build ships +# hundreds of codecs; AIPerf only encodes VP8/VP9 with Vorbis or Opus, so +# --disable-everything plus this allowlist keeps the set minimal, and +# --disable-autodetect stops configure linking stray dev packages. The artifact +# still exceeds the lists: muxers add aac_adtstoasc (not an AAC codec), and +# --enable-ffmpeg adds 18 filters. Verify with `ffmpeg -encoders`, not `-codecs`. ARG FFMPEG_VERSION=8.1.2 RUN wget https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz \ && tar -xf ffmpeg-${FFMPEG_VERSION}.tar.xz \ @@ -117,8 +123,26 @@ RUN wget https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz \ --disable-nonfree \ --enable-shared \ --disable-static \ + --disable-autodetect \ + --enable-zlib \ --enable-libvorbis \ --enable-libvpx \ + --enable-libopus \ + --disable-everything \ + --disable-v4l2-m2m \ + --disable-devices \ + --disable-network \ + --disable-programs \ + --enable-ffmpeg \ + --enable-ffprobe \ + --enable-encoder=libvpx_vp8,libvpx_vp9,libvorbis,libopus \ + --enable-decoder=rawvideo,png,pcm_s16le,pcm_s24le,pcm_s32le,pcm_u8 \ + --enable-demuxer=rawvideo,image2,wav,matroska,mov \ + --enable-muxer=webm,matroska,mp4 \ + --enable-parser=png,vp9,opus,vorbis \ + --enable-protocol=file,pipe \ + --enable-filter=scale,format,aformat,aresample,anull,null,copy \ + --enable-bsf=vp9_superframe \ --disable-doc \ --disable-htmlpages \ --disable-manpages \ @@ -136,19 +160,22 @@ RUN wget https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz \ && cp -P /usr/lib/*/libvorbis.so* /usr/lib/*/libvorbisenc.so* /opt/ffmpeg/lib/ 2>/dev/null || \ cp -P /usr/lib/libvorbis.so* /usr/lib/libvorbisenc.so* /opt/ffmpeg/lib/ 2>/dev/null || { echo "Error: libvorbis.so not found"; exit 1; } \ && cp -P /usr/lib/*/libogg.so* /opt/ffmpeg/lib/ 2>/dev/null || \ - cp -P /usr/lib/libogg.so* /opt/ffmpeg/lib/ 2>/dev/null || { echo "Error: libogg.so not found"; exit 1; } + cp -P /usr/lib/libogg.so* /opt/ffmpeg/lib/ 2>/dev/null || { echo "Error: libogg.so not found"; exit 1; } \ + && cp -P /usr/lib/*/libopus.so* /opt/ffmpeg/lib/ 2>/dev/null || \ + cp -P /usr/lib/libopus.so* /opt/ffmpeg/lib/ 2>/dev/null || { echo "Error: libopus.so not found"; exit 1; } # Collect copyright files for packages whose files we explicitly copy into the runtime. # `dpkg -S` resolves paths against the dpkg database, which only tracks files at -# their ORIGINAL locations. /opt/ffmpeg/lib/libvpx.so*, libvorbis.so*, libogg.so* -# were copied from /usr/lib/, so querying /opt/ffmpeg/lib/ returns nothing for -# them — we must query the /usr/lib/ source paths instead. /bin/bash is still -# at its dpkg-tracked location. +# their ORIGINAL locations. /opt/ffmpeg/lib/libvpx.so*, libvorbis.so*, libogg.so*, +# libopus.so* were copied from /usr/lib/, so querying /opt/ffmpeg/lib/ returns +# nothing for them — we must query the /usr/lib/ source paths instead. /bin/bash +# is still at its dpkg-tracked location. RUN { dpkg -S /bin/bash 2>/dev/null; \ for f in /usr/lib/*/libvpx.so* /usr/lib/libvpx.so* \ /usr/lib/*/libvorbis.so* /usr/lib/libvorbis.so* \ /usr/lib/*/libvorbisenc.so* /usr/lib/libvorbisenc.so* \ - /usr/lib/*/libogg.so* /usr/lib/libogg.so*; do \ + /usr/lib/*/libogg.so* /usr/lib/libogg.so* \ + /usr/lib/*/libopus.so* /usr/lib/libopus.so*; do \ [ -e "$f" ] && dpkg -S "$f" 2>/dev/null; \ done; \ } | awk -F: '{print $1}' \ @@ -271,7 +298,7 @@ COPY --from=python-licenses /opt/licenses/python/ATTRIBUTIONS-Python.md /license # Copy bash with executable permissions preserved using --chmod COPY --from=env-builder --chown=1000:1000 --chmod=755 /bin/bash /bin/bash -# Copy ffmpeg binaries and libraries (includes libvpx) +# Copy ffmpeg binaries and libraries (includes libvpx, libvorbis, libogg, libopus) COPY --from=env-builder --chown=1000:1000 /opt/ffmpeg /opt/ffmpeg ENV PATH="/opt/ffmpeg/bin${PATH:+:${PATH}}" ENV LD_LIBRARY_PATH="/opt/ffmpeg/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" diff --git a/docs/cli-options.md b/docs/cli-options.md index 1c55109faa..a641fd6b39 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -887,7 +887,7 @@ Algorithm for generating synthetic video content. Different types produce differ #### `--video-format` `` -Container format for generated video files. Supports `webm` (VP9, recommended, BSD-licensed) and `mp4` (H.264/H.265, widely compatible). Format choice affects compatibility, file size, and encoding options. Use `webm` for open-source workflows, `mp4` for maximum compatibility. +Container format for generated video files. Supports `webm` (VP9, recommended, BSD-licensed) and `mp4` (widely compatible container, VP9 by default). Format choice affects compatibility, file size, and encoding options. `mp4` is the more portable container, but note that the default VP9 video and Opus audio are less widely supported by players than H.264/AAC would be. **Choices:** @@ -898,7 +898,7 @@ Container format for generated video files. Supports `webm` (VP9, recommended, B #### `--video-codec` `` -The video codec to use for encoding. Common options: libvpx-vp9 (CPU, BSD-licensed, default for WebM), libx264 (CPU, GPL-licensed, widely compatible), libx265 (CPU, GPL-licensed, smaller files), h264_nvenc (NVIDIA GPU), hevc_nvenc (NVIDIA GPU, smaller files). Any FFmpeg-supported codec can be used. +The video codec to use for encoding. Common options: libvpx-vp9 (CPU, BSD-licensed, default), libvpx (CPU, BSD-licensed, VP8). Any codec the local FFmpeg supports can be used, but the AIPerf container ships a minimal FFmpeg build limited to VP8/VP9 video with Vorbis/Opus audio; codecs such as libx264 or h264_nvenc require an FFmpeg build that includes them.
_Default: `libvpx-vp9`_ #### `--video-audio-sample-rate` `` @@ -915,15 +915,15 @@ Number of audio channels to embed in generated video files. 0 = disabled (no aud #### `--video-audio-codec` `` -Audio codec for the embedded audio track. If not specified, auto-selects based on video format: aac for MP4, libvorbis for WebM. Options: aac, libvorbis, libopus. +Audio codec for the embedded audio track. If not specified, auto-selects based on video format: libopus for MP4, libvorbis for WebM. Options: libvorbis, libopus, aac. The AIPerf container ships only libvorbis and libopus; aac requires an FFmpeg build that includes an AAC encoder. libopus always encodes at 48 kHz, so any --video-audio-sample-rate is resampled during muxing. **Choices:** | | | | |-------|:-------:|-------------| -| `aac` | | AAC codec. Default for MP4 containers. | +| `aac` | | AAC codec. Not built into the AIPerf container's FFmpeg; selecting it requires an FFmpeg build that includes an AAC encoder. | | `libvorbis` | | Vorbis codec. Default for WebM containers. | -| `libopus` | | Opus codec. Alternative for WebM containers. | +| `libopus` | | Opus codec. Default for MP4 containers. Always encodes at 48 kHz regardless of the requested sample rate. | #### `--video-audio-depth` `` @@ -2422,7 +2422,7 @@ Algorithm for generating synthetic video content. Different types produce differ #### `--video-format` `` -Container format for generated video files. Supports `webm` (VP9, recommended, BSD-licensed) and `mp4` (H.264/H.265, widely compatible). Format choice affects compatibility, file size, and encoding options. Use `webm` for open-source workflows, `mp4` for maximum compatibility. +Container format for generated video files. Supports `webm` (VP9, recommended, BSD-licensed) and `mp4` (widely compatible container, VP9 by default). Format choice affects compatibility, file size, and encoding options. `mp4` is the more portable container, but note that the default VP9 video and Opus audio are less widely supported by players than H.264/AAC would be. **Choices:** @@ -2433,7 +2433,7 @@ Container format for generated video files. Supports `webm` (VP9, recommended, B #### `--video-codec` `` -The video codec to use for encoding. Common options: libvpx-vp9 (CPU, BSD-licensed, default for WebM), libx264 (CPU, GPL-licensed, widely compatible), libx265 (CPU, GPL-licensed, smaller files), h264_nvenc (NVIDIA GPU), hevc_nvenc (NVIDIA GPU, smaller files). Any FFmpeg-supported codec can be used. +The video codec to use for encoding. Common options: libvpx-vp9 (CPU, BSD-licensed, default), libvpx (CPU, BSD-licensed, VP8). Any codec the local FFmpeg supports can be used, but the AIPerf container ships a minimal FFmpeg build limited to VP8/VP9 video with Vorbis/Opus audio; codecs such as libx264 or h264_nvenc require an FFmpeg build that includes them.
_Default: `libvpx-vp9`_ #### `--video-audio-sample-rate` `` @@ -2450,15 +2450,15 @@ Number of audio channels to embed in generated video files. 0 = disabled (no aud #### `--video-audio-codec` `` -Audio codec for the embedded audio track. If not specified, auto-selects based on video format: aac for MP4, libvorbis for WebM. Options: aac, libvorbis, libopus. +Audio codec for the embedded audio track. If not specified, auto-selects based on video format: libopus for MP4, libvorbis for WebM. Options: libvorbis, libopus, aac. The AIPerf container ships only libvorbis and libopus; aac requires an FFmpeg build that includes an AAC encoder. libopus always encodes at 48 kHz, so any --video-audio-sample-rate is resampled during muxing. **Choices:** | | | | |-------|:-------:|-------------| -| `aac` | | AAC codec. Default for MP4 containers. | +| `aac` | | AAC codec. Not built into the AIPerf container's FFmpeg; selecting it requires an FFmpeg build that includes an AAC encoder. | | `libvorbis` | | Vorbis codec. Default for WebM containers. | -| `libopus` | | Opus codec. Alternative for WebM containers. | +| `libopus` | | Opus codec. Default for MP4 containers. Always encodes at 48 kHz regardless of the requested sample rate. | #### `--video-audio-depth` `` diff --git a/docs/genai-perf-feature-comparison.md b/docs/genai-perf-feature-comparison.md index 2636588cb5..dbc0b9042b 100644 --- a/docs/genai-perf-feature-comparison.md +++ b/docs/genai-perf-feature-comparison.md @@ -313,7 +313,7 @@ This comparison matrix shows the supported CLI options between GenAI-Perf and AI | **Video Width/Height** | `--video-width --video-height` | ❌ | ✅ | Resolution in pixels (both or neither) | | **Video Synth Type** | `--video-synth-type`
`{moving_shapes,grid_clock,noise}` | ❌ | ✅ | Synthetic content generator | | **Video Format** | `--video-format`
`{webm,mp4}` | ❌ | ✅ | Container format | -| **Video Codec** | `--video-codec` | ❌ | ✅ | Any FFmpeg-supported codec (libvpx-vp9, libx264, h264_nvenc, ...) | +| **Video Codec** | `--video-codec` | ❌ | ✅ | Any codec the local FFmpeg supports (libvpx-vp9 default; the AIPerf container ships VP8/VP9 only) | | **Embedded Audio Track** | `--video-audio-num-channels --video-audio-sample-rate --video-audio-codec --video-audio-depth` | ❌ | ✅ | Optional audio mux for video clips | | **Download Video Content** | `--download-video-content` | ❌ | ✅ | Include video download time in request latency | diff --git a/docs/tutorials/synthetic-dataset.md b/docs/tutorials/synthetic-dataset.md index 480019578f..86410db5aa 100644 --- a/docs/tutorials/synthetic-dataset.md +++ b/docs/tutorials/synthetic-dataset.md @@ -183,7 +183,7 @@ aiperf profile \ Video generation is fully documented in [Synthetic Video Generation](synthetic-video.md). Key points: - **Synthesis Types**: `moving_shapes` (animated geometry), `grid_clock` (grid with animation), or `noise` (random pixels) -- **Codecs**: CPU (`libvpx-vp9`, `libx264`, `libx265`) or GPU (`h264_nvenc`, `hevc_nvenc`) +- **Codecs**: `libvpx-vp9` (default) and `libvpx`; other FFmpeg codecs such as `libx264` or `h264_nvenc` require an FFmpeg build that includes them (not in the AIPerf container) - **Formats**: WebM (default) or MP4 **Prerequisite:** Video generation requires FFmpeg. For installations, see [Synthetic Video Tutorial](synthetic-video.md#installing-ffmpeg). diff --git a/docs/tutorials/synthetic-video.md b/docs/tutorials/synthetic-video.md index a886c56ef8..b2f06bfe04 100644 --- a/docs/tutorials/synthetic-video.md +++ b/docs/tutorials/synthetic-video.md @@ -197,7 +197,17 @@ Features: ### Video Codec Selection -Choose encoding codec based on your hardware and requirements: +Choose encoding codec based on your hardware and requirements. + +> [!IMPORTANT] +> AIPerf invokes whichever `ffmpeg` binary is on `PATH`. The container bundles a +> minimal build at `/opt/ffmpeg` supporting only VP8/VP9 video with Vorbis or +> Opus audio; a PyPI install uses the FFmpeg you have installed. +> +> Codecs outside that set — `libx264`, `libx265`, `h264_nvenc`, `hevc_nvenc` and +> `aac` — remain valid options, but encoding with them fails inside the +> container. To use them, run AIPerf with your own FFmpeg on `PATH`, or rebuild +> the container's FFmpeg with them enabled. #### CPU Encoding (Default) @@ -214,13 +224,14 @@ aiperf profile \ ``` **Available CPU Codecs:** -- `libvpx-vp9`: VP9 encoding, BSD-licensed (default, WebM format) -- `libx264`: H.264 encoding, GPL-licensed, widely compatible (MP4 format) -- `libx265`: H.265 encoding, GPL-licensed, smaller file sizes, slower encoding (MP4 format) +- `libvpx-vp9`: VP9 encoding, BSD-licensed (default, WebM and MP4 formats) +- `libvpx`: VP8 encoding, BSD-licensed +- `libx264`: H.264 encoding, GPL-licensed, widely compatible (MP4 format; not in the AIPerf container) +- `libx265`: H.265 encoding, GPL-licensed, smaller file sizes, slower encoding (MP4 format; not in the AIPerf container) #### GPU Encoding (NVIDIA) -For faster encoding with NVIDIA GPUs: +For faster encoding with NVIDIA GPUs, using an FFmpeg built with NVENC: ```bash aiperf profile \ @@ -233,7 +244,7 @@ aiperf profile \ --request-count 50 ``` -**Available NVIDIA GPU Codecs:** +**Available NVIDIA GPU Codecs** (require an FFmpeg built with NVENC; not in the AIPerf container): - `h264_nvenc`: H.264 GPU encoding - `hevc_nvenc`: H.265 GPU encoding, smaller files @@ -273,7 +284,7 @@ aiperf profile \ --request-count 20 ``` -This generates videos with a mono, 44.1 kHz audio track using an auto-selected codec (libvorbis for WebM, aac for MP4). +This generates videos with a mono audio track using an auto-selected codec (libvorbis for WebM, libopus for MP4). #### Audio Parameters @@ -281,7 +292,7 @@ This generates videos with a mono, 44.1 kHz audio track using an auto-selected c |---|---|---|---| | `--video-audio-num-channels` | `int` | `0` | 0 = disabled, 1 = mono, 2 = stereo | | `--video-audio-sample-rate` | `float` | `44.1` | Sample rate in kHz (8-96) | -| `--video-audio-codec` | `string` | auto | Audio codec (`aac`, `libvorbis`, `libopus`) | +| `--video-audio-codec` | `string` | auto | Audio codec (`libvorbis`, `libopus`, `aac`; `aac` is not in the AIPerf container) | | `--video-audio-depth` | `int` | `16` | Bit depth per sample (8, 16, 24, or 32) | #### Audio Codec Selection @@ -291,7 +302,14 @@ When `--video-audio-codec` is not specified, the codec is automatically selected | Video Format | Auto-Selected Audio Codec | |---|---| | WebM | `libvorbis` (Vorbis) | -| MP4 | `aac` (AAC) | +| MP4 | `libopus` (Opus) | + +> [!NOTE] +> `libopus` always encodes at 48 kHz. Whatever `--video-audio-sample-rate` you +> request is resampled to 48 kHz during muxing, so the audio stream in the +> generated file reports 48 kHz — this applies to every rate, including ones +> Opus itself supports such as 16 kHz. Use `libvorbis` if the track must keep +> its requested rate. You can override the auto-selection with an explicit codec: @@ -327,7 +345,7 @@ aiperf profile \ 1. A Gaussian noise audio signal is generated matching the video duration 2. The audio is encoded as 16-bit PCM WAV 3. FFmpeg muxes the video and audio streams together using `-shortest` to ensure duration alignment -4. The audio codec converts the WAV data to the target format (AAC, Vorbis, or Opus) +4. The audio codec converts the WAV data to the target format (Vorbis or Opus) 5. The resulting video+audio file is base64-encoded for API requests The audio generation uses a deterministic RNG seed (`dataset.video.audio`), so videos with audio are reproducible across runs when using `--random-seed`. @@ -337,7 +355,7 @@ The audio generation uses a deterministic RNG seed (`dataset.video.audio`), so v Factors affecting audio contribution to file size: - **Sample rate**: 48 kHz produces ~9% more data than 44.1 kHz - **Channels**: Stereo (2) doubles audio data compared to mono (1) -- **Codec**: Vorbis and Opus provide better compression than AAC at lower bitrates +- **Codec**: Opus generally compresses better than Vorbis at lower bitrates - **Duration**: Audio size scales linearly with video duration For most benchmarking scenarios, the audio track adds minimal overhead compared to the video stream. @@ -377,7 +395,7 @@ aiperf profile \ --video-height 1080 \ --video-fps 8 \ --video-duration 10.0 \ - --video-codec h264_nvenc \ + --video-codec libvpx-vp9 \ --concurrency 2 \ --request-count 20 ``` @@ -441,9 +459,9 @@ aiperf profile \ --video-fps 8 \ --video-duration 10.0 \ --video-format mp4 \ - --video-codec libx264 \ + --video-codec libvpx-vp9 \ --video-audio-num-channels 2 \ - --video-audio-sample-rate 44.1 \ + --video-audio-sample-rate 48 \ --concurrency 2 \ --request-count 20 ``` @@ -498,7 +516,7 @@ aiperf profile \ --video-width 640 \ --video-height 480 \ --video-format mp4 \ - --video-codec libx264 \ + --video-codec libvpx-vp9 \ --request-count 20 ``` @@ -515,8 +533,8 @@ This allows seamless integration with vision-language model APIs that accept bas ### Encoding Performance -- **CPU codecs** (`libvpx-vp9`, `libx264`, `libx265`): Slower but universally available -- **GPU codecs** (`h264_nvenc`, `hevc_nvenc`): Much faster, requires NVIDIA GPU +- **CPU codecs** (`libvpx-vp9`, `libvpx`): Slower but always available, including in the AIPerf container +- **GPU codecs** (`h264_nvenc`, `hevc_nvenc`): Much faster, requires an NVIDIA GPU and an FFmpeg built with NVENC - Higher resolution and frame rates increase encoding time ### Video Size Impact @@ -563,7 +581,7 @@ Error: Encoder 'h264_nvenc' not found Solutions: 1. Verify NVIDIA GPU is available: `nvidia-smi` 2. Check FFmpeg was compiled with NVENC support: `ffmpeg -encoders | grep nvenc` -3. Fall back to CPU codec: `--video-codec libvpx-vp9 --video-format webm` or `--video-codec libx264 --video-format mp4` +3. Fall back to CPU codec: `--video-codec libvpx-vp9 --video-format webm` or `--video-codec libvpx-vp9 --video-format mp4` ### Out of Memory @@ -595,7 +613,7 @@ All video-related parameters at a glance: |---|---|---| | `--video-audio-num-channels` | `0` | 0 = disabled, 1 = mono, 2 = stereo | | `--video-audio-sample-rate` | `44.1` | Sample rate in kHz (8-96) | -| `--video-audio-codec` | auto | Audio codec (`aac`, `libvorbis`, `libopus`) | +| `--video-audio-codec` | auto | Audio codec (`libvorbis`, `libopus`, `aac`; `aac` is not in the AIPerf container) | | `--video-audio-depth` | `16` | Bit depth per sample (8, 16, 24, or 32) | ## Summary diff --git a/pyproject.toml b/pyproject.toml index 6da315f417..e614263c13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -266,4 +266,4 @@ filterwarnings = [ [tool.codespell] skip = "*.pyc,*build*,tests/unit/transports/test_aiohttp_sse.py,tests/integration/assets/canary_reference_inputs.json,src/aiperf/server_metrics/units.py,src/aiperf/api/static/dashboard.html,src/aiperf/config/schema/aiperf-config.schema.json" -ignore-words-list = "timeslice,timeslices,optiona,disjointness,concurency,strat,clen" +ignore-words-list = "timeslice,timeslices,optiona,disjointness,concurency,strat,clen,anull" diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index b4043d36d8..44cf4a23eb 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -674,13 +674,15 @@ class VideoAudioCodec(CaseInsensitiveStrEnum): """Audio codecs for embedding audio in synthetic video files.""" AAC = "aac" - """AAC codec. Default for MP4 containers.""" + """AAC codec. Not built into the AIPerf container's FFmpeg; selecting it + requires an FFmpeg build that includes an AAC encoder.""" LIBVORBIS = "libvorbis" """Vorbis codec. Default for WebM containers.""" LIBOPUS = "libopus" - """Opus codec. Alternative for WebM containers.""" + """Opus codec. Default for MP4 containers. Always encodes at 48 kHz + regardless of the requested sample rate.""" class VideoSynthType(CaseInsensitiveStrEnum): diff --git a/src/aiperf/config/dataset/video.py b/src/aiperf/config/dataset/video.py index 6c8a93700d..9919801311 100644 --- a/src/aiperf/config/dataset/video.py +++ b/src/aiperf/config/dataset/video.py @@ -27,7 +27,7 @@ VIDEO_AUDIO_CODEC_MAP: dict[VideoFormat, VideoAudioCodec] = { VideoFormat.WEBM: VideoAudioCodec.LIBVORBIS, - VideoFormat.MP4: VideoAudioCodec.AAC, + VideoFormat.MP4: VideoAudioCodec.LIBOPUS, } @@ -91,8 +91,12 @@ def validate_config(self) -> Self: default=None, description="Audio codec for the embedded audio track. " "If not specified, auto-selects based on video format: " - "aac for MP4, libvorbis for WebM. " - "Options: aac, libvorbis, libopus.", + "libopus for MP4, libvorbis for WebM. " + "Options: libvorbis, libopus, aac. The AIPerf container ships only " + "libvorbis and libopus; aac requires an FFmpeg build that " + "includes an AAC encoder. " + "libopus always encodes at 48 kHz, so any " + "--video-audio-sample-rate is resampled during muxing.", ), ] @@ -185,7 +189,7 @@ class VideoConfig(BaseConfig): default=VideoFormat.WEBM, description="Container format for generated video files. " "webm: VP9 codec, BSD-licensed, recommended for open-source workflows. " - "mp4: H.264/H.265, widely compatible. " + "mp4: widely compatible container, VP9 by default. " "Format affects compatibility, file size, and encoding options.", ), ] @@ -195,10 +199,12 @@ class VideoConfig(BaseConfig): Field( default="libvpx-vp9", description="Video codec for encoding. " - "Common options: libvpx-vp9 (CPU, BSD-licensed, default for WebM), " - "libx264 (CPU, GPL, widely compatible), libx265 (CPU, GPL, smaller files), " - "h264_nvenc (NVIDIA GPU), hevc_nvenc (NVIDIA GPU, smaller files). " - "Any FFmpeg-supported codec can be used.", + "Common options: libvpx-vp9 (CPU, BSD-licensed, default), " + "libvpx (CPU, BSD-licensed, VP8). " + "Any codec the local FFmpeg supports can be used, but the AIPerf " + "container ships a minimal FFmpeg build limited to VP8/VP9 video " + "with Vorbis/Opus audio; codecs such as libx264 or h264_nvenc " + "require an FFmpeg build that includes them.", ), ] diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 3a2baab304..ea12ea6418 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -1658,9 +1658,11 @@ def url(self) -> str: video_format: Annotated[ VideoFormat, Field( - description="Container format for generated video files. Supports `webm` (VP9, recommended, BSD-licensed) and `mp4` (H.264/H.265, widely compatible). " + description="Container format for generated video files. Supports `webm` (VP9, recommended, BSD-licensed) and `mp4` (widely compatible container, VP9 by default). " "Format choice affects compatibility, file size, and encoding options. " - "Use `webm` for open-source workflows, `mp4` for maximum compatibility.", + "`mp4` is the more portable container, but note that the default VP9 " + "video and Opus audio are less widely supported by players than " + "H.264/AAC would be.", ), CLIParameter( name=("--video-format",), @@ -1673,11 +1675,12 @@ def url(self) -> str: Field( description=( "The video codec to use for encoding. Common options: " - "libvpx-vp9 (CPU, BSD-licensed, default for WebM), " - "libx264 (CPU, GPL-licensed, widely compatible), " - "libx265 (CPU, GPL-licensed, smaller files), " - "h264_nvenc (NVIDIA GPU), hevc_nvenc (NVIDIA GPU, smaller files). " - "Any FFmpeg-supported codec can be used." + "libvpx-vp9 (CPU, BSD-licensed, default), " + "libvpx (CPU, BSD-licensed, VP8). " + "Any codec the local FFmpeg supports can be used, but the AIPerf " + "container ships a minimal FFmpeg build limited to VP8/VP9 video " + "with Vorbis/Opus audio; codecs such as libx264 or h264_nvenc " + "require an FFmpeg build that includes them." ), ), CLIParameter( @@ -1722,8 +1725,12 @@ def url(self) -> str: Field( description="Audio codec for the embedded audio track. " "If not specified, auto-selects based on video format: " - "aac for MP4, libvorbis for WebM. " - "Options: aac, libvorbis, libopus.", + "libopus for MP4, libvorbis for WebM. " + "Options: libvorbis, libopus, aac. The AIPerf container ships only " + "libvorbis and libopus; aac requires an FFmpeg build that " + "includes an AAC encoder. " + "libopus always encodes at 48 kHz, so any " + "--video-audio-sample-rate is resampled during muxing.", ), CLIParameter( name=("--video-audio-codec",), diff --git a/tests/component_integration/media/test_video.py b/tests/component_integration/media/test_video.py index 59f8d4bb75..91aa17afb6 100644 --- a/tests/component_integration/media/test_video.py +++ b/tests/component_integration/media/test_video.py @@ -5,7 +5,7 @@ import shutil import pytest -from pytest import approx +from pytest import approx, param from tests.component_integration.conftest import ( ComponentIntegrationTestDefaults as defaults, @@ -99,18 +99,20 @@ class TestVideoAudio: @pytest.mark.slow @pytest.mark.parametrize( - "video_format,video_codec,expected_audio_codec", + "video_format,video_codec,expected_audio_codec,expected_sample_rate", [ - ("webm", "libvpx-vp9", "vorbis"), - ("mp4", "libx264", "aac"), + param("webm", "libvpx-vp9", "vorbis", 44100, id="webm-vorbis"), + # libopus always encodes at 48 kHz, whatever rate is requested. + param("mp4", "libvpx-vp9", "opus", 48000, id="mp4-opus"), ], - ) + ) # fmt: skip def test_video_audio_auto_codec( self, cli: AIPerfCLI, video_format: str, video_codec: str, expected_audio_codec: str, + expected_sample_rate: int, ): """Verify auto-selected audio codec matches video format.""" result = cli.run_sync( @@ -143,7 +145,7 @@ def test_video_audio_auto_codec( assert details.has_audio, f"Expected audio in {video_format} video" assert details.audio_codec == expected_audio_codec assert details.audio_channels == 1 - assert details.audio_sample_rate == 44100 + assert details.audio_sample_rate == expected_sample_rate @pytest.mark.slow def test_video_audio_stereo(self, cli: AIPerfCLI): diff --git a/tests/integration/test_video.py b/tests/integration/test_video.py index 0fdfb4a3ae..ce53a8356b 100644 --- a/tests/integration/test_video.py +++ b/tests/integration/test_video.py @@ -32,7 +32,7 @@ class TestVideo: "video_format,video_codec,check_fragmentation", [ ("webm", "libvpx-vp9", False), - ("mp4", "libx264", True), + ("mp4", "libvpx-vp9", True), ], ) async def test_video_generation_parameters( @@ -83,12 +83,13 @@ async def test_video_generation_parameters( ) @pytest.mark.parametrize( - "video_format,video_codec,expected_audio_codec", + "video_format,video_codec,expected_audio_codec,expected_sample_rate", [ - ("webm", "libvpx-vp9", "vorbis"), - ("mp4", "libx264", "aac"), + param("webm", "libvpx-vp9", "vorbis", 44100, id="webm-vorbis"), + # libopus always encodes at 48 kHz, whatever rate is requested. + param("mp4", "libvpx-vp9", "opus", 48000, id="mp4-opus"), ], - ) + ) # fmt: skip async def test_video_with_audio_embeds_correct_stream( self, cli: AIPerfCLI, @@ -96,6 +97,7 @@ async def test_video_with_audio_embeds_correct_stream( video_format: str, video_codec: str, expected_audio_codec: str, + expected_sample_rate: int, ): """Verify video+audio muxing produces correct audio stream per format.""" width, height, fps, duration = 320, 240, 4, 2.0 @@ -133,7 +135,7 @@ async def test_video_with_audio_embeds_correct_stream( assert details.has_audio, f"Expected audio stream in {video_format} video" assert details.audio_codec == expected_audio_codec assert details.audio_channels == 1 - assert details.audio_sample_rate == 44100 + assert details.audio_sample_rate == expected_sample_rate async def test_video_without_audio_has_no_audio_stream( self, @@ -198,7 +200,7 @@ def _seeded_rng(self): "video_format,video_codec", [ param(VideoFormat.WEBM, "libvpx-vp9", id="webm-vp9"), - param(VideoFormat.MP4, "libx264", id="mp4-h264"), + param(VideoFormat.MP4, "libvpx-vp9", id="mp4-vp9"), ], ) # fmt: skip def test_synthetic_video_records_container_duration( diff --git a/tests/scripts/chaos/local_cases.py b/tests/scripts/chaos/local_cases.py index 7ccd1f931c..76ae5e577d 100644 --- a/tests/scripts/chaos/local_cases.py +++ b/tests/scripts/chaos/local_cases.py @@ -779,7 +779,7 @@ def profile_full( "--video-audio-num-channels", "1", "--video-audio-codec", - "aac", + "libopus", "--video-audio-depth", "16", ), @@ -1169,7 +1169,7 @@ def profile_full( "--video-audio-num-channels", "1", "--video-audio-codec", - "aac", + "libopus", "--video-audio-depth", "16", ), diff --git a/tests/unit/dataset/generator/test_video_generator.py b/tests/unit/dataset/generator/test_video_generator.py index 2bbc241c9e..862abe1ddd 100644 --- a/tests/unit/dataset/generator/test_video_generator.py +++ b/tests/unit/dataset/generator/test_video_generator.py @@ -224,7 +224,7 @@ def test_create_video_with_pipes_fallback(self, base_config): @pytest.mark.parametrize( "video_format,codec,expected_movflags", [ - (VideoFormat.MP4, "libx264", "faststart"), + (VideoFormat.MP4, "libvpx-vp9", "faststart"), (VideoFormat.WEBM, "libvpx-vp9", None), ], ) @@ -377,7 +377,7 @@ def test_generate_audio_data_channels(self, channels): "video_format,expected_codec", [ (VideoFormat.WEBM, VideoAudioCodec.LIBVORBIS), - (VideoFormat.MP4, VideoAudioCodec.AAC), + (VideoFormat.MP4, VideoAudioCodec.LIBOPUS), ], ) def test_resolve_audio_codec_auto_select(self, video_format, expected_codec): @@ -388,7 +388,7 @@ def test_resolve_audio_codec_auto_select(self, video_format, expected_codec): duration=0.5, fps=2, format=video_format, - codec="libvpx-vp9" if video_format == VideoFormat.WEBM else "libx264", + codec="libvpx-vp9", synth_type=VideoSynthType.MOVING_SHAPES, audio=VideoAudioConfig(channels=1, codec=None), ) @@ -414,7 +414,8 @@ def test_resolve_audio_codec_unsupported_format_raises(self): generator._resolve_audio_codec() @pytest.mark.parametrize( - "explicit_codec", [VideoAudioCodec.LIBOPUS, VideoAudioCodec.AAC] + "explicit_codec", + [VideoAudioCodec.LIBOPUS, VideoAudioCodec.LIBVORBIS, VideoAudioCodec.AAC], ) def test_resolve_audio_codec_explicit_override(self, explicit_codec): """Explicit codec beats auto-select.""" diff --git a/tests/unit/dataset/loader/conftest.py b/tests/unit/dataset/loader/conftest.py index 8ba9384ccf..993709bd7f 100644 --- a/tests/unit/dataset/loader/conftest.py +++ b/tests/unit/dataset/loader/conftest.py @@ -393,7 +393,7 @@ def _create_video(name: str = "test_video.mp4"): # Use ffmpeg to create video ( ffmpeg.input(f"{temp_frame_dir}/frame_%03d.png", framerate=1) - .output(str(dest_path), vcodec="libx264", pix_fmt="yuv420p", t=1) + .output(str(dest_path), vcodec="libvpx-vp9", pix_fmt="yuv420p", t=1) .overwrite_output() .run(quiet=True) ) @@ -403,10 +403,11 @@ def _create_video(name: str = "test_video.mp4"): Path(temp_frame_dir).rmdir() except (ImportError, Exception): - # Fallback: create a minimal valid MP4 file - # This is a minimal MP4 with just headers (won't play but is valid for testing) + # Fallback: a minimal MP4 header stub with no encoded frames — it + # won't play, and loader tests only need a file that base64-encodes. + # Branded vp09 to match the libvpx-vp9 path above, not H.264. minimal_mp4 = bytes.fromhex( - "000000186674797069736f6d0000020069736f6d69736f32617663310000" + "000000186674797069736f6d0000020069736f6d69736f32767030390000" "0008667265650000002c6d6461740000001c6d6f6f7600000000006d7668" "6400000000000000000000000000000001000000" ) From 499ae8f11eb93e6c723415161ad5cd80866b6cee Mon Sep 17 00:00:00 2001 From: Anant Sharma Date: Fri, 14 Aug 2026 16:09:47 -0400 Subject: [PATCH 023/119] chore: raise aiohttp minimum to 3.14.3 (#1277) Signed-off-by: Anant Sharma (cherry picked from commit 9e96b499d1e942e1b338010fa5d99d9d9c96e074) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e614263c13..51ccc5585e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ authors = [ requires-python = ">=3.11,<3.14" dependencies = [ "aiofiles~=24.1.0", - "aiohttp~=3.13.3", + "aiohttp>=3.14.3,<4", "cyclopts>=4,<5", "fastapi>=0.115,<1", "ffmpeg-python~=0.2.0", From 036ec7b2e0c71975ffb46b0916453c8c80eb7c09 Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:57:31 -0700 Subject: [PATCH 024/119] fix: align adaptive-scale request_error_rate SLA with exported metric unit (#1240) Signed-off-by: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> (cherry picked from commit 260d00f5e90505c74a5db8c503118111211c7f99) --- docs/tutorials/adaptive-scale.md | 16 +++- .../timing/strategies/adaptive_scale_sla.py | 36 +++++++- .../timing/strategies/test_adaptive_scale.py | 85 ++++++++++++++++++- 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/docs/tutorials/adaptive-scale.md b/docs/tutorials/adaptive-scale.md index d3dbc9f94b..8f852a76f3 100644 --- a/docs/tutorials/adaptive-scale.md +++ b/docs/tutorials/adaptive-scale.md @@ -176,6 +176,20 @@ Adaptive scale supports these SLA metric families in this release: | Error rate | `error_rate`, `request_error_rate` | `avg`, `min`, `max` | | Cancellation rate | `cancellation_rate`, `request_cancellation_rate` | `avg`, `min`, `max` | +Thresholds use each metric's own unit, and the three rate metrics above do not +share one: + +| Metric family | Threshold unit | `le: 1` means | +| --- | --- | --- | +| Error rate | percentage points in `[0, 100]`, matching the exported `request_error_rate` metric; denominator is successes + errors, so cancellations are excluded | 1% | +| Success rate | fraction in `[0, 1]`; denominator is all window requests | 100% | +| Cancellation rate | fraction in `[0, 1]`; denominator is all window requests | 100% | + +Error rate changed to percentage points so that adaptive scale and the exported +`request_error_rate` metric agree; before that, `error_rate: {avg: {le: 0.05}}` +meant 5% and now means 0.05%. Existing configs written against the old fraction +scale must be multiplied by 100. + Quality goodput and goodput ratio require at least one request-quality filter, such as `request_latency`, `time_to_first_token`, or `inter_token_latency`, so the controller can decide which successful requests count as quality-passing. ## YAML-only configuration @@ -215,7 +229,7 @@ benchmark: sustain_duration: 5m sla: request_latency: {p95: {le: 500}} - error_rate: {avg: {le: 0.05}} + error_rate: {avg: {le: 5}} # percentage points: 5% ``` Each adaptive phase runs its own controller and writes its own phase-scoped event and summary files. Use `adaptive_scale_manifest.json` to discover all adaptive phases in the run and locate their artifacts. For general `name` / `kind` rules and repeated warmup or profiling windows, see [Multi-Phase Workflows](multi-phase-workflows.md). diff --git a/src/aiperf/timing/strategies/adaptive_scale_sla.py b/src/aiperf/timing/strategies/adaptive_scale_sla.py index a07a99ec9e..87e0b1a876 100644 --- a/src/aiperf/timing/strategies/adaptive_scale_sla.py +++ b/src/aiperf/timing/strategies/adaptive_scale_sla.py @@ -4,6 +4,7 @@ from __future__ import annotations +import logging import math from typing import TYPE_CHECKING @@ -16,6 +17,8 @@ from aiperf.config.sweep.adaptive import SLAFilter +_LOGGER = logging.getLogger(__name__) + LATENCY_STATS = { "avg", "min", @@ -222,11 +225,26 @@ def goodput_ratio_value( @staticmethod def error_rate_value(stats: WindowStats, stat: str) -> float: + """Window error rate in percentage points, matching the exported metric. + + The exported ``request_error_rate`` metric is ``100 * errors / + (successes + errors)``, so the adaptive-scale evaluator uses the same + unit, and the same successes+errors denominator shape: a threshold of + ``1`` means 1%, and cancellations are excluded. + + The counts are window-local and not identical to the exported metric's + inputs: ``WindowStats.errors`` also counts requests that completed + without the required latency data, which the exported metric credits as + successes; pre-sustain requests are dropped from the window entirely; + and this returns ``0.0`` on zero completions where the exported metric + raises ``NoMetricValue``. + """ match stat: case "avg" | "min" | "max": - if stats.total == 0: + completed = len(stats.samples) + stats.errors + if completed == 0: return 0.0 - return stats.errors / stats.total + return 100.0 * stats.errors / completed raise ValueError(f"Unsupported error_rate SLA stat: {stat}") @staticmethod @@ -325,6 +343,20 @@ def validate_single_filter(sla: SLAFilter) -> None: ) if sla.stat not in allowed_stats: raise ValueError(f"Unsupported {metric_name} SLA stat: {sla.stat}") + if sla.metric_tag in {"error_rate", "request_error_rate"}: + if not 0.0 <= sla.threshold <= 100.0: + raise ValueError( + "error_rate SLA threshold is in percentage points and must be " + f"within [0, 100], got {sla.threshold}" + ) + if 0.0 < sla.threshold < 1.0: + _LOGGER.warning( + "error_rate SLA threshold %s is below 1 percentage point. " + "Adaptive-scale error_rate thresholds are percentage points " + "(1 means 1%%), not fractions; a config written for the old " + "0-1 fraction scale should be multiplied by 100.", + sla.threshold, + ) @staticmethod def _stat_family_for_metric(metric_tag: str) -> tuple[set[str], str]: diff --git a/tests/unit/timing/strategies/test_adaptive_scale.py b/tests/unit/timing/strategies/test_adaptive_scale.py index 9a0425756c..f384a4934e 100644 --- a/tests/unit/timing/strategies/test_adaptive_scale.py +++ b/tests/unit/timing/strategies/test_adaptive_scale.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import logging import re import time from pathlib import Path @@ -1199,7 +1200,7 @@ async def test_all_error_rate_sla_window_evaluates_without_successes(tmp_path) - metric_tag="error_rate", stat="avg", op="le", - threshold=1.0, + threshold=100.0, ) strategy = _strategy(tmp_path, adaptive_sla_filters=[error_sla]) strategy._window_errors = 2 @@ -1215,7 +1216,7 @@ async def test_all_error_rate_sla_window_evaluates_without_successes(tmp_path) - window = next(event for event in events if event["event"] == "adaptive_window") assert window["reason"] == "SLA window evaluated" assert window["sla_passed"] is True - assert window["sla_values"] == {"error_rate:avg:le:1": 1.0} + assert window["sla_values"] == {"error_rate:avg:le:100": 100.0} assert events[-1]["event"] == "adaptive_decision" @@ -1859,7 +1860,7 @@ def test_sla_evaluator_rate_metric_aliases_and_failures() -> None: assert evaluator.value( SLAFilter(metric_tag="request_error_rate", stat="avg", op="le", threshold=1), stats, - ) == pytest.approx(0.25) + ) == pytest.approx(100.0 / 3.0) assert evaluator.value( SLAFilter( metric_tag="request_cancellation_rate", stat="max", op="le", threshold=1 @@ -1908,13 +1909,40 @@ def test_sla_evaluator_supports_ttft_error_and_cancellation_rate() -> None: assert evaluator.value( SLAFilter(metric_tag="error_rate", stat="avg", op="le", threshold=0.5), stats, - ) == pytest.approx(0.25) + ) == pytest.approx(100.0 / 3.0) assert evaluator.value( SLAFilter(metric_tag="cancellation_rate", stat="avg", op="le", threshold=0.5), stats, ) == pytest.approx(0.25) +def test_error_rate_sla_matches_exported_metric_unit_and_denominator() -> None: + """request_error_rate must be percentage points over completed requests. + + The exported metric is ``100 * errors / (request_count + errors)``; the + adaptive-scale evaluator previously returned ``errors / total`` (a 0-1 + ratio whose denominator also included cancelled requests), so + ``request_error_rate:avg:le:1`` allowed a 100% error rate instead of 1%. + """ + from aiperf.timing.strategies.adaptive_scale_sla import AdaptiveScaleSLAEvaluator + from aiperf.timing.strategies.adaptive_scale_types import WindowStats + + evaluator = AdaptiveScaleSLAEvaluator() + stats = WindowStats( + samples=[100_000_000, 200_000_000], + errors=1, + cancelled=1, + elapsed_sec=2.0, + ) + + sla = SLAFilter(metric_tag="request_error_rate", stat="avg", op="le", threshold=1) + value = evaluator.value(sla, stats) + + # 100 * 1 / (2 successes + 1 error); the cancellation is excluded. + assert value == pytest.approx(100.0 / 3.0) + assert not evaluator.passes([sla], {evaluator.key(sla): value}) + + def test_missing_ttft_sample_fails_lower_is_better_sla() -> None: from aiperf.timing.strategies.adaptive_scale_sla import AdaptiveScaleSLAEvaluator from aiperf.timing.strategies.adaptive_scale_types import WindowStats @@ -2025,3 +2053,52 @@ def test_build_backend_rejects_invalid_bounds_and_unknown_variable(tmp_path) -> concurrency_manager=MagicMock(), config=config, ) + + +def test_error_rate_sla_threshold_out_of_percentage_range_is_rejected() -> None: + """error_rate thresholds are percentage points, so [0, 100] is the domain.""" + from aiperf.timing.strategies.adaptive_scale_sla import AdaptiveScaleSLAEvaluator + + evaluator = AdaptiveScaleSLAEvaluator() + + for threshold in (-1.0, 101.0): + with pytest.raises(ValueError, match="percentage points"): + evaluator.validate_single_filter( + SLAFilter( + metric_tag="error_rate", + stat="avg", + op="le", + threshold=threshold, + ) + ) + + for threshold in (0.0, 100.0): + evaluator.validate_single_filter( + SLAFilter( + metric_tag="request_error_rate", + stat="avg", + op="le", + threshold=threshold, + ) + ) + + +def test_error_rate_sla_fraction_style_threshold_warns(caplog) -> None: + """A pre-change fraction threshold such as 0.05 is accepted but flagged.""" + from aiperf.timing.strategies.adaptive_scale_sla import AdaptiveScaleSLAEvaluator + + evaluator = AdaptiveScaleSLAEvaluator() + + with caplog.at_level(logging.WARNING): + evaluator.validate_single_filter( + SLAFilter(metric_tag="error_rate", stat="avg", op="le", threshold=0.05) + ) + + assert "percentage points" in caplog.text + + caplog.clear() + with caplog.at_level(logging.WARNING): + evaluator.validate_single_filter( + SLAFilter(metric_tag="error_rate", stat="avg", op="le", threshold=5.0) + ) + assert caplog.text == "" From bb3e310670f390399559c308745ab4ff25a34fb5 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:40:47 -0700 Subject: [PATCH 025/119] feat(metrics): add --per-chunk-usage to fix TPS/user inflation from bundled first streamed chunk (#1279) Signed-off-by: lkomali (cherry picked from commit 324bb05773b3f99743c6516018f3c30cfe33de0b) --- docs/cli-options.md | 10 ++ docs/metrics-reference.md | 5 +- .../common/models/model_endpoint_info.py | 6 ++ src/aiperf/common/models/record_models.py | 33 ++++++ src/aiperf/config/endpoint.py | 43 ++++++++ .../config/flags/_converter_endpoint.py | 1 + src/aiperf/config/flags/_section_fields.py | 1 + src/aiperf/config/flags/cli_config.py | 20 ++++ src/aiperf/endpoints/openai_chat.py | 35 +++--- .../types/inter_token_latency_metric.py | 51 ++++++++- src/aiperf/records/inference_result_parser.py | 12 +++ tests/aiperf_mock_server/models.py | 13 +++ tests/aiperf_mock_server/utils.py | 99 +++++++++++------ .../test_per_chunk_usage_metric.py | 89 +++++++++++++++ .../test_converter_endpoint_template_paths.py | 24 +++++ tests/unit/config/test_endpoint.py | 41 +++++++ .../endpoints/test_openai_chat_completions.py | 35 ++++++ tests/unit/metrics/conftest.py | 2 + .../test_inter_token_latency_metric.py | 102 ++++++++++++++++++ .../records/test_inference_result_parser.py | 91 ++++++++++++++++ 20 files changed, 666 insertions(+), 47 deletions(-) create mode 100644 tests/component_integration/test_per_chunk_usage_metric.py diff --git a/docs/cli-options.md b/docs/cli-options.md index a641fd6b39..6fe6fbb631 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -410,6 +410,11 @@ Use the legacy 'max_tokens' field instead of 'max_completion_tokens' in request Use server-reported token counts from API usage fields instead of client-side tokenization. When enabled, tokenizers are still loaded (needed for dataset generation) but tokenizer.encode() is not called for computing metrics. Token count fields will be None if the server does not provide usage information. For OpenAI-compatible streaming endpoints (chat/completions), stream_options.include_usage is automatically configured when this flag is enabled.
_Flag (no value required)_ +#### `--per-chunk-usage` + +Request per-chunk token usage on streaming responses by setting stream_options.continuous_usage_stats, so the server reports cumulative usage on every chunk instead of only the final one. This lets inter-token latency subtract the first content chunk's real token count (fixing TPS/user inflation when a server bundles multiple tokens into the first streamed chunk). Requires a server that supports continuous_usage_stats (e.g. vLLM, TRT-LLM); strict OpenAI rejects it. Requires --use-server-token-count. +
_Flag (no value required)_ + #### `--connection-reuse-strategy` `` Transport connection reuse strategy. 'pooled' (default): connections are pooled and reused across all requests. 'never': new connection for each request, closed after response. 'sticky-user-sessions': connection persists across turns of a multi-turn conversation, closed on final turn (enables sticky load balancing). @@ -1945,6 +1950,11 @@ Use the legacy 'max_tokens' field instead of 'max_completion_tokens' in request Use server-reported token counts from API usage fields instead of client-side tokenization. When enabled, tokenizers are still loaded (needed for dataset generation) but tokenizer.encode() is not called for computing metrics. Token count fields will be None if the server does not provide usage information. For OpenAI-compatible streaming endpoints (chat/completions), stream_options.include_usage is automatically configured when this flag is enabled.
_Flag (no value required)_ +#### `--per-chunk-usage` + +Request per-chunk token usage on streaming responses by setting stream_options.continuous_usage_stats, so the server reports cumulative usage on every chunk instead of only the final one. This lets inter-token latency subtract the first content chunk's real token count (fixing TPS/user inflation when a server bundles multiple tokens into the first streamed chunk). Requires a server that supports continuous_usage_stats (e.g. vLLM, TRT-LLM); strict OpenAI rejects it. Requires --use-server-token-count. +
_Flag (no value required)_ + #### `--connection-reuse-strategy` `` Transport connection reuse strategy. 'pooled' (default): connections are pooled and reused across all requests. 'never': new connection for each request, closed after response. 'sticky-user-sessions': connection persists across turns of a multi-turn conversation, closed on final turn (enables sticky load balancing). diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index 4ac5fe79b8..4cb2341fb4 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -331,6 +331,7 @@ Measures the average time between consecutive tokens during generation, excludin **Formula:** ```python +# Default (all runs): assume one token in the first chunk. # Calculate in nanoseconds, then convert to seconds inter_token_latency_ns = (request_latency_ns - time_to_first_token_ns) / (output_sequence_length - 1) @@ -342,7 +343,9 @@ inter_token_latency_ms = inter_token_latency_ns / 1e6 ``` **Notes:** -- Requires an output sequence length of at least 2 tokens and valid `time_to_first_token`, `request_latency`, and `output_sequence_length` metrics. +- By default the divisor is `output_sequence_length - 1`, which is exact for servers that stream one token per chunk. This is the formula every run uses unless `--per-chunk-usage` is enabled. +- **Opt-in first-chunk correction (`--per-chunk-usage`).** The decode window (`request_latency - time_to_first_token`) covers only the tokens that arrive *after* the first content chunk. When a server bundles several tokens into the first streamed chunk (for example TRT-LLM's `stream-interval`), assuming a single token over-counts the decode tokens and inflates Output Token Throughput Per User (`1 / ITL`). Passing `--per-chunk-usage` (which also requires `--use-server-token-count`, a `chat` endpoint, and `--streaming`) makes the server report cumulative per-chunk usage via `stream_options.continuous_usage_stats`; the divisor then becomes `output_sequence_length - tokens_in_first_content_chunk`, where that count is the server-reported cumulative `completion_tokens` (reasoning included, matching OSL which is `output + reasoning = completion`) through the first content chunk. Requires a server that honors `continuous_usage_stats` (e.g. vLLM, TRT-LLM); otherwise the divisor stays `output_sequence_length - 1`. +- Requires an output sequence length of at least 2, plus valid `time_to_first_token`, `request_latency`, and `output_sequence_length` metrics. If the first-chunk count is inconsistent with OSL (>= OSL), the divisor degrades to `output_sequence_length - 1` and a one-time warning is logged rather than dropping the metric. - ITL is generated-token-normalized decode duration. It is not the total wall-clock decode duration. - Compare runs at equivalent output lengths, or inspect Decode Duration and Output Sequence Length alongside ITL. - Streaming chunks can contain multiple tokens. ITL uses token count, while ICL uses chunk arrival timestamps. diff --git a/src/aiperf/common/models/model_endpoint_info.py b/src/aiperf/common/models/model_endpoint_info.py index a9ba57a5f2..9229502bae 100644 --- a/src/aiperf/common/models/model_endpoint_info.py +++ b/src/aiperf/common/models/model_endpoint_info.py @@ -122,6 +122,11 @@ def _redact_headers(self, value: list[tuple[str, str]]) -> list[tuple[str, str]] default=EndpointDefaults.USE_SERVER_TOKEN_COUNT, description="Use server-reported token counts from API usage fields instead of client-side tokenization.", ) + per_chunk_usage: bool = Field( + default=EndpointDefaults.PER_CHUNK_USAGE, + description="Request per-chunk token usage via stream_options.continuous_usage_stats so " + "the server reports cumulative usage on every streamed chunk.", + ) connection_reuse_strategy: ConnectionReuseStrategy = Field( default=EndpointDefaults.CONNECTION_REUSE_STRATEGY, description="Transport connection reuse strategy.", @@ -212,6 +217,7 @@ def from_run(cls, run: BenchmarkRun) -> ModelEndpointInfo: api_key=ep.api_key, use_legacy_max_tokens=ep.use_legacy_max_tokens, use_server_token_count=ep.use_server_token_count, + per_chunk_usage=ep.per_chunk_usage, connection_reuse_strategy=ep.connection_reuse, download_video_content=ep.download_video_content, request_content_type=ep.request_content_type, diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 5dcebb5347..a0d7e8018b 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -1437,6 +1437,29 @@ def find_last_non_empty_usage(responses: list[ParsedResponse]) -> Usage | None: return None +def first_content_chunk_completion_tokens( + responses: list[ParsedResponse], +) -> int | None: + """Return the cumulative ``completion_tokens`` reported on the first content + chunk (the chunk whose arrival defines TTFT). + + Requires per-chunk (``continuous_usage_stats``) usage: it walks forward to the + first response carrying content (``data`` is not None) and reads its usage's + ``completion_tokens``. This is deliberately the raw completion count + (reasoning included), matching ``OutputSequenceLengthMetric`` -- OSL is + ``output + reasoning`` = ``completion_tokens`` -- so inter-token latency + subtracts operands in the same unit. Because per-chunk usage is cumulative, + that value is the number of tokens delivered through the first content chunk. + Returns ``None`` when no content chunk carries usage (e.g. the server only + reports the final total), so callers fall back to assuming one token in the + first chunk. + """ + for response in responses: + if response.data and response.usage: + return response.usage.completion_tokens + return None + + @dataclass(slots=True) class ParsedResponse: """Parsed response from a inference client.""" @@ -1502,6 +1525,16 @@ class TokenCounts: reasoning: int | None = None """The number of reasoning tokens. None if token count could not be calculated or the model does not support reasoning.""" + first_content_chunk_tokens: int | None = None + """The number of tokens delivered in the first content chunk -- the chunk whose + arrival defines TTFT -- taken from the server's per-chunk usage (cumulative + ``completion_tokens`` at that chunk). Inter-token latency subtracts this from the + decode-token count rather than assuming exactly one token arrived first, which + corrects the TPS/user inflation a server produces when it bundles multiple tokens + into the first streamed chunk (e.g. TRT-LLM ``stream-interval``). ``None`` unless + ``--per-chunk-usage`` is set and the server reports per-chunk usage; inter-token + latency then falls back to assuming one token in the first chunk.""" + @dataclass(slots=True) class MediaCounts: diff --git a/src/aiperf/config/endpoint.py b/src/aiperf/config/endpoint.py index 5ea37ea782..a39d6079ad 100644 --- a/src/aiperf/config/endpoint.py +++ b/src/aiperf/config/endpoint.py @@ -53,6 +53,7 @@ class EndpointDefaults: API_KEY = None USE_LEGACY_MAX_TOKENS = False USE_SERVER_TOKEN_COUNT = False + PER_CHUNK_USAGE = False CONNECTION_REUSE_STRATEGY = ConnectionReuseStrategy.POOLED DOWNLOAD_VIDEO_CONTENT = False REQUEST_CONTENT_TYPE = None @@ -242,6 +243,21 @@ def _redact_api_key(self, value: str | None) -> str | None: ), ] + per_chunk_usage: Annotated[ + bool, + Field( + default=False, + description="Request per-chunk token usage on streaming responses by " + "setting stream_options.continuous_usage_stats. When true, the server " + "reports cumulative usage on every chunk (not just the final one), which " + "lets inter-token latency subtract the first content chunk's real token " + "count instead of assuming one token. Requires a server that supports " + "continuous_usage_stats (e.g. vLLM, TRT-LLM); strict OpenAI rejects it. " + "Requires use_server_token_count so OSL and the first-chunk count share " + "the server source.", + ), + ] + template: Annotated[ TemplateConfig | None, Field( @@ -504,6 +520,33 @@ def _validate_uuid_and_strip(self) -> Self: raise ValueError("--uuid-and-strip requires endpoint type 'chat'") return self + @model_validator(mode="after") + def _validate_per_chunk_usage(self) -> Self: + """Require server token counts when per-chunk usage is requested. + + The first content chunk's token count is read only from server-reported + per-chunk usage (in the server-token-count path). Without + ``--use-server-token-count`` the client-tokenization path never populates + it, so ``--per-chunk-usage`` would silently have no effect on inter-token + latency.""" + if self.per_chunk_usage and not self.use_server_token_count: + raise ValueError( + "--per-chunk-usage requires --use-server-token-count " + "(the first-chunk token count comes from server-reported per-chunk usage)" + ) + if self.per_chunk_usage and self.type != EndpointType.CHAT: + raise ValueError( + "--per-chunk-usage requires endpoint type 'chat' " + "(continuous_usage_stats is only injected on the chat endpoint)" + ) + if self.per_chunk_usage and not self.streaming: + raise ValueError( + "--per-chunk-usage requires --streaming " + "(continuous_usage_stats and inter-token latency apply only to " + "streaming responses)" + ) + return self + @model_validator(mode="after") def _validate_wait_for_model_coherent(self) -> Self: """Reject configurations where probe sub-options are set to non-default diff --git a/src/aiperf/config/flags/_converter_endpoint.py b/src/aiperf/config/flags/_converter_endpoint.py index cee2871b23..b93d4ff110 100644 --- a/src/aiperf/config/flags/_converter_endpoint.py +++ b/src/aiperf/config/flags/_converter_endpoint.py @@ -68,6 +68,7 @@ def _endpoint_template_fallback(endpoint: dict[str, Any]) -> None: "transport": "transport", "use_legacy_max_tokens": "use_legacy_max_tokens", "use_server_token_count": "use_server_token_count", + "per_chunk_usage": "per_chunk_usage", "connection_reuse_strategy": "connection_reuse", "download_video_content": "download_video_content", "request_content_type": "request_content_type", diff --git a/src/aiperf/config/flags/_section_fields.py b/src/aiperf/config/flags/_section_fields.py index 0e939b38f7..adc1a144ce 100644 --- a/src/aiperf/config/flags/_section_fields.py +++ b/src/aiperf/config/flags/_section_fields.py @@ -29,6 +29,7 @@ "transport", "endpoint_type", "url_selection_strategy", + "per_chunk_usage", "urls", "use_legacy_max_tokens", "use_server_token_count", diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index ea12ea6418..dccbe4667d 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -329,6 +329,26 @@ class CLIConfig(BaseConfig): ), ] = EndpointDefaults.USE_SERVER_TOKEN_COUNT + per_chunk_usage: Annotated[ + bool, + Field( + description=( + "Request per-chunk token usage on streaming responses by setting " + "stream_options.continuous_usage_stats, so the server reports " + "cumulative usage on every chunk instead of only the final one. " + "This lets inter-token latency subtract the first content chunk's " + "real token count (fixing TPS/user inflation when a server bundles " + "multiple tokens into the first streamed chunk). Requires a server " + "that supports continuous_usage_stats (e.g. vLLM, TRT-LLM); strict " + "OpenAI rejects it. Requires --use-server-token-count." + ), + ), + CLIParameter( + name=("--per-chunk-usage",), + group=Groups.ENDPOINT, + ), + ] = EndpointDefaults.PER_CHUNK_USAGE + connection_reuse_strategy: Annotated[ ConnectionReuseStrategy, Field( diff --git a/src/aiperf/endpoints/openai_chat.py b/src/aiperf/endpoints/openai_chat.py index c2c8c099f5..e7385b7cd0 100644 --- a/src/aiperf/endpoints/openai_chat.py +++ b/src/aiperf/endpoints/openai_chat.py @@ -71,11 +71,15 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: if extra_body: payload.update(extra_body) + # per_chunk_usage implies use_server_token_count (enforced by the endpoint + # config validator), so gating on use_server_token_count alone is sufficient. if ( model_endpoint.endpoint.streaming and model_endpoint.endpoint.use_server_token_count ): - self._ensure_include_usage(payload) + self._ensure_include_usage( + payload, continuous=model_endpoint.endpoint.per_chunk_usage + ) self.trace(lambda: f"Formatted payload: {payload}") return payload @@ -145,18 +149,25 @@ def _extend_image_parts( ) @staticmethod - def _ensure_include_usage(payload: dict[str, Any]) -> None: - """Force ``stream_options.include_usage = True`` while preserving any - author-supplied stream_options keys (and any explicit ``include_usage`` - the author already set).""" - if "stream_options" not in payload: - payload["stream_options"] = {"include_usage": True} + def _ensure_include_usage( + payload: dict[str, Any], *, continuous: bool = False + ) -> None: + """Force ``stream_options.include_usage = True`` (and, when ``continuous``, + ``stream_options.continuous_usage_stats = True``) while preserving any + author-supplied stream_options keys (and any explicit values the author + already set). + + ``continuous_usage_stats`` asks the server to report cumulative usage on + every streamed chunk, not just the final one. It is a vLLM/TRT-LLM + extension (strict OpenAI rejects it), so it is only injected when the + caller opts in via ``--per-chunk-usage``. + """ + stream_options = payload.setdefault("stream_options", {}) + if not isinstance(stream_options, dict): return - if ( - isinstance(payload["stream_options"], dict) - and "include_usage" not in payload["stream_options"] - ): - payload["stream_options"]["include_usage"] = True + stream_options.setdefault("include_usage", True) + if continuous: + stream_options.setdefault("continuous_usage_stats", True) def parse_response( self, response: InferenceServerResponse diff --git a/src/aiperf/metrics/types/inter_token_latency_metric.py b/src/aiperf/metrics/types/inter_token_latency_metric.py index 2aa0f05242..8adff3afa7 100644 --- a/src/aiperf/metrics/types/inter_token_latency_metric.py +++ b/src/aiperf/metrics/types/inter_token_latency_metric.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.enums import MetricFlags, MetricTimeUnit from aiperf.common.exceptions import NoMetricValue from aiperf.common.models import ParsedResponseRecord @@ -13,13 +14,24 @@ from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.ttft_metric import TTFTMetric +_logger = AIPerfLogger(__name__) + class InterTokenLatencyMetric(BaseRecordMetric[float]): """ Post Processor for calculating Inter Token Latency (ITL) metric. Formula: - Inter Token Latency = (Request Latency - Time to First Token) / (Output Sequence Length - 1) + Inter Token Latency = (Request Latency - Time to First Token) / (Output Sequence Length - First Content Chunk Tokens) + + The decode window ``Request Latency - Time to First Token`` covers the tokens + that arrive AFTER the first content chunk, so the divisor subtracts that chunk's + token count rather than assuming exactly one token arrived first. When a server + bundles several tokens into the first streamed chunk (e.g. TRT-LLM + ``stream-interval``), assuming one token over-counts the decode tokens and + inflates TPS/user (``1 / ITL``). The first-chunk count comes from the server's + per-chunk usage (``--per-chunk-usage``); when it is unavailable the divisor falls + back to subtracting one, which is exact for servers that stream one token per chunk. """ tag = "inter_token_latency" @@ -38,6 +50,9 @@ class InterTokenLatencyMetric(BaseRecordMetric[float]): OutputSequenceLengthMetric.tag, } + # Warn only once per process when the first-chunk count is inconsistent with OSL. + _mismatch_warned: bool = False + def _parse_record( self, record: ParsedResponseRecord, @@ -50,10 +65,42 @@ def _parse_record( if osl < 2: # type: ignore raise NoMetricValue(f"Output sequence length must be at least 2, got {osl}") + # Subtract the first content chunk's real output-token count (the chunk that + # set TTFT) instead of a hard-coded 1, so a server that bundles the first + # chunk cannot inflate the decode-token count. The count is populated only + # under --per-chunk-usage from server per-chunk usage; None (or a + # non-positive lagging value) means "not reported", so fall back to + # assuming one token in the first chunk -- the legacy `osl - 1`. + first_chunk_tokens = ( + record.token_counts.first_content_chunk_tokens + if record.token_counts is not None + else None + ) + if first_chunk_tokens is None: + # Not reported (no --per-chunk-usage, or server sent no per-chunk usage): + # legacy divisor, no warning -- this is the expected default path. + decode_tokens = osl - 1 # type: ignore + else: + decode_tokens = osl - first_chunk_tokens # type: ignore + # A server-reported count that is non-positive or >= OSL is inconsistent + # (e.g. per-chunk usage lagging a chunk, or the whole response in one + # chunk). Degrade to `osl - 1` and warn once rather than silently + # degrading a value the server actually reported. + if first_chunk_tokens <= 0 or decode_tokens < 1: + if not type(self)._mismatch_warned: + type(self)._mismatch_warned = True + _logger.warning( + lambda: f"Inter-token latency: server-reported first content chunk " + f"token count ({first_chunk_tokens}) is inconsistent with output " + f"sequence length ({osl}); falling back to (OSL - 1). Check " + f"--per-chunk-usage server support." + ) + decode_tokens = osl - 1 # type: ignore + ttft = record_metrics.get_or_raise(TTFTMetric) request_latency = record_metrics.get_or_raise(RequestLatencyMetric) - return (request_latency - ttft) / (osl - 1) # type: ignore + return (request_latency - ttft) / decode_tokens # type: ignore class FullResponseInterTokenLatencyMetric(BaseRecordMetric[float]): diff --git a/src/aiperf/records/inference_result_parser.py b/src/aiperf/records/inference_result_parser.py index 6337905d8f..0c815bc390 100644 --- a/src/aiperf/records/inference_result_parser.py +++ b/src/aiperf/records/inference_result_parser.py @@ -25,6 +25,7 @@ TokenCounts, ToolCallResponseData, find_last_non_empty_usage, + first_content_chunk_completion_tokens, ) from aiperf.common.scenario import is_context_overflow_response from aiperf.common.tokenizer import Tokenizer @@ -515,10 +516,21 @@ async def _compute_server_token_counts( usage.completion_tokens, reasoning_token_count ) + # Only read the first content chunk's count when the user opted into + # per-chunk usage. Otherwise a server that volunteers usage on a + # content-bearing chunk would change (or delete) inter-token latency with + # no flag set -- the opt-in contract the docs describe must be enforced + # here, not just at payload-injection time. + first_chunk_tokens = ( + first_content_chunk_completion_tokens(responses) + if self.run.cfg.endpoint.per_chunk_usage + else None + ) token_counts = TokenCounts( input=input_token_count, reasoning=reasoning_token_count, output=output_token_count, + first_content_chunk_tokens=first_chunk_tokens, ) # Warn if server provided no usage information diff --git a/tests/aiperf_mock_server/models.py b/tests/aiperf_mock_server/models.py index 7072a330c0..70df116d40 100644 --- a/tests/aiperf_mock_server/models.py +++ b/tests/aiperf_mock_server/models.py @@ -53,12 +53,25 @@ class BaseCompletionRequest(BaseModel): max_tokens: int | None = None ignore_eos: bool = False min_tokens: int | None = None + mock_first_chunk_tokens: int = Field( + default=1, + description="Test knob: bundle the first N output tokens into the first " + "streamed chunk to emulate a server (e.g. TRT-LLM stream-interval) whose " + "first content chunk carries more than one token. Defaults to 1.", + ) @property def include_usage(self) -> bool: """Check if usage statistics should be included in streaming response.""" return bool(self.stream_options and self.stream_options.get("include_usage")) + @property + def continuous_usage_stats(self) -> bool: + """Check if per-chunk (cumulative) usage should be reported on every chunk.""" + return bool( + self.stream_options and self.stream_options.get("continuous_usage_stats") + ) + class ChatCompletionRequest(BaseCompletionRequest): """Request model for chat completion endpoints.""" diff --git a/tests/aiperf_mock_server/utils.py b/tests/aiperf_mock_server/utils.py index 09c6aabaa1..71c8a113ec 100644 --- a/tests/aiperf_mock_server/utils.py +++ b/tests/aiperf_mock_server/utils.py @@ -371,6 +371,12 @@ class RequestCtx: latency_sim: LatencySimulator """Latency simulator for TTFT and ITL timing.""" + continuous_usage: bool = False + """Emit cumulative usage on every streamed chunk (continuous_usage_stats).""" + + first_chunk_tokens: int = 1 + """Number of output tokens to bundle into the first streamed content chunk.""" + @property def tokens(self) -> list[str]: return self.tokenized.tokens @@ -424,6 +430,8 @@ def make_ctx( isl=tokenized.prompt_token_count, osl=len(tokenized.tokens), ), + continuous_usage=getattr(request, "continuous_usage_stats", False), + first_chunk_tokens=max(1, getattr(request, "mock_first_chunk_tokens", 1)), ) @@ -493,6 +501,26 @@ def _sse(data: dict[str, Any]) -> bytes: return _SSE_DATA_PREFIX + orjson.dumps(data) + _SSE_NEWLINES +def _bundle_first_chunk(tokens: list[str], first_chunk_tokens: int) -> list[list[str]]: + """Group output tokens into streamed chunks, bundling the first + ``first_chunk_tokens`` tokens into the first chunk; the rest stream one per + chunk. ``first_chunk_tokens=1`` reproduces one-token-per-chunk streaming.""" + if first_chunk_tokens <= 1 or len(tokens) <= 1: + return [[t] for t in tokens] + n = min(first_chunk_tokens, len(tokens)) + return [tokens[:n], *([t] for t in tokens[n:])] + + +def _partial_usage(ctx: "RequestCtx", completion_tokens: int) -> dict[str, Any]: + """Cumulative per-chunk usage in OpenAI shape (continuous_usage_stats).""" + prompt = ctx.usage.get("prompt_tokens", 0) + return { + "prompt_tokens": prompt, + "completion_tokens": completion_tokens, + "total_tokens": prompt + completion_tokens, + } + + async def stream_chat_completion( ctx: RequestCtx, endpoint: str, include_usage: bool ) -> AsyncGenerator[bytes, None]: @@ -501,47 +529,54 @@ async def stream_chat_completion( try: # Stream reasoning tokens first (if any) + completion_so_far = 0 for token in ctx.reasoning_content_tokens: await ctx.latency_sim.wait_for_next_token() record_streamed_token(endpoint, ctx.model) - yield _sse( - { - "id": ctx.request_id, - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": ctx.model, - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "reasoning_content": token}, - } - ], - } - ) - - # Stream output tokens - num_tokens = len(ctx.tokens) - for i, token in enumerate(ctx.tokens): - await ctx.latency_sim.wait_for_next_token() - record_streamed_token(endpoint, ctx.model) + completion_so_far += 1 + chunk: dict[str, Any] = { + "id": ctx.request_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": ctx.model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "reasoning_content": token}, + } + ], + } + if ctx.continuous_usage: + chunk["usage"] = _partial_usage(ctx, completion_so_far) + yield _sse(chunk) - delta: dict[str, Any] = {"content": token} - if i == 0 and not has_reasoning: + # Stream output tokens, bundling the first chunk when requested. + groups = _bundle_first_chunk(ctx.tokens, ctx.first_chunk_tokens) + num_groups = len(groups) + for gi, group in enumerate(groups): + for _ in group: + await ctx.latency_sim.wait_for_next_token() + record_streamed_token(endpoint, ctx.model) + completion_so_far += len(group) + + delta: dict[str, Any] = {"content": "".join(group)} + if gi == 0 and not has_reasoning: delta["role"] = "assistant" choice: dict[str, Any] = {"index": 0, "delta": delta} - if i == num_tokens - 1: + if gi == num_groups - 1: choice["finish_reason"] = ctx.finish_reason - yield _sse( - { - "id": ctx.request_id, - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": ctx.model, - "choices": [choice], - } - ) + chunk = { + "id": ctx.request_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": ctx.model, + "choices": [choice], + } + if ctx.continuous_usage: + chunk["usage"] = _partial_usage(ctx, completion_so_far) + yield _sse(chunk) # Final usage chunk (if requested) if include_usage: diff --git a/tests/component_integration/test_per_chunk_usage_metric.py b/tests/component_integration/test_per_chunk_usage_metric.py new file mode 100644 index 0000000000..6f01b83c64 --- /dev/null +++ b/tests/component_integration/test_per_chunk_usage_metric.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Component integration test: ``--per-chunk-usage`` drives the full path end to +end against the mock server -- ``continuous_usage_stats`` is requested, the mock +reports cumulative per-chunk usage, the first content chunk is bundled, and +inter-token latency is emitted from the server-reported first-chunk count. + +The mock is told to bundle the first three output tokens into the first streamed +chunk (``mock_first_chunk_tokens:3``), emulating a TRT-LLM ``stream-interval`` +server. ``test-model`` avoids the mock's reasoning path so the bundled output +chunk is genuinely the first content chunk. +""" + +import math + +import pytest + +from aiperf.common.models import RawRecordInfo +from tests.component_integration.conftest import ( + ComponentIntegrationTestDefaults as defaults, +) +from tests.harness.utils import AIPerfCLI + +FIRST_CHUNK_TOKENS = 3 + + +def _first_content_chunk_usage(raw_record: RawRecordInfo) -> int | None: + """Cumulative completion_tokens on the first streamed chunk carrying content.""" + for response in raw_record.responses: + chunk = response.get_json() + if not chunk: + continue + choices = chunk.get("choices") or [] + delta = (choices[0].get("delta") if choices else {}) or {} + if delta.get("content"): + usage = chunk.get("usage") or {} + return usage.get("completion_tokens") + return None + + +@pytest.mark.component_integration +class TestPerChunkUsageMetric: + """The --per-chunk-usage path works end to end and feeds ITL the real + first-content-chunk token count from the server's per-chunk usage.""" + + def test_per_chunk_usage_bundled_first_chunk(self, cli: AIPerfCLI) -> None: + result = cli.run_sync( + f""" + aiperf profile \ + --model test-model --tokenizer {defaults.tokenizer} \ + --endpoint-type chat --streaming \ + --per-chunk-usage --use-server-token-count \ + --extra-inputs ignore_eos:true \ + --extra-inputs mock_first_chunk_tokens:{FIRST_CHUNK_TOKENS} \ + --synthetic-input-tokens-mean 50 --output-tokens-mean 20 \ + --request-count {defaults.request_count} \ + --concurrency {defaults.concurrency} \ + --workers-max {defaults.workers_max} \ + --export-level raw \ + --ui {defaults.ui} + """, + timeout=defaults.timeout, + ) + assert result.request_count == defaults.request_count + + # ITL is emitted and finite/positive. + itl = getattr(result.json, "inter_token_latency", None) + assert itl is not None, "inter_token_latency missing from JSON export" + itl_avg = itl["avg"] if isinstance(itl, dict) else itl.avg + assert math.isfinite(itl_avg) and itl_avg > 0, ( + f"ITL not finite/positive: {itl_avg}" + ) + + # Deterministic proof the fix's input is correct: continuous_usage_stats + # was requested and the mock reported the first content chunk carrying the + # bundled count (3), which is what ITL subtracts. + assert result.raw_records, "raw records must be exported" + checked = 0 + for rec in result.raw_records: + stream_options = (rec.payload or {}).get("stream_options") or {} + assert stream_options.get("continuous_usage_stats") is True, ( + f"continuous_usage_stats not requested: {stream_options}" + ) + first_chunk = _first_content_chunk_usage(rec) + assert first_chunk == FIRST_CHUNK_TOKENS, ( + f"first content chunk usage should be {FIRST_CHUNK_TOKENS}, got {first_chunk}" + ) + checked += 1 + assert checked > 0, "no raw records inspected" diff --git a/tests/unit/config/test_converter_endpoint_template_paths.py b/tests/unit/config/test_converter_endpoint_template_paths.py index 86fc703427..deadb78053 100644 --- a/tests/unit/config/test_converter_endpoint_template_paths.py +++ b/tests/unit/config/test_converter_endpoint_template_paths.py @@ -28,7 +28,9 @@ from aiperf.config.flags._converter_endpoint import ( _endpoint_template_fallback, _endpoint_template_from_extra, + build_endpoint, ) +from aiperf.config.flags.cli_config import CLIConfig from aiperf.plugin.enums import EndpointType @@ -272,3 +274,25 @@ def test_non_utf8_file_returns_none(self, tmp_path: Path) -> None: bad.write_bytes(b"\xff\xfe binary template body") assert safe_read_template_path(str(bad)) is None + + +class TestEndpointFieldPropagation: + """Regression: fields set on CLIConfig must reach the endpoint config. + + A field can be wired on CLIConfig, the endpoint model, and the converter map + yet still be silently dropped if it is missing from ``ENDPOINT_FIELDS`` (which + ``build_endpoint`` intersects against ``model_fields_set``). These tests + exercise the full CLI-to-endpoint hop so such a gap fails loudly. + """ + + def test_per_chunk_usage_propagates_to_endpoint(self) -> None: + cli = CLIConfig(per_chunk_usage=True, use_server_token_count=True) + endpoint = build_endpoint(cli) + + assert endpoint["per_chunk_usage"] is True + assert endpoint["use_server_token_count"] is True + + def test_per_chunk_usage_absent_when_not_set(self) -> None: + endpoint = build_endpoint(CLIConfig()) + + assert "per_chunk_usage" not in endpoint diff --git a/tests/unit/config/test_endpoint.py b/tests/unit/config/test_endpoint.py index 3b10bd16d5..48ae9e1bb8 100644 --- a/tests/unit/config/test_endpoint.py +++ b/tests/unit/config/test_endpoint.py @@ -51,6 +51,47 @@ def test_uuid_and_strip_rejects_non_chat_endpoint() -> None: ) +def test_per_chunk_usage_requires_server_token_count() -> None: + with pytest.raises(ValueError, match="requires --use-server-token-count"): + EndpointConfig( + urls=["http://localhost:8000"], + per_chunk_usage=True, + use_server_token_count=False, + ) + + +def test_per_chunk_usage_requires_chat_endpoint() -> None: + with pytest.raises(ValueError, match="requires endpoint type 'chat'"): + EndpointConfig( + urls=["http://localhost:8000"], + type=EndpointType.COMPLETIONS, + per_chunk_usage=True, + use_server_token_count=True, + ) + + +def test_per_chunk_usage_requires_streaming() -> None: + with pytest.raises(ValueError, match="requires --streaming"): + EndpointConfig( + urls=["http://localhost:8000"], + type=EndpointType.CHAT, + per_chunk_usage=True, + use_server_token_count=True, + streaming=False, + ) + + +def test_per_chunk_usage_with_server_token_count_is_valid() -> None: + endpoint = EndpointConfig( + urls=["http://localhost:8000"], + type=EndpointType.CHAT, + per_chunk_usage=True, + use_server_token_count=True, + streaming=True, + ) + assert endpoint.per_chunk_usage is True + + def _make_model_endpoint( base_url: str, transport: TransportType | None = None ) -> ModelEndpointInfo: diff --git a/tests/unit/endpoints/test_openai_chat_completions.py b/tests/unit/endpoints/test_openai_chat_completions.py index 73870b7d74..1773e779d3 100644 --- a/tests/unit/endpoints/test_openai_chat_completions.py +++ b/tests/unit/endpoints/test_openai_chat_completions.py @@ -234,6 +234,41 @@ def test_stream_options_auto_configuration( assert payload["stream_options"] == expected_stream_options endpoint.build_messages(turns) + @pytest.mark.parametrize( + "streaming,per_chunk_usage,expected", + [ + # per_chunk_usage implies use_server_token_count (validator), so these + # are the only reachable states; use_server_token_count is always True. + (True, True, {"include_usage": True, "continuous_usage_stats": True}), + (True, False, {"include_usage": True}), + (False, True, None), + ], + ids=["on", "server_count_only", "non_streaming_no_injection"], + ) # fmt: skip + def test_per_chunk_usage_injects_continuous_usage_stats( + self, + model_endpoint, + sample_conversations, + streaming, + per_chunk_usage, + expected, + ): + """--per-chunk-usage adds stream_options.continuous_usage_stats (on top of + include_usage) on streaming server-token-count requests; server-token-count + alone adds only include_usage; non-streaming injects nothing.""" + endpoint = ChatEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = streaming + model_endpoint.endpoint.use_server_token_count = True + model_endpoint.endpoint.per_chunk_usage = per_chunk_usage + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + if expected is None: + assert "stream_options" not in payload + else: + assert payload["stream_options"] == expected + def test_create_messages_with_system_message( self, model_endpoint, sample_conversations ): diff --git a/tests/unit/metrics/conftest.py b/tests/unit/metrics/conftest.py index 5068a3f850..acabff0806 100644 --- a/tests/unit/metrics/conftest.py +++ b/tests/unit/metrics/conftest.py @@ -60,6 +60,7 @@ def create_record( responses: list[int] | None = None, input_tokens: int | None = None, output_tokens_per_response: int = 1, + first_content_chunk_tokens: int | None = None, error: ErrorDetails | None = None, ) -> ParsedResponseRecord: """ @@ -98,6 +99,7 @@ def create_record( input=input_tokens, output=len(responses) * output_tokens_per_response, reasoning=None, + first_content_chunk_tokens=first_content_chunk_tokens, ), ) diff --git a/tests/unit/metrics/test_inter_token_latency_metric.py b/tests/unit/metrics/test_inter_token_latency_metric.py index e972efc9ae..5be7acec6c 100644 --- a/tests/unit/metrics/test_inter_token_latency_metric.py +++ b/tests/unit/metrics/test_inter_token_latency_metric.py @@ -141,3 +141,105 @@ def test_requires_dependencies(self) -> None: FullResponseInterTokenLatencyMetric().parse_record( create_record(), MetricRecordDict() ) + def test_inter_token_latency_subtracts_bundled_first_chunk(self): + """When the server reports the first content chunk carried N tokens, ITL + divides the decode window by (OSL - N), not (OSL - 1), so a server that + bundles the first chunk cannot inflate TPS/user.""" + record = create_record( + start_ns=100, + responses=[120, 200], + output_tokens_per_response=3, + first_content_chunk_tokens=3, # first chunk carried 3 tokens, not 1 + ) + + metric_results = run_simple_metrics_pipeline( + [record], + RequestLatencyMetric.tag, + TTFTMetric.tag, + OutputSequenceLengthMetric.tag, + InterTokenLatencyMetric.tag, + ) + + # ttft=20, latency=100, OSL=6, first chunk=3 + # ITL = (100 - 20) / (6 - 3) = 26.6667, vs the inflated (100-20)/(6-1)=16 + assert metric_results[InterTokenLatencyMetric.tag] == approx([80.0 / 3.0]) + + def test_inter_token_latency_first_chunk_one_matches_legacy(self): + """A one-token first chunk reproduces the legacy (OSL - 1) formula exactly.""" + record = create_record( + start_ns=100, + responses=[120, 200], + output_tokens_per_response=3, + first_content_chunk_tokens=1, + ) + + metric_results = run_simple_metrics_pipeline( + [record], + RequestLatencyMetric.tag, + TTFTMetric.tag, + OutputSequenceLengthMetric.tag, + InterTokenLatencyMetric.tag, + ) + + assert metric_results[InterTokenLatencyMetric.tag] == approx([16.0]) + + def test_inter_token_latency_absent_first_chunk_falls_back_to_one(self): + """When per-chunk usage is unavailable (None), ITL falls back to (OSL - 1).""" + record = create_record( + start_ns=100, + responses=[120, 200], + output_tokens_per_response=3, + first_content_chunk_tokens=None, + ) + + metric_results = run_simple_metrics_pipeline( + [record], + RequestLatencyMetric.tag, + TTFTMetric.tag, + OutputSequenceLengthMetric.tag, + InterTokenLatencyMetric.tag, + ) + + assert metric_results[InterTokenLatencyMetric.tag] == approx([16.0]) + + def test_inter_token_latency_count_mismatch_degrades_to_legacy(self): + """When the first-chunk count >= OSL (a count inconsistency, e.g. reasoning + unit mismatch or all output in one chunk), ITL degrades to the legacy + (OSL - 1) divisor and is still emitted rather than silently dropped.""" + record = create_record( + start_ns=100, + responses=[120, 200], + output_tokens_per_response=3, + first_content_chunk_tokens=6, # first-chunk count == OSL (inconsistent) + ) + + metric_results = run_simple_metrics_pipeline( + [record], + RequestLatencyMetric.tag, + TTFTMetric.tag, + OutputSequenceLengthMetric.tag, + InterTokenLatencyMetric.tag, + ) + + # Degrades to (100 - 20) / (6 - 1) = 16.0 instead of dropping the metric. + assert metric_results[InterTokenLatencyMetric.tag] == approx([16.0]) + + def test_inter_token_latency_zero_first_chunk_falls_back_to_legacy(self): + """A non-positive first-chunk count (e.g. per-chunk usage lagging a chunk) + is treated as 'not reported' and falls back to (OSL - 1), not OSL - 0.""" + record = create_record( + start_ns=100, + responses=[120, 200], + output_tokens_per_response=3, + first_content_chunk_tokens=0, + ) + + metric_results = run_simple_metrics_pipeline( + [record], + RequestLatencyMetric.tag, + TTFTMetric.tag, + OutputSequenceLengthMetric.tag, + InterTokenLatencyMetric.tag, + ) + + assert metric_results[InterTokenLatencyMetric.tag] == approx([16.0]) diff --git a/tests/unit/records/test_inference_result_parser.py b/tests/unit/records/test_inference_result_parser.py index ba6fe5ea3a..395b0dc197 100644 --- a/tests/unit/records/test_inference_result_parser.py +++ b/tests/unit/records/test_inference_result_parser.py @@ -47,6 +47,14 @@ def server_token_parser(setup_inference_parser): return setup_inference_parser +@pytest.fixture +def per_chunk_usage_parser(setup_inference_parser): + """Parser with server token count AND per-chunk usage enabled.""" + setup_inference_parser.run.cfg.endpoint.use_server_token_count = True + setup_inference_parser.run.cfg.endpoint.per_chunk_usage = True + return setup_inference_parser + + def make_parsed_response( text: str = "output", perf_ns: int = 1000, @@ -325,6 +333,89 @@ async def test_missing_usage_returns_none( assert result.token_counts.output is None assert result.token_counts.reasoning is None + async def test_first_content_chunk_from_per_chunk_usage( + self, per_chunk_usage_parser, request_record + ): + """With --per-chunk-usage, first_content_chunk_tokens is the cumulative + output-token count on the first content chunk -- the real bundled-first-chunk + count ITL subtracts.""" + setup_parser_responses( + per_chunk_usage_parser, + [ + make_parsed_response(text="hello world", completion_tokens=20), + make_parsed_response(text=" more", completion_tokens=50), + ], + ) + + result = await per_chunk_usage_parser.process_valid_record(request_record) + + assert result.token_counts.first_content_chunk_tokens == 20 + assert result.token_counts.output == 50 # final total, unchanged + + async def test_first_content_chunk_is_completion_unit_for_reasoning( + self, per_chunk_usage_parser, request_record + ): + """The first-chunk count is raw completion_tokens (reasoning included), + matching OSL (= output + reasoning = completion). For a reasoning model this + keeps the ITL operands in the same unit: OSL = output(20) + reasoning(32) = + 52 = final completion, first chunk = 32, decode = 52 - 32 = 20 (positive).""" + setup_parser_responses( + per_chunk_usage_parser, + [ + make_parsed_response( + text="answer", completion_tokens=32, reasoning_tokens=32 + ), + make_parsed_response( + text=" more", completion_tokens=52, reasoning_tokens=32 + ), + ], + ) + + result = await per_chunk_usage_parser.process_valid_record(request_record) + + # Raw completion (reasoning included), same unit as OSL. Not 0. + assert result.token_counts.first_content_chunk_tokens == 32 + # OSL = output + reasoning = 20 + 32 = 52; first chunk (32) < OSL, decode = 20. + assert result.token_counts.output == 20 + assert result.token_counts.reasoning == 32 + + async def test_first_content_chunk_not_read_without_per_chunk_usage( + self, server_token_parser, request_record + ): + """The read path is gated: with --use-server-token-count but NOT + --per-chunk-usage, first_content_chunk_tokens stays None even when the + server volunteers usage on a content chunk, so ITL is unchanged from main.""" + setup_parser_responses( + server_token_parser, + [ + make_parsed_response(text="hello world", completion_tokens=20), + make_parsed_response(text=" more", completion_tokens=50), + ], + ) + + result = await server_token_parser.process_valid_record(request_record) + + assert result.token_counts.first_content_chunk_tokens is None + assert result.token_counts.output == 50 + + async def test_first_content_chunk_none_when_only_final_usage( + self, per_chunk_usage_parser, request_record + ): + """Even with --per-chunk-usage, if only the final chunk carries usage, + first_content_chunk_tokens is None so ITL falls back to subtracting one.""" + setup_parser_responses( + per_chunk_usage_parser, + [ + make_parsed_response(text="hello", include_usage=False), + make_parsed_response(text="", completion_tokens=50), # final usage-only + ], + ) + + result = await per_chunk_usage_parser.process_valid_record(request_record) + + assert result.token_counts.first_content_chunk_tokens is None + assert result.token_counts.output == 50 + async def test_partial_usage(self, server_token_parser, request_record): """Partial usage information is handled correctly.""" setup_parser_responses( From 657b43db525474b8002544a9700d1a73e08c9591 Mon Sep 17 00:00:00 2001 From: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:27:03 -0700 Subject: [PATCH 026/119] fix(spec-decode): align vLLM adapter with #48915 review-updated wire format (#1282) (cherry picked from commit 810fd8bdd40a1c35b64d487b3b8487f0a71a0f6b) --- docs/reference/spec-decode-acceptance.md | 43 ++++--- docs/tutorials/spec-decode-metrics.md | 23 ++-- src/aiperf/common/models/record_models.py | 4 +- .../common/models/spec_decode_models.py | 5 +- src/aiperf/dataset/mmap_cache.py | 19 ++- src/aiperf/endpoints/base_endpoint.py | 37 +++--- src/aiperf/endpoints/openai_chat.py | 35 ++++-- src/aiperf/endpoints/openai_completions.py | 36 ++++-- src/aiperf/plugin/plugins.yaml | 7 +- src/aiperf/records/inference_result_parser.py | 5 +- src/aiperf/spec_decode/vllm_adapter.py | 58 +++++++-- tests/unit/dataset/test_mmap_cache.py | 2 +- .../endpoints/test_openai_chat_completions.py | 80 ++++++++++++- .../unit/endpoints/test_openai_completions.py | 79 ++++++++++++- .../endpoints/test_spec_decode_capture.py | 52 ++++----- tests/unit/spec_decode/test_vllm_adapter.py | 110 +++++++++++++----- 16 files changed, 435 insertions(+), 160 deletions(-) diff --git a/docs/reference/spec-decode-acceptance.md b/docs/reference/spec-decode-acceptance.md index a73be0f193..068900d845 100644 --- a/docs/reference/spec-decode-acceptance.md +++ b/docs/reference/spec-decode-acceptance.md @@ -66,7 +66,7 @@ class SpecDecodeAdapterProtocol(Protocol): ```mermaid flowchart LR - R["Raw response
choices[].speculative_decoding_stats"] + R["Raw response
metrics.speculative_decoding"] P["ParsedResponse
.spec_decode_stats (raw dict)"] A["Engine adapter
(auto-detected)"] N["SpecDecodeAcceptanceRecord
(engine-neutral)"] @@ -76,22 +76,32 @@ flowchart LR ## The vLLM adapter -`VLLMSpecDecodeAdapter` reads vLLM's per-choice `speculative_decoding_stats` -object, emitted when the server runs with `--per-request-spec-decode-stats` +`VLLMSpecDecodeAdapter` reads vLLM's response-root `metrics.speculative_decoding` +object, emitted when the server runs with `--per-request-spec-decode-metrics` (`summary` or `detailed`). The field names and shape track vLLM PR [#48915](https://github.com/vllm-project/vllm/pull/48915); its *Per-Request Acceptance Metrics* feature doc is the authoritative wire-format reference. It is present on chat and completions, streaming and non-streaming; in streaming it -rides the finish-reason chunk's choice, which AIPerf already parses, so no extra -endpoint code is needed. +rides the trailing `include_usage` chunk (empty `choices`) at the response root. + +Because vLLM emits that trailing chunk only when `stream_options.include_usage` +is set, AIPerf requests it on **every** streaming run -- not just when +`--use-server-token-count` is on, which would otherwise silently drop the +metrics. The chunk carries no content, so it is excluded from timing metrics via +`ParsedResponseRecord.content_responses`, and token counting still follows the +`use_server_token_count` config rather than the presence of `usage`. To opt out, +set the field explicitly: `--extra-inputs '{"stream_options": +{"include_usage": false}}'`. The wire object maps to the record one-to-one, except: -- **Histogram keys are JSON strings** and are int-cast into the record. -- **`completion_tokens`** comes from the response `usage`, not the payload. In - streaming that usage rides the trailing `include_usage` chunk, which AIPerf - auto-injects only when `endpoint.use_server_token_count` is enabled; otherwise - (or whenever the server omits usage) `completion_tokens` stays `None`. +- **`acceptance_histogram` is a dense `list[int]`** -- index `j` holds the number + of verify steps that accepted exactly `j` draft tokens (length + `num_spec_tokens + 1`). AIPerf inflates it into the record's sparse + `{j: count}` map, dropping zero-count buckets. +- **`completion_tokens`** is copied from the response `usage` (not the payload) + so the record trace carries it next to acceptance; no metric consumes it yet. + `None` when the server omits usage. - **`num_draft_tokens`** is vLLM's post-adjustment count: drafts invalidated by structured-output/grammar constraints are already subtracted server-side. - **`num_spec_tokens`** is always present (the configured `num_speculative_tokens`); @@ -112,10 +122,13 @@ The wire object maps to the record one-to-one, except: one bad response cannot abort a run. Records whose aggregate counts contradict each other (histogram not summing to `num_spec_steps`, etc.) are rejected the same way. -- **`n > 1`**: when a request produces multiple sequences, each choice carries - its own per-sequence stats, but `completion_tokens` is request-level. Rather - than mix one sequence's acceptance with all sequences' token count, the record - is suppressed (`None`) for `n > 1`, mirroring how per-request timing metrics - are suppressed for multi-sequence requests. +- **`n > 1`**: vLLM populates `metrics.speculative_decoding` only when the stats + are attributable to a single generation stream, leaving it `null` otherwise -- + for `n > 1` on both endpoints, and additionally for **multi-prompt** requests + on completions (`prompt: ["a", "b"]`), where timestamps would span prompts. + Because the object is at the response root -- not per-choice -- AIPerf simply + reads it as present or absent; no client-side suppression is needed, and such + requests yield no record. AIPerf sends one prompt per request, so in practice + only `n > 1` triggers this. - **Behind Dynamo** the custom field is currently stripped, so this path is direct-to-vLLM only. diff --git a/docs/tutorials/spec-decode-metrics.md b/docs/tutorials/spec-decode-metrics.md index b696d25ccd..13e85dd986 100644 --- a/docs/tutorials/spec-decode-metrics.md +++ b/docs/tutorials/spec-decode-metrics.md @@ -34,31 +34,34 @@ of the metrics reference. ## Prerequisites -- A vLLM server running **speculative decoding** with **per-request stats enabled** via - `--per-request-spec-decode-stats summary` (or `detailed`). The field shape tracks vLLM +- A vLLM server running **speculative decoding** with **per-request metrics enabled** via + `--per-request-spec-decode-metrics summary` (or `detailed`). The field shape tracks vLLM [PR #48915](https://github.com/vllm-project/vllm/pull/48915) -- confirm your vLLM build includes it. - **Direct-to-vLLM only.** Behind Dynamo the custom stats field is currently stripped, so the per-request path is unavailable there (use the server-scrape path instead). -- Streaming works out of the box; per-request `completion_tokens` is only populated when - the server also returns usage (enable server token counting if you want it). +- Streaming works out of the box. In streaming, vLLM sends the metrics on the trailing + usage chunk, so AIPerf requests `stream_options.include_usage` on every streaming run; + no extra flag is needed. --- -## Start a vLLM server with per-request spec-decode stats +## Start a vLLM server with per-request spec-decode metrics This example uses a Llama-3.1-8B target with a Llama-3.2-1B draft model and a 5-token draft -budget. `--per-request-spec-decode-stats` is the flag that makes vLLM attach acceptance -stats to each response choice: +budget. `--per-request-spec-decode-metrics` is the flag that makes vLLM attach acceptance +metrics to each response at the root `metrics.speculative_decoding` object (single-sequence +requests only): ```bash docker run --gpus all -p 8000:8000 vllm/vllm-openai:latest \ --model meta-llama/Llama-3.1-8B-Instruct \ --speculative-config '{"model": "meta-llama/Llama-3.2-1B-Instruct", "num_speculative_tokens": 5, "method": "draft_model"}' \ - --per-request-spec-decode-stats summary + --per-request-spec-decode-metrics summary ``` -Verify the server is ready: +Verify the server is ready and emitting the field (look for `metrics.speculative_decoding` +in the response): ```bash curl -s localhost:8000/v1/chat/completions \ @@ -186,7 +189,7 @@ If the Spec Decode section, histogram, and `spec_decode_*` fields are all absent the expected clean-degradation behavior -- not an error. Common causes: - speculative decoding is off, or the requests had no verify steps; -- the server was not started with `--per-request-spec-decode-stats`; +- the server was not started with `--per-request-spec-decode-metrics`; - the server is behind Dynamo, which strips the custom field (use the [server-scrape path](speed-bench.md) instead); - the vLLM build predates [PR #48915](https://github.com/vllm-project/vllm/pull/48915). diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index a0d7e8018b..8970e24727 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -1500,8 +1500,8 @@ class ParsedResponse: """Additional metadata from the response useful for analysis (rate limits, content filters, etc.).""" spec_decode_stats: dict[str, Any] | None = None - """Raw per-choice speculative-decoding payload captured from the wire (e.g. - vLLM's ``choices[].speculative_decoding_stats``), or None when absent. Left + """Raw speculative-decoding payload captured from the response root (e.g. + vLLM's ``metrics.speculative_decoding``), or None when absent. Left uninterpreted here; a ``SpecDecodeAdapterProtocol`` converts it into the engine-neutral ``SpecDecodeAcceptanceRecord`` at record-assembly time.""" diff --git a/src/aiperf/common/models/spec_decode_models.py b/src/aiperf/common/models/spec_decode_models.py index 04cdcfb9d6..4b6a139e9a 100644 --- a/src/aiperf/common/models/spec_decode_models.py +++ b/src/aiperf/common/models/spec_decode_models.py @@ -51,8 +51,9 @@ class SpecDecodeAcceptanceRecord(AIPerfBaseModel): ) acceptance_histogram: dict[NonNegativeInt, NonNegativeInt] = Field( description="Sparse map from accepted draft count j to the number of " - "verification steps that accepted exactly j draft tokens. Keys are " - "integers (int-cast from the string JSON object keys on the wire); " + "verification steps that accepted exactly j draft tokens. Engine " + "adapters populate it (e.g. vLLM inflates its dense on-the-wire " + "list[int], dropping zero-count buckets); keys are integers and " "zero-count buckets are omitted. Excludes the bonus token.", ) num_accepted_draft_tokens: int = Field( diff --git a/src/aiperf/dataset/mmap_cache.py b/src/aiperf/dataset/mmap_cache.py index 0ee0286b74..b37fd4e075 100644 --- a/src/aiperf/dataset/mmap_cache.py +++ b/src/aiperf/dataset/mmap_cache.py @@ -78,6 +78,13 @@ # Version 5 fixed the Conversation.metadata() projection of per-turn # theoretical prefix-cache block counts for realtime infinite-cache hit rate. MANIFEST_VERSION = ( + # v27: streaming chat/completions payloads now always carry + # stream_options.include_usage, not just when use_server_token_count is set + # (vLLM only emits the trailing usage chunk -- which carries per-request + # spec-decode metrics -- when it is present). Under PREFORMAT_PAYLOADS the + # payload bytes are baked at build time, and the key's preformat_endpoint + # dict is unchanged for a streaming + use_server_token_count=False run, so + # a warm entry would keep serving payloads without the field. # v26: weka inter-turn delays are now always end-to-start # (t_k - (t_{k-1} + api_{k-1}), floored at 0) instead of start-to-start. # The use_end_to_start_delays flag was removed, so Turn.delay decodes to @@ -174,7 +181,7 @@ # v10: merge of the flattened-agent-splitting lineage and the # tool-shaping lineage (boundary-cut overhang strip; shaping decided at # first emission so reset re-emits reproduce the first-sent shape). - 26 + 27 ) MANIFEST_FILENAME = "manifest.json" INPUTS_JSON_FILENAME = "inputs.json" @@ -783,10 +790,12 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: # When preformat_payloads is on, endpoint.format_payload() bakes the # stream flag, endpoint.extra, the max_tokens-vs-max_completion_tokens # field name, and (for streaming OpenAI-compatible endpoints) - # stream_options.include_usage from use_server_token_count into the - # stored bytes, so those knobs must key the cache or a warm entry serves - # bytes the run never asked for (e.g. "stream": true). Gated on the flag - # so the common non-preformat key stays stable. + # stream_options.include_usage into the stored bytes, so those knobs + # must key the cache or a warm entry serves bytes the run never asked + # for (e.g. "stream": true). include_usage now follows streaming alone, + # but use_server_token_count still keys the cache because it selects + # the token-count source baked into the run. Gated on the flag so the + # common non-preformat key stays stable. "preformat_endpoint": ( { "streaming": cfg.endpoint.streaming, diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index fadbfc4f9f..2888805e5f 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -107,30 +107,23 @@ def process_responses( @staticmethod def extract_spec_decode_stats(json_obj: dict[str, Any]) -> dict[str, Any] | None: - """Capture the raw ``choices[0]`` speculative-decoding payload, if any. - - vLLM attaches ``speculative_decoding_stats`` per choice (the finish-reason - chunk in streaming). Captured verbatim and uninterpreted so a - ``SpecDecodeAdapterProtocol`` owns the engine-specific interpretation - downstream; None when absent. - - A response with more than one choice is an ``n > 1`` non-streaming - request; its record is suppressed (None) because a single per-request - record can't attribute request-level usage to one sequence -- reporting - one choice's acceptance alongside all choices' token count would be a - mixed, misleading record. (``n > 1`` streaming is suppressed in the - parser, where each sequence's stats arrive on separate chunks.) + """Capture the raw speculative-decoding payload from the response root. + + vLLM nests it at ``metrics.speculative_decoding`` -- on the response body + non-streaming, or on the trailing usage chunk (empty ``choices``) + streaming -- when the server runs with ``--per-request-spec-decode-metrics``. + Captured verbatim and uninterpreted so a ``SpecDecodeAdapterProtocol`` + owns the engine-specific interpretation downstream; None when absent. + + vLLM populates it only for single-sequence requests (``n == 1``), leaving + it ``null`` otherwise, so no client-side ``n > 1`` suppression is needed: + a mixed per-request record can never arise here. """ - choices = json_obj.get("choices") - if ( - not isinstance(choices, list) - or not choices - or not isinstance(choices[0], dict) - ): - return None - if len(choices) > 1: + metrics = json_obj.get("metrics") + if not isinstance(metrics, dict): return None - return choices[0].get("speculative_decoding_stats") + stats = metrics.get("speculative_decoding") + return stats if isinstance(stats, dict) else None def build_assistant_turn(self, record: RequestRecord) -> Turn | None: """Build a Turn representing the assistant response for context replay. diff --git a/src/aiperf/endpoints/openai_chat.py b/src/aiperf/endpoints/openai_chat.py index e7385b7cd0..e393918e47 100644 --- a/src/aiperf/endpoints/openai_chat.py +++ b/src/aiperf/endpoints/openai_chat.py @@ -71,12 +71,19 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: if extra_body: payload.update(extra_body) - # per_chunk_usage implies use_server_token_count (enforced by the endpoint - # config validator), so gating on use_server_token_count alone is sufficient. - if ( - model_endpoint.endpoint.streaming - and model_endpoint.endpoint.use_server_token_count - ): + # Read the merged payload, not endpoint.streaming: the extras above can + # override "stream", and a server rejects stream_options when stream is + # false ("Stream options can only be defined when stream=True"). + if payload.get("stream"): + # Requested for every streaming run, not just server-token-count + # ones: vLLM rides per-request metrics (including + # metrics.speculative_decoding) on the trailing usage chunk and + # only emits that chunk when include_usage is set, so gating it on + # an unrelated flag would silently drop those metrics. Authors who + # want it off can set stream_options.include_usage explicitly. + # continuous_usage_stats stays opt-in regardless: per_chunk_usage + # implies use_server_token_count (enforced by the endpoint config + # validator), so widening this gate cannot turn it on by itself. self._ensure_include_usage( payload, continuous=model_endpoint.endpoint.per_chunk_usage ) @@ -162,12 +169,20 @@ def _ensure_include_usage( extension (strict OpenAI rejects it), so it is only injected when the caller opts in via ``--per-chunk-usage``. """ - stream_options = payload.setdefault("stream_options", {}) - if not isinstance(stream_options, dict): + stream_options = payload.get("stream_options") + if stream_options is None: + stream_options = {} + elif not isinstance(stream_options, dict): return - stream_options.setdefault("include_usage", True) + # Copy rather than mutate: the payload merge aliases endpoint.extra / + # turn.extra_body, which are long-lived config reused across every + # request, so an in-place edit would rewrite the author's config and + # leak into every subsequent request. + merged = {**stream_options} + merged.setdefault("include_usage", True) if continuous: - stream_options.setdefault("continuous_usage_stats", True) + merged.setdefault("continuous_usage_stats", True) + payload["stream_options"] = merged def parse_response( self, response: InferenceServerResponse diff --git a/src/aiperf/endpoints/openai_completions.py b/src/aiperf/endpoints/openai_completions.py index 3f2977052c..f16181fc44 100644 --- a/src/aiperf/endpoints/openai_completions.py +++ b/src/aiperf/endpoints/openai_completions.py @@ -69,18 +69,30 @@ def format_payload(self, request_info: RequestInfo) -> RequestOutputT: if turn.extra_body: payload.update(turn.extra_body) - if ( - model_endpoint.endpoint.streaming - and model_endpoint.endpoint.use_server_token_count - ): - # Automatically set stream_options to include usage when using server token counts - if "stream_options" not in payload: - payload["stream_options"] = {"include_usage": True} - elif ( - isinstance(payload["stream_options"], dict) - and "include_usage" not in payload["stream_options"] - ): - payload["stream_options"]["include_usage"] = True + # Read the merged payload, not endpoint.streaming: the extras above can + # override "stream", and a server rejects stream_options when stream is + # false ("Stream options can only be defined when stream=True"). + if payload.get("stream"): + # Requested for every streaming run, not just server-token-count + # ones: vLLM rides per-request metrics (including + # metrics.speculative_decoding) on the trailing usage chunk and + # only emits that chunk when include_usage is set, so gating it on + # an unrelated flag would silently drop those metrics. Authors who + # want it off can set stream_options.include_usage explicitly. + stream_options = payload.get("stream_options") + # An explicit null parses straight from the CLI + # (--extra-inputs '{"stream_options": null}'); treat it as absent so + # this endpoint agrees with chat instead of silently skipping. + if stream_options is None: + stream_options = {} + if isinstance(stream_options, dict): + # Copy rather than mutate: the payload merge aliases + # endpoint.extra / turn.extra_body, which are long-lived config + # reused across every request, so an in-place edit would rewrite + # the author's config and leak into every subsequent request. + merged = {**stream_options} + merged.setdefault("include_usage", True) + payload["stream_options"] = merged self.trace(lambda: f"Formatted payload: {payload}") return payload diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index bf07c81a16..616740cbff 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -1566,10 +1566,11 @@ spec_decode_adapter: vllm: class: aiperf.spec_decode.vllm_adapter:VLLMSpecDecodeAdapter description: | - Reads vLLM's per-choice ``speculative_decoding_stats`` (emitted when the - server runs with ``--per-request-spec-decode-stats``) and fills the + Reads vLLM's response-root ``metrics.speculative_decoding`` (emitted when + the server runs with ``--per-request-spec-decode-metrics``) and fills the engine-neutral SpecDecodeAcceptanceRecord. Handles chat and completions, - streaming and non-streaming; int-casts the string histogram keys. + streaming and non-streaming; inflates the dense ``acceptance_histogram`` + list into the record's sparse map. metadata: {} accuracy_grader: diff --git a/src/aiperf/records/inference_result_parser.py b/src/aiperf/records/inference_result_parser.py index 0c815bc390..0a5683cd69 100644 --- a/src/aiperf/records/inference_result_parser.py +++ b/src/aiperf/records/inference_result_parser.py @@ -340,8 +340,9 @@ def _extract_spec_decode_acceptance( ``n > 1`` streaming request, where each sequence's stats ride its own finish chunk): the per-request record can't attribute request-level ``completion_tokens`` to a single sequence, so a mixed record is worse - than none. ``n > 1`` non-streaming is suppressed upstream in - ``BaseEndpoint.extract_spec_decode_stats``. + than none. ``n > 1`` non-streaming needs no client-side guard: vLLM + populates ``metrics.speculative_decoding`` only for single-sequence + requests and leaves it null otherwise. Counts payloads by truthiness (not ``is not None``) to match the adapter's ``_find_spec_decode_payload``: an empty ``{}`` is treated as diff --git a/src/aiperf/spec_decode/vllm_adapter.py b/src/aiperf/spec_decode/vllm_adapter.py index 067869a5a2..32d60637bf 100644 --- a/src/aiperf/spec_decode/vllm_adapter.py +++ b/src/aiperf/spec_decode/vllm_adapter.py @@ -18,11 +18,11 @@ ENGINE = "vllm" -# Keys that identify a payload as vLLM's ``speculative_decoding_stats`` shape. +# Keys that identify a payload as vLLM's ``speculative_decoding`` shape. # ``can_adapt`` checks for these so that, once other engines populate the same # raw slot, this adapter claims only its own payloads (auto-detection) rather # than greedily matching on mere presence. Both are always emitted by vLLM, -# including the zero-step case (``acceptance_histogram`` is ``{}`` but present). +# including the zero-step case (``acceptance_histogram`` is all-zero but present). _VLLM_SIGNATURE_KEYS = ("acceptance_histogram", "num_spec_steps") @@ -37,9 +37,9 @@ def _find_spec_decode_payload( ) -> dict[str, Any] | None: """Return the last non-empty ``spec_decode_stats`` payload across responses. - vLLM attaches the payload to a single choice: the finish-reason chunk in - streaming, or the sole choice non-streaming. Walking from the end mirrors - ``find_last_non_empty_usage`` and tolerates either layout. + vLLM attaches the payload once per request at the response root: on the body + non-streaming, or on the trailing usage chunk streaming. Walking from the end + mirrors ``find_last_non_empty_usage`` and tolerates either layout. """ for response in reversed(responses): stats = response.spec_decode_stats @@ -49,12 +49,15 @@ def _find_spec_decode_payload( class VLLMSpecDecodeAdapter: - """Fills the acceptance record from vLLM's ``speculative_decoding_stats``. + """Fills the acceptance record from vLLM's ``metrics.speculative_decoding``. - Reads the per-choice ``speculative_decoding_stats`` object emitted by vLLM - when the server runs with ``--per-request-spec-decode-stats``. Present on - chat and completions, streaming and non-streaming. The object's histogram - keys are JSON strings, so they are int-cast into the neutral record. + Reads the response-root ``metrics.speculative_decoding`` object emitted by + vLLM when the server runs with ``--per-request-spec-decode-metrics`` + (``summary`` or ``detailed``). Present on chat and completions, streaming and + non-streaming. Its ``acceptance_histogram`` is a dense ``list[int]`` (index j + holds the number of steps that accepted exactly j draft tokens), inflated + into the neutral record's sparse ``{j: count}`` map with zero-count buckets + dropped. The field names and shape track vLLM PR https://github.com/vllm-project/vllm/pull/48915; its per-request @@ -74,8 +77,41 @@ def adapt( return None try: + # vLLM sends a dense ``list[int]`` (index j -> step count); inflate + # into the neutral record's sparse map, dropping zero-count buckets. + # Validate the shape AND every element before filtering: a str/dict + # is iterable (enumerate would build a bucket per character/key), + # and filtering on truthiness first would silently drop a falsey + # malformed entry (None/False/0.0) while keeping a truthy one that + # coerces to zero ("0"), which would violate the record's + # zero-buckets-omitted invariant. ``type(...) is int`` rather than + # isinstance so bools -- an int subclass -- are rejected too. + raw_histogram = payload["acceptance_histogram"] + if not isinstance(raw_histogram, list) or not all( + type(count) is int for count in raw_histogram + ): + raise TypeError( + "acceptance_histogram must be a list of ints, got " + f"{raw_histogram!r}" + ) + # Length is num_spec_tokens + 1 (one bucket per accepted count from + # 0..k). Enforcing it rejects a payload whose bucket indices exceed + # the draft budget -- j > k is physically impossible, yet satisfies + # the record's arithmetic validators, so it would otherwise be + # reported as a real acceptance length. Doubles as a tripwire if the + # still-unmerged upstream PR changes the histogram shape. + # Optional on the record, so only enforce when the server sent it. + num_spec_tokens = payload.get("num_spec_tokens") + if ( + num_spec_tokens is not None + and len(raw_histogram) != num_spec_tokens + 1 + ): + raise ValueError( + f"acceptance_histogram has {len(raw_histogram)} buckets, " + f"expected num_spec_tokens + 1 = {num_spec_tokens + 1}" + ) histogram = { - int(j): count for j, count in payload["acceptance_histogram"].items() + j: count for j, count in enumerate(raw_histogram) if count != 0 } usage = find_last_non_empty_usage(responses) return SpecDecodeAcceptanceRecord( diff --git a/tests/unit/dataset/test_mmap_cache.py b/tests/unit/dataset/test_mmap_cache.py index c4ec89ae7e..0abaa01673 100644 --- a/tests/unit/dataset/test_mmap_cache.py +++ b/tests/unit/dataset/test_mmap_cache.py @@ -574,7 +574,7 @@ def test_lookup_rejects_pre_overlap_frontier_manifest(self, tmp_path: Path) -> N raw["version"] = 22 manifest_path.write_bytes(orjson.dumps(raw)) - assert mmap_cache.MANIFEST_VERSION == 26 + assert mmap_cache.MANIFEST_VERSION == 27 assert mmap_cache.lookup("pre-overlap-frontier", compressed=False) is None def test_lookup_compressed_mismatch_returns_none(self, tmp_path: Path) -> None: diff --git a/tests/unit/endpoints/test_openai_chat_completions.py b/tests/unit/endpoints/test_openai_chat_completions.py index 1773e779d3..3545db5846 100644 --- a/tests/unit/endpoints/test_openai_chat_completions.py +++ b/tests/unit/endpoints/test_openai_chat_completions.py @@ -3,7 +3,7 @@ import pytest from aiperf.common.enums import ModelSelectionStrategy -from aiperf.common.models import Image, Text, Turn +from aiperf.common.models import Conversation, Image, Text, Turn from aiperf.common.models.model_endpoint_info import ( EndpointInfo, ModelEndpointInfo, @@ -71,6 +71,9 @@ def test_format_payload_with_max_tokens_and_streaming( "model": "test-model", "stream": True, "max_completion_tokens": 42, + # Requested on every streaming run so vLLM emits the trailing + # usage chunk that carries per-request metrics. + "stream_options": {"include_usage": True}, } assert payload == expected_payload @@ -197,12 +200,16 @@ def test_create_messages_audio_format_error( (True, True, None, {"include_usage": True}), # Don't add when not streaming (False, True, None, None), - # Don't add when flag disabled - (True, False, None, None), - # Don't add when neither enabled + # Add for any streaming run, even without server token counts: + # vLLM only emits the metrics-bearing trailing usage chunk when + # include_usage is set, and per-request spec-decode metrics ride it. + (True, False, None, {"include_usage": True}), + # Don't add when not streaming (False, False, None, None), # Preserve user's include_usage=False (True, True, {"stream_options": {"include_usage": False}}, {"include_usage": False}), + # Preserve the opt-out without server token counts too + (True, False, {"stream_options": {"include_usage": False}}, {"include_usage": False}), # Merge with user's other options (True, True, {"stream_options": {"continuous_updates": True}}, {"continuous_updates": True, "include_usage": True}), ], @@ -269,6 +276,71 @@ def test_per_chunk_usage_injects_continuous_usage_stats( else: assert payload["stream_options"] == expected + def test_format_payload_does_not_mutate_endpoint_extra( + self, + model_endpoint: ModelEndpointInfo, + sample_conversations: dict[str, Conversation], + ) -> None: + """Auto-adding include_usage must not write back into endpoint.extra. + + The payload merge aliases the shared config dict, so an in-place edit + would rewrite the author's config mid-run and leak into every + subsequent request. + """ + endpoint = ChatEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = True + model_endpoint.endpoint.use_server_token_count = False + model_endpoint.endpoint.extra = [ + ("stream_options", {"continuous_updates": True}) + ] + + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + assert payload["stream_options"] == { + "continuous_updates": True, + "include_usage": True, + } + assert model_endpoint.endpoint.extra == [ + ("stream_options", {"continuous_updates": True}) + ] + + def test_format_payload_stream_override_suppresses_stream_options( + self, + model_endpoint: ModelEndpointInfo, + sample_conversations: dict[str, Conversation], + ) -> None: + """extra_body may turn streaming off; stream_options must follow it. + + A server rejects stream_options when stream is false, so the injection + decision reads the merged payload rather than the endpoint config. + """ + endpoint = ChatEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = True + turns[-1].extra_body = {"stream": False} + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + assert payload["stream"] is False + assert "stream_options" not in payload + + def test_format_payload_explicit_null_stream_options_still_injects( + self, + model_endpoint: ModelEndpointInfo, + sample_conversations: dict[str, Conversation], + ) -> None: + """An explicit null parses from the CLI; treat it as absent, not as opt-out.""" + endpoint = ChatEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = True + model_endpoint.endpoint.extra = [("stream_options", None)] + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + assert payload["stream_options"] == {"include_usage": True} + def test_create_messages_with_system_message( self, model_endpoint, sample_conversations ): diff --git a/tests/unit/endpoints/test_openai_completions.py b/tests/unit/endpoints/test_openai_completions.py index 7b6772e21a..bddd9a1712 100644 --- a/tests/unit/endpoints/test_openai_completions.py +++ b/tests/unit/endpoints/test_openai_completions.py @@ -5,6 +5,7 @@ from pytest import param from aiperf.common.enums import CreditPhase, ModelSelectionStrategy +from aiperf.common.models import Conversation from aiperf.common.models.model_endpoint_info import ( EndpointInfo, ModelEndpointInfo, @@ -67,6 +68,9 @@ def test_format_payload_with_extra_options( "stream": True, "max_tokens": 50, "ignore_eos": True, + # Requested on every streaming run so vLLM emits the trailing + # usage chunk that carries per-request metrics. + "stream_options": {"include_usage": True}, } assert payload == expected_payload @@ -120,12 +124,16 @@ def test_format_payload_extra_body_prompt_overrides_in_every_phase( (True, True, None, {"include_usage": True}), # Don't add when not streaming (False, True, None, None), - # Don't add when flag disabled - (True, False, None, None), - # Don't add when neither enabled + # Add for any streaming run, even without server token counts: + # vLLM only emits the metrics-bearing trailing usage chunk when + # include_usage is set, and per-request spec-decode metrics ride it. + (True, False, None, {"include_usage": True}), + # Don't add when not streaming (False, False, None, None), # Preserve user's include_usage=False (True, True, [("stream_options", {"include_usage": False})], {"include_usage": False}), + # Preserve the opt-out without server token counts too + (True, False, [("stream_options", {"include_usage": False})], {"include_usage": False}), # Merge with user's other options (True, True, [("stream_options", {"continuous_updates": True})], {"continuous_updates": True, "include_usage": True}), ], @@ -155,3 +163,68 @@ def test_stream_options_auto_configuration( else: assert "stream_options" in payload assert payload["stream_options"] == expected_stream_options + + def test_format_payload_does_not_mutate_endpoint_extra( + self, + model_endpoint: ModelEndpointInfo, + sample_conversations: dict[str, Conversation], + ) -> None: + """Auto-adding include_usage must not write back into endpoint.extra. + + The payload merge aliases the shared config dict, so an in-place edit + would rewrite the author's config mid-run and leak into every + subsequent request. + """ + endpoint = CompletionsEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = True + model_endpoint.endpoint.use_server_token_count = False + model_endpoint.endpoint.extra = [ + ("stream_options", {"continuous_updates": True}) + ] + + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + assert payload["stream_options"] == { + "continuous_updates": True, + "include_usage": True, + } + assert model_endpoint.endpoint.extra == [ + ("stream_options", {"continuous_updates": True}) + ] + + def test_format_payload_stream_override_suppresses_stream_options( + self, + model_endpoint: ModelEndpointInfo, + sample_conversations: dict[str, Conversation], + ) -> None: + """extra_body may turn streaming off; stream_options must follow it. + + A server rejects stream_options when stream is false, so the injection + decision reads the merged payload rather than the endpoint config. + """ + endpoint = CompletionsEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = True + turns[-1].extra_body = {"stream": False} + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + assert payload["stream"] is False + assert "stream_options" not in payload + + def test_format_payload_explicit_null_stream_options_still_injects( + self, + model_endpoint: ModelEndpointInfo, + sample_conversations: dict[str, Conversation], + ) -> None: + """An explicit null parses from the CLI; treat it as absent, not as opt-out.""" + endpoint = CompletionsEndpoint(model_endpoint) + turns = [sample_conversations["session_1"].turns[0]] + model_endpoint.endpoint.streaming = True + model_endpoint.endpoint.extra = [("stream_options", None)] + request_info = create_request_info(turns=turns, model_endpoint=model_endpoint) + payload = endpoint.format_payload(request_info) + + assert payload["stream_options"] == {"include_usage": True} diff --git a/tests/unit/endpoints/test_spec_decode_capture.py b/tests/unit/endpoints/test_spec_decode_capture.py index 4a210b7e5c..bf6d07b81e 100644 --- a/tests/unit/endpoints/test_spec_decode_capture.py +++ b/tests/unit/endpoints/test_spec_decode_capture.py @@ -3,10 +3,10 @@ """Tests that chat/completions endpoints capture the raw spec-decode payload. -``parse_response`` must lift ``choices[0].speculative_decoding_stats`` onto the -``ParsedResponse`` uninterpreted, including on a streaming finish-reason chunk -whose delta is empty (which otherwise carries no data and would be dropped), and -must not choke on a malformed non-list ``choices``. +``parse_response`` must lift ``metrics.speculative_decoding`` from the response +root onto the ``ParsedResponse`` uninterpreted -- including on a streaming +trailing usage chunk whose ``choices`` is empty (it carries no content and would +otherwise be dropped) -- and must not choke on a malformed non-list ``choices``. """ from typing import Any, TypeVar @@ -30,7 +30,7 @@ STATS = { "mean_acceptance_length": 1.5, "draft_acceptance_rate": 0.25, - "acceptance_histogram": {"0": 2, "2": 2}, + "acceptance_histogram": [2, 0, 2, 0], "num_spec_steps": 4, "num_accepted_draft_tokens": 4, "num_draft_tokens": 16, @@ -83,27 +83,23 @@ def test_parse_response_non_streaming_captures_stats( { "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop", - "speculative_decoding_stats": STATS, } ], + "metrics": {"speculative_decoding": STATS}, } parsed = chat_endpoint.parse_response(_mock_response(json_obj)) assert parsed is not None assert parsed.spec_decode_stats == STATS - def test_parse_response_streaming_empty_delta_retains_stats( + def test_parse_response_streaming_usage_chunk_retains_stats( self, chat_endpoint: ChatEndpoint ) -> None: - """The finish chunk has no content but must still surface the stats.""" + """The trailing usage chunk has empty choices but carries stats at root.""" json_obj = { "object": "chat.completion.chunk", - "choices": [ - { - "delta": {}, - "finish_reason": "stop", - "speculative_decoding_stats": STATS, - } - ], + "choices": [], + "usage": {"completion_tokens": 50}, + "metrics": {"speculative_decoding": STATS}, } parsed = chat_endpoint.parse_response(_mock_response(json_obj)) assert parsed is not None @@ -128,16 +124,15 @@ def test_parse_response_malformed_choices_returns_none( json_obj = {"object": "error", "choices": {"0": {"message": {}}}} assert chat_endpoint.parse_response(_mock_response(json_obj)) is None - def test_parse_response_multi_choice_suppresses_stats( + def test_parse_response_metrics_without_spec_decode_yields_none( self, chat_endpoint: ChatEndpoint ) -> None: - """n > 1 (multiple stats-bearing choices): stats suppressed, not mixed.""" + """``metrics`` present (timing only, or ``n > 1`` where vLLM nulls the + ``speculative_decoding`` sub-object): no spec-decode payload captured.""" json_obj = { "object": "chat.completion", - "choices": [ - {"message": {"content": "a"}, "speculative_decoding_stats": STATS}, - {"message": {"content": "b"}, "speculative_decoding_stats": STATS}, - ], + "choices": [{"message": {"content": "hi"}}], + "metrics": {"time_to_first_token_ms": 12.0, "speculative_decoding": None}, } parsed = chat_endpoint.parse_response(_mock_response(json_obj)) assert parsed is not None @@ -150,24 +145,21 @@ def test_parse_response_non_streaming_captures_stats( ) -> None: json_obj = { "object": "text_completion", - "choices": [{"text": "hi", "speculative_decoding_stats": STATS}], + "choices": [{"text": "hi"}], + "metrics": {"speculative_decoding": STATS}, } parsed = completions_endpoint.parse_response(_mock_response(json_obj)) assert parsed is not None assert parsed.spec_decode_stats == STATS - def test_parse_response_streaming_empty_text_retains_stats( + def test_parse_response_streaming_usage_chunk_retains_stats( self, completions_endpoint: CompletionsEndpoint ) -> None: json_obj = { "object": "text_completion", - "choices": [ - { - "text": "", - "finish_reason": "stop", - "speculative_decoding_stats": STATS, - } - ], + "choices": [], + "usage": {"completion_tokens": 50}, + "metrics": {"speculative_decoding": STATS}, } parsed = completions_endpoint.parse_response(_mock_response(json_obj)) assert parsed is not None diff --git a/tests/unit/spec_decode/test_vllm_adapter.py b/tests/unit/spec_decode/test_vllm_adapter.py index d44f39eead..03352c5341 100644 --- a/tests/unit/spec_decode/test_vllm_adapter.py +++ b/tests/unit/spec_decode/test_vllm_adapter.py @@ -4,13 +4,14 @@ """Unit tests for the vLLM per-request spec-decode adapter. Covers the engine-neutral ``SpecDecodeAcceptanceRecord`` filled from vLLM's -per-choice ``speculative_decoding_stats`` payload across the shapes the ticket -calls out: present, absent, zero-step, and fully-rejected, in both streaming -and non-streaming layouts, plus the detailed per-step arrays and malformed +root ``metrics.speculative_decoding`` payload across the shapes the ticket calls +out: present, absent, zero-step, and fully-rejected, in both streaming and +non-streaming layouts, plus the detailed per-step arrays and malformed degradation. The sample payloads mirror the wire format from vLLM PR -https://github.com/vllm-project/vllm/pull/48915. +https://github.com/vllm-project/vllm/pull/48915: ``acceptance_histogram`` is a +dense ``list[int]`` (index j -> step count, length num_spec_tokens + 1). """ from collections.abc import Callable @@ -23,28 +24,28 @@ from aiperf.common.models import ParsedResponse, SpecDecodeAcceptanceRecord from aiperf.spec_decode.vllm_adapter import VLLMSpecDecodeAdapter -# A representative vLLM ``summary`` payload (histogram keys are JSON strings). -# 39 steps accepted 0 drafts, 1 step accepted 1, 3 steps accepted 3: -# num_spec_steps = 39 + 1 + 3 = 43 -# num_accepted_draft = 0*39 + 1*1 + 3*3 = 10 -# mean_acceptance_len = 1 + 10 / 43 = 1.2325... -# draft_acceptance = 10 / 129 = 0.0775... +# A representative vLLM ``summary`` payload. The dense acceptance_histogram +# [39, 1, 0, 3] means 39 steps accepted 0 drafts, 1 accepted 1, 3 accepted 3: +# num_spec_steps = 39 + 1 + 0 + 3 = 43 +# num_accepted_draft = 0*39 + 1*1 + 2*0 + 3*3 = 10 +# mean_acceptance_len = 1 + 10 / 43 = 1.2325... +# draft_acceptance = 10 / 129 = 0.0775... SUMMARY_PAYLOAD: dict[str, Any] = { "mean_acceptance_length": 1.2325581395348837, "draft_acceptance_rate": 0.07751937984496124, - "acceptance_histogram": {"0": 39, "1": 1, "3": 3}, + "acceptance_histogram": [39, 1, 0, 3], "num_spec_steps": 43, "num_accepted_draft_tokens": 10, "num_draft_tokens": 129, "num_spec_tokens": 3, } -# A self-consistent ``detailed`` payload: 4 steps, accepted [0,1,3,0] -> -# histogram {0:2, 1:1, 3:1} and 4 accepted; drafted [3,3,3,3] -> 12 proposed. +# A self-consistent ``detailed`` payload: 4 steps, accepted [0,1,3,0] -> dense +# histogram [2, 1, 0, 1] and 4 accepted; drafted [3,3,3,3] -> 12 proposed. DETAILED_PAYLOAD: dict[str, Any] = { "mean_acceptance_length": 2.0, "draft_acceptance_rate": 4 / 12, - "acceptance_histogram": {"0": 2, "1": 1, "3": 1}, + "acceptance_histogram": [2, 1, 0, 1], "num_spec_steps": 4, "num_accepted_draft_tokens": 4, "num_draft_tokens": 12, @@ -73,15 +74,15 @@ def _non_streaming(payload: dict[str, Any]) -> list[ParsedResponse]: def _streaming(payload: dict[str, Any]) -> list[ParsedResponse]: - """Streaming layout: content chunk, terminal stats chunk, usage-only chunk. + """Streaming layout: content chunk, then the trailing usage chunk. - Mirrors vLLM: acceptance stats ride the finish-reason chunk's choice while - the trailing ``include_usage`` chunk carries usage on empty choices. + Mirrors vLLM: the ``metrics.speculative_decoding`` payload and the usage both + ride the trailing ``include_usage`` chunk (empty choices) at the response + root. """ return [ _response(), - _response(spec_decode_stats=payload), - _response(usage={"completion_tokens": 50}), + _response(spec_decode_stats=payload, usage={"completion_tokens": 50}), ] @@ -105,7 +106,8 @@ def test_adapt_summary_payload_fills_record( assert record.engine == "vllm" assert record.mean_acceptance_length == pytest.approx(1.2325581395348837) assert record.draft_acceptance_rate == pytest.approx(0.07751937984496124) - # Histogram string keys are int-cast into the neutral record. + # Index 2 is absent because the neutral record stays sparse: engines + # may report any bucket layout, so zero counts carry no information. assert record.acceptance_histogram == {0: 39, 1: 1, 3: 3} assert record.num_spec_steps == 43 assert record.num_accepted_draft_tokens == 10 @@ -155,7 +157,7 @@ def test_adapt_zero_step_payload_fills_record(self) -> None: payload = { "mean_acceptance_length": 1.0, "draft_acceptance_rate": 0.0, - "acceptance_histogram": {}, + "acceptance_histogram": [0, 0, 0, 0], "num_spec_steps": 0, "num_accepted_draft_tokens": 0, "num_draft_tokens": 0, @@ -176,7 +178,7 @@ def test_adapt_fully_rejected_payload_fills_record(self) -> None: payload = { "mean_acceptance_length": 1.0, "draft_acceptance_rate": 0.0, - "acceptance_histogram": {"0": 20}, + "acceptance_histogram": [20, 0, 0, 0], "num_spec_steps": 20, "num_accepted_draft_tokens": 0, "num_draft_tokens": 60, @@ -250,16 +252,68 @@ def test_adapt_no_usage_leaves_completion_tokens_none(self) -> None: # Signature keys present (so can_adapt matches) but the rest of the # required body is missing. param( - {"acceptance_histogram": {"0": 1}, "num_spec_steps": 1}, + {"acceptance_histogram": [0, 1], "num_spec_steps": 1}, id="signature_only_missing_rest", ), param( - {**SUMMARY_PAYLOAD, "acceptance_histogram": {"x": 1}}, - id="non_integer_histogram_key", + {**SUMMARY_PAYLOAD, "acceptance_histogram": [39, 1, "x", 3]}, + id="non_integer_histogram_element", ), param( - {**SUMMARY_PAYLOAD, "acceptance_histogram": [1, 2, 3]}, - id="histogram_wrong_type", + # A dict is the pre-rework wire shape; the current list-based + # parser must reject it rather than emit a garbled record. + {**SUMMARY_PAYLOAD, "acceptance_histogram": {"0": 39, "1": 1, "3": 3}}, + id="histogram_dict_not_list", + ), + param( + # A str is iterable, so without an explicit type check enumerate + # would build a bucket per character. + {**SUMMARY_PAYLOAD, "acceptance_histogram": "303"}, + id="histogram_str_not_list", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": 5}, + id="histogram_int_not_list", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": None}, + id="histogram_none_not_list", + ), + param( + # Falsey malformed elements must not slip through the + # zero-bucket filter and yield a valid-looking record. + {**SUMMARY_PAYLOAD, "acceptance_histogram": [39, 1, None, 3]}, + id="histogram_falsey_none_element", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": [39, 1, False, 3]}, + id="histogram_falsey_bool_element", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": [39, 1, 0.0, 3]}, + id="histogram_falsey_float_element", + ), + param( + # Truthy but coerces to zero -- would land a 0 bucket in a map + # that promises zero buckets are omitted. + {**SUMMARY_PAYLOAD, "acceptance_histogram": [39, 1, "0", 3]}, + id="histogram_str_zero_element", + ), + param( + # Length must be num_spec_tokens + 1; a longer list puts a + # bucket at j > k, which is impossible yet satisfies every + # arithmetic invariant the record checks. + { + **SUMMARY_PAYLOAD, + "acceptance_histogram": [0, 0, 0, 0, 1], + "num_spec_steps": 1, + "num_accepted_draft_tokens": 4, + }, + id="histogram_longer_than_num_spec_tokens", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": [39, 1, 3]}, + id="histogram_shorter_than_num_spec_tokens", ), ], ) # fmt: skip @@ -287,7 +341,7 @@ def test_adapt_last_payload_wins_across_chunks(self) -> None: def test_adapt_negative_count_payload_degrades_to_none(self) -> None: """A signature-matching payload with a negative count is rejected by the record's ge=0 constraints, and the adapter degrades to None.""" - bad = {**SUMMARY_PAYLOAD, "acceptance_histogram": {"0": -1}} + bad = {**SUMMARY_PAYLOAD, "acceptance_histogram": [-1, 1, 0, 3]} responses = [_response(spec_decode_stats=bad)] assert VLLMSpecDecodeAdapter.can_adapt(responses) is True assert VLLMSpecDecodeAdapter.adapt(responses) is None From 08a1094db0a7d34a3ceb26321d7fd3ca8250bc46 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 21 Aug 2026 13:00:56 -0700 Subject: [PATCH 027/119] fix(enums): make != the negation of == on case-insensitive str enums (#1314) Signed-off-by: Anthony Casagrande (cherry picked from commit e5ebe915df38958b2a7d36eb40bb1d407e991ead) --- src/aiperf/common/enums/base_enums.py | 27 +++- src/aiperf/plugin/extensible_enums.py | 27 +++- tests/unit/common/enums/test_base_enums.py | 109 ++++++++++++++++ tests/unit/plugin/test_extensible_enums.py | 138 +++++++++++++++++++++ 4 files changed, 289 insertions(+), 12 deletions(-) diff --git a/src/aiperf/common/enums/base_enums.py b/src/aiperf/common/enums/base_enums.py index 62b36bcf4e..68cc5f0aa0 100644 --- a/src/aiperf/common/enums/base_enums.py +++ b/src/aiperf/common/enums/base_enums.py @@ -31,11 +31,12 @@ def __init__(self: Self, *args: object) -> None: def _norm_value(self: Self) -> str: # Lazy fallback for members created outside Enum construction # (e.g. dynamically registered custom members), which skip __init__. - norm = self.__dict__.get("_norm_value_cache") - if norm is None: + try: + return self._norm_value_cache + except AttributeError: norm = _normalize_name(self.value) self._norm_value_cache = norm - return norm + return norm def __str__(self) -> str: return self.value @@ -60,12 +61,26 @@ def __eq__(self: Self, other: object) -> bool: ) return super().__eq__(other) + def __ne__(self: Self, other: object) -> bool: + """Negate __eq__, forwarding NotImplemented to the other operand. + + Required explicitly: Python only derives __ne__ from __eq__ via + object.__ne__, and str.__ne__ sits between this class and object in the + MRO. Without this, != compares raw values and disagrees with the + normalizing __eq__ above, making `a == b` and `a != b` both true. + """ + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + def __hash__(self: Self) -> int: - norm_hash = self.__dict__.get("_norm_hash_cache") - if norm_hash is None: + try: + return self._norm_hash_cache + except AttributeError: norm_hash = hash(self._norm_value()) self._norm_hash_cache = norm_hash - return norm_hash + return norm_hash @classmethod def _missing_(cls, value): diff --git a/src/aiperf/plugin/extensible_enums.py b/src/aiperf/plugin/extensible_enums.py index 271e4e8964..7789d9d19e 100644 --- a/src/aiperf/plugin/extensible_enums.py +++ b/src/aiperf/plugin/extensible_enums.py @@ -110,11 +110,12 @@ def __repr__(self) -> str: def _norm_value(self: Self) -> str: # Lazily cached: members can be created dynamically via register(), # so there is no single construction point to precompute this. - norm = self.__dict__.get("_norm_value_cache") - if norm is None: + try: + return self._norm_value_cache + except AttributeError: norm = _normalize_name(self.value) self._norm_value_cache = norm - return norm + return norm def __eq__(self: Self, other: object) -> bool: if self is other: @@ -127,12 +128,26 @@ def __eq__(self: Self, other: object) -> bool: return self._norm_value() == _normalize_name(other.value) return super().__eq__(other) + def __ne__(self: Self, other: object) -> bool: + """Negate __eq__, forwarding NotImplemented to the other operand. + + Required explicitly: Python only derives __ne__ from __eq__ via + object.__ne__, and str.__ne__ sits between this class and object in the + MRO. Without this, != compares raw values and disagrees with the + normalizing __eq__ above, making `a == b` and `a != b` both true. + """ + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + def __hash__(self: Self) -> int: - norm_hash = self.__dict__.get("_norm_hash_cache") - if norm_hash is None: + try: + return self._norm_hash_cache + except AttributeError: norm_hash = hash(self._norm_value()) self._norm_hash_cache = norm_hash - return norm_hash + return norm_hash @property def name(self) -> str: diff --git a/tests/unit/common/enums/test_base_enums.py b/tests/unit/common/enums/test_base_enums.py index 090d18fb18..6e80b21308 100644 --- a/tests/unit/common/enums/test_base_enums.py +++ b/tests/unit/common/enums/test_base_enums.py @@ -6,6 +6,7 @@ import pytest from pydantic import BaseModel, ValidationError +from pytest import param from aiperf.common.enums.base_enums import CaseInsensitiveStrEnum, _normalize_name @@ -354,3 +355,111 @@ class IntEnum(Enum): X = 1 assert SampleEnum.ALPHA != IntEnum.X + + +# ============================================================================= +# __ne__ / __eq__ Symmetry Tests +# ============================================================================= + + +class TestNotEqualMirrorsEqual: + """`!=` must be the exact negation of `==` for every operand shape. + + Regression guard: `str.__ne__` sits between these classes and `object` in + the MRO, so an inherited `__ne__` silently bypasses the normalizing + `__eq__` and makes `a == b` and `a != b` both True. + """ + + @pytest.mark.parametrize( + "other", + [ + "alpha", + "ALPHA", + "Alpha", + "aLpHa", + "beta", + "nonexistent", + 123, + None, + [], + SampleEnum.ALPHA, + SampleEnum.BETA, + ], + ) # fmt: skip + def test_ne_is_negation_of_eq(self: Self, other: object) -> None: + """`member != other` equals `not (member == other)` for any operand.""" + assert (SampleEnum.ALPHA != other) is not (SampleEnum.ALPHA == other) # noqa: SIM300 - enum must stay on the left; that is the operand order under test + + @pytest.mark.parametrize( + "other", + ["foo_bar", "foo-bar", "FOO_BAR", "FOO-BAR", "Foo-Bar", "baz"], + ) # fmt: skip + def test_ne_is_negation_of_eq_across_dash_forms(self: Self, other: object) -> None: + """Dash/underscore/case variants negate consistently.""" + assert (SampleEnum.FOO_BAR != other) is not (SampleEnum.FOO_BAR == other) # noqa: SIM300 - enum must stay on the left; that is the operand order under test + + @pytest.mark.parametrize( + "other", + ["foo_bar", "foo-bar", "FOO_BAR", "FOO-BAR", "Foo-Bar"], + ) # fmt: skip + def test_ne_false_for_normalized_match(self: Self, other: str) -> None: + """A normalized match is never `!=`.""" + assert (SampleEnum.FOO_BAR != other) is False # noqa: SIM300 - enum must stay on the left; that is the operand order under test + + @pytest.mark.parametrize( + "other", + ["my_value", "my-value", "MY_VALUE", "MY-VALUE"], + ) # fmt: skip + def test_ne_false_for_dash_valued_enum(self: Self, other: str) -> None: + """Dash-valued members are not `!=` their underscore spellings.""" + assert (DashValueEnum.MY_VALUE != other) is False # noqa: SIM300 - enum must stay on the left; that is the operand order under test + + @pytest.mark.parametrize( + "other", + ["foo_bar", "foo-bar", "FOO_BAR", "FOO-BAR", "Foo-Bar"], + ) # fmt: skip + def test_ne_false_with_string_on_the_left(self: Self, other: str) -> None: + """Reflected `!=` also routes through the normalizing comparison.""" + assert (other != SampleEnum.FOO_BAR) is False + + def test_ne_true_for_different_member(self: Self) -> None: + """Genuinely different members stay `!=`.""" + assert (SampleEnum.ALPHA != SampleEnum.BETA) is True + + def test_ne_false_across_enums_with_same_normalized_value(self: Self) -> None: + """Cross-enum members sharing a normalized value are not `!=`.""" + + class EnumA(CaseInsensitiveStrEnum): + ITEM = "foo_bar" + + class EnumB(CaseInsensitiveStrEnum): + ITEM = "foo-bar" + + assert (EnumA.ITEM != EnumB.ITEM) is False + + def test_ne_true_for_non_string_types(self: Self) -> None: + """Non-string, non-enum operands remain `!=`.""" + assert (SampleEnum.ALPHA != 123) is True + assert (SampleEnum.ALPHA != None) is True # noqa: E711 - testing __ne__ with None + assert (SampleEnum.ALPHA != []) is True + + @pytest.mark.parametrize( + "other", + [ + param(123, id="int"), + param(None, id="none"), + param([], id="list"), + param(4.5, id="float"), + ], + ) # fmt: skip + def test_ne_forwards_notimplemented_for_unsupported_operands( + self: Self, other: object + ) -> None: + """__ne__ defers to the other operand instead of asserting inequality. + + The operator-level check above cannot see this: `!= 123` is True either + way, whether __ne__ forwards NotImplemented or wrongly hardcodes True. + Only the direct call distinguishes them, and the difference matters for + any third-party type whose reflected __eq__ should decide the result. + """ + assert SampleEnum.ALPHA.__ne__(other) is NotImplemented diff --git a/tests/unit/plugin/test_extensible_enums.py b/tests/unit/plugin/test_extensible_enums.py index 5b4aa81662..58aa89894c 100644 --- a/tests/unit/plugin/test_extensible_enums.py +++ b/tests/unit/plugin/test_extensible_enums.py @@ -6,6 +6,7 @@ import pytest from pydantic import BaseModel, ValidationError +from pytest import param from aiperf.plugin.extensible_enums import ( ExtensibleStrEnum, @@ -249,6 +250,25 @@ def test_norm_value_cached_lazily( assert hash(ext) == hash("ext") assert ext.__dict__.get("_norm_hash_cache") == hash("ext") + def test_norm_value_populated_on_never_used_extension(self: Self) -> None: + """A registered member that has never been compared still normalizes. + + Distinct from the pop-and-recompute case above: register() creates the + member via str.__new__ so the cache attribute is absent, not stale. + """ + + class TestEnum(ExtensibleStrEnum): + BASE = "base" + + TestEnum.register("FOO_BAR", "foo_bar") + ext = TestEnum.FOO_BAR + assert "_norm_value_cache" not in ext.__dict__ + assert "_norm_hash_cache" not in ext.__dict__ + + assert ext == "FOO-BAR" + assert hash(ext) == hash("foo_bar") + assert ext.__dict__.get("_norm_value_cache") == "foo_bar" + # ============================================================================= # Registration Tests @@ -712,3 +732,121 @@ class EnumB(ExtensibleStrEnum): assert EnumA.ITEM == "foo_bar" assert EnumB.ITEM == "foo-bar" assert EnumB.ITEM == "foo_bar" + + +# ============================================================================= +# __ne__ / __eq__ Symmetry Tests +# ============================================================================= + + +class TestNotEqualMirrorsEqual: + """`!=` must be the exact negation of `==` for every operand shape. + + Regression guard: `str.__ne__` sits between this class and `object` in the + MRO, so an inherited `__ne__` silently bypasses the normalizing `__eq__` + and makes `a == b` and `a != b` both True. + """ + + @pytest.fixture + def enum_with_underscores(self) -> type[ExtensibleStrEnum]: + class UnderscoreEnum(ExtensibleStrEnum): + FOO_BAR = "foo_bar" + BAZ_QUX = "baz_qux" + + return UnderscoreEnum + + @pytest.mark.parametrize( + "other", + [ + "alpha", + "ALPHA", + "Alpha", + "aLpHa", + "beta", + "nonexistent", + 123, + None, + [], + SampleEnum.ALPHA, + SampleEnum.BETA, + ], + ) # fmt: skip + def test_ne_is_negation_of_eq(self: Self, other: object) -> None: + """`member != other` equals `not (member == other)` for any operand.""" + assert (SampleEnum.ALPHA != other) is not (SampleEnum.ALPHA == other) # noqa: SIM300 - enum must stay on the left; that is the operand order under test + + @pytest.mark.parametrize( + "other", + ["foo_bar", "foo-bar", "FOO_BAR", "FOO-BAR", "Foo-Bar"], + ) # fmt: skip + def test_ne_false_for_normalized_match( + self: Self, enum_with_underscores: type[ExtensibleStrEnum], other: str + ) -> None: + """A normalized match is never `!=`.""" + assert (enum_with_underscores.FOO_BAR != other) is False # noqa: SIM300 - enum must stay on the left; that is the operand order under test + + @pytest.mark.parametrize( + "other", + ["foo_bar", "foo-bar", "FOO_BAR", "FOO-BAR", "Foo-Bar"], + ) # fmt: skip + def test_ne_false_with_string_on_the_left( + self: Self, enum_with_underscores: type[ExtensibleStrEnum], other: str + ) -> None: + """Reflected `!=` also routes through the normalizing comparison.""" + assert (other != enum_with_underscores.FOO_BAR) is False + + def test_ne_true_for_different_member( + self: Self, enum_with_underscores: type[ExtensibleStrEnum] + ) -> None: + """Genuinely different members stay `!=`.""" + assert (enum_with_underscores.FOO_BAR != enum_with_underscores.BAZ_QUX) is True + + def test_ne_false_for_registered_extension_spellings(self: Self) -> None: + """Runtime-registered extensions negate consistently too.""" + + class TestEnum(ExtensibleStrEnum): + BASE = "base" + + TestEnum.register("FOO_BAR", "foo_bar") + + assert (TestEnum.FOO_BAR != "foo-bar") is False + assert (TestEnum.FOO_BAR != "FOO-BAR") is False + assert (TestEnum.FOO_BAR != "base") is True + + def test_ne_false_across_enums_with_same_normalized_value(self: Self) -> None: + """Cross-enum members sharing a normalized value are not `!=`.""" + + class EnumA(ExtensibleStrEnum): + ITEM = "foo_bar" + + class EnumB(ExtensibleStrEnum): + ITEM = "foo-bar" + + assert (EnumA.ITEM != EnumB.ITEM) is False + + def test_ne_true_for_non_string_types(self: Self) -> None: + """Non-string, non-enum operands remain `!=`.""" + assert (SampleEnum.ALPHA != 123) is True + assert (SampleEnum.ALPHA != None) is True # noqa: E711 - testing __ne__ with None + assert (SampleEnum.ALPHA != []) is True + + @pytest.mark.parametrize( + "other", + [ + param(123, id="int"), + param(None, id="none"), + param([], id="list"), + param(4.5, id="float"), + ], + ) # fmt: skip + def test_ne_forwards_notimplemented_for_unsupported_operands( + self: Self, other: object + ) -> None: + """__ne__ defers to the other operand instead of asserting inequality. + + The operator-level check above cannot see this: `!= 123` is True either + way, whether __ne__ forwards NotImplemented or wrongly hardcodes True. + Only the direct call distinguishes them, and the difference matters for + any third-party type whose reflected __eq__ should decide the result. + """ + assert SampleEnum.ALPHA.__ne__(other) is NotImplemented From 2c6e3d01ac35977acc54e00701fec6a4eb5caa08 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Fri, 21 Aug 2026 13:19:19 -0700 Subject: [PATCH 028/119] fix(dataset): report smallest peak context when all traces are rejected (#1287) Signed-off-by: Shimi Bandiel (cherry picked from commit bfe33151de75426710e51ca054823aa91342cebc) --- src/aiperf/dataset/loader/selection.py | 32 ++++++++++++++++- .../loader/semianalysis_cc_traces_weka.py | 12 ++++--- src/aiperf/dataset/loader/tracelab_trace.py | 12 ++++--- src/aiperf/dataset/loader/weka_trace.py | 13 +++---- .../loader/test_selection_filter_then_cap.py | 36 ++++++++++++++++++- 5 files changed, 87 insertions(+), 18 deletions(-) diff --git a/src/aiperf/dataset/loader/selection.py b/src/aiperf/dataset/loader/selection.py index b99df1d787..02d23532e2 100644 --- a/src/aiperf/dataset/loader/selection.py +++ b/src/aiperf/dataset/loader/selection.py @@ -30,6 +30,8 @@ class SelectionStats: scanned: int = 0 rejected_by_maxctx: int = 0 largest_observed: int = 0 + smallest_observed: int = 0 + """The minimum peak context among scanned candidates.""" eligible: int = 0 loaded: int = 0 @@ -51,6 +53,8 @@ def filter_then_cap( stats.scanned += 1 if peak > stats.largest_observed: stats.largest_observed = peak + if stats.scanned == 1 or peak < stats.smallest_observed: + stats.smallest_observed = peak if max_context_length is not None and peak > max_context_length: stats.rejected_by_maxctx += 1 continue @@ -77,13 +81,39 @@ def log_selection_summary( f"(--max-context-length={max_context_length}), " f"eligible {stats.eligible:,}, loaded {stats.loaded:,} " f"(--num-dataset-entries={num_dataset_entries}), " - f"largest_observed_peak_context {stats.largest_observed:,}" + f"largest_observed_peak_context {stats.largest_observed:,}, " + f"smallest_observed_peak_context {stats.smallest_observed:,}" ) ) +def raise_on_empty_selection( + stats: SelectionStats, + *, + source: str, + num_dataset_entries: int | None, + max_context_length: int | None, +) -> None: + """Raise DatasetLoaderError when no candidates are selected, with a size hint.""" + from aiperf.common.exceptions import DatasetLoaderError + + msg = ( + f"No eligible traces in {source} after " + f"filter-then-cap (scanned {stats.scanned}, " + f"--max-context-length={max_context_length}, " + f"--num-dataset-entries={num_dataset_entries})." + ) + if stats.smallest_observed > 0: + msg += ( + f" Smallest trace requires {stats.smallest_observed} tokens; " + f"raise --max-context-length to at least that (e.g. --max-context-length {stats.smallest_observed}) to admit any trace." + ) + raise DatasetLoaderError(msg) + + __all__ = [ "SelectionStats", "filter_then_cap", "log_selection_summary", + "raise_on_empty_selection", ] diff --git a/src/aiperf/dataset/loader/semianalysis_cc_traces_weka.py b/src/aiperf/dataset/loader/semianalysis_cc_traces_weka.py index 1c0f8f75c7..304d710e6b 100644 --- a/src/aiperf/dataset/loader/semianalysis_cc_traces_weka.py +++ b/src/aiperf/dataset/loader/semianalysis_cc_traces_weka.py @@ -189,11 +189,13 @@ def _candidates() -> Any: max_context_length=max_ctx, ) if not kept_pairs: - raise DatasetLoaderError( - f"No eligible traces in {self.hf_dataset_name} after " - f"filter-then-cap (scanned {stats.scanned}, " - f"--max-context-length={max_ctx}, " - f"--num-dataset-entries={num_entries})." + from aiperf.dataset.loader.selection import raise_on_empty_selection + + raise_on_empty_selection( + stats, + source=self.hf_dataset_name, + num_dataset_entries=num_entries, + max_context_length=max_ctx, ) out = {} diff --git a/src/aiperf/dataset/loader/tracelab_trace.py b/src/aiperf/dataset/loader/tracelab_trace.py index 332e69bde6..c178181a33 100644 --- a/src/aiperf/dataset/loader/tracelab_trace.py +++ b/src/aiperf/dataset/loader/tracelab_trace.py @@ -368,11 +368,13 @@ def _select(self, traces: dict[str, WekaTrace]) -> dict[str, list[WekaTrace]]: max_context_length=max_ctx, ) if not kept: - raise DatasetLoaderError( - f"No eligible TraceLab traces in '{self.filename}' after " - f"filter-then-cap (scanned {stats.scanned}, " - f"--max-context-length={max_ctx}, " - f"--num-dataset-entries={num_entries})." + from aiperf.dataset.loader.selection import raise_on_empty_selection + + raise_on_empty_selection( + stats, + source=f"{self.filename}", + num_dataset_entries=num_entries, + max_context_length=max_ctx, ) return {tid: [trace] for tid, trace in kept} diff --git a/src/aiperf/dataset/loader/weka_trace.py b/src/aiperf/dataset/loader/weka_trace.py index f60459b4c4..e943a8fce5 100644 --- a/src/aiperf/dataset/loader/weka_trace.py +++ b/src/aiperf/dataset/loader/weka_trace.py @@ -990,7 +990,6 @@ def _select_traces_filter_then_cap( max_context_length: int | None, ) -> dict[str, list[WekaTrace]]: """Filter by peak context, then keep the first N eligible traces.""" - from aiperf.common.exceptions import DatasetLoaderError from aiperf.dataset.loader.selection import ( filter_then_cap, log_selection_summary, @@ -1018,11 +1017,13 @@ def _candidates() -> Any: max_context_length=max_context_length, ) if not kept_pairs: - raise DatasetLoaderError( - f"All traces rejected by filter-then-cap " - f"(scanned {stats.scanned}, " - f"--max-context-length={max_context_length}, " - f"--num-dataset-entries={num_dataset_entries})." + from aiperf.dataset.loader.selection import raise_on_empty_selection + + raise_on_empty_selection( + stats, + source="weka_trace", + num_dataset_entries=num_dataset_entries, + max_context_length=max_context_length, ) return {trace_id: wekas for trace_id, wekas in kept_pairs} diff --git a/tests/unit/dataset/loader/test_selection_filter_then_cap.py b/tests/unit/dataset/loader/test_selection_filter_then_cap.py index 253746e4f9..ac863ebcb4 100644 --- a/tests/unit/dataset/loader/test_selection_filter_then_cap.py +++ b/tests/unit/dataset/loader/test_selection_filter_then_cap.py @@ -11,6 +11,7 @@ import pytest from pytest import param +from aiperf.common.exceptions import DatasetLoaderError from aiperf.dataset.loader.selection import SelectionStats, filter_then_cap from aiperf.dataset.loader.weka_trace import WekaTraceLoader from tests.unit.dataset.loader.conftest import make_weka_run @@ -49,6 +50,7 @@ def test_filter_then_cap_skips_rejects_until_n_eligible() -> None: assert stats.eligible == _NUM_ENTRIES assert stats.loaded == _NUM_ENTRIES assert stats.largest_observed == _OVER_INPUT + _OUTPUT + assert stats.smallest_observed == _UNDER_INPUT + _OUTPUT def test_filter_then_cap_both_none_keeps_scan_order() -> None: @@ -57,7 +59,9 @@ def test_filter_then_cap_both_none_keeps_scan_order() -> None: candidates, num_dataset_entries=None, max_context_length=None ) assert kept == [0, 1, 2, 3, 4] - assert stats == SelectionStats(scanned=5, eligible=5, loaded=5, largest_observed=1) + assert stats == SelectionStats( + scanned=5, eligible=5, loaded=5, largest_observed=1, smallest_observed=1 + ) def _weka_trace_dict(index: int) -> dict: @@ -144,3 +148,33 @@ def test_weka_file_loader_filter_then_cap( max_context_length=max_ctx, ) assert list(data.keys()) == expected_ids + + +def test_weka_file_loader_all_rejected_raises_with_smallest_observed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _write_weka_dir(tmp_path / "weka") + # Set max_context_length very small so all are rejected (smallest trace peak context is 110) + run = make_weka_run( + model_names=["m"], + tokenizer_name="t", + max_context_length=5, + entries=None, + ) + loader = _make_loader(root, run, monkeypatch) + data = loader.load_dataset() + + with pytest.raises(DatasetLoaderError) as exc_info: + loader._select_traces_filter_then_cap( + data, + num_dataset_entries=None, + max_context_length=5, + ) + + err_msg = str(exc_info.value) + assert "No eligible traces in weka_trace" in err_msg + assert ( + "Smallest trace requires 110 tokens; raise --max-context-length to at least that (e.g. --max-context-length 110) to admit any trace." + in err_msg + ) From 4ba45490a9306983fa9130257f6059cad4781ee9 Mon Sep 17 00:00:00 2001 From: Michael Feil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:22:24 -0700 Subject: [PATCH 029/119] feat(http): send session affinity header by default (#1312) Signed-off-by: Michael Feil <63565275+michaelfeil@users.noreply.github.com> (cherry picked from commit dd3f09b0c34710470444bad17c9e7050c1cd694a) --- docs/benchmark-modes/trace-replay.md | 3 +++ docs/environment-variables.md | 1 + src/aiperf/common/environment.py | 4 ++++ src/aiperf/transports/base_transports.py | 3 +++ tests/unit/transports/test_base_transport.py | 3 ++- 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/benchmark-modes/trace-replay.md b/docs/benchmark-modes/trace-replay.md index 946cf33428..1faad5275c 100644 --- a/docs/benchmark-modes/trace-replay.md +++ b/docs/benchmark-modes/trace-replay.md @@ -111,6 +111,9 @@ aiperf profile \ AIPerf refuses parameter sweeps (e.g. `--concurrency 1,2,4`) against an auto-promoted trace; either pin a single value or pass `--no-fixed-schedule` to keep your sweep semantics. +AIPerf sends the stable replay-session ID in both `X-Correlation-ID` and +`X-Session-Affinity` so session-based traffic can be routed consistently. + ### Large Baseten Parquet Traces The `baseten_trace` loader reads trace rows and sampled-session metadata in diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 2ccf049469..ac5070cd94 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -166,6 +166,7 @@ HTTP client socket and connection configuration. Controls low-level socket optio | `AIPERF_HTTP_IP_VERSION` | `'4'` | one of: '4' / '6' / 'auto' | IP version for HTTP socket connections. Options: '4' (AF_INET, default), '6' (AF_INET6), or 'auto' (AF_UNSPEC, system chooses). | | `AIPERF_HTTP_TRUST_ENV` | `False` | — | Trust environment variables for HTTP client configuration. When enabled, aiohttp will read proxy settings from HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables. | | `AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID` | `False` | — | Also send X-Session-ID with the stable X-Correlation-ID value. This transport setting is the supported way to enable generic HTTP session affinity. It is ADDITIVE (both headers are sent); --session-header only RENAMES the single correlation header. | +| `AIPERF_HTTP_X_SESSION_AFFINITY_FROM_CORRELATION_ID` | `True` | — | Also send X-Session-Affinity with the stable X-Correlation-ID value. | | `AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID` | `False` | — | Also send X-SMG-Routing-Key with the stable X-Correlation-ID value. This transport setting is the supported affinity path for the SGLang Model Gateway manual routing policy. | | `AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID` | `False` | — | Also send X-Dynamo-Session-ID with the stable X-Correlation-ID value, plus X-Dynamo-Parent-Session-ID on subagent children. This transport setting is the supported affinity path for a Dynamo frontend running --router-session-affinity-ttl-secs, pinning every turn of a session to the replica holding its KV prefix. | | `AIPERF_HTTP_VIDEO_POLL_INTERVAL` | `0.1` | ≥ 0.001, ≤ 10.0 | Interval in seconds between status polls for async video generation jobs. Lower values provide faster completion detection but increase server load. Applies to the aiohttp transport. | diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index ed1b4e50e8..3d19d7d835 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -836,6 +836,10 @@ class _HTTPSettings(BaseSettings): "affinity. It is ADDITIVE (both headers are sent); --session-header only " "RENAMES the single correlation header.", ) + X_SESSION_AFFINITY_FROM_CORRELATION_ID: bool = Field( + default=True, + description="Also send X-Session-Affinity with the stable X-Correlation-ID value.", + ) X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: bool = Field( default=False, description="Also send X-SMG-Routing-Key with the stable X-Correlation-ID value. " diff --git a/src/aiperf/transports/base_transports.py b/src/aiperf/transports/base_transports.py index ecb99e3ef7..0d8a5a3de1 100644 --- a/src/aiperf/transports/base_transports.py +++ b/src/aiperf/transports/base_transports.py @@ -147,6 +147,9 @@ def build_headers(self, request_info: RequestInfo) -> dict[str, str]: if Environment.HTTP.X_SESSION_ID_FROM_CORRELATION_ID: _remove_headers_case_insensitive(headers, ("X-Session-ID",)) headers["X-Session-ID"] = request_info.x_correlation_id + if Environment.HTTP.X_SESSION_AFFINITY_FROM_CORRELATION_ID: + _remove_headers_case_insensitive(headers, ("X-Session-Affinity",)) + headers["X-Session-Affinity"] = request_info.x_correlation_id # Additive SGLang Model Gateway routing key: co-locate a session's # requests on one worker via the stable correlation ID. if Environment.HTTP.X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: diff --git a/tests/unit/transports/test_base_transport.py b/tests/unit/transports/test_base_transport.py index f00d1bbc4d..4c518b3a9a 100644 --- a/tests/unit/transports/test_base_transport.py +++ b/tests/unit/transports/test_base_transport.py @@ -515,13 +515,14 @@ def _make_request_info( ) def test_default_sends_x_correlation_id(self): - """Without --session-header, X-Correlation-ID is used as before.""" + """Default requests send correlation and additive affinity headers.""" transport = self._make_transport() request_info = self._make_request_info(transport) headers = transport.build_headers(request_info) assert headers["X-Correlation-ID"] == "conv-uuid-123" + assert headers["X-Session-Affinity"] == "conv-uuid-123" assert "X-Session-ID" not in headers def test_turn_headers_preserve_external_session_identity(self): From dc0721d1e3e0e8294ada46b52c1973f6172ae5f0 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:01:54 -0700 Subject: [PATCH 030/119] fix(cache-bust): add WARMUP_ISOLATION_* targets; remove unconditional warmup prefix (#1256) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 1b34de0637b4128e8d2cd1662c9e8285428ba91a) --- docs/cli-options.md | 8 +- docs/index.yml | 2 + docs/reference/cache-bust.md | 354 ++++++++++++++++++ src/aiperf/common/constants.py | 3 - src/aiperf/common/enums/enums.py | 2 + src/aiperf/config/config.py | 87 +++++ src/aiperf/config/dataset/content.py | 27 +- src/aiperf/config/flags/cli_config.py | 14 +- src/aiperf/credit/issuer.py | 20 +- src/aiperf/endpoints/openai_chat.py | 31 +- src/aiperf/endpoints/openai_completions.py | 12 - src/aiperf/timing/phase/runner.py | 27 +- src/aiperf/timing/strategies/cache_bust.py | 35 +- src/aiperf/workers/worker.py | 56 +-- ..._config_cache_bust_lockdown_adversarial.py | 168 ++++++++- .../common/enums/test_cache_bust_target.py | 14 + tests/unit/credit/test_issuer.py | 166 +++++++- .../endpoints/test_completions_endpoint.py | 84 ----- .../endpoints/test_message_merging_fixes.py | 47 --- .../unit/timing/strategies/test_cache_bust.py | 37 ++ tests/unit/workers/test_worker.py | 89 +---- .../test_worker_cache_bust_injection.py | 120 ++++++ 22 files changed, 1046 insertions(+), 357 deletions(-) create mode 100644 docs/reference/cache-bust.md diff --git a/docs/cli-options.md b/docs/cli-options.md index 6fe6fbb631..7144ac2492 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -678,8 +678,8 @@ Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare so #### `--cache-bust` `` -Where (and how) to inject a per-conversation cache-bust marker. Prefix variants prepend at token 0 (most aggressive); suffix variants append after existing content. 'none' disables the feature (default). -
_Choices: [`none`, `system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`]_ +Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (system_prefix, system_suffix, first_turn_prefix, first_turn_suffix) — inject a per-trajectory unique SHA-256 digest marker that is identical across warmup and profiling, so warmup KV-cache work transfers to profiling while preventing cross-trajectory cache sharing. Prefix variants prepend at token 0; suffix variants append after existing content. (2) Warmup-isolation targets (warmup_isolation_system, warmup_isolation_first_turn) — inject a constant '[warmup]' marker only during the WARMUP phase; profiling sees no marker (fully cold start or system-pre-warmed). Incompatible with agentic_replay timing mode. 'none' disables the feature (default). +
_Choices: [`none`, `system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`, `warmup_isolation_system`, `warmup_isolation_first_turn`]_
_Default: `none`_ ### Prefix Prompt @@ -2218,8 +2218,8 @@ Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare so #### `--cache-bust` `` -Where (and how) to inject a per-conversation cache-bust marker. Prefix variants prepend at token 0 (most aggressive); suffix variants append after existing content. 'none' disables the feature (default). -
_Choices: [`none`, `system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`]_ +Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (system_prefix, system_suffix, first_turn_prefix, first_turn_suffix) — inject a per-trajectory unique SHA-256 digest marker that is identical across warmup and profiling, so warmup KV-cache work transfers to profiling while preventing cross-trajectory cache sharing. Prefix variants prepend at token 0; suffix variants append after existing content. (2) Warmup-isolation targets (warmup_isolation_system, warmup_isolation_first_turn) — inject a constant '[warmup]' marker only during the WARMUP phase; profiling sees no marker (fully cold start or system-pre-warmed). Incompatible with agentic_replay timing mode. 'none' disables the feature (default). +
_Choices: [`none`, `system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`, `warmup_isolation_system`, `warmup_isolation_first_turn`]_
_Default: `none`_ ### Prefix Prompt diff --git a/docs/index.yml b/docs/index.yml index a4fbda72d5..d25ab45eb8 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -245,6 +245,8 @@ navigation: path: reference/json-export-schema.md - page: HTTP API Endpoints path: reference/api-endpoints.md + - page: Cache-Bust Targets + path: reference/cache-bust.md - page: YAML Config Roadmap path: reference/yaml-config-roadmap.md - page: Mocking a ROCm Environment diff --git a/docs/reference/cache-bust.md b/docs/reference/cache-bust.md new file mode 100644 index 0000000000..f010e76b50 --- /dev/null +++ b/docs/reference/cache-bust.md @@ -0,0 +1,354 @@ +--- +sidebar-title: Cache-Bust Targets +--- + +# Cache-Bust Targets + +The `--cache-bust` flag (CLI) or `cache_bust.target` (YAML) controls how AIPerf mutates outgoing +request payloads to defeat the server's KV-cache prefix matching. This is essential when you need +per-trajectory isolation or want precise control over how much warmup KV-cache work carries over +into the profiling phase. + +## Configuration + +**CLI:** +```bash +aiperf profile --cache-bust warmup_isolation_first_turn ... +``` + +**YAML (`cache_bust.target` inside a dataset's `prompts` block):** +```yaml +datasets: + - name: main + type: synthetic + prompts: + isl: 512 + osl: 128 + cache_bust: + target: warmup_isolation_first_turn + prefix_prompts: + shared_system_length: 512 +``` + +## Behavior Table + +| Target | Marker | Per-trajectory unique | Injection point | Warmup→Profiling KV cache | Cross-trajectory isolation | +|---|---|---|---|---|---| +| `none` | — | — | — | Full warmup priming | None | +| `system_prefix` | `[rid:xxx]\n\n` | Yes (SHA-256 digest) | System message (token 0) | Full warmup priming (shared marker) | Yes | +| `system_suffix` | `\n\n[rid:xxx]` | Yes (SHA-256 digest) | System message (end) | Full warmup priming (shared marker) | Yes | +| `first_turn_prefix` | `[rid:xxx]\n\n` | Yes (SHA-256 digest) | First user turn (token 0) | System prompt pre-warmed; user cold | Yes | +| `first_turn_suffix` | `\n\n[rid:xxx]` | Yes (SHA-256 digest) | First user turn (end) | System prompt pre-warmed; user cold | Yes | +| `warmup_isolation_system` | `[warmup]\n\n` | No (constant) | System message (token 0) | None — fully cold start | N/A | +| `warmup_isolation_first_turn` | `[warmup]\n\n` | No (constant) | First user turn (token 0) | System prompt pre-warmed; user cold | N/A | + +## `none` (default) + +Cache-bust is disabled. No marker is injected. The server's KV-cache prefix matching operates +without interference: warmup turns prime the cache, and profiling requests for the same trajectory +benefit from that warm state. + +**Wire payload (chat, warmup and profiling identical):** + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"} + ] +} +``` + +## RID-Based Targets + +`system_prefix`, `system_suffix`, `first_turn_prefix`, and `first_turn_suffix` each inject a +per-trajectory unique marker derived from a SHA-256 digest of the benchmark ID, recycle pass, +trajectory index, and base trace ID. The same trajectory always receives the same digest — across +warmup and profiling — so warmup KV-cache work transfers to profiling for that trajectory. +Different trajectories (different lanes or recycle passes) receive distinct digests, preventing +cross-trajectory cache sharing. + +The marker format is `[rid:<12-hex-chars>]`. Prefix variants append `\n\n` after the marker; +suffix variants prepend `\n\n` before it. + +### `system_prefix` + +Injects `[rid:xxx]\n\n` at the very beginning of the system message content (token 0). + +```json +{ + "messages": [ + {"role": "system", "content": "[rid:a3f9b2c1d4e5]\n\nYou are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"} + ] +} +``` + +Both warmup and profiling payloads are identical (same digest for the same trajectory). + +### `system_suffix` + +Injects `\n\n[rid:xxx]` at the end of the system message content. + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant.\n\n[rid:a3f9b2c1d4e5]"}, + {"role": "user", "content": "What is the capital of France?"} + ] +} +``` + +Both warmup and profiling payloads are identical (same digest for the same trajectory). + +### `first_turn_prefix` + +Injects `[rid:xxx]\n\n` at the very beginning of the first user turn content (token 0). The +system message is untouched, so system-prompt tokens are always prefix-cached across trajectories +while per-trajectory isolation starts at the first user turn. + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "[rid:a3f9b2c1d4e5]\n\nWhat is the capital of France?"} + ] +} +``` + +Both warmup and profiling payloads are identical (same digest for the same trajectory). + +### `first_turn_suffix` + +Injects `\n\n[rid:xxx]` at the end of the first user turn content. The system message is untouched. + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?\n\n[rid:a3f9b2c1d4e5]"} + ] +} +``` + +Both warmup and profiling payloads are identical (same digest for the same trajectory). + +## Warmup-Isolation Targets + +`warmup_isolation_system` and `warmup_isolation_first_turn` use a constant marker +(`[warmup]\n\n`) during the WARMUP phase. During PROFILING, no marker is injected (the field is +`None`). Because the warmup and profiling payloads differ, the server cannot reuse warmup KV-cache +entries for profiling requests — warmup work is deliberately discarded. + +These targets are **phase-aware**: the marker is present only during warmup and absent during +profiling. This is the opposite of the RID targets, which are phase-agnostic (same marker in both +phases) so warmup primes profiling. + +> **Incompatibility with `agentic_replay`:** These targets cannot be used with the +> `agentic_replay` timing mode. In agentic replay, the same session object spans the +> WARMUP→PROFILING boundary and Turn objects mutated during warmup would carry the +> `[warmup]` marker into profiling credits. Use `cache_bust=none` for `agentic_replay` +> workloads, or one of the RID-based targets for per-trajectory isolation. + +### `warmup_isolation_system` + +Injects `[warmup]\n\n` at the very beginning of the system message during WARMUP. During +PROFILING the payload is clean — no marker anywhere. + +> **Synthetic datasets only — requires a shared system prompt.** +> For synthetic datasets, `warmup_isolation_system` is rejected at config validation time +> if no system message is statically present. This means `--shared-system-prompt-length` +> must be set. If you are using `--user-context-prompt-length` or `--num-prefix-prompts` +> without a shared system prompt, switch to `warmup_isolation_first_turn` instead. +> +> File and public datasets are not checked statically — system-message presence depends +> on the dataset content and cannot be determined at config time. If the dataset does not +> produce a system message, the marker will silently fall through to the first user turn. + +**WARMUP phase:** + +```json +{ + "messages": [ + {"role": "system", "content": "[warmup]\n\nYou are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"} + ] +} +``` + +**PROFILING phase:** + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"} + ] +} +``` + +The warmup marker poisons every prefix cache token from position 0, so profiling sees a fully +cold start — no benefit from warmup at any level. + +### `warmup_isolation_first_turn` + +Injects `[warmup]\n\n` at the very beginning of the first user turn during WARMUP. The system +message is untouched in both phases. + +**WARMUP phase:** + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "[warmup]\n\nWhat is the capital of France?"} + ] +} +``` + +**PROFILING phase:** + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"} + ] +} +``` + +Because the system message is identical in both phases, the server can prefix-cache system-prompt +tokens during warmup. However, the diverging first user turn means user-turn tokens onward are +cold in profiling. + +## Scenario Summary + +The table below maps each combination of prefix prompt mode and cache-bust target to the resulting +warmup→profiling cache behavior. Use it to pick the right pair of flags for your scenario. + +| Prefix prompt mode | Flags | `--cache-bust` | Profiling system cache | Profiling user cache | Valid? | +|---|---|---|---|---|---| +| None | _(no prefix flags)_ | `none` | Pre-warmed | Pre-warmed | ✅ | +| None | _(no prefix flags)_ | `warmup_isolation_first_turn` | Pre-warmed | Cold | ✅ | +| None | _(no prefix flags)_ | `warmup_isolation_system` | — | — | ❌ no system message | +| Shared system | `--shared-system-prompt-length N` | `none` | Pre-warmed | Pre-warmed | ✅ | +| Shared system | `--shared-system-prompt-length N` | `warmup_isolation_system` | Cold | Cold | ✅ fully cold start | +| Shared system | `--shared-system-prompt-length N` | `warmup_isolation_first_turn` | Pre-warmed | Cold | ✅ system pre-warmed | +| User context | `--user-context-prompt-length N` | `none` | N/A (no system) | Pre-warmed | ✅ | +| User context | `--user-context-prompt-length N` | `warmup_isolation_first_turn` | N/A (no system) | Context pre-warmed; query cold | ✅ | +| User context | `--user-context-prompt-length N` | `warmup_isolation_system` | — | — | ❌ no system message | +| Prefix pool | `--num-prefix-prompts N --prefix-prompt-length M` | `none` | N/A (no system) | Pre-warmed | ✅ | +| Prefix pool | `--num-prefix-prompts N --prefix-prompt-length M` | `warmup_isolation_first_turn` | N/A (no system) | Cold (prefix + query) | ✅ | +| Prefix pool | `--num-prefix-prompts N --prefix-prompt-length M` | `warmup_isolation_system` | — | — | ❌ no system message | + +**Key:** "Pre-warmed" = profiling requests can reuse KV-cache entries from warmup for those tokens. +"Cold" = warmup marker differs from profiling payload; server cannot reuse warmup cache entries. +"N/A (no system)" = configuration produces no system message; system cache column does not apply. + +## Prefix Prompt Compatibility + +The warmup-isolation targets interact with the three prefix prompt modes differently. + +### Shared system prompt (`--shared-system-prompt-length`) + +A synthetic system message is created and shared identically across all sessions. + +| Target | Warmup system msg | Profiling system msg | Warmup user msg | Profiling user msg | +|---|---|---|---|---| +| `warmup_isolation_system` | `[warmup]\n\n` | `` (clean) | `` | `` | +| `warmup_isolation_first_turn` | `` | `` | `[warmup]\n\n` | `` (clean) | + +`warmup_isolation_system` makes profiling fully cold (system poisoned during warmup). +`warmup_isolation_first_turn` keeps the system prompt pre-warmed; only user-turn tokens are cold. + +### User context prefix (`--user-context-prompt-length`) + +Each session gets a unique per-session context message prepended as a **user-role** message +before the main user prompt. No system message is created. + +| Target | Warmup msg 1 (context) | Warmup msg 2 (prompt) | Profiling msg 1 | Profiling msg 2 | +|---|---|---|---|---| +| `warmup_isolation_system` | — | — | — | — | +| `warmup_isolation_first_turn` | `` (clean) | `[warmup]\n\n` | `` | `` (clean) | + +`warmup_isolation_system` is **rejected** at config validation time because there is no system +message slot to target. Use `warmup_isolation_first_turn` instead. + +`warmup_isolation_first_turn` injects into the main user prompt (the second user message), not +the per-session context prefix. This means the context prefix tokens are pre-warmed during +profiling; only the main query tokens are cold. + +### Prefix prompt pool (`--num-prefix-prompts` / `--prefix-prompt-length`) + +The prefix pool content is concatenated directly into the user message — there is no system +message. + +| Target | Warmup user msg | Profiling user msg | +|---|---|---| +| `warmup_isolation_system` | — | — | +| `warmup_isolation_first_turn` | `[warmup]\n\n` | `` (clean) | + +`warmup_isolation_system` is **rejected** at config validation time (no system message slot). + +`warmup_isolation_first_turn` injects at the start of the combined prefix+prompt user message, +making the entire content cold in profiling — including prefix pool tokens. + +## Choosing a Target + +```mermaid +flowchart TD + A{Warmup-to-profiling\ncache transfer desired?} -->|Yes| B{Cross-trajectory\nisolation needed?} + A -->|No| C{System prompt\npre-warmed?} + + B -->|No| D["`**none**`"] + B -->|Yes| E{Isolate from\ntoken 0?} + + E -->|Yes — system prompt\nvaries per trajectory| F["`**system_prefix**\n**system_suffix**`"] + E -->|No — share system-prompt\ncache across trajectories| G["`**first_turn_prefix**\n**first_turn_suffix**`"] + + C -->|No — fully cold start| H["`**warmup_isolation_system**`"] + C -->|Yes — system prompt\nalready resident| I["`**warmup_isolation_first_turn**`"] +``` + +### `none` + +Warmup primes the full KV cache for the profiling phase. Best for latency benchmarks where KV +cache hit rate is itself a variable under test, or when you want to measure a fully warmed +steady-state server. + +### RID targets (`system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`) + +Each trajectory receives a unique digest marker. Trajectories cannot share each other's cached +prefixes. Within a single trajectory, warmup still primes the profiling phase because the digest +is phase-agnostic — the same marker appears in both warmup and profiling turns for that +trajectory. Use these when running multi-trajectory agentic workloads where cross-session cache +contamination would skew latency numbers. + +Choose the `system_*` variants when the system prompt itself must be unique per trajectory. +Choose the `first_turn_*` variants when you want the system prompt pre-cached across all +trajectories (cheaper amortized cost) but each trajectory's user path isolated from others. + +> **Note:** RID targets do not isolate warmup from profiling — the same digest appears in both +> phases, so warmup primes the profiling cache for each trajectory. There is no single target that +> provides both per-trajectory isolation and a cold profiling start; if you need a fully cold +> profiling phase, use `warmup_isolation_system` or `warmup_isolation_first_turn` instead (which +> offer no per-trajectory isolation). + +### `warmup_isolation_system` + +Profiling sees a fully cold start — no KV cache benefit from warmup. Use this to measure +cold-cache throughput or to simulate the first ever request to a freshly started server. + +### `warmup_isolation_first_turn` + +The system prompt is pre-warmed (as if already resident from prior real traffic), but each user +turn arrives cold. This models a common production deployment where a shared system prompt is +always in cache while individual user queries are novel. Use this to benchmark the realistic +latency of new user queries against a pre-warmed system prompt. + +## Mutual Exclusivity + +Pass only one `--cache-bust` value per run. The `warmup_isolation_*` targets and the RID targets +are mutually exclusive by design: warmup-isolation targets are phase-aware (marker present only +during WARMUP) while RID targets are phase-agnostic (marker identical in both phases). Combining +them in a single run is not supported. diff --git a/src/aiperf/common/constants.py b/src/aiperf/common/constants.py index 7419466592..f8e3399254 100644 --- a/src/aiperf/common/constants.py +++ b/src/aiperf/common/constants.py @@ -3,8 +3,6 @@ import platform as _platform -from aiperf.common.enums.enums import CreditPhase - # Platform detection — evaluated once at import time. IS_WINDOWS: bool = _platform.system() == "Windows" IS_MACOS: bool = _platform.system() == "Darwin" @@ -18,7 +16,6 @@ NANOS_PER_MILLIS = 1_000_000 MILLIS_PER_SECOND = 1000 BYTES_PER_MIB = 1024 * 1024 -WARMUP_SYSTEM_MESSAGE_PREFIX = CreditPhase.WARMUP STAT_KEYS = [ "avg", diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index 44cf4a23eb..8e17af4db4 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -256,6 +256,8 @@ class CacheBustTarget(CaseInsensitiveStrEnum): SYSTEM_SUFFIX = "system_suffix" FIRST_TURN_PREFIX = "first_turn_prefix" FIRST_TURN_SUFFIX = "first_turn_suffix" + WARMUP_ISOLATION_SYSTEM = "warmup_isolation_system" + WARMUP_ISOLATION_FIRST_TURN = "warmup_isolation_first_turn" class PromptCorpus(CaseInsensitiveStrEnum): diff --git a/src/aiperf/config/config.py b/src/aiperf/config/config.py index 20fc2c489a..9f4f1713f2 100644 --- a/src/aiperf/config/config.py +++ b/src/aiperf/config/config.py @@ -565,6 +565,93 @@ def validate_cache_bust_compatibility(self) -> Self: return self + @model_validator(mode="after") + def validate_warmup_isolation_not_agentic_replay(self) -> Self: + """Reject WARMUP_ISOLATION_* targets when timing mode resolves to AGENTIC_REPLAY. + + ``_inject_marker_at_first_user`` mutates Turn objects in-place inside + ``session.turn_list``. In agentic replay the same session object spans + the WARMUP→PROFILING boundary (the session is never evicted at phase + transition), so a Turn mutated with ``[warmup]`` during WARMUP carries + that mutation into PROFILING — even though the PROFILING credit has + ``cache_bust_marker=None`` and ``_apply_cache_bust`` returns early. + + Resolution logic mirrors ``validate_agentic_cache_warmup``: resolve the + scenario's declared timing_mode when a scenario is present (scenario + stamps phases post-construction); otherwise inspect the phases directly. + """ + from aiperf.common.enums import CacheBustTarget + from aiperf.plugin.enums import TimingMode + from aiperf.timing.config import _is_agentic_replay + + target = self.get_cache_bust_target() + _WARMUP_ISOLATION = ( + CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, + ) + if target not in _WARMUP_ISOLATION: + return self + + profiling_phases = self.get_profiling_phases() + is_agentic = False + if self.scenario is not None: + from aiperf.common.scenario.registry import get_scenario + + is_agentic = ( + get_scenario(self.scenario).timing_mode == TimingMode.AGENTIC_REPLAY + ) + else: + is_agentic = _is_agentic_replay(profiling_phases) + + if is_agentic: + raise ValueError( + "cache_bust targets warmup_isolation_system and " + "warmup_isolation_first_turn are not compatible with agentic_replay " + "timing mode: Turn objects mutated during warmup persist into profiling " + "sessions. Use cache_bust=none for agentic_replay workloads or one of " + "the RID-based targets (system_prefix, first_turn_prefix, etc.) for " + "per-trajectory isolation." + ) + return self + + @model_validator(mode="after") + def validate_warmup_isolation_system_requires_shared_system_prompt(self) -> Self: + """Reject warmup_isolation_system when no system message is statically present. + + ``warmup_isolation_system`` injects ``[warmup]`` into the system message slot + during warmup. When no system message exists in the dataset config the marker + silently falls through to the user turn, making it indistinguishable from + ``warmup_isolation_first_turn``. + + This check is scoped to synthetic datasets where system-message presence is + statically known. File and trace datasets may carry system messages inside + their content, which cannot be inspected at config time. + """ + from aiperf.common.enums import CacheBustTarget, DatasetType + + if self.get_cache_bust_target() != CacheBustTarget.WARMUP_ISOLATION_SYSTEM: + return self + + dataset = self.get_default_dataset() + if dataset.type != DatasetType.SYNTHETIC: + return self + + prefix_prompts = getattr(dataset, "prefix_prompts", None) + has_shared_system = ( + prefix_prompts is not None + and getattr(prefix_prompts, "shared_system_length", None) is not None + ) + if not has_shared_system: + raise ValueError( + "cache_bust=warmup_isolation_system requires a shared system prompt, " + "but no shared_system_length is configured on the synthetic dataset. " + "The marker would silently fall through to the user turn, making it " + "indistinguishable from warmup_isolation_first_turn. " + "Either set prefix_prompts.shared_system_length (--shared-system-prompt-length) " + "or switch to cache_bust=warmup_isolation_first_turn." + ) + return self + @model_validator(mode="after") def validate_agentic_cache_warmup(self) -> Self: """Restrict accelerated cache warmup to the agentic_replay timing mode. diff --git a/src/aiperf/config/dataset/content.py b/src/aiperf/config/dataset/content.py index 2d077d8cd2..b90ae4130e 100644 --- a/src/aiperf/config/dataset/content.py +++ b/src/aiperf/config/dataset/content.py @@ -59,15 +59,24 @@ def _parse_image_source(value: object) -> object: class CacheBustConfig(BaseConfig): - """Per-conversation cache-bust marker injected into the prompt. - - Prefix variants diverge at token 0 (defeats KV-cache prefix matching for - the entire prompt -- recommended when shared_system_length is large). - Suffix variants append after existing content (lighter bust; preserves - leading-prefix caching). Marker is deterministic from - (benchmark_id, recycle_pass, trajectory_index) -- reproducible across - reruns. Same marker for all turns within a conversation; fresh marker - on each recycle of a trace_id. + """Per-run cache-bust marker configuration. + + Two families of targets: + + RID targets (``system_prefix``, ``system_suffix``, ``first_turn_prefix``, + ``first_turn_suffix``): inject a per-trajectory unique marker derived from a + SHA-256 digest of (benchmark_id, recycle_pass, trajectory_index, trace_id). + The digest is phase-agnostic — warmup and profiling turns for the same + trajectory share the same marker — so warmup KV-cache work transfers to + profiling. Different trajectories (different lanes or recycle passes) receive + distinct markers. Prefix variants diverge at token 0; suffix variants append + after existing content. + + Warmup-isolation targets (``warmup_isolation_system``, + ``warmup_isolation_first_turn``): inject a constant ``[warmup]`` marker only + during the WARMUP phase; during PROFILING the marker is ``None`` so the + profiling request arrives clean. These targets are phase-aware and do NOT use + a per-trajectory digest. Incompatible with ``agentic_replay`` timing mode. """ model_config = ConfigDict(extra="forbid") diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index dccbe4667d..6aacca795a 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -1145,9 +1145,17 @@ def url(self) -> str: CacheBustTarget, Field( description=( - "Where (and how) to inject a per-conversation cache-bust marker. " - "Prefix variants prepend at token 0 (most aggressive); " - "suffix variants append after existing content. " + "Where (and how) to inject a cache-bust marker. Two families: " + "(1) RID targets (system_prefix, system_suffix, first_turn_prefix, " + "first_turn_suffix) — inject a per-trajectory unique SHA-256 digest " + "marker that is identical across warmup and profiling, so warmup KV-cache " + "work transfers to profiling while preventing cross-trajectory cache sharing. " + "Prefix variants prepend at token 0; suffix variants append after existing " + "content. " + "(2) Warmup-isolation targets (warmup_isolation_system, " + "warmup_isolation_first_turn) — inject a constant '[warmup]' marker only " + "during the WARMUP phase; profiling sees no marker (fully cold start or " + "system-pre-warmed). Incompatible with agentic_replay timing mode. " "'none' disables the feature (default)." ), ), diff --git a/src/aiperf/credit/issuer.py b/src/aiperf/credit/issuer.py index e3ba0a2094..86a1c92237 100644 --- a/src/aiperf/credit/issuer.py +++ b/src/aiperf/credit/issuer.py @@ -21,11 +21,15 @@ from msgspec.structs import replace as _struct_replace from aiperf.common.aiperf_logger import AIPerfLogger -from aiperf.common.enums import CreditPhase +from aiperf.common.enums import CacheBustTarget, CreditPhase from aiperf.common.phase import phase_runtime_key from aiperf.credit.dispatch import ChildDispatchResult, TurnAdmission from aiperf.credit.structs import Credit, TurnToSend from aiperf.timing.replay_dependencies import ReplayIssueGate +from aiperf.timing.strategies.cache_bust import ( + WARMUP_ISOLATION_MARKER, + WARMUP_ISOLATION_TARGETS, +) from aiperf.timing.url_samplers import URLSelectionStrategyProtocol if TYPE_CHECKING: @@ -81,6 +85,7 @@ def __init__( session_tree_registry: SessionTreeRegistry | None = None, session_tree_registry_enabled: bool | None = None, replay_barrier: ReplayBarrierCoordinator | None = None, + cache_bust_target: CacheBustTarget = CacheBustTarget.NONE, ) -> None: """Initialize credit issuer. @@ -125,6 +130,7 @@ def __init__( ) else None ) + self._cache_bust_target = cache_bust_target self._issuing_stopped = False self._max_tokens_override: int | None = None self._turn_admission: Callable[[TurnToSend], TurnAdmission | bool] | None = None @@ -461,6 +467,14 @@ async def _issue_credit_internal(self, turn: TurnToSend) -> bool: else None ) + cache_bust_marker = turn.cache_bust_marker + cache_bust_target_for_credit = turn.cache_bust_target + if self._cache_bust_target in WARMUP_ISOLATION_TARGETS: + cache_bust_target_for_credit = self._cache_bust_target + cache_bust_marker = ( + WARMUP_ISOLATION_MARKER if self._phase == CreditPhase.WARMUP else None + ) + credit = Credit( id=credit_index, phase=self._phase, @@ -481,8 +495,8 @@ async def _issue_credit_internal(self, turn: TurnToSend) -> bool: counts_toward_phase_target=turn.counts_toward_phase_target, has_forks=turn.has_forks, branch_mode=turn.branch_mode, - cache_bust_marker=turn.cache_bust_marker, - cache_bust_target=turn.cache_bust_target, + cache_bust_marker=cache_bust_marker, + cache_bust_target=cache_bust_target_for_credit, max_tokens_override=turn.max_tokens_override, ) diff --git a/src/aiperf/endpoints/openai_chat.py b/src/aiperf/endpoints/openai_chat.py index e393918e47..67e0133cfe 100644 --- a/src/aiperf/endpoints/openai_chat.py +++ b/src/aiperf/endpoints/openai_chat.py @@ -5,7 +5,6 @@ from typing import Any -from aiperf.common.enums import CreditPhase from aiperf.common.models import ( BaseResponseData, Image, @@ -102,15 +101,8 @@ def _format_messages( and isinstance(rendered[0], dict) and rendered[0].get("role") == "system" ) - if request_info.system_message: - if first_is_system and request_info.credit_phase == CreditPhase.WARMUP: - rendered = ChatEndpoint._prepend_system_message( - rendered, request_info.system_message - ) - elif not first_is_system: - messages.append( - {"role": "system", "content": request_info.system_message} - ) + if request_info.system_message and not first_is_system: + messages.append({"role": "system", "content": request_info.system_message}) if request_info.user_context_message: messages.append( {"role": "user", "content": request_info.user_context_message} @@ -118,25 +110,6 @@ def _format_messages( messages.extend(rendered) return messages - @staticmethod - def _prepend_system_message( - rendered: list[dict[str, Any]], system_message: str - ) -> list[dict[str, Any]]: - """Prepend ``system_message`` to the leading rendered system message.""" - first = dict(rendered[0]) - content = first.get("content") - if isinstance(content, str): - first["content"] = ( - f"{system_message}\n{content}" if content else system_message - ) - elif isinstance(content, list): - first["content"] = [{"type": "text", "text": system_message}, *content] - elif content is None: - first["content"] = system_message - else: - first["content"] = f"{system_message}\n{content}" - return [first, *rendered[1:]] - def _extend_image_parts( self, parts: list[dict[str, Any]], images: list[Image] ) -> None: diff --git a/src/aiperf/endpoints/openai_completions.py b/src/aiperf/endpoints/openai_completions.py index f16181fc44..76cfbac263 100644 --- a/src/aiperf/endpoints/openai_completions.py +++ b/src/aiperf/endpoints/openai_completions.py @@ -3,8 +3,6 @@ from __future__ import annotations -from aiperf.common.constants import WARMUP_SYSTEM_MESSAGE_PREFIX -from aiperf.common.enums import CacheBustTarget, CreditPhase from aiperf.common.models import ( BaseResponseData, InferenceServerResponse, @@ -39,16 +37,6 @@ def format_payload(self, request_info: RequestInfo) -> RequestOutputT: prompts = [ content for text in turn.texts for content in text.contents if content ] - # Skipped when cache-bust owns prefix isolation: its markers are - # warmup-coherent by design (warmup primes the prefix profiling hits), - # and prefixing in front of the shared marker would break that prime. - # See ``Worker._system_message_for_phase``. - if request_info.credit_phase == CreditPhase.WARMUP and ( - request_info.cache_bust_target in (None, CacheBustTarget.NONE) - ): - prompts = [ - f"{WARMUP_SYSTEM_MESSAGE_PREFIX}\n{prompt}" for prompt in prompts - ] extra = model_endpoint.endpoint.extra or [] diff --git a/src/aiperf/timing/phase/runner.py b/src/aiperf/timing/phase/runner.py index 52ccf72383..2cbaadd57f 100644 --- a/src/aiperf/timing/phase/runner.py +++ b/src/aiperf/timing/phase/runner.py @@ -237,6 +237,14 @@ def _root_idle_gap_cap_seconds(self) -> float | None: cap = getattr(dataset, "trace_idle_gap_cap_seconds", None) return float(cap) if isinstance(cap, int | float) else None + def _resolve_cache_bust_target(self) -> CacheBustTarget: + """Return the active cache-bust target, or NONE when no run is attached.""" + return ( + self._run.cfg.get_cache_bust_target() + if self._run is not None + else CacheBustTarget.NONE + ) + def _build_credit_issuer( self, url_selection_strategy: URLSelectionStrategyProtocol | None ) -> CreditIssuer: @@ -262,6 +270,7 @@ def _build_credit_issuer( or self._cache_warmup_enabled ), replay_barrier=self._replay_barrier, + cache_bust_target=self._resolve_cache_bust_target(), ) def _maybe_construct_branch_orchestrator( @@ -293,11 +302,7 @@ def _maybe_construct_branch_orchestrator( return sticky_router = getattr(self._credit_router, "sticky_router", None) benchmark_id = self._run.benchmark_id if self._run is not None else "unknown" - cache_bust_target = ( - self._run.cfg.get_cache_bust_target() - if self._run is not None - else CacheBustTarget.NONE - ) + cache_bust_target = self._resolve_cache_bust_target() self._branch_orchestrator = BranchOrchestrator( conversation_source=conversation_source, credit_issuer=self._credit_issuer, @@ -549,11 +554,13 @@ def _release_tree_slots(self) -> None: return released = self._session_tree_registry.release_all(self._phase_key) self.info( - lambda: f"Session-tree slots for phase {self._config.phase}: " - f"peak_open={self._session_tree_registry.peak_open} " - f"(target concurrency {self._config.concurrency}); " - f"released {released} still-open at teardown; " - f"late_events={self._session_tree_registry.late_events}" + lambda: ( + f"Session-tree slots for phase {self._config.phase}: " + f"peak_open={self._session_tree_registry.peak_open} " + f"(target concurrency {self._config.concurrency}); " + f"released {released} still-open at teardown; " + f"late_events={self._session_tree_registry.late_events}" + ) ) async def _run_strategy( diff --git a/src/aiperf/timing/strategies/cache_bust.py b/src/aiperf/timing/strategies/cache_bust.py index 4cfe6c6bfe..5fe5aa2b9e 100644 --- a/src/aiperf/timing/strategies/cache_bust.py +++ b/src/aiperf/timing/strategies/cache_bust.py @@ -25,6 +25,12 @@ _SUFFIX_SEP = "::" _UNSET = object() +WARMUP_ISOLATION_MARKER = "[warmup]\n\n" +WARMUP_ISOLATION_TARGETS = ( + CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, +) + def base_trace_id(conversation_id: str) -> str: """Strip any descendant suffix (``::sa:``/``::fa:``/``:sN``) to the root trace id. @@ -90,11 +96,19 @@ def build_cache_bust_marker( ) -> str | None: """Render the marker text for the given inputs and target position. - The digest tuple is intentionally phase-agnostic. Spec requires - "warmup-coherent" markers: a trajectory's warmup turn ``k_i`` and its - first profiling turn ``k_i+1`` must share the same marker so warmup - KV-cache work transfers to profiling. Adding phase to the digest - would defeat that — keep it out. + Two paths depending on ``target``: + + (a) RID-digest targets (``SYSTEM_PREFIX``, ``SYSTEM_SUFFIX``, + ``FIRST_TURN_PREFIX``, ``FIRST_TURN_SUFFIX``): the digest tuple is + intentionally phase-agnostic. Spec requires "warmup-coherent" markers: + a trajectory's warmup turn ``k_i`` and its first profiling turn + ``k_i+1`` must share the same marker so warmup KV-cache work transfers + to profiling. Adding phase to the digest would defeat that — keep it out. + + (b) ``WARMUP_ISOLATION_*`` targets: return the constant + ``WARMUP_ISOLATION_MARKER`` string. No digest is computed. The + phase-aware gate (emit during WARMUP, suppress during PROFILING) lives + in ``CreditIssuer._issue_credit_internal``, not here. Returns ``None`` when target is NONE so callers can unconditionally pass the result through into ``Credit.cache_bust_marker: str | None``. Returning @@ -102,6 +116,8 @@ def build_cache_bust_marker( """ if target == CacheBustTarget.NONE: return None + if target in WARMUP_ISOLATION_TARGETS: + return WARMUP_ISOLATION_MARKER unique_str = f"{benchmark_id}:{recycle_pass}:{trajectory_index}:{trace_id}" digest = hashlib.sha256(unique_str.encode()).hexdigest()[:_DIGEST_LEN] @@ -120,11 +136,16 @@ def estimate_marker_token_cost( """Average token count of the cache-bust marker for a given target. Tokenizes ``samples`` distinct markers and rounds the mean to an int. - Returns 0 for ``CacheBustTarget.NONE``. The 12-hex digest dominates - the variance, so a handful of samples is enough. + Returns 0 for ``CacheBustTarget.NONE`` and for ``WARMUP_ISOLATION_*`` + targets (their marker is only injected during WARMUP; PROFILING credits + carry ``None``, so there is no profiling-phase token cost to estimate). + The 12-hex digest dominates the variance for RID targets, so a handful + of samples is enough. """ if target == CacheBustTarget.NONE: return 0 + if target in WARMUP_ISOLATION_TARGETS: + return 0 total = 0 for i in range(samples): diff --git a/src/aiperf/workers/worker.py b/src/aiperf/workers/worker.py index a062073cc6..3d5ddb9bd8 100644 --- a/src/aiperf/workers/worker.py +++ b/src/aiperf/workers/worker.py @@ -9,13 +9,12 @@ from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.base_component_service import BaseComponentService -from aiperf.common.constants import BYTES_PER_MIB, WARMUP_SYSTEM_MESSAGE_PREFIX +from aiperf.common.constants import BYTES_PER_MIB from aiperf.common.enums import ( CacheBustTarget, CommAddress, CommandType, ConversationBranchMode, - CreditPhase, MemoryMapFormat, MessageType, ) @@ -146,7 +145,10 @@ def _apply_cache_bust_to_system_message( """ if not marker or target == CacheBustTarget.NONE or system_message is None: return system_message - if target == CacheBustTarget.SYSTEM_PREFIX: + if target in ( + CacheBustTarget.SYSTEM_PREFIX, + CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + ): return marker + system_message if target == CacheBustTarget.SYSTEM_SUFFIX: return system_message + marker @@ -432,10 +434,16 @@ def _apply_cache_bust( is_prefix = target in ( CacheBustTarget.SYSTEM_PREFIX, CacheBustTarget.FIRST_TURN_PREFIX, + CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, ) prefix_turns = _effective_prefix_turns(session) - if target in (CacheBustTarget.SYSTEM_PREFIX, CacheBustTarget.SYSTEM_SUFFIX): + if target in ( + CacheBustTarget.SYSTEM_PREFIX, + CacheBustTarget.SYSTEM_SUFFIX, + CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + ): return _apply_system_target_cache_bust( prefix_turns, all_turns=session.turn_list, @@ -462,7 +470,7 @@ def _apply_system_target_cache_bust( target: CacheBustTarget, is_prefix: bool, ) -> str | None: - """Inject a ``SYSTEM_PREFIX`` / ``SYSTEM_SUFFIX`` marker for one credit. + """Inject a ``SYSTEM_PREFIX`` / ``SYSTEM_SUFFIX`` / ``WARMUP_ISOLATION_SYSTEM`` marker for one credit. ``prefix_turns`` is the effective wire prefix slice (see :func:`_effective_prefix_turns`). Four sub-paths, ordered to match how the @@ -1082,13 +1090,7 @@ async def _process_credit_with_session( system_message = _apply_cache_bust( session, credit, - self._system_message_for_phase( - system_message=session.conversation.system_message, - phase=credit.phase, - cache_bust_target=credit.cache_bust_target - if credit.cache_bust_marker is not None - else None, - ), + session.conversation.system_message, ) self._maybe_warn_cache_bust_silent_drop(session, credit) @@ -1389,36 +1391,6 @@ def _release_and_evict_for_terminal( else: self.session_manager.evict(x_correlation_id) - @staticmethod - def _system_message_for_phase( - *, - system_message: str | None, - phase: CreditPhase, - cache_bust_target: CacheBustTarget | None, - ) -> str | None: - """Prefix warmup system messages so warmup cannot reuse profiling's - prefix cache — unless cache-bust already owns prefix isolation. - - Cache-bust is deliberately warmup-coherent: ``build_cache_bust_marker`` - keeps the phase out of its digest and the marker ledger survives the - WARMUP -> PROFILING boundary, so a trajectory's warmup turn ``k_i`` and - its first profiling turn ``k_i+1`` share one marker and warmup PRIMES - the cache profiling then hits. Injecting this prefix in front of that - shared marker diverges the two prefixes at token 0, so warmup primes an - entry profiling never touches — the round-trip is spent for nothing. - - Any non-NONE target already guarantees warmup cannot collide with - another trajectory's prefix (the marker digests per trajectory tree), so - standing down here costs no isolation. - """ - if phase != CreditPhase.WARMUP: - return system_message - if cache_bust_target not in (None, CacheBustTarget.NONE): - return system_message - if not system_message: - return WARMUP_SYSTEM_MESSAGE_PREFIX - return f"{WARMUP_SYSTEM_MESSAGE_PREFIX}\n{system_message}" - def _maybe_warn_cache_bust_silent_drop( self, session: UserSession, diff --git a/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py b/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py index d3488390d4..adef711018 100644 --- a/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py +++ b/tests/unit/common/config/test_user_config_cache_bust_lockdown_adversarial.py @@ -21,7 +21,12 @@ def _build( endpoint_type: EndpointType = EndpointType.CHAT, timing_mode: TimingMode | None = None, ) -> BenchmarkConfig: - """Construct a v2 BenchmarkConfig for an endpoint capability combination.""" + """Construct a v2 BenchmarkConfig for an endpoint capability combination. + + warmup_isolation_system requires a shared system prompt; this helper adds + one automatically so callers testing timing-mode or endpoint compatibility + don't need to repeat that setup. + """ phase: dict[str, Any] = { "name": "profiling", "type": "concurrency", @@ -38,20 +43,22 @@ def _build( "response_field": "text", } + dataset: dict[str, Any] = { + "name": "main", + "type": "synthetic", + "prompts": { + "isl": 128, + "osl": 16, + "cache_bust": {"target": target}, + }, + } + if target == CacheBustTarget.WARMUP_ISOLATION_SYSTEM: + dataset["prefix_prompts"] = {"shared_system_length": 512} + body: dict[str, Any] = { "models": ["test-model"], "endpoint": endpoint, - "datasets": [ - { - "name": "main", - "type": "synthetic", - "prompts": { - "isl": 128, - "osl": 16, - "cache_bust": {"target": target}, - }, - } - ], + "datasets": [dataset], "phases": [phase], } return BenchmarkConfig.model_validate(body) @@ -60,7 +67,17 @@ def _build( _NON_NONE_CACHE_BUST_TARGETS: list[CacheBustTarget] = [ t for t in CacheBustTarget if t != CacheBustTarget.NONE ] +_WARMUP_ISOLATION_TARGETS: list[CacheBustTarget] = [ + CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, +] +_RID_CACHE_BUST_TARGETS: list[CacheBustTarget] = [ + t for t in _NON_NONE_CACHE_BUST_TARGETS if t not in _WARMUP_ISOLATION_TARGETS +] _TIMING_MODES: list[TimingMode] = list(TimingMode) +_NON_AGENTIC_TIMING_MODES: list[TimingMode] = [ + t for t in TimingMode if t != TimingMode.AGENTIC_REPLAY +] _INCOMPATIBLE_ENDPOINT_TYPES: list[EndpointType] = [ e for e in EndpointType @@ -68,18 +85,43 @@ def _build( ] -@pytest.mark.parametrize("timing_mode", _TIMING_MODES) +@pytest.mark.parametrize("timing_mode", _NON_AGENTIC_TIMING_MODES) @pytest.mark.parametrize("target", _NON_NONE_CACHE_BUST_TARGETS) -def test_cache_bust_accepted_with_every_timing_mode( +def test_cache_bust_accepted_with_non_agentic_timing_mode( timing_mode: TimingMode, target: CacheBustTarget ) -> None: - """Every timing mode accepts cache-bust with a structured chat endpoint.""" + """Every non-agentic-replay timing mode accepts cache-bust with a structured chat endpoint.""" cfg = _build( target=target, endpoint_type=EndpointType.CHAT, timing_mode=timing_mode ) assert cfg.get_cache_bust_target() == target +@pytest.mark.parametrize("timing_mode", _TIMING_MODES) +@pytest.mark.parametrize("target", _RID_CACHE_BUST_TARGETS) +def test_rid_cache_bust_accepted_with_every_timing_mode( + timing_mode: TimingMode, target: CacheBustTarget +) -> None: + """RID-based cache-bust targets are accepted with every timing mode.""" + cfg = _build( + target=target, endpoint_type=EndpointType.CHAT, timing_mode=timing_mode + ) + assert cfg.get_cache_bust_target() == target + + +@pytest.mark.parametrize("target", _WARMUP_ISOLATION_TARGETS) +def test_warmup_isolation_rejected_with_agentic_replay( + target: CacheBustTarget, +) -> None: + """WARMUP_ISOLATION_* targets are rejected when timing_mode=agentic_replay.""" + with pytest.raises(ValueError, match="not compatible with agentic_replay"): + _build( + target=target, + endpoint_type=EndpointType.CHAT, + timing_mode=TimingMode.AGENTIC_REPLAY, + ) + + @pytest.mark.parametrize("endpoint_type", _INCOMPATIBLE_ENDPOINT_TYPES) @pytest.mark.parametrize("target", _NON_NONE_CACHE_BUST_TARGETS) def test_cache_bust_rejected_with_unsupported_endpoint( @@ -134,6 +176,102 @@ def test_cache_bust_none_passes_all_endpoint_types(endpoint_type: EndpointType) assert cfg.endpoint.type == endpoint_type +def _build_with_prefix( + *, + target: CacheBustTarget, + shared_system_length: int | None = None, + user_context_length: int | None = None, + pool_size: int | None = None, + prefix_length: int | None = None, +) -> BenchmarkConfig: + """Build a BenchmarkConfig with prefix_prompts configured.""" + prefix_prompts: dict[str, Any] = {} + if shared_system_length is not None: + prefix_prompts["shared_system_length"] = shared_system_length + if user_context_length is not None: + prefix_prompts["user_context_length"] = user_context_length + if pool_size is not None: + prefix_prompts["pool_size"] = pool_size + if prefix_length is not None: + prefix_prompts["length"] = prefix_length + + dataset: dict[str, Any] = { + "name": "main", + "type": "synthetic", + "prompts": { + "isl": 128, + "osl": 16, + "cache_bust": {"target": target}, + }, + } + if prefix_prompts: + dataset["prefix_prompts"] = prefix_prompts + + body: dict[str, Any] = { + "models": ["test-model"], + "endpoint": {"urls": [_URL], "type": EndpointType.CHAT}, + "datasets": [dataset], + "phases": [ + { + "name": "profiling", + "type": "concurrency", + "concurrency": 1, + "requests": 10, + } + ], + } + return BenchmarkConfig.model_validate(body) + + +def test_warmup_isolation_system_without_shared_system_prompt_rejected() -> None: + """warmup_isolation_system is rejected when no shared_system_length is configured.""" + with pytest.raises(ValueError, match="requires a shared system prompt"): + _build_with_prefix(target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM) + + +def test_warmup_isolation_system_with_user_context_only_rejected() -> None: + """warmup_isolation_system is rejected when only user_context_length is set (no system msg).""" + with pytest.raises(ValueError, match="requires a shared system prompt"): + _build_with_prefix( + target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + user_context_length=256, + ) + + +def test_warmup_isolation_system_with_prefix_pool_only_rejected() -> None: + """warmup_isolation_system is rejected when only a prefix pool is configured (no system msg).""" + with pytest.raises(ValueError, match="requires a shared system prompt"): + _build_with_prefix( + target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + pool_size=4, + prefix_length=256, + ) + + +def test_warmup_isolation_system_with_shared_system_prompt_accepted() -> None: + """warmup_isolation_system is accepted when shared_system_length is set.""" + cfg = _build_with_prefix( + target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + shared_system_length=512, + ) + assert cfg.get_cache_bust_target() == CacheBustTarget.WARMUP_ISOLATION_SYSTEM + + +def test_warmup_isolation_first_turn_without_shared_system_prompt_accepted() -> None: + """warmup_isolation_first_turn is accepted with any prefix config (targets user turn).""" + cfg = _build_with_prefix( + target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, + user_context_length=256, + ) + assert cfg.get_cache_bust_target() == CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN + + +def test_warmup_isolation_first_turn_with_no_prefix_accepted() -> None: + """warmup_isolation_first_turn is accepted even without any prefix config.""" + cfg = _build_with_prefix(target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN) + assert cfg.get_cache_bust_target() == CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN + + def test_unsafe_override_does_not_bypass_cache_bust_endpoint_validation() -> None: """unsafe_override does not bypass endpoint capability validation.""" body: dict[str, Any] = { diff --git a/tests/unit/common/enums/test_cache_bust_target.py b/tests/unit/common/enums/test_cache_bust_target.py index a4e1970f8d..db8d915c35 100644 --- a/tests/unit/common/enums/test_cache_bust_target.py +++ b/tests/unit/common/enums/test_cache_bust_target.py @@ -28,3 +28,17 @@ def test_cache_bust_target_case_insensitive(raw, expected): def test_cache_bust_target_default_is_none(): assert CacheBustTarget.NONE.value == "none" + + +def test_warmup_isolation_system_in_cache_bust_target(): + assert ( + CacheBustTarget("warmup_isolation_system") + == CacheBustTarget.WARMUP_ISOLATION_SYSTEM + ) + + +def test_warmup_isolation_first_turn_in_cache_bust_target(): + assert ( + CacheBustTarget("warmup_isolation_first_turn") + == CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN + ) diff --git a/tests/unit/credit/test_issuer.py b/tests/unit/credit/test_issuer.py index f6ab354e42..6657bd557a 100644 --- a/tests/unit/credit/test_issuer.py +++ b/tests/unit/credit/test_issuer.py @@ -106,14 +106,22 @@ def make_turn( conversation_id: str = "conv1", turn_index: int = 0, num_turns: int = 1, + cache_bust_marker: str | None = None, + cache_bust_target=None, ) -> TurnToSend: """Create a TurnToSend for testing.""" - return TurnToSend( + + kwargs: dict = dict( conversation_id=conversation_id, x_correlation_id=f"corr-{conversation_id}", turn_index=turn_index, num_turns=num_turns, ) + if cache_bust_marker is not None: + kwargs["cache_bust_marker"] = cache_bust_marker + if cache_bust_target is not None: + kwargs["cache_bust_target"] = cache_bust_target + return TurnToSend(**kwargs) # ============================================================================= @@ -923,3 +931,159 @@ async def test_no_url_strategy_means_none_url_index( sent_credit = mock_router.send_credit.call_args.kwargs["credit"] assert sent_credit.url_index is None + + +def _make_warmup_isolation_issuer( + phase, + cache_bust_target, + mock_stop_checker, + mock_progress, + mock_concurrency, + mock_router, + mock_cancellation, + mock_lifecycle, +) -> CreditIssuer: + """Build a CreditIssuer with explicit phase and cache_bust_target.""" + return CreditIssuer( + phase=phase, + cache_bust_target=cache_bust_target, + stop_checker=mock_stop_checker, + progress=mock_progress, + concurrency_manager=mock_concurrency, + credit_router=mock_router, + cancellation_policy=mock_cancellation, + lifecycle=mock_lifecycle, + ) + + +class TestWarmupIsolationMarkerEmission: + """CreditIssuer emits WARMUP_ISOLATION_MARKER during WARMUP and None during PROFILING.""" + + @pytest.mark.asyncio + async def test_credit_issuer_warmup_isolation_system_sets_marker_in_warmup( + self, + mock_stop_checker, + mock_progress, + mock_concurrency, + mock_router, + mock_cancellation, + mock_lifecycle, + ): + """WARMUP_ISOLATION_SYSTEM credits carry the marker during WARMUP phase.""" + from aiperf.common.enums import CacheBustTarget, CreditPhase + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + issuer = _make_warmup_isolation_issuer( + phase=CreditPhase.WARMUP, + cache_bust_target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + mock_stop_checker=mock_stop_checker, + mock_progress=mock_progress, + mock_concurrency=mock_concurrency, + mock_router=mock_router, + mock_cancellation=mock_cancellation, + mock_lifecycle=mock_lifecycle, + ) + await issuer.issue_credit(make_turn()) + + sent_credit = mock_router.send_credit.call_args.kwargs["credit"] + assert sent_credit.cache_bust_marker == WARMUP_ISOLATION_MARKER + assert sent_credit.cache_bust_target == CacheBustTarget.WARMUP_ISOLATION_SYSTEM + + @pytest.mark.asyncio + async def test_credit_issuer_warmup_isolation_system_clears_marker_in_profiling( + self, + mock_stop_checker, + mock_progress, + mock_concurrency, + mock_router, + mock_cancellation, + mock_lifecycle, + ): + """WARMUP_ISOLATION_SYSTEM credits carry no marker during PROFILING phase.""" + from aiperf.common.enums import CacheBustTarget, CreditPhase + + issuer = _make_warmup_isolation_issuer( + phase=CreditPhase.PROFILING, + cache_bust_target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + mock_stop_checker=mock_stop_checker, + mock_progress=mock_progress, + mock_concurrency=mock_concurrency, + mock_router=mock_router, + mock_cancellation=mock_cancellation, + mock_lifecycle=mock_lifecycle, + ) + await issuer.issue_credit(make_turn()) + + sent_credit = mock_router.send_credit.call_args.kwargs["credit"] + assert sent_credit.cache_bust_marker is None + + @pytest.mark.asyncio + async def test_credit_issuer_warmup_isolation_first_turn_sets_marker_in_warmup( + self, + mock_stop_checker, + mock_progress, + mock_concurrency, + mock_router, + mock_cancellation, + mock_lifecycle, + ): + """WARMUP_ISOLATION_FIRST_TURN credits carry the marker during WARMUP phase.""" + from aiperf.common.enums import CacheBustTarget, CreditPhase + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + issuer = _make_warmup_isolation_issuer( + phase=CreditPhase.WARMUP, + cache_bust_target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, + mock_stop_checker=mock_stop_checker, + mock_progress=mock_progress, + mock_concurrency=mock_concurrency, + mock_router=mock_router, + mock_cancellation=mock_cancellation, + mock_lifecycle=mock_lifecycle, + ) + await issuer.issue_credit(make_turn()) + + sent_credit = mock_router.send_credit.call_args.kwargs["credit"] + assert sent_credit.cache_bust_marker == WARMUP_ISOLATION_MARKER + assert ( + sent_credit.cache_bust_target == CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN + ) + + @pytest.mark.asyncio + async def test_credit_issuer_non_warmup_isolation_target_unchanged( + self, + mock_stop_checker, + mock_progress, + mock_concurrency, + mock_router, + mock_cancellation, + mock_lifecycle, + ): + """Non-WARMUP_ISOLATION targets pass through the turn's marker unchanged. + + The issuer is configured with NONE (no WARMUP_ISOLATION override), so the + marker already on the TurnToSend must survive into the Credit unmodified. + Using a real non-None marker makes the assertion meaningful — if the issuer + were to clobber it with None the test would catch the regression. + """ + from aiperf.common.enums import CacheBustTarget, CreditPhase + + issuer = _make_warmup_isolation_issuer( + phase=CreditPhase.WARMUP, + cache_bust_target=CacheBustTarget.NONE, + mock_stop_checker=mock_stop_checker, + mock_progress=mock_progress, + mock_concurrency=mock_concurrency, + mock_router=mock_router, + mock_cancellation=mock_cancellation, + mock_lifecycle=mock_lifecycle, + ) + turn = make_turn( + cache_bust_marker="[rid:abc123def456]", + cache_bust_target=CacheBustTarget.SYSTEM_PREFIX, + ) + await issuer.issue_credit(turn) + + sent_credit = mock_router.send_credit.call_args.kwargs["credit"] + assert sent_credit.cache_bust_marker == "[rid:abc123def456]" + assert sent_credit.cache_bust_target == CacheBustTarget.SYSTEM_PREFIX diff --git a/tests/unit/endpoints/test_completions_endpoint.py b/tests/unit/endpoints/test_completions_endpoint.py index ce54a5118e..6b6e736880 100644 --- a/tests/unit/endpoints/test_completions_endpoint.py +++ b/tests/unit/endpoints/test_completions_endpoint.py @@ -2,10 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from pytest import param -from aiperf.common.constants import WARMUP_SYSTEM_MESSAGE_PREFIX -from aiperf.common.enums import CacheBustTarget, CreditPhase from aiperf.common.models import Text, Turn from aiperf.endpoints.openai_completions import CompletionsEndpoint from aiperf.plugin.enums import EndpointType @@ -61,87 +58,6 @@ def test_format_payload_multiple_prompts(self, endpoint, model_endpoint): assert len(payload["prompt"]) == 3 assert payload["prompt"] == ["Prompt 1", "Prompt 2", "Prompt 3"] - def test_format_payload_warmup_prefixes_prompts_with_warmup_marker( - self, endpoint, model_endpoint - ): - turn = Turn( - texts=[Text(contents=["Prompt 1", "Prompt 2"])], - model="completion-model", - ) - request_info = create_request_info( - model_endpoint=model_endpoint, - turns=[turn], - credit_phase=CreditPhase.WARMUP, - system_message="should not be used", - ) - - payload = endpoint.format_payload(request_info) - - assert payload["prompt"] == [ - f"{WARMUP_SYSTEM_MESSAGE_PREFIX}\nPrompt 1", - f"{WARMUP_SYSTEM_MESSAGE_PREFIX}\nPrompt 2", - ] - - def test_format_payload_warmup_single_prompt_prefixed_string( - self, endpoint, model_endpoint - ): - turn = Turn(texts=[Text(contents=["Prompt 1"])], model="completion-model") - request_info = create_request_info( - model_endpoint=model_endpoint, - turns=[turn], - credit_phase=CreditPhase.WARMUP, - ) - - payload = endpoint.format_payload(request_info) - - assert payload["prompt"] == f"{WARMUP_SYSTEM_MESSAGE_PREFIX}\nPrompt 1" - - @pytest.mark.parametrize( - "target", - [ - param(CacheBustTarget.SYSTEM_PREFIX, id="system_prefix"), - param(CacheBustTarget.SYSTEM_SUFFIX, id="system_suffix"), - param(CacheBustTarget.FIRST_TURN_PREFIX, id="first_turn_prefix"), - param(CacheBustTarget.FIRST_TURN_SUFFIX, id="first_turn_suffix"), - ], - ) # fmt: skip - def test_format_payload_warmup_skips_prefix_when_cache_bust_active( - self, endpoint, model_endpoint, target - ): - """Cache-bust markers are warmup-coherent, so warmup primes the prefix - profiling hits. Prefixing in front of the shared marker would diverge - the two at token 0 and prime an entry profiling never touches. - """ - turn = Turn(texts=[Text(contents=["Prompt 1"])], model="completion-model") - request_info = create_request_info( - model_endpoint=model_endpoint, - turns=[turn], - credit_phase=CreditPhase.WARMUP, - cache_bust_target=target, - ) - - payload = endpoint.format_payload(request_info) - - assert payload["prompt"] == "Prompt 1" - - def test_format_payload_warmup_prefixes_when_cache_bust_target_none_enum( - self, endpoint, model_endpoint - ): - """``CacheBustTarget.NONE`` means cache-bust is off, so warmup still - needs its own prefix isolation. - """ - turn = Turn(texts=[Text(contents=["Prompt 1"])], model="completion-model") - request_info = create_request_info( - model_endpoint=model_endpoint, - turns=[turn], - credit_phase=CreditPhase.WARMUP, - cache_bust_target=CacheBustTarget.NONE, - ) - - payload = endpoint.format_payload(request_info) - - assert payload["prompt"] == f"{WARMUP_SYSTEM_MESSAGE_PREFIX}\nPrompt 1" - def test_format_payload_filters_empty_prompts(self, endpoint, model_endpoint): """Test that empty strings are filtered from prompts.""" turn = Turn( diff --git a/tests/unit/endpoints/test_message_merging_fixes.py b/tests/unit/endpoints/test_message_merging_fixes.py index 7ada1640ba..cb56841b2a 100644 --- a/tests/unit/endpoints/test_message_merging_fixes.py +++ b/tests/unit/endpoints/test_message_merging_fixes.py @@ -11,7 +11,6 @@ import orjson import pytest -from aiperf.common.enums import CreditPhase from aiperf.common.models import ( RequestRecord, Text, @@ -354,52 +353,6 @@ def test_authored_leading_system_wins(self, chat_endpoint): assert len(systems) == 1 assert systems[0]["content"] == "authored system" - def test_warmup_marker_merges_into_raw_messages_leading_system(self, chat_endpoint): - authored = [ - {"role": "system", "content": "authored system"}, - {"role": "user", "content": "hi"}, - ] - turn = Turn(role="user", raw_messages=authored) - request_info = create_request_info( - model_endpoint=chat_endpoint.model_endpoint, - turns=[turn], - credit_phase=CreditPhase.WARMUP, - system_message="warmup", - ) - payload = chat_endpoint.format_payload(request_info) - - systems = [m for m in payload["messages"] if m["role"] == "system"] - assert len(systems) == 1 - assert systems[0]["content"] == "warmup\nauthored system" - assert authored[0]["content"] == "authored system" - - def test_warmup_marker_merges_into_raw_messages_leading_system_list_content( - self, chat_endpoint - ): - authored = [ - { - "role": "system", - "content": [{"type": "text", "text": "authored system"}], - }, - {"role": "user", "content": "hi"}, - ] - turn = Turn(role="user", raw_messages=authored) - request_info = create_request_info( - model_endpoint=chat_endpoint.model_endpoint, - turns=[turn], - credit_phase=CreditPhase.WARMUP, - system_message="warmup", - ) - payload = chat_endpoint.format_payload(request_info) - - systems = [m for m in payload["messages"] if m["role"] == "system"] - assert len(systems) == 1 - assert systems[0]["content"] == [ - {"type": "text", "text": "warmup"}, - {"type": "text", "text": "authored system"}, - ] - assert authored[0]["content"] == [{"type": "text", "text": "authored system"}] - # --------------------------------------------------------------------------- # raw_messages=[] is an explicit zero-message delta, not a synthesis request diff --git a/tests/unit/timing/strategies/test_cache_bust.py b/tests/unit/timing/strategies/test_cache_bust.py index 6d165246e9..54adfc41ed 100644 --- a/tests/unit/timing/strategies/test_cache_bust.py +++ b/tests/unit/timing/strategies/test_cache_bust.py @@ -2,11 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 import re +from unittest.mock import MagicMock import pytest from aiperf.common.enums import CacheBustTarget from aiperf.timing.strategies.cache_bust import ( + WARMUP_ISOLATION_MARKER, build_cache_bust_marker, estimate_marker_token_cost, ) @@ -179,6 +181,41 @@ def encode(self, text: str, **_kwargs): assert tok.calls == 4 +def test_build_cache_bust_marker_warmup_isolation_system_returns_constant(): + result = build_cache_bust_marker( + "bid", 0, 0, "trace", target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM + ) + assert result == WARMUP_ISOLATION_MARKER + + +def test_build_cache_bust_marker_warmup_isolation_first_turn_returns_constant(): + result = build_cache_bust_marker( + "bid", 0, 0, "trace", target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN + ) + assert result == WARMUP_ISOLATION_MARKER + + +def test_build_cache_bust_marker_warmup_isolation_is_constant_not_per_trajectory(): + """Same inputs with different trajectory indices must produce the same marker.""" + a = build_cache_bust_marker( + "bid", 0, 0, "trace_a", target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM + ) + b = build_cache_bust_marker( + "bid", 1, 5, "trace_b", target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM + ) + assert a == b + + +def test_estimate_marker_token_cost_warmup_isolation_returns_zero(): + tok = MagicMock() + assert estimate_marker_token_cost(CacheBustTarget.WARMUP_ISOLATION_SYSTEM, tok) == 0 + assert ( + estimate_marker_token_cost(CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, tok) + == 0 + ) + tok.encode.assert_not_called() + + def test_estimate_marker_token_cost_rounds_to_int(): """Variable token counts across samples round to a clean int.""" diff --git a/tests/unit/workers/test_worker.py b/tests/unit/workers/test_worker.py index 19144e2784..b9d6d56ef7 100644 --- a/tests/unit/workers/test_worker.py +++ b/tests/unit/workers/test_worker.py @@ -7,7 +7,7 @@ import pytest from pytest import param -from aiperf.common.enums import CacheBustTarget, CreditPhase +from aiperf.common.enums import CreditPhase from aiperf.common.models import ( Conversation, ErrorDetails, @@ -461,93 +461,6 @@ async def test_inter_token_latency_is_none_for_negative_timing( ) -class TestWarmupSystemMessage: - def test_profiling_preserves_system_message(self): - assert ( - Worker._system_message_for_phase( - system_message="existing system", - phase=CreditPhase.PROFILING, - cache_bust_target=None, - ) - == "existing system" - ) - - def test_warmup_sets_system_message_when_missing(self): - assert ( - Worker._system_message_for_phase( - system_message=None, - phase=CreditPhase.WARMUP, - cache_bust_target=None, - ) - == "warmup" - ) - - def test_warmup_prefixes_existing_system_message(self): - assert ( - Worker._system_message_for_phase( - system_message="existing system", - phase=CreditPhase.WARMUP, - cache_bust_target=None, - ) - == "warmup\nexisting system" - ) - - def test_warmup_prefixes_when_cache_bust_target_none_enum(self): - """``CacheBustTarget.NONE`` is cache-bust-disabled, so the prefix applies.""" - assert ( - Worker._system_message_for_phase( - system_message="existing system", - phase=CreditPhase.WARMUP, - cache_bust_target=CacheBustTarget.NONE, - ) - == "warmup\nexisting system" - ) - - @pytest.mark.parametrize( - "target", - [ - param(CacheBustTarget.SYSTEM_PREFIX, id="system_prefix"), - param(CacheBustTarget.SYSTEM_SUFFIX, id="system_suffix"), - param(CacheBustTarget.FIRST_TURN_PREFIX, id="first_turn_prefix"), - param(CacheBustTarget.FIRST_TURN_SUFFIX, id="first_turn_suffix"), - ], - ) # fmt: skip - def test_warmup_skips_prefix_when_cache_bust_active(self, target): - """Cache-bust markers are warmup-coherent: warmup primes the prefix - profiling hits, so prefixing in front of the shared marker would break - the prime. Every non-NONE target already isolates per trajectory tree. - """ - assert ( - Worker._system_message_for_phase( - system_message="existing system", - phase=CreditPhase.WARMUP, - cache_bust_target=target, - ) - == "existing system" - ) - - @pytest.mark.parametrize( - "target", - [ - param(CacheBustTarget.SYSTEM_PREFIX, id="system_prefix"), - param(CacheBustTarget.FIRST_TURN_PREFIX, id="first_turn_prefix"), - ], - ) # fmt: skip - def test_warmup_leaves_system_message_none_when_cache_bust_active(self, target): - """No synthetic system message is invented under cache-bust — a - warmup-only ``system`` role would diverge the message array from - profiling's even before content is compared. - """ - assert ( - Worker._system_message_for_phase( - system_message=None, - phase=CreditPhase.WARMUP, - cache_bust_target=target, - ) - is None - ) - - @pytest.mark.asyncio class TestCreateRequestInfo: async def test_create_request_info_overrides_only_outgoing_turn(self, mock_worker): diff --git a/tests/unit/workers/test_worker_cache_bust_injection.py b/tests/unit/workers/test_worker_cache_bust_injection.py index de6e47972c..a8bdf34583 100644 --- a/tests/unit/workers/test_worker_cache_bust_injection.py +++ b/tests/unit/workers/test_worker_cache_bust_injection.py @@ -1703,3 +1703,123 @@ def test_seeded_resume_with_buried_reset_suffix(): ) assert session.turn_list[1].raw_messages[0]["content"] == "u1" + _SUFFIX_MARKER assert session.turn_list[0].raw_messages[0]["content"] == "u0" + + +# WARMUP_ISOLATION_* target routing. +# During WARMUP the CreditIssuer sets cache_bust_marker=WARMUP_ISOLATION_MARKER. +# During PROFILING it sets cache_bust_marker=None, so _apply_cache_bust returns +# early (the `not marker` gate) before any routing. + + +def test_apply_cache_bust_warmup_isolation_system_injects_into_system_message(): + """WARMUP_ISOLATION_SYSTEM with a conversation system_message must prefix the string.""" + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + raw = [{"role": "user", "content": "hi"}] + session = _make_session(raw) + credit = _make_credit( + target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + marker=WARMUP_ISOLATION_MARKER, + ) + + out = _apply_cache_bust(session, credit, system_message="You are helpful.") + + assert out == WARMUP_ISOLATION_MARKER + "You are helpful." + # Raw user turn must NOT be touched. + assert session.turn_list[-1].raw_messages[0]["content"] == "hi" + + +def test_apply_cache_bust_warmup_isolation_system_falls_back_to_raw_system_role(): + """WARMUP_ISOLATION_SYSTEM with no conversation system_message injects into the raw system-role message.""" + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + raw = [{"role": "system", "content": "rules"}, {"role": "user", "content": "hi"}] + session = _make_session(raw) + credit = _make_credit( + target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + marker=WARMUP_ISOLATION_MARKER, + ) + + out = _apply_cache_bust(session, credit, system_message=None) + + assert out is None + assert ( + session.turn_list[-1].raw_messages[0]["content"] + == WARMUP_ISOLATION_MARKER + "rules" + ) + assert session.turn_list[-1].raw_messages[1]["content"] == "hi" + + +def test_apply_cache_bust_warmup_isolation_system_falls_back_to_first_user_when_no_system(): + """WARMUP_ISOLATION_SYSTEM with no system anywhere falls back to the first user turn.""" + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + raw = [{"role": "user", "content": "hi"}] + session = _make_session(raw) + credit = _make_credit( + target=CacheBustTarget.WARMUP_ISOLATION_SYSTEM, + marker=WARMUP_ISOLATION_MARKER, + ) + + out = _apply_cache_bust(session, credit, system_message=None) + + assert out is None + assert ( + session.turn_list[-1].raw_messages[0]["content"] + == WARMUP_ISOLATION_MARKER + "hi" + ) + + +def test_apply_cache_bust_warmup_isolation_first_turn_injects_into_first_user_turn(): + """WARMUP_ISOLATION_FIRST_TURN must prefix the first user turn, ignoring any system_message.""" + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + raw = [{"role": "user", "content": "Hello"}] + session = _make_session(raw) + credit = _make_credit( + target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, + marker=WARMUP_ISOLATION_MARKER, + ) + + out = _apply_cache_bust(session, credit, system_message="sys") + + # System message returned unchanged. + assert out == "sys" + assert ( + session.turn_list[-1].raw_messages[0]["content"] + == WARMUP_ISOLATION_MARKER + "Hello" + ) + + +def test_apply_cache_bust_warmup_isolation_first_turn_with_synthetic_turn(): + """WARMUP_ISOLATION_FIRST_TURN on a synthetic Turn (raw_messages=None) mutates Text.contents[0].""" + from aiperf.timing.strategies.cache_bust import WARMUP_ISOLATION_MARKER + + turn = Turn(raw_messages=None, texts=[Text(contents=["Hello"])]) + session = _make_synthetic_session(turn) + credit = _make_credit( + target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, + marker=WARMUP_ISOLATION_MARKER, + ) + + out = _apply_cache_bust(session, credit, system_message=None) + + assert out is None + assert ( + session.turn_list[-1].texts[0].contents[0] == WARMUP_ISOLATION_MARKER + "Hello" + ) + + +def test_apply_cache_bust_warmup_isolation_profiling_no_marker_is_noop(): + """When marker is None (PROFILING phase) the early-exit gate fires and no injection occurs.""" + raw = [{"role": "user", "content": "hi"}] + session = _make_session(raw) + credit = _make_credit( + target=CacheBustTarget.WARMUP_ISOLATION_FIRST_TURN, + marker=None, + ) + + out = _apply_cache_bust(session, credit, system_message="sys") + + assert out == "sys" + assert session.turn_list[-1].raw_messages[0]["content"] == "hi" From 896783f4352f7a61f0ba8c813a0957970856139e Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:51:42 -0700 Subject: [PATCH 031/119] docs(cache-bust): add reference link to cache-bust.md in --cache-bust description (#1264) (cherry picked from commit 6ed4823d127b3a6d12c63fb8c2ca5eff13f9ba23) --- .agents/skills/aiperf-code-review/SKILL.md | 1 + docs/cli-options.md | 4 ++-- src/aiperf/config/flags/cli_config.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.agents/skills/aiperf-code-review/SKILL.md b/.agents/skills/aiperf-code-review/SKILL.md index d0d1954138..3370f56e14 100644 --- a/.agents/skills/aiperf-code-review/SKILL.md +++ b/.agents/skills/aiperf-code-review/SKILL.md @@ -17,6 +17,7 @@ description: Review the current branch against origin/main, capture findings in Requirements: - Treat `artifacts/code-review.md` as a living document. Update it in place if it already exists. + - **CLI docs check:** If the diff adds or removes any CLI parameter (i.e., any `Annotated[..., CLIParameter(...)]` field or `--flag` name in `src/aiperf/config/flags/cli_config.py`), verify that `docs/cli-options.md` was regenerated to reflect those changes. The file is auto-generated by `make generate-cli-docs`; a missing regeneration is a blocking finding. The description field inside `Field(description=...)` is the source of truth for what appears in the docs — changes there also require a regen. - For each finding, record: - status: `Confirmed`, `Partially confirmed`, or `Not confirmed` - source-level evidence with exact file paths and line references diff --git a/docs/cli-options.md b/docs/cli-options.md index 7144ac2492..126a28fdf7 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -678,7 +678,7 @@ Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare so #### `--cache-bust` `` -Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (system_prefix, system_suffix, first_turn_prefix, first_turn_suffix) — inject a per-trajectory unique SHA-256 digest marker that is identical across warmup and profiling, so warmup KV-cache work transfers to profiling while preventing cross-trajectory cache sharing. Prefix variants prepend at token 0; suffix variants append after existing content. (2) Warmup-isolation targets (warmup_isolation_system, warmup_isolation_first_turn) — inject a constant '[warmup]' marker only during the WARMUP phase; profiling sees no marker (fully cold start or system-pre-warmed). Incompatible with agentic_replay timing mode. 'none' disables the feature (default). +Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (system_prefix, system_suffix, first_turn_prefix, first_turn_suffix) — inject a per-trajectory unique SHA-256 digest marker that is identical across warmup and profiling, so warmup KV-cache work transfers to profiling while preventing cross-trajectory cache sharing. Prefix variants prepend at token 0; suffix variants append after existing content. (2) Warmup-isolation targets (warmup_isolation_system, warmup_isolation_first_turn) — inject a constant '[warmup]' marker only during the WARMUP phase; profiling sees no marker (fully cold start or system-pre-warmed). Incompatible with agentic_replay timing mode. 'none' disables the feature (default). See [cache-bust.md](reference/cache-bust.md) for detailed semantics, trade-offs, and examples.
_Choices: [`none`, `system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`, `warmup_isolation_system`, `warmup_isolation_first_turn`]_
_Default: `none`_ @@ -2218,7 +2218,7 @@ Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare so #### `--cache-bust` `` -Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (system_prefix, system_suffix, first_turn_prefix, first_turn_suffix) — inject a per-trajectory unique SHA-256 digest marker that is identical across warmup and profiling, so warmup KV-cache work transfers to profiling while preventing cross-trajectory cache sharing. Prefix variants prepend at token 0; suffix variants append after existing content. (2) Warmup-isolation targets (warmup_isolation_system, warmup_isolation_first_turn) — inject a constant '[warmup]' marker only during the WARMUP phase; profiling sees no marker (fully cold start or system-pre-warmed). Incompatible with agentic_replay timing mode. 'none' disables the feature (default). +Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (system_prefix, system_suffix, first_turn_prefix, first_turn_suffix) — inject a per-trajectory unique SHA-256 digest marker that is identical across warmup and profiling, so warmup KV-cache work transfers to profiling while preventing cross-trajectory cache sharing. Prefix variants prepend at token 0; suffix variants append after existing content. (2) Warmup-isolation targets (warmup_isolation_system, warmup_isolation_first_turn) — inject a constant '[warmup]' marker only during the WARMUP phase; profiling sees no marker (fully cold start or system-pre-warmed). Incompatible with agentic_replay timing mode. 'none' disables the feature (default). See [cache-bust.md](reference/cache-bust.md) for detailed semantics, trade-offs, and examples.
_Choices: [`none`, `system_prefix`, `system_suffix`, `first_turn_prefix`, `first_turn_suffix`, `warmup_isolation_system`, `warmup_isolation_first_turn`]_
_Default: `none`_ diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 6aacca795a..7eb111e57e 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -1156,7 +1156,8 @@ def url(self) -> str: "warmup_isolation_first_turn) — inject a constant '[warmup]' marker only " "during the WARMUP phase; profiling sees no marker (fully cold start or " "system-pre-warmed). Incompatible with agentic_replay timing mode. " - "'none' disables the feature (default)." + "'none' disables the feature (default). " + "See [cache-bust.md](reference/cache-bust.md) for detailed semantics, trade-offs, and examples." ), ), CLIParameter( From ba08695bf048545ae3974dcfc5f4138c60cd4361 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:34:54 -0700 Subject: [PATCH 032/119] feat(dataset): add --system-prompt/--system-prompt-file for verbatim system prompts (#1268) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 88242293b552db96b90b2e3999bbfa93488c994f) --- docs/cli-options.md | 16 + docs/reference/cache-bust.md | 26 +- docs/tutorials/custom-dataset.md | 54 +++ src/aiperf/common/constants.py | 11 + src/aiperf/config/config.py | 67 +++- src/aiperf/config/dataset/config.py | 8 +- src/aiperf/config/dataset/system_prompt.py | 111 ++++++ src/aiperf/config/flags/_converter_dataset.py | 4 + src/aiperf/config/flags/cli_config.py | 40 ++ src/aiperf/config/loader/helpers.py | 10 + src/aiperf/dataset/composer/base.py | 76 +++- src/aiperf/dataset/mmap_cache.py | 22 +- src/aiperf/endpoints/anthropic_messages.py | 25 +- src/aiperf/endpoints/openai_chat.py | 42 +- .../unit/config/test_system_prompt_config.py | 264 +++++++++++++ .../composer/test_custom_system_prompt.py | 360 ++++++++++++++++++ tests/unit/dataset/test_mmap_cache.py | 67 ++++ .../endpoints/test_message_merging_fixes.py | 50 ++- .../unit/endpoints/test_messages_endpoint.py | 42 +- 19 files changed, 1260 insertions(+), 35 deletions(-) create mode 100644 src/aiperf/config/dataset/system_prompt.py create mode 100644 tests/unit/config/test_system_prompt_config.py create mode 100644 tests/unit/dataset/composer/test_custom_system_prompt.py diff --git a/docs/cli-options.md b/docs/cli-options.md index 126a28fdf7..8d1733b61a 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -684,6 +684,14 @@ Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (sy ### Prefix Prompt +#### `--system-prompt` `` + +Verbatim system prompt text, identical across every conversation. Sent as a system-role message ahead of all turns. Works with both synthetic and file/public datasets; when the dataset already carries its own system message, this text is prepended to it. Tokens are additive: `--isl` continues to size the generated user prompt only. Mutually exclusive with `--system-prompt-file`, `--shared-system-prompt-length`, and `--num-prefix-prompts`/`--prefix-prompt-length`. + +#### `--system-prompt-file` `` + +Path to a UTF-8 text file holding the verbatim system prompt. Preferred over `--system-prompt` for real production prompts, which are long enough that shell quoting mangles them. Read once at startup, so a missing or unreadable file fails immediately rather than mid-run. Mutually exclusive with `--system-prompt`, `--shared-system-prompt-length`, and `--num-prefix-prompts`/`--prefix-prompt-length`. + #### `--prompt-prefix-pool-size`, `--prefix-prompt-pool-size`, `--num-prefix-prompts` `` Number of distinct prefix prompts to generate for K-V cache testing. Each prefix is prepended to user prompts, simulating cached context scenarios. Prefixes randomly selected from pool per request. Set to 0 to disable prefix prompts. Mutually exclusive with `--shared-system-prompt-length`/`--user-context-prompt-length`. @@ -2224,6 +2232,14 @@ Where (and how) to inject a cache-bust marker. Two families: (1) RID targets (sy ### Prefix Prompt +#### `--system-prompt` `` + +Verbatim system prompt text, identical across every conversation. Sent as a system-role message ahead of all turns. Works with both synthetic and file/public datasets; when the dataset already carries its own system message, this text is prepended to it. Tokens are additive: `--isl` continues to size the generated user prompt only. Mutually exclusive with `--system-prompt-file`, `--shared-system-prompt-length`, and `--num-prefix-prompts`/`--prefix-prompt-length`. + +#### `--system-prompt-file` `` + +Path to a UTF-8 text file holding the verbatim system prompt. Preferred over `--system-prompt` for real production prompts, which are long enough that shell quoting mangles them. Read once at startup, so a missing or unreadable file fails immediately rather than mid-run. Mutually exclusive with `--system-prompt`, `--shared-system-prompt-length`, and `--num-prefix-prompts`/`--prefix-prompt-length`. + #### `--prompt-prefix-pool-size`, `--prefix-prompt-pool-size`, `--num-prefix-prompts` `` Number of distinct prefix prompts to generate for K-V cache testing. Each prefix is prepended to user prompts, simulating cached context scenarios. Prefixes randomly selected from pool per request. Set to 0 to disable prefix prompts. Mutually exclusive with `--shared-system-prompt-length`/`--user-context-prompt-length`. diff --git a/docs/reference/cache-bust.md b/docs/reference/cache-bust.md index f010e76b50..6e12a8f33f 100644 --- a/docs/reference/cache-bust.md +++ b/docs/reference/cache-bust.md @@ -157,8 +157,9 @@ PROFILING the payload is clean — no marker anywhere. > **Synthetic datasets only — requires a shared system prompt.** > For synthetic datasets, `warmup_isolation_system` is rejected at config validation time -> if no system message is statically present. This means `--shared-system-prompt-length` -> must be set. If you are using `--user-context-prompt-length` or `--num-prefix-prompts` +> if no system message is statically present. This means either +> `--shared-system-prompt-length` or `--system-prompt`/`--system-prompt-file` must be set. +> If you are using `--user-context-prompt-length` or `--num-prefix-prompts` > without a shared system prompt, switch to `warmup_isolation_first_turn` instead. > > File and public datasets are not checked statically — system-message presence depends @@ -261,6 +262,27 @@ A synthetic system message is created and shared identically across all sessions `warmup_isolation_system` makes profiling fully cold (system poisoned during warmup). `warmup_isolation_first_turn` keeps the system prompt pre-warmed; only user-turn tokens are cold. +### Verbatim system prompt (`--system-prompt` / `--system-prompt-file`) + +A user-supplied system message, identical across all sessions, works with synthetic **and** +file/public datasets. + +It is **mutually exclusive** with `--shared-system-prompt-length`: both fill the same +system-message slot, so setting both is rejected at config-validation time rather than one +silently taking precedence. There is no ordering between them — the run fails before any +request is sent. The same exclusivity applies to +`--num-prefix-prompts`/`--prefix-prompt-length`. + +For marker routing the verbatim prompt behaves exactly like +`--shared-system-prompt-length`: the `system_*` and `warmup_isolation_system` targets land +on it rather than falling through to the first user turn. + +One difference in token accounting: the synthetic system prompt is shrunk by the marker's +token cost so its wire length still matches the configured `--shared-system-prompt-length`. +Verbatim text has no target length to compensate against, so the marker is simply **additive** +— consistent with the system prompt's own tokens, which sit on top of `--isl` rather than +inside it. + ### User context prefix (`--user-context-prompt-length`) Each session gets a unique per-session context message prepended as a **user-role** message diff --git a/docs/tutorials/custom-dataset.md b/docs/tutorials/custom-dataset.md index bb01de8fe2..acb876d018 100644 --- a/docs/tutorials/custom-dataset.md +++ b/docs/tutorials/custom-dataset.md @@ -466,6 +466,60 @@ benchmark: --- +## Adding a System Prompt + +`--system-prompt` and `--system-prompt-file` attach a fixed system message to every +conversation. Unlike `--shared-system-prompt-length`, which generates synthetic filler of a +target token length, these take the **exact text** — so prefix-cache hit rates and TTFT +reflect the system prompt your deployment actually sends. + +They work with every dataset kind: synthetic, file-based, and public. + +```bash +# Inline, for short prompts +aiperf profile -m Qwen/Qwen3-0.6B --url http://localhost:8000 \ + --input-file ./data.jsonl \ + --system-prompt "You are a terse assistant." + +# From a file, for real production prompts +aiperf profile -m Qwen/Qwen3-0.6B --url http://localhost:8000 \ + --input-file ./data.jsonl \ + --system-prompt-file ./prod_system.txt +``` + +Or in YAML: + +```yaml +dataset: + type: file + path: ./data.jsonl + system_prompt_file: ./prod_system.txt +``` + +Behavior worth knowing: + +- **Tokens are additive.** With `--isl 1000` and a 350-token system prompt, the request + carries roughly 1350 tokens: `--isl` continues to size the generated user prompt only. +- **A dataset's own system message is kept.** If the dataset already authors one, your text + is prepended to it, separated by a blank line, and both are sent as a single system + message. Repeated system roles are mishandled by many OpenAI-compatible servers. +- **The file is read once at startup**, so a missing or unreadable path fails immediately + rather than mid-benchmark. Paths containing a symlinked component are rejected — note that + on macOS this includes anything under `/tmp` or `$TMPDIR`, since `/var` is itself a symlink. +- **Supported on `chat`, `responses`, `messages`, and `chat_embeddings`.** Endpoints with no + system role (`completions`, `embeddings`, the rankings endpoints) reject the option at + startup rather than silently dropping it. +- **Mutually exclusive** with `--shared-system-prompt-length` and with + `--num-prefix-prompts`/`--prefix-prompt-length`, all of which fill the same slot. Setting + two of them is rejected at startup rather than one taking precedence, so a misconfigured + run fails before sending any requests. It does combine with + `--user-context-prompt-length` for a two-tier shared/per-session structure. + +> **Note:** `--num-prefix-prompts` and `--prefix-prompt-length` apply only to synthetic +> datasets — they are dropped for file and public datasets. `--system-prompt` is not. + +--- + ## Related - [Multi-Turn Conversations](multi-turn.md) - Multi-turn conversation benchmarking diff --git a/src/aiperf/common/constants.py b/src/aiperf/common/constants.py index f8e3399254..060898dfe8 100644 --- a/src/aiperf/common/constants.py +++ b/src/aiperf/common/constants.py @@ -36,3 +36,14 @@ GOOD_REQUEST_COUNT_TAG = "good_request_count" """GoodRequestCount metric tag""" + +SYSTEM_PROMPT_JOIN_SEP = "\n\n" +"""Separator joining a verbatim ``--system-prompt`` to a dataset's own system message. + +Shared by the two places that perform the merge -- the composer, for datasets +whose system message was hoisted to ``Conversation.system_message``, and the chat +endpoint, for datasets that leave an unhoisted ``role: system`` in +``turn.raw_messages``. The two must agree: a run can hit either path depending on +the loader, and a drift between them would silently change request bytes and +prefix-cache behavior. +""" diff --git a/src/aiperf/config/config.py b/src/aiperf/config/config.py index 9f4f1713f2..5a2d55b6e7 100644 --- a/src/aiperf/config/config.py +++ b/src/aiperf/config/config.py @@ -565,6 +565,60 @@ def validate_cache_bust_compatibility(self) -> Self: return self + @model_validator(mode="after") + def validate_system_prompt_compatibility(self) -> Self: + """Reject a verbatim system prompt that would conflict or be silently dropped. + + A custom system prompt is the content-valued replacement for + ``prefix_prompts.shared_system_length``: both fill + ``Conversation.system_message``. It therefore inherits that field's + exclusivity with the prefix pool rather than defining its own rule. + + Endpoints opt into receiving a system message via the + ``consumes_system_message`` plugin metadata flag. On the others the text + would never reach the wire, so the benchmark would silently measure + something other than what was asked for. + """ + from aiperf.plugin import plugins + from aiperf.plugin.enums import PluginType + + if self.get_system_prompt() is None: + return self + + prefix_prompts = getattr(self.get_default_dataset(), "prefix_prompts", None) + if prefix_prompts is not None: + if prefix_prompts.shared_system_length is not None: + raise ValueError( + "--system-prompt/--system-prompt-file and " + "--shared-system-prompt-length are mutually exclusive: both fill " + "the system message. Use the first for verbatim text, the second " + "for synthetic filler of a target token length." + ) + if prefix_prompts.pool_size or prefix_prompts.length: + raise ValueError( + "--system-prompt/--system-prompt-file and " + "--num-prefix-prompts/--prefix-prompt-length are mutually " + "exclusive, matching --shared-system-prompt-length, which fills " + "the same system-message slot." + ) + + endpoint_metadata = plugins.get_endpoint_metadata(self.endpoint.type) + if not endpoint_metadata.consumes_system_message: + supported = ", ".join( + sorted( + entry.name + for entry, _ in plugins.iter_all(PluginType.ENDPOINT) + if plugins.get_endpoint_metadata(entry.name).consumes_system_message + ) + ) + raise ValueError( + f"--system-prompt/--system-prompt-file is not supported by endpoint " + f"type '{self.endpoint.type}' (no system role), so the text would " + f"never reach the wire. Supported endpoint types: {supported}." + ) + + return self + @model_validator(mode="after") def validate_warmup_isolation_not_agentic_replay(self) -> Self: """Reject WARMUP_ISOLATION_* targets when timing mode resolves to AGENTIC_REPLAY. @@ -623,6 +677,10 @@ def validate_warmup_isolation_system_requires_shared_system_prompt(self) -> Self silently falls through to the user turn, making it indistinguishable from ``warmup_isolation_first_turn``. + A verbatim ``--system-prompt``/``--system-prompt-file`` satisfies this + requirement exactly as ``shared_system_length`` does: both statically + guarantee a system message, so the marker has a real slot to land in. + This check is scoped to synthetic datasets where system-message presence is statically known. File and trace datasets may carry system messages inside their content, which cannot be inspected at config time. @@ -636,6 +694,9 @@ def validate_warmup_isolation_system_requires_shared_system_prompt(self) -> Self if dataset.type != DatasetType.SYNTHETIC: return self + if self.get_system_prompt() is not None: + return self + prefix_prompts = getattr(dataset, "prefix_prompts", None) has_shared_system = ( prefix_prompts is not None @@ -644,10 +705,12 @@ def validate_warmup_isolation_system_requires_shared_system_prompt(self) -> Self if not has_shared_system: raise ValueError( "cache_bust=warmup_isolation_system requires a shared system prompt, " - "but no shared_system_length is configured on the synthetic dataset. " + "but no shared_system_length or verbatim system prompt is configured " + "on the synthetic dataset. " "The marker would silently fall through to the user turn, making it " "indistinguishable from warmup_isolation_first_turn. " - "Either set prefix_prompts.shared_system_length (--shared-system-prompt-length) " + "Either set prefix_prompts.shared_system_length (--shared-system-prompt-length), " + "pass --system-prompt/--system-prompt-file, " "or switch to cache_bust=warmup_isolation_first_turn." ) return self diff --git a/src/aiperf/config/dataset/config.py b/src/aiperf/config/dataset/config.py index 4dba0cd2ef..382bfdcd5c 100644 --- a/src/aiperf/config/dataset/config.py +++ b/src/aiperf/config/dataset/config.py @@ -29,7 +29,6 @@ DatasetFormat, DatasetType, ) -from aiperf.config.base import BaseConfig from aiperf.config.dataset.content import ( AudioConfig, CacheBustConfig, @@ -39,6 +38,7 @@ PromptSelectionConfig, RankingsConfig, ) +from aiperf.config.dataset.system_prompt import SystemPromptMixin from aiperf.config.dataset.trace import ( SynthesisConfig, ) @@ -85,7 +85,7 @@ # Dataset type variants using discriminated unions -class SyntheticDataset(BaseConfig): +class SyntheticDataset(SystemPromptMixin): """ Synthetic dataset configuration. @@ -281,7 +281,7 @@ def _validate_turns_at_least_one(self) -> SyntheticDataset: return self -class FileDataset(BaseConfig): +class FileDataset(SystemPromptMixin): """ File-based dataset configuration. @@ -690,7 +690,7 @@ def _warn_large_inline_records(self) -> FileDataset: return self -class PublicDataset(BaseConfig): +class PublicDataset(SystemPromptMixin): """ Public dataset configuration. diff --git a/src/aiperf/config/dataset/system_prompt.py b/src/aiperf/config/dataset/system_prompt.py new file mode 100644 index 0000000000..83bae0615b --- /dev/null +++ b/src/aiperf/config/dataset/system_prompt.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verbatim system-prompt configuration shared by every dataset variant. + +``SyntheticDataset``, ``FileDataset``, and ``PublicDataset`` each inherit +``SystemPromptMixin`` so a user-supplied system prompt works identically +regardless of where the conversations came from. + +This is the content-valued counterpart to +``PrefixPromptConfig.shared_system_length``: both fill +``Conversation.system_message``, one by naming a token length for synthetic +filler, the other by naming the exact text. They are mutually exclusive +(enforced in ``AIPerfConfig``, which can see both blocks at once). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Self + +from pydantic import Field, PrivateAttr, model_validator + +from aiperf.common.path_safety import safe_read_template_path +from aiperf.config.base import BaseConfig + + +class SystemPromptMixin(BaseConfig): + """Adds ``system_prompt`` / ``system_prompt_file`` to a dataset variant.""" + + system_prompt: Annotated[ + str | None, + Field( + default=None, + description="Verbatim system prompt text, identical across every conversation. " + "Emitted as a system-role message ahead of all turns. " + "When the dataset already carries its own system message, this text is " + "prepended to it rather than replacing it. " + "Tokens are additive: --isl continues to size the generated user prompt only. " + "Mutually exclusive with system_prompt_file and with " + "prefix_prompts.shared_system_length/pool_size/length.", + ), + ] + + system_prompt_file: Annotated[ + Path | None, + Field( + default=None, + description="Path to a UTF-8 text file holding the verbatim system prompt. " + "Preferred over system_prompt for real production prompts, which are long " + "enough that shell quoting mangles them. Read once at config-validation " + "time so a missing or unreadable file fails at startup. " + "Mutually exclusive with system_prompt.", + ), + ] + + # Resolved text for whichever source was configured. Populated once during + # validation so conversation-building never re-reads the file. + _resolved_system_prompt: str | None = PrivateAttr(default=None) + + @model_validator(mode="after") + def _resolve_system_prompt(self) -> Self: + """Reject conflicting/empty sources and cache the resolved text. + + Reading here rather than at conversation-build time means an unreadable + path is a startup error instead of a mid-benchmark one, and the file is + read exactly once instead of once per conversation. + """ + if self.system_prompt is not None and self.system_prompt_file is not None: + raise ValueError( + "system_prompt (--system-prompt) and system_prompt_file " + "(--system-prompt-file) are mutually exclusive; set exactly one." + ) + + if self.system_prompt is not None: + if not self.system_prompt.strip(): + raise ValueError( + "system_prompt (--system-prompt) is empty or whitespace-only. " + "Omit the option entirely to run without a system prompt." + ) + self._resolved_system_prompt = self.system_prompt + return self + + if self.system_prompt_file is not None: + text = safe_read_template_path(str(self.system_prompt_file)) + if text is None: + raise ValueError( + f"system_prompt_file (--system-prompt-file) could not be read: " + f"{self.system_prompt_file}. Expected a readable regular UTF-8 " + "text file with no symlinked path component." + ) + if not text.strip(): + raise ValueError( + f"system_prompt_file (--system-prompt-file) is empty or " + f"whitespace-only: {self.system_prompt_file}. Omit the option " + "entirely to run without a system prompt." + ) + self._resolved_system_prompt = text + return self + + return self + + @property + def resolved_system_prompt(self) -> str | None: + """The verbatim system prompt text, or ``None`` when unconfigured. + + Resolved from whichever of ``system_prompt`` / ``system_prompt_file`` + was set. Composers read this rather than the raw fields so the file + vs inline distinction stays confined to validation. + """ + return self._resolved_system_prompt diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index ca38de7194..ea7c4342ee 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -266,6 +266,10 @@ def _parse_dataset_filters(values: list[str]) -> dict[str, str]: ("open_loop_strict", "open_loop_strict", False), ("omit_kv_hints", "omit_kv_hints", False), ("force_min_tokens", "force_min_tokens", False), + # SystemPromptMixin fields live on all three dataset variants, so unlike the + # synthetic-only subtables below they must NOT be popped per-type. + ("system_prompt", "system_prompt", False), + ("system_prompt_file", "system_prompt_file", False), ) diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 7eb111e57e..936a82b84c 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -1169,6 +1169,46 @@ def url(self) -> str: ############################################################################## # Prefix Prompt ############################################################################## + system_prompt: Annotated[ + str | None, + Field( + default=None, + description=( + "Verbatim system prompt text, identical across every conversation.\n" + "Sent as a system-role message ahead of all turns. Works with both\n" + "synthetic and file/public datasets; when the dataset already carries\n" + "its own system message, this text is prepended to it.\n" + "Tokens are additive: `--isl` continues to size the generated user prompt only.\n" + "Mutually exclusive with `--system-prompt-file`, " + "`--shared-system-prompt-length`, and `--num-prefix-prompts`/`--prefix-prompt-length`." + ), + ), + CLIParameter( + name=("--system-prompt",), + group=Groups.PREFIX_PROMPT, + ), + ] = None + + system_prompt_file: Annotated[ + Path | None, + Field( + default=None, + description=( + "Path to a UTF-8 text file holding the verbatim system prompt.\n" + "Preferred over `--system-prompt` for real production prompts, which are\n" + "long enough that shell quoting mangles them. Read once at startup, so a\n" + "missing or unreadable file fails immediately rather than mid-run.\n" + "Mutually exclusive with `--system-prompt`, " + "`--shared-system-prompt-length`, and " + "`--num-prefix-prompts`/`--prefix-prompt-length`." + ), + ), + CLIParameter( + name=("--system-prompt-file",), + group=Groups.PREFIX_PROMPT, + ), + ] = None + prompt_prefix_pool_size: Annotated[ int, Field( diff --git a/src/aiperf/config/loader/helpers.py b/src/aiperf/config/loader/helpers.py index 05afb82647..e464647ab7 100644 --- a/src/aiperf/config/loader/helpers.py +++ b/src/aiperf/config/loader/helpers.py @@ -109,6 +109,16 @@ def get_prompt_corpus(self) -> PromptCorpus | None: return None return getattr(prompts, "corpus", None) + def get_system_prompt(self) -> str | None: + """Resolve the active dataset's verbatim system prompt text, if any. + + Reads ``SystemPromptMixin.resolved_system_prompt``, which every dataset + variant carries and which already collapsed the + ``system_prompt`` / ``system_prompt_file`` distinction during + validation. Returns ``None`` when the user configured neither. + """ + return getattr(self.get_default_dataset(), "resolved_system_prompt", None) + # ========================================================================== # CONVENIENCE PROPERTIES # ========================================================================== diff --git a/src/aiperf/dataset/composer/base.py b/src/aiperf/dataset/composer/base.py index a1a74a41d3..25d41ea27e 100644 --- a/src/aiperf/dataset/composer/base.py +++ b/src/aiperf/dataset/composer/base.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from aiperf.common import random_generator as rng +from aiperf.common.constants import SYSTEM_PROMPT_JOIN_SEP from aiperf.common.enums import ConversationContextMode, ModelSelectionStrategy from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.common.models import Conversation, Turn @@ -213,6 +214,14 @@ def _init_isl_compensation( else None ) has_synthetic_system_prompt = configured_shared_sys_len is not None + # A verbatim --system-prompt fills the same system-message slot, so + # SYSTEM_* markers land on it instead of falling through to the first + # user turn. Unlike the synthetic prompt it cannot be shrunk to absorb + # the marker cost (component (c) below): the text is the user's, so the + # marker is simply additive -- matching the additive ISL model for the + # system prompt itself. + has_custom_system_prompt = self.run.cfg.get_system_prompt() is not None + has_system_prompt = has_synthetic_system_prompt or has_custom_system_prompt is_system_target = cache_bust_target in ( CacheBustTarget.SYSTEM_PREFIX, CacheBustTarget.SYSTEM_SUFFIX, @@ -228,12 +237,9 @@ def _init_isl_compensation( # Component (a) routing: mirrors ``worker._apply_cache_bust``'s fallback: # SYSTEM_* + system prompt -> marker on system msg; otherwise (incl. # SYSTEM_* with no system message, and FIRST_TURN_*) -> first user turn. - marker_on_shared_system_prompt = ( - is_system_target and has_synthetic_system_prompt - ) + marker_on_system_prompt = is_system_target and has_system_prompt marker_on_first_user_turn = ( - cache_bust_target != CacheBustTarget.NONE - and not marker_on_shared_system_prompt + cache_bust_target != CacheBustTarget.NONE and not marker_on_system_prompt ) self._first_turn_cache_bust_marker_tokens = ( self._cache_bust_marker_tokens if marker_on_first_user_turn else 0 @@ -257,7 +263,11 @@ def _init_isl_compensation( # cost so its wire length still matches ``--shared-system-prompt-length``. # Compensate via a model_copy passed to PromptGenerator — never mutate # the user-facing config in place. - if marker_on_shared_system_prompt and configured_shared_sys_len is not None: + # + # The ``configured_shared_sys_len`` guard keeps this synthetic-only: a + # verbatim --system-prompt has no target length to compensate against, + # so its marker stays additive. + if marker_on_system_prompt and configured_shared_sys_len is not None: compensated_shared_sys_len = max( 1, configured_shared_sys_len - self._cache_bust_marker_tokens ) @@ -501,6 +511,43 @@ def _finalize_conversations(self, conversations: list[Conversation]) -> None: """ self._inject_context_prompts(conversations) + def _inject_custom_system_prompt(self, conversations: list[Conversation]) -> None: + """Prepend the user's verbatim ``--system-prompt`` to every conversation. + + Prepends rather than assigns so a dataset that authored its own system + message keeps it: loaders such as ``multi_turn`` hoist a leading system + turn into ``conversation.system_message`` before this runs, and + discarding that would drop content the trace depends on. The two are + joined into a single system message rather than emitted as two, since + repeated system roles are mishandled by many OpenAI-compatible servers. + + Datasets that leave an unhoisted ``role: system`` inside + ``turn.raw_messages`` are merged at the endpoint layer instead; the + composer does not reach into pre-rendered message arrays. + + Mutually exclusive with ``prefix_prompts.shared_system_length`` (enforced + in config validation), so this never races the synthetic system prompt + injected below. + """ + system_prompt = self.run.cfg.get_system_prompt() + if system_prompt is None: + return + + self.debug( + lambda: ( + f"Injecting custom system prompt into {len(conversations)} conversations" + ) + ) + for conversation in conversations: + if conversation.system_message: + conversation.system_message = ( + f"{system_prompt}" + f"{SYSTEM_PROMPT_JOIN_SEP}" + f"{conversation.system_message}" + ) + else: + conversation.system_message = system_prompt + def _inject_context_prompts(self, conversations: list[Conversation]) -> None: """Inject shared system and user context prompts into conversations. @@ -510,6 +557,12 @@ def _inject_context_prompts(self, conversations: list[Conversation]) -> None: Args: conversations: List of conversations to inject prompts into """ + # Runs ahead of both guards below: a verbatim system prompt needs + # neither a tokenizer nor ``prefix_prompts``, and file/public datasets + # carry neither. Gating it behind them is exactly why those datasets + # get no system prompt today. + self._inject_custom_system_prompt(conversations) + if self.prompt_generator is None: return @@ -541,7 +594,9 @@ def _inject_context_prompts(self, conversations: list[Conversation]) -> None: if shared_system_prompt: conversation.system_message = shared_system_prompt self.trace( - lambda conv=conversation: f"Set system_message on conversation {conv.session_id}" + lambda conv=conversation: ( + f"Set system_message on conversation {conv.session_id}" + ) ) # Set user context prompt (unique per session) @@ -551,9 +606,10 @@ def _inject_context_prompts(self, conversations: list[Conversation]) -> None: ) conversation.user_context_message = user_context self.trace( - lambda idx=session_index, - conv=conversation: f"Set user_context_message for session {idx} " - f"(conversation {conv.session_id})" + lambda idx=session_index, conv=conversation: ( + f"Set user_context_message for session {idx} " + f"(conversation {conv.session_id})" + ) ) @staticmethod diff --git a/src/aiperf/dataset/mmap_cache.py b/src/aiperf/dataset/mmap_cache.py index b37fd4e075..a3032c0a8c 100644 --- a/src/aiperf/dataset/mmap_cache.py +++ b/src/aiperf/dataset/mmap_cache.py @@ -763,8 +763,21 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: if records else None ) + # The composer bakes the verbatim system prompt into + # ``Conversation.system_message``, which round-trips through the stored mmap + # (model_dump_json on write, model_validate_json on read). A cache HIT skips + # the composer entirely, so two runs differing only in + # --system-prompt/--system-prompt-file must NOT share an entry or the second + # silently replays the first one's prompt. Hashed rather than inlined because + # production system prompts run to many KB. + # + # The key is OMITTED (not set to None) when no system prompt is configured: + # ``compute_cache_key`` orjson-dumps this payload, which would serialize an + # explicit None as ``null`` and shift every existing key, needlessly cold- + # starting warm caches for runs that never touch the feature. + system_prompt = cfg.get_system_prompt() - return { + payload: dict[str, object] = { "num_dataset_entries": getattr(dataset, "entries", None), "dataset_sampling_strategy": str(getattr(dataset, "sampling", "")), "custom_dataset_type": ( @@ -863,6 +876,13 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: "entries_explicit": getattr(dataset, "entries_explicit", False), } + if system_prompt is not None: + payload["system_prompt_sha256"] = hashlib.sha256( + system_prompt.encode("utf-8") + ).hexdigest() + + return payload + def compute_cache_key_from_run(run: BenchmarkRun) -> str | None: """Build a cache key for ``run`` or return None when caching is unsafe. diff --git a/src/aiperf/endpoints/anthropic_messages.py b/src/aiperf/endpoints/anthropic_messages.py index cd36270311..f34c54b718 100644 --- a/src/aiperf/endpoints/anthropic_messages.py +++ b/src/aiperf/endpoints/anthropic_messages.py @@ -497,11 +497,20 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: if model_endpoint.endpoint.streaming: payload["stream"] = True - # raw_system (Turn-level list-of-blocks) wins over the - # conversation-level system_message string. Lets callers attach - # cache_control / Anthropic-specific extensions per-block. + # raw_system (Turn-level list-of-blocks) carries per-block + # cache_control / Anthropic-specific extensions, so it is preserved + # verbatim. A conversation-level system_message is prepended as a + # leading text block rather than dropped -- Anthropic's `system` is + # already a block list, so this needs no merging of adjacent text and + # keeps a verbatim --system-prompt at token 0. if raw_system is not None: - payload["system"] = raw_system + if request_info.system_message: + payload["system"] = [ + {"type": "text", "text": request_info.system_message}, + *raw_system, + ] + else: + payload["system"] = raw_system elif request_info.system_message: payload["system"] = request_info.system_message @@ -777,9 +786,11 @@ def _parse_streaming_event( case EventType.ERROR: error_detail = json_obj.get("error", {}) self.warning( - lambda: f"Anthropic streaming error: " - f"type={error_detail.get('type')}, " - f"message={error_detail.get('message')}" + lambda: ( + f"Anthropic streaming error: " + f"type={error_detail.get('type')}, " + f"message={error_detail.get('message')}" + ) ) return None diff --git a/src/aiperf/endpoints/openai_chat.py b/src/aiperf/endpoints/openai_chat.py index 67e0133cfe..10a9399d71 100644 --- a/src/aiperf/endpoints/openai_chat.py +++ b/src/aiperf/endpoints/openai_chat.py @@ -5,6 +5,7 @@ from typing import Any +from aiperf.common.constants import SYSTEM_PROMPT_JOIN_SEP from aiperf.common.models import ( BaseResponseData, Image, @@ -20,6 +21,21 @@ from aiperf.endpoints.base_endpoint import BaseEndpoint +def _prepend_system_text(prefix: str, content: Any) -> Any: + """Return ``content`` with ``prefix`` prepended, preserving its shape. + + System-message content is normally a plain string, but the OpenAI schema + also permits a list of content parts; a raw-payload dataset may author + either. Returns a new object in both cases -- callers pass content that + aliases reusable turn state. + """ + if isinstance(content, list): + return [{"type": "text", "text": prefix}, *content] + if isinstance(content, str) and content: + return f"{prefix}{SYSTEM_PROMPT_JOIN_SEP}{content}" + return prefix + + class ChatEndpoint(BaseEndpoint): """OpenAI Chat Completions endpoint. @@ -94,15 +110,35 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: def _format_messages( request_info: RequestInfo, rendered: list[dict[str, Any]] ) -> list[dict[str, Any]]: - """Build chat messages with RequestInfo-level prompts applied.""" + """Build chat messages with RequestInfo-level prompts applied. + + When the dataset already rendered a leading system message, the + conversation-level ``system_message`` is merged into it rather than + dropped, so a verbatim ``--system-prompt`` still reaches the wire on + raw-payload/DAG datasets. The two are joined into one message because + repeated system roles are mishandled by many OpenAI-compatible servers. + """ messages: list[dict[str, Any]] = [] first_is_system = ( bool(rendered) and isinstance(rendered[0], dict) and rendered[0].get("role") == "system" ) - if request_info.system_message and not first_is_system: - messages.append({"role": "system", "content": request_info.system_message}) + if request_info.system_message: + if first_is_system: + # Copy rather than mutate: ``rendered`` aliases the turn's + # raw_messages, which are reused across credits in a session, so + # an in-place edit would restack the prefix on every replay. + merged = dict(rendered[0]) + merged["content"] = _prepend_system_text( + request_info.system_message, merged.get("content") + ) + messages.append(merged) + rendered = rendered[1:] + else: + messages.append( + {"role": "system", "content": request_info.system_message} + ) if request_info.user_context_message: messages.append( {"role": "user", "content": request_info.user_context_message} diff --git a/tests/unit/config/test_system_prompt_config.py b/tests/unit/config/test_system_prompt_config.py new file mode 100644 index 0000000000..e0e15bccb0 --- /dev/null +++ b/tests/unit/config/test_system_prompt_config.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validation gates for the verbatim ``--system-prompt`` / ``--system-prompt-file``. + +The feature is the content-valued replacement for +``prefix_prompts.shared_system_length``: both fill ``Conversation.system_message``, +so it inherits that field's exclusivity rules rather than defining its own. +""" + +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError +from pytest import param + +from aiperf.common.enums import DatasetType +from aiperf.config.config import BenchmarkConfig +from aiperf.config.dataset import FileDataset, PublicDataset, SyntheticDataset +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.flags.converter import convert_cli_to_aiperf +from aiperf.plugin.enums import PublicDatasetType + +SYSTEM_TEXT = "You are a production assistant." + + +@pytest.fixture +def prompt_file(tmp_path: Path) -> Path: + """A readable system-prompt file. + + ``tmp_path`` resolves through /private on macOS, so it has no symlinked + parent and survives ``safe_read_template_path``'s symlink rejection. + """ + path = tmp_path / "system.txt" + path.write_text(SYSTEM_TEXT) + return path + + +def _cli(**kwargs: Any) -> CLIConfig: + return CLIConfig( + model_names=["mock-model"], + url="http://localhost:8000", + request_count=1, + **kwargs, + ) + + +def _benchmark(**kwargs: Any) -> BenchmarkConfig: + return convert_cli_to_aiperf(_cli(**kwargs)).benchmark + + +# --------------------------------------------------------------------------- +# Mixin-level resolution +# --------------------------------------------------------------------------- + + +class TestSystemPromptResolution: + def test_inline_text_resolves(self) -> None: + dataset = SyntheticDataset( + name="main", type=DatasetType.SYNTHETIC, system_prompt=SYSTEM_TEXT + ) + assert dataset.resolved_system_prompt == SYSTEM_TEXT + + def test_file_resolves(self, prompt_file: Path) -> None: + dataset = SyntheticDataset( + name="main", + type=DatasetType.SYNTHETIC, + system_prompt_file=str(prompt_file), + ) + assert dataset.resolved_system_prompt == SYSTEM_TEXT + + def test_unset_resolves_to_none(self) -> None: + dataset = SyntheticDataset(name="main", type=DatasetType.SYNTHETIC) + assert dataset.resolved_system_prompt is None + + def test_file_read_once_at_validation_time(self, prompt_file: Path) -> None: + """Later edits to the file do not change an already-validated config.""" + dataset = SyntheticDataset( + name="main", + type=DatasetType.SYNTHETIC, + system_prompt_file=str(prompt_file), + ) + prompt_file.write_text("mutated after validation") + assert dataset.resolved_system_prompt == SYSTEM_TEXT + + @pytest.mark.parametrize( + "dataset_cls,extra", + [ + param(SyntheticDataset, {"type": DatasetType.SYNTHETIC}, id="synthetic"), + param( + FileDataset, + {"type": DatasetType.FILE, "path": "/etc/hosts"}, + id="file", + ), + param( + PublicDataset, + { + "type": DatasetType.PUBLIC, + "dataset": PublicDatasetType.SHAREGPT, + }, + id="public", + ), + ], + ) # fmt: skip + def test_every_dataset_variant_carries_the_field( + self, dataset_cls: type, extra: dict[str, Any] + ) -> None: + dataset = dataset_cls(name="main", system_prompt=SYSTEM_TEXT, **extra) + assert dataset.resolved_system_prompt == SYSTEM_TEXT + + +class TestSystemPromptRejections: + def test_text_and_file_are_mutually_exclusive(self, prompt_file: Path) -> None: + with pytest.raises(ValidationError, match="mutually exclusive"): + SyntheticDataset( + name="main", + type=DatasetType.SYNTHETIC, + system_prompt=SYSTEM_TEXT, + system_prompt_file=str(prompt_file), + ) + + @pytest.mark.parametrize( + "text", + [ + param("", id="empty"), + param(" ", id="spaces"), + param("\n\t ", id="whitespace"), + ], + ) # fmt: skip + def test_blank_text_rejected(self, text: str) -> None: + with pytest.raises(ValidationError, match="empty or whitespace-only"): + SyntheticDataset( + name="main", type=DatasetType.SYNTHETIC, system_prompt=text + ) + + def test_blank_file_rejected(self, tmp_path: Path) -> None: + blank = tmp_path / "blank.txt" + blank.write_text(" \n") + with pytest.raises(ValidationError, match="empty or whitespace-only"): + SyntheticDataset( + name="main", + type=DatasetType.SYNTHETIC, + system_prompt_file=str(blank), + ) + + def test_missing_file_rejected_at_startup(self, tmp_path: Path) -> None: + with pytest.raises(ValidationError, match="could not be read"): + SyntheticDataset( + name="main", + type=DatasetType.SYNTHETIC, + system_prompt_file=str(tmp_path / "does_not_exist.txt"), + ) + + +# --------------------------------------------------------------------------- +# BenchmarkConfig-level compatibility gates +# --------------------------------------------------------------------------- + + +class TestSystemPromptCompatibility: + def test_resolves_through_cli_for_synthetic(self, prompt_file: Path) -> None: + benchmark = _benchmark( + prompt_input_tokens_mean=100, system_prompt_file=str(prompt_file) + ) + assert benchmark.get_system_prompt() == SYSTEM_TEXT + + def test_resolves_through_cli_for_file_dataset( + self, prompt_file: Path, tmp_path: Path + ) -> None: + data = tmp_path / "data.jsonl" + data.write_text('{"text": "hello"}\n') + benchmark = _benchmark( + input_file=str(data), system_prompt_file=str(prompt_file) + ) + assert benchmark.get_system_prompt() == SYSTEM_TEXT + + def test_resolves_through_cli_for_public_dataset(self, prompt_file: Path) -> None: + benchmark = _benchmark( + public_dataset="sharegpt", system_prompt_file=str(prompt_file) + ) + assert benchmark.get_system_prompt() == SYSTEM_TEXT + + def test_conflicts_with_shared_system_prompt_length(self) -> None: + with pytest.raises(ValidationError, match="shared-system-prompt-length"): + _benchmark( + prompt_input_tokens_mean=100, + system_prompt=SYSTEM_TEXT, + prompt_prefix_shared_system_length=200, + ) + + def test_conflicts_with_prefix_pool(self) -> None: + """Inherited from shared_system_length, which fills the same slot.""" + with pytest.raises(ValidationError, match="num-prefix-prompts"): + _benchmark( + prompt_input_tokens_mean=100, + system_prompt=SYSTEM_TEXT, + prompt_prefix_pool_size=50, + prompt_prefix_length=100, + ) + + def test_allowed_with_user_context_length(self) -> None: + """The two-tier shape, with verbatim text for the shared tier.""" + benchmark = _benchmark( + prompt_input_tokens_mean=100, + system_prompt=SYSTEM_TEXT, + prompt_prefix_user_context_length=64, + ) + assert benchmark.get_system_prompt() == SYSTEM_TEXT + + @pytest.mark.parametrize( + "endpoint_type", + [ + param("completions", id="completions"), + param("embeddings", id="embeddings"), + param("nim_rankings", id="rankings"), + ], + ) # fmt: skip + def test_rejected_on_endpoints_without_a_system_role( + self, endpoint_type: str + ) -> None: + with pytest.raises(ValidationError, match="not supported by endpoint"): + _benchmark( + prompt_input_tokens_mean=100, + system_prompt=SYSTEM_TEXT, + endpoint_type=endpoint_type, + ) + + @pytest.mark.parametrize( + "endpoint_type", + [ + param("chat", id="chat"), + param("responses", id="responses"), + param("messages", id="messages"), + param("chat_embeddings", id="chat_embeddings"), + ], + ) # fmt: skip + def test_allowed_on_endpoints_that_consume_system_message( + self, endpoint_type: str + ) -> None: + benchmark = _benchmark( + prompt_input_tokens_mean=100, + system_prompt=SYSTEM_TEXT, + endpoint_type=endpoint_type, + ) + assert benchmark.get_system_prompt() == SYSTEM_TEXT + + def test_satisfies_warmup_isolation_system_requirement(self) -> None: + """A verbatim prompt is a real system message, so the marker has a slot.""" + benchmark = _benchmark( + prompt_input_tokens_mean=100, + system_prompt=SYSTEM_TEXT, + cache_bust="warmup_isolation_system", + ) + assert benchmark.get_system_prompt() == SYSTEM_TEXT + + def test_warmup_isolation_system_still_rejected_without_any_system_prompt( + self, + ) -> None: + with pytest.raises(ValidationError, match="requires a shared system prompt"): + _benchmark( + prompt_input_tokens_mean=100, + cache_bust="warmup_isolation_system", + ) diff --git a/tests/unit/dataset/composer/test_custom_system_prompt.py b/tests/unit/dataset/composer/test_custom_system_prompt.py new file mode 100644 index 0000000000..c71233b0a1 --- /dev/null +++ b/tests/unit/dataset/composer/test_custom_system_prompt.py @@ -0,0 +1,360 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Injection of the verbatim ``--system-prompt`` into composed conversations. + +Covers the three composers that build conversations (synthetic, custom/file, +public) and the prepend-vs-assign branch that keeps a dataset's own authored +system message intact. +""" + +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import orjson +import pytest + +from aiperf.common.models import Conversation, Text, Turn +from aiperf.common.tokenizer import Tokenizer +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.dataset.composer.custom import CustomDatasetComposer +from aiperf.dataset.composer.public import PublicDatasetComposer +from aiperf.dataset.composer.synthetic import SyntheticDatasetComposer +from aiperf.endpoints.openai_chat import ChatEndpoint +from aiperf.plugin.enums import ( + DatasetSamplingStrategy, + EndpointType, + PublicDatasetType, +) +from tests.unit.dataset.composer.conftest import make_run +from tests.unit.endpoints.conftest import create_model_endpoint, create_request_info + +SYSTEM_TEXT = "You are a production assistant." + + +@pytest.fixture +def prompt_file(tmp_path: Path) -> Path: + path = tmp_path / "system.txt" + path.write_text(SYSTEM_TEXT) + return path + + +@pytest.fixture +def multi_turn_file(tmp_path: Path) -> Path: + """A dataset whose first turn is a system message the loader hoists.""" + path = tmp_path / "data.jsonl" + path.write_text( + '{"type": "multi_turn", "session_id": "s1", "turns": [' + '{"role": "system", "text": "Dataset system."}, ' + '{"role": "user", "text": "hi"}]}\n' + ) + return path + + +def _synthetic_cli(**kwargs: Any) -> CLIConfig: + return CLIConfig( + model_names=["test-model"], + conversation_num_dataset_entries=3, + prompt_input_tokens_mean=10, + prompt_output_tokens_mean=5, + **kwargs, + ) + + +class TestSyntheticComposerSystemPrompt: + def test_sets_system_message_on_every_conversation( + self, prompt_file: Path, mock_tokenizer: Tokenizer + ) -> None: + run = make_run(_synthetic_cli(system_prompt_file=str(prompt_file))) + conversations = SyntheticDatasetComposer( + run=run, tokenizer=mock_tokenizer + ).create_dataset() + + assert len(conversations) == 3 + assert all(c.system_message == SYSTEM_TEXT for c in conversations) + + def test_absent_without_the_flag(self, mock_tokenizer: Tokenizer) -> None: + run = make_run(_synthetic_cli()) + conversations = SyntheticDatasetComposer( + run=run, tokenizer=mock_tokenizer + ).create_dataset() + + assert all(c.system_message is None for c in conversations) + + def test_inline_text_matches_file_text( + self, prompt_file: Path, mock_tokenizer: Tokenizer + ) -> None: + from_file = SyntheticDatasetComposer( + run=make_run(_synthetic_cli(system_prompt_file=str(prompt_file))), + tokenizer=mock_tokenizer, + ).create_dataset() + from_text = SyntheticDatasetComposer( + run=make_run(_synthetic_cli(system_prompt=SYSTEM_TEXT)), + tokenizer=mock_tokenizer, + ).create_dataset() + + assert from_file[0].system_message == from_text[0].system_message + + +class TestCustomComposerSystemPrompt: + def test_prepends_to_authored_system_message( + self, prompt_file: Path, multi_turn_file: Path, mock_tokenizer: Tokenizer + ) -> None: + """The dataset's own system message is kept, not replaced.""" + run = make_run( + CLIConfig( + model_names=["test-model"], + input_file=str(multi_turn_file), + system_prompt_file=str(prompt_file), + ) + ) + conversations = CustomDatasetComposer( + run=run, tokenizer=mock_tokenizer + ).create_dataset() + + assert conversations[0].system_message == f"{SYSTEM_TEXT}\n\nDataset system." + + def test_authored_system_message_untouched_without_the_flag( + self, multi_turn_file: Path, mock_tokenizer: Tokenizer + ) -> None: + run = make_run( + CLIConfig(model_names=["test-model"], input_file=str(multi_turn_file)) + ) + conversations = CustomDatasetComposer( + run=run, tokenizer=mock_tokenizer + ).create_dataset() + + assert conversations[0].system_message == "Dataset system." + + def test_assigns_when_dataset_has_no_system_message( + self, prompt_file: Path, tmp_path: Path, mock_tokenizer: Tokenizer + ) -> None: + data = tmp_path / "plain.jsonl" + data.write_text('{"text": "hello"}\n') + run = make_run( + CLIConfig( + model_names=["test-model"], + input_file=str(data), + system_prompt_file=str(prompt_file), + ) + ) + conversations = CustomDatasetComposer( + run=run, tokenizer=mock_tokenizer + ).create_dataset() + + assert conversations[0].system_message == SYSTEM_TEXT + + def test_injected_without_a_tokenizer( + self, prompt_file: Path, tmp_path: Path + ) -> None: + """Verbatim text needs no tokenizer, unlike the synthetic prefix path.""" + data = tmp_path / "plain.jsonl" + data.write_text('{"text": "hello"}\n') + run = make_run( + CLIConfig( + model_names=["test-model"], + input_file=str(data), + system_prompt_file=str(prompt_file), + ) + ) + conversations = CustomDatasetComposer(run=run, tokenizer=None).create_dataset() + + assert conversations[0].system_message == SYSTEM_TEXT + + +class TestSpeedBenchSystemPrompt: + """SPEED-Bench keeps its leading system message as a dispatched turn. + + ``SpeedBenchLoader._hoist_leading_system_message`` is False, so unlike + ``multi_turn`` the dataset's system message never reaches + ``conversation.system_message``. The composer therefore has nothing to + prepend to, and the merge has to happen at the endpoint layer instead -- + the one production loader that exercises that path. + """ + + DATASET_SYSTEM = "SPEED-Bench system message." + + @pytest.fixture + def speed_bench_file(self, tmp_path: Path) -> Path: + row = { + "question_id": "speed-coding-1".ljust(32, "0"), + "category": "coding", + "messages": [ + {"role": "system", "content": self.DATASET_SYSTEM}, + {"role": "user", "content": "Write a function to reverse a string."}, + ], + } + path = tmp_path / "speedbench.jsonl" + path.write_bytes(orjson.dumps(row) + b"\n") + return path + + def _compose( + self, + speed_bench_file: Path, + prompt_file: Path | None, + tokenizer: Tokenizer | None, + ) -> list[Conversation]: + kwargs = {} + if prompt_file is not None: + kwargs["system_prompt_file"] = str(prompt_file) + run = make_run( + CLIConfig( + model_names=["test-model"], + input_file=str(speed_bench_file), + custom_dataset_type="speed_bench_coding", + **kwargs, + ) + ) + return CustomDatasetComposer(run=run, tokenizer=tokenizer).create_dataset() + + @staticmethod + def _format(conversation: Conversation, turns: list[Turn]) -> list[dict[str, Any]]: + endpoint = ChatEndpoint(model_endpoint=create_model_endpoint(EndpointType.CHAT)) + request_info = create_request_info( + model_endpoint=endpoint.model_endpoint, + turns=turns, + system_message=conversation.system_message, + ) + return endpoint.format_payload(request_info)["messages"] + + def test_composer_does_not_merge_unhoisted_system_turn( + self, speed_bench_file: Path, prompt_file: Path, mock_tokenizer: Tokenizer + ) -> None: + """The dataset's system message stays a turn; only the custom text is lifted.""" + conversation = self._compose(speed_bench_file, prompt_file, mock_tokenizer)[0] + + assert conversation.system_message == SYSTEM_TEXT + assert self.DATASET_SYSTEM not in conversation.system_message + + def test_endpoint_merges_into_one_system_message( + self, speed_bench_file: Path, prompt_file: Path, mock_tokenizer: Tokenizer + ) -> None: + conversation = self._compose(speed_bench_file, prompt_file, mock_tokenizer)[0] + messages = self._format(conversation, conversation.turns) + + systems = [m for m in messages if m["role"] == "system"] + assert len(systems) == 1 + assert systems[0]["content"] == f"{SYSTEM_TEXT}\n\n{self.DATASET_SYSTEM}" + + def test_prefix_not_duplicated_across_repeated_formats( + self, speed_bench_file: Path, prompt_file: Path, mock_tokenizer: Tokenizer + ) -> None: + """Turn state is shared across credits, so the merge must not restack. + + Regression guard for the copy-in-_format_messages: mutating the rendered + system message in place would compound the prefix on every later request + in the same session. + """ + conversation = self._compose(speed_bench_file, prompt_file, mock_tokenizer)[0] + + for _ in range(3): + messages = self._format(conversation, conversation.turns) + systems = [m for m in messages if m["role"] == "system"] + assert len(systems) == 1 + assert systems[0]["content"].count(SYSTEM_TEXT) == 1 + + def test_dataset_system_turn_untouched_without_the_flag( + self, speed_bench_file: Path, mock_tokenizer: Tokenizer + ) -> None: + conversation = self._compose(speed_bench_file, None, mock_tokenizer)[0] + messages = self._format(conversation, conversation.turns) + + assert conversation.system_message is None + systems = [m for m in messages if m["role"] == "system"] + assert len(systems) == 1 + assert systems[0]["content"] == self.DATASET_SYSTEM + + +@pytest.mark.asyncio +class TestPublicComposerSystemPrompt: + """Public (HF-backed) datasets reach the same injection point. + + ``PublicDatasetComposer.create_dataset_async`` calls + ``_finalize_conversations`` like the other composers, but it is the only one + on the async path. The loader is mocked so this stays offline. + """ + + @staticmethod + def _public_cli(**kwargs: Any) -> CLIConfig: + return CLIConfig( + model_names=["test-model"], + conversation_num_dataset_entries=2, + public_dataset=PublicDatasetType.AIMO, + **kwargs, + ) + + @staticmethod + async def _compose( + cli: CLIConfig, conversations: list[Conversation] + ) -> list[Conversation]: + mock_loader = AsyncMock() + mock_loader.load_dataset = AsyncMock(return_value={"dataset": []}) + mock_loader.convert_to_conversations = AsyncMock(return_value=conversations) + + mock_loader_class = MagicMock() + mock_loader_class.get_preferred_sampling_strategy.return_value = ( + DatasetSamplingStrategy.SEQUENTIAL + ) + mock_loader_class.return_value = mock_loader + + composer = PublicDatasetComposer(run=make_run(cli), tokenizer=None) + with ( + patch( + "aiperf.dataset.composer.public.plugins.get_class", + return_value=mock_loader_class, + ), + patch( + "aiperf.dataset.composer.public.plugins.get_public_dataset_loader_metadata", + return_value=MagicMock( + hf_dataset_name="test/dataset", + hf_split="train", + hf_subset=None, + prompt_column="problem", + multi_turn=False, + streaming=False, + is_trace=False, + ), + ), + ): + return await composer.create_dataset_async() + + @staticmethod + def _conversations(system_message: str | None = None) -> list[Conversation]: + return [ + Conversation( + session_id=f"conv-{i}", + system_message=system_message, + turns=[Turn(texts=[Text(contents=[f"What is {i} + {i}?"])])], + ) + for i in range(2) + ] + + async def test_sets_system_message_on_every_conversation( + self, prompt_file: Path + ) -> None: + result = await self._compose( + self._public_cli(system_prompt_file=str(prompt_file)), + self._conversations(), + ) + + assert len(result) == 2 + assert all(c.system_message == SYSTEM_TEXT for c in result) + + async def test_absent_without_the_flag(self) -> None: + result = await self._compose(self._public_cli(), self._conversations()) + + assert all(c.system_message is None for c in result) + + async def test_prepends_to_loader_supplied_system_message( + self, prompt_file: Path + ) -> None: + """A public loader that already set system_message keeps its text.""" + result = await self._compose( + self._public_cli(system_prompt_file=str(prompt_file)), + self._conversations(system_message="Dataset system."), + ) + + assert all( + c.system_message == f"{SYSTEM_TEXT}\n\nDataset system." for c in result + ) diff --git a/tests/unit/dataset/test_mmap_cache.py b/tests/unit/dataset/test_mmap_cache.py index 0abaa01673..0ae71dacc6 100644 --- a/tests/unit/dataset/test_mmap_cache.py +++ b/tests/unit/dataset/test_mmap_cache.py @@ -117,6 +117,73 @@ def _key(**synthesis_kw: float) -> str | None: assert base is not None assert len({base, speedup, out_mult, prefix_mult}) == 4 + def test_key_changes_with_system_prompt(self, tmp_path: Path) -> None: + # The composer bakes the verbatim system prompt into + # Conversation.system_message, which round-trips through the stored mmap. + # A cache HIT skips the composer entirely, so two runs differing only in + # --system-prompt must NOT share an entry -- otherwise the second run + # silently replays the first one's prompt. + from aiperf.plugin.enums import CustomDatasetType + + trace = _write_input_file( + tmp_path, + b'{"session_id": "s1", "timestamp": 0, "input_length": 8}\n', + ) + prompt_a = tmp_path / "a.txt" + prompt_a.write_text("Prompt A.") + prompt_b = tmp_path / "b.txt" + prompt_b.write_text("Prompt B, different content.") + + def _key(**system_kw: str) -> str | None: + run = make_run_from_cli( + CLIConfig( + model_names=["test-model"], + input_file=str(trace), + custom_dataset_type=CustomDatasetType.MOONCAKE_TRACE, + **system_kw, + ) + ) + return mmap_cache.compute_cache_key_from_run(run) + + none_key = _key() + a_key = _key(system_prompt_file=str(prompt_a)) + b_key = _key(system_prompt_file=str(prompt_b)) + assert none_key is not None + assert len({none_key, a_key, b_key}) == 3 + + # Identical text through either source composes identical bytes, so the + # two spellings must collapse onto one entry. + assert a_key == _key(system_prompt="Prompt A.") + + def test_key_omits_system_prompt_field_when_unset(self, tmp_path: Path) -> None: + # Emitting an explicit None would serialize as ``null`` through + # orjson.dumps and shift every pre-existing key, needlessly cold-starting + # warm caches for runs that never touch the feature. + from aiperf.plugin.enums import CustomDatasetType + + trace = _write_input_file( + tmp_path, + b'{"session_id": "s1", "timestamp": 0, "input_length": 8}\n', + ) + + def _payload(**system_kw: str) -> dict: + run = make_run_from_cli( + CLIConfig( + model_names=["test-model"], + input_file=str(trace), + custom_dataset_type=CustomDatasetType.MOONCAKE_TRACE, + **system_kw, + ) + ) + return mmap_cache._settings_payload_from_run(run) + + unset = _payload() + configured = _payload(system_prompt="Prompt A.") + assert "system_prompt_sha256" not in unset + assert "system_prompt_sha256" in configured + # Every other key is untouched by the addition. + assert set(unset) == set(configured) - {"system_prompt_sha256"} + def test_key_changes_with_preformat_payloads( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/endpoints/test_message_merging_fixes.py b/tests/unit/endpoints/test_message_merging_fixes.py index cb56841b2a..a6eae84166 100644 --- a/tests/unit/endpoints/test_message_merging_fixes.py +++ b/tests/unit/endpoints/test_message_merging_fixes.py @@ -332,11 +332,16 @@ def test_failed_stream_returns_none_even_with_partial_items( # --------------------------------------------------------------------------- # P3 #11: chat de-dups leading system when raw_messages already starts with system +# +# The de-dup invariant (never emit two system messages) still holds; what the +# single message CONTAINS changed. The conversation-level system_message used to +# be discarded outright, which silently dropped a verbatim --system-prompt on +# raw-payload/DAG datasets. It is now prepended into the authored message. # --------------------------------------------------------------------------- class TestChatDeDupsLeadingSystem: - def test_authored_leading_system_wins(self, chat_endpoint): + def test_authored_leading_system_merges_with_request_info(self, chat_endpoint): authored = [ {"role": "system", "content": "authored system"}, {"role": "user", "content": "hi"}, @@ -348,11 +353,52 @@ def test_authored_leading_system_wins(self, chat_endpoint): system_message="request_info system", ) payload = chat_endpoint.format_payload(request_info) - # Exactly one system message, and it's the authored one. + # Still exactly one system message, now carrying both parts with the + # conversation-level prompt first (token 0 for prefix-cache purposes). + systems = [m for m in payload["messages"] if m["role"] == "system"] + assert len(systems) == 1 + assert systems[0]["content"] == "request_info system\n\nauthored system" + + def test_authored_leading_system_untouched_without_request_info( + self, chat_endpoint + ): + """No conversation-level prompt -> the authored message passes through.""" + authored = [ + {"role": "system", "content": "authored system"}, + {"role": "user", "content": "hi"}, + ] + turn = Turn(role="user", raw_messages=authored) + request_info = create_request_info( + model_endpoint=chat_endpoint.model_endpoint, + turns=[turn], + ) + payload = chat_endpoint.format_payload(request_info) systems = [m for m in payload["messages"] if m["role"] == "system"] assert len(systems) == 1 assert systems[0]["content"] == "authored system" + def test_authored_leading_system_not_restacked_across_calls(self, chat_endpoint): + """Formatting twice must not stack the prefix onto shared turn state. + + ``raw_messages`` is reused across credits within a session, so the merge + copies the message rather than mutating it in place. + """ + authored = [ + {"role": "system", "content": "authored system"}, + {"role": "user", "content": "hi"}, + ] + turn = Turn(role="user", raw_messages=authored) + request_info = create_request_info( + model_endpoint=chat_endpoint.model_endpoint, + turns=[turn], + system_message="request_info system", + ) + chat_endpoint.format_payload(request_info) + payload = chat_endpoint.format_payload(request_info) + systems = [m for m in payload["messages"] if m["role"] == "system"] + assert systems[0]["content"] == "request_info system\n\nauthored system" + assert authored[0]["content"] == "authored system" + # --------------------------------------------------------------------------- # raw_messages=[] is an explicit zero-message delta, not a synthesis request diff --git a/tests/unit/endpoints/test_messages_endpoint.py b/tests/unit/endpoints/test_messages_endpoint.py index 7df4de526d..69dd431531 100644 --- a/tests/unit/endpoints/test_messages_endpoint.py +++ b/tests/unit/endpoints/test_messages_endpoint.py @@ -1117,7 +1117,44 @@ def endpoint(self): def model_endpoint(self): return create_model_endpoint(EndpointType.MESSAGES) - def test_raw_system_overrides_string_system_message(self, endpoint, model_endpoint): + def test_raw_system_preserved_and_system_message_prepended( + self, endpoint, model_endpoint + ): + """A conversation-level system_message leads; raw_system blocks follow. + + The string used to be discarded, which silently dropped a verbatim + --system-prompt. Anthropic's ``system`` is already a block list, so the + string becomes a leading text block instead -- raw_system's authored + blocks (and their cache_control) still round-trip untouched. + """ + system_blocks = [ + { + "type": "text", + "text": "you are concise", + "cache_control": {"type": "ephemeral"}, + } + ] + turn = Turn( + texts=[Text(contents=["hi"])], + raw_system=system_blocks, + ) + request_info = create_request_info( + model_endpoint=model_endpoint, + turns=[turn], + system_message="custom system prompt", + ) + payload = endpoint.format_payload(request_info) + assert payload["system"] == [ + {"type": "text", "text": "custom system prompt"}, + *system_blocks, + ] + # Cache_control round-trips verbatim on the authored block. + assert payload["system"][1]["cache_control"] == {"type": "ephemeral"} + + def test_raw_system_passes_through_without_system_message( + self, endpoint, model_endpoint + ): + """No conversation-level prompt -> raw_system is emitted unchanged.""" system_blocks = [ { "type": "text", @@ -1132,12 +1169,9 @@ def test_raw_system_overrides_string_system_message(self, endpoint, model_endpoi request_info = create_request_info( model_endpoint=model_endpoint, turns=[turn], - system_message="this string should be ignored", ) payload = endpoint.format_payload(request_info) assert payload["system"] == system_blocks - # Cache_control round-trips verbatim. - assert payload["system"][0]["cache_control"] == {"type": "ephemeral"} def test_raw_system_falls_back_to_system_message_when_unset( self, endpoint, model_endpoint From 8b5954171b652ac71ee3ce12a1820f8fed6290e3 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:10:50 -0700 Subject: [PATCH 033/119] feat(dataset): random range ratio + PromptCorpus.RANDOM aligned with vLLM bench (#850) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 94fee7338b2cdddf0d69c10526d9f3f81afa64dd) --- docs/cli-options.md | 32 + docs/index.yml | 2 + docs/reference/isl-budget-compensation.md | 9 +- docs/reference/isl-distribution-examples.html | 124 ++++ docs/reference/isl-example-gpt2.svg | 11 + docs/reference/isl-example-llama.svg | 11 + docs/reference/prompt-corpus.md | 186 +++++ docs/reference/validating-isl-distribution.md | 269 +++++++ src/aiperf/common/enums/__init__.py | 2 + src/aiperf/common/enums/enums.py | 73 ++ .../common/models/sequence_distribution.py | 592 +++++++++++++++- src/aiperf/common/random_generator.py | 27 +- src/aiperf/common/tokenizer.py | 103 +++ src/aiperf/config/dataset/config.py | 28 +- src/aiperf/config/dataset/content.py | 149 +++- src/aiperf/config/flags/_converter_dataset.py | 42 ++ src/aiperf/config/flags/cli_config.py | 49 ++ src/aiperf/dataset/composer/base.py | 169 +++-- src/aiperf/dataset/composer/synthetic.py | 61 +- .../dataset/generator/coding_content.py | 72 +- src/aiperf/dataset/generator/corpus.py | 18 +- src/aiperf/dataset/generator/prompt.py | 350 ++++++++- src/aiperf/dataset/protocols.py | 65 ++ tests/aiperf_mock_server/request_recorder.py | 13 +- .../models/test_sequence_distribution.py | 666 ++++++++++++++++++ tests/unit/common/test_random_generator.py | 48 ++ tests/unit/common/test_tokenizer.py | 148 ++++ .../test_converter_random_corpus_style.py | 65 ++ .../config/test_prompt_config_range_ratio.py | 245 +++++++ .../dataset/composer/test_base_composer.py | 11 +- .../composer/test_degenerate_range_ratio.py | 108 +++ .../composer/test_isl_budget_compensation.py | 115 ++- .../composer/test_synthetic_composer.py | 19 +- .../generator/test_prompt_generator.py | 506 ++++++++++++- .../unit/property/test_cli_help_references.py | 86 +++ tests/unit/property/test_finite_invariants.py | 5 + 36 files changed, 4325 insertions(+), 154 deletions(-) create mode 100644 docs/reference/isl-distribution-examples.html create mode 100644 docs/reference/isl-example-gpt2.svg create mode 100644 docs/reference/isl-example-llama.svg create mode 100644 docs/reference/validating-isl-distribution.md create mode 100644 tests/unit/config/test_converter_random_corpus_style.py create mode 100644 tests/unit/config/test_prompt_config_range_ratio.py create mode 100644 tests/unit/dataset/composer/test_degenerate_range_ratio.py create mode 100644 tests/unit/property/test_cli_help_references.py diff --git a/docs/cli-options.md b/docs/cli-options.md index 8d1733b61a..9da10cea6b 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -673,6 +673,7 @@ Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare so |-------|:-------:|-------------| | `sonnet` | | Shakespeare sonnets (default). Classic prose for filler text. | | `coding` | | Realistic coding content: code, bash output, JSON, error tracebacks, git diffs. | +| `random` | | Random tokens drawn from the full tokenizer vocabulary minus special tokens. Matches vLLM bench's RandomDataset token-generation strategy. | ### Cache Bust @@ -735,6 +736,21 @@ Token block size for hash-based prompt synthesis when dataset entries carry `has Distribution of (ISL, OSL) pairs with probabilities for mixed workload simulation. Format: `ISL,OSL:prob;ISL,OSL:prob` (semicolons separate pairs, probabilities are percentages 0-100 that must sum to 100). Supports optional stddev: `ISL|stddev,OSL|stddev:prob`. Examples: `128,64:25;512,128:50;1024,256:25` or with variance: `256|10,128|5:40;512|20,256|10:60`. Also supports bracket `[(256,128):40,(512,256):60]` and JSON formats. +#### `--random-range-ratio` `` + +Sample ISL and OSL uniformly from a ratio-defined integer window around the configured means. The window is computed from `--random-corpus-style` (defaults to `vllm`): vllm style → `[floor(mean*(1-r)), ceil(mean*(1+r))]` (symmetric); sglang style → `[max(1, int(mean*r)), mean]` (lower-bounded). Accepts a single float applied to both ISL and OSL. The JSON object form `{"input": 0.3, "output": 0.5}` for independent values is accepted only under vllm style; sglang style applies one ratio to both and requires a plain float. Requires both `--isl` and `--osl` to be set explicitly — there is no default mean for either. Mutually exclusive with `--seq-dist`. Under vllm style with a tokenizer configured, the ISL mean is automatically reduced by `tokenizer.num_special_tokens_to_add(pair=False)` so `--isl` represents total server-side input tokens, matching `vllm bench serve` semantics; sglang style applies no such adjustment. + +#### `--random-corpus-style` `` + +Benchmark style for RANDOM corpus generation. Controls range ratio formula, token pool composition, and other per-tool behaviors. `vllm` (default) mirrors `vllm bench serve`: symmetric range window, non-special token pool (special tokens excluded). `sglang` mirrors `sglang.benchmark.serving` run with `--dataset-name random-ids`: lower-bounded range window, full `range(vocab_size)` token pool (no exclusion). Applies in two independent places: token-pool selection whenever `--prompt-corpus random` is set (including with no `--random-range-ratio`, where the pool is the only thing the style selects), and the range-window formula, special-token accounting and RNG algorithm whenever `--random-range-ratio` is set — the latter regardless of corpus. + +**Choices:** + +| | | | +|-------|:-------:|-------------| +| `vllm` | _default_ | vllm bench serve semantics: symmetric window ``[floor(mean*(1-r)), ceil(mean*(1+r))]``. r=0 is fixed at mean; larger r widens the window on both sides. r must be in [0, 1). Special tokens excluded from the sampling pool. BOS subtracted from ISL mean. | +| `sglang` | | sglang ``benchmark.serving`` semantics under ``--dataset-name random-ids``: lower-bounded window ``[max(1, int(mean*r)), mean]``. r=0 allows full variability [1, mean]; r=1 fixes length at mean. Full vocab_size range used for token sampling (no special-token exclusion). No BOS adjustment. The default ``--dataset-name random`` is a different algorithm upstream (repeat/truncate ShareGPT token ids) and is not what this style mirrors. | + ### Output Sequence Length (OSL) #### `--prompt-output-tokens-mean`, `--output-tokens-mean`, `--osl` `` @@ -2221,6 +2237,7 @@ Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare so |-------|:-------:|-------------| | `sonnet` | | Shakespeare sonnets (default). Classic prose for filler text. | | `coding` | | Realistic coding content: code, bash output, JSON, error tracebacks, git diffs. | +| `random` | | Random tokens drawn from the full tokenizer vocabulary minus special tokens. Matches vLLM bench's RandomDataset token-generation strategy. | ### Cache Bust @@ -2283,6 +2300,21 @@ Token block size for hash-based prompt synthesis when dataset entries carry `has Distribution of (ISL, OSL) pairs with probabilities for mixed workload simulation. Format: `ISL,OSL:prob;ISL,OSL:prob` (semicolons separate pairs, probabilities are percentages 0-100 that must sum to 100). Supports optional stddev: `ISL|stddev,OSL|stddev:prob`. Examples: `128,64:25;512,128:50;1024,256:25` or with variance: `256|10,128|5:40;512|20,256|10:60`. Also supports bracket `[(256,128):40,(512,256):60]` and JSON formats. +#### `--random-range-ratio` `` + +Sample ISL and OSL uniformly from a ratio-defined integer window around the configured means. The window is computed from `--random-corpus-style` (defaults to `vllm`): vllm style → `[floor(mean*(1-r)), ceil(mean*(1+r))]` (symmetric); sglang style → `[max(1, int(mean*r)), mean]` (lower-bounded). Accepts a single float applied to both ISL and OSL. The JSON object form `{"input": 0.3, "output": 0.5}` for independent values is accepted only under vllm style; sglang style applies one ratio to both and requires a plain float. Requires both `--isl` and `--osl` to be set explicitly — there is no default mean for either. Mutually exclusive with `--seq-dist`. Under vllm style with a tokenizer configured, the ISL mean is automatically reduced by `tokenizer.num_special_tokens_to_add(pair=False)` so `--isl` represents total server-side input tokens, matching `vllm bench serve` semantics; sglang style applies no such adjustment. + +#### `--random-corpus-style` `` + +Benchmark style for RANDOM corpus generation. Controls range ratio formula, token pool composition, and other per-tool behaviors. `vllm` (default) mirrors `vllm bench serve`: symmetric range window, non-special token pool (special tokens excluded). `sglang` mirrors `sglang.benchmark.serving` run with `--dataset-name random-ids`: lower-bounded range window, full `range(vocab_size)` token pool (no exclusion). Applies in two independent places: token-pool selection whenever `--prompt-corpus random` is set (including with no `--random-range-ratio`, where the pool is the only thing the style selects), and the range-window formula, special-token accounting and RNG algorithm whenever `--random-range-ratio` is set — the latter regardless of corpus. + +**Choices:** + +| | | | +|-------|:-------:|-------------| +| `vllm` | _default_ | vllm bench serve semantics: symmetric window ``[floor(mean*(1-r)), ceil(mean*(1+r))]``. r=0 is fixed at mean; larger r widens the window on both sides. r must be in [0, 1). Special tokens excluded from the sampling pool. BOS subtracted from ISL mean. | +| `sglang` | | sglang ``benchmark.serving`` semantics under ``--dataset-name random-ids``: lower-bounded window ``[max(1, int(mean*r)), mean]``. r=0 allows full variability [1, mean]; r=1 fixes length at mean. Full vocab_size range used for token sampling (no special-token exclusion). No BOS adjustment. The default ``--dataset-name random`` is a different algorithm upstream (repeat/truncate ShareGPT token ids) and is not what this style mirrors. | + ### Output Sequence Length (OSL) #### `--prompt-output-tokens-mean`, `--output-tokens-mean`, `--osl` `` diff --git a/docs/index.yml b/docs/index.yml index d25ab45eb8..b3a8b534c6 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -229,6 +229,8 @@ navigation: path: reference/isl-tokenization.md - page: Prompt Corpus Selection path: reference/prompt-corpus.md + - page: Validating ISL/OSL Distributions with the Mock Server + path: reference/validating-isl-distribution.md - page: ISL Budget Compensation Derivation path: reference/isl-budget-compensation.md - page: Conversation Context Mode diff --git a/docs/reference/isl-budget-compensation.md b/docs/reference/isl-budget-compensation.md index cbf5569d89..331528751d 100644 --- a/docs/reference/isl-budget-compensation.md +++ b/docs/reference/isl-budget-compensation.md @@ -19,8 +19,9 @@ We split the total wire-token cost into three components, each compensated at a | Component | What it represents | Where AIPerf compensates | | --- | --- | --- | | **(a) Cache-bust marker** | Hex-string injected into a message to defeat KV cache reuse. | Reduce either the first user turn's bare prompt OR the synthetic shared system prompt by the marker's token cost, depending on where the worker actually places the marker. | -| **(b) Chat-template wrapping** | Role headers, end-of-turn tokens, BOS, and the assistant-prompt suffix that the model server's tokenizer adds on top of the bare content. | Subtract from every user turn's bare prompt — first turn pays the per-request fixed cost + per-message wrap; later turns pay only the per-message wrap. | +| **(b) Chat-template wrapping** | Role headers, end-of-turn tokens, and the assistant-prompt suffix that the model server's tokenizer adds on top of the bare content. BOS is handled by (d), not here. | Subtract from every user turn's bare prompt — first turn pays the per-request fixed cost + per-message wrap; later turns pay only the per-message wrap. | | **(c) System message length when marker lands on system** | When `--cache-bust system_*` lands the marker on the synthetic shared system prompt, the prompt's wire length grows by the marker token cost. | Reduce the synthetic shared system prompt length by the marker cost so the wire system message still matches `--shared-system-prompt-length`. | +| **(d) Special tokens (BOS et al.)** | Tokens the server prepends on encode, counted by `tokenizer.num_special_tokens_to_add()`. | Subtracted per ISL path, never via the chat-template terms: `vllm` style folds them into the window bounds; `sglang` style shifts each drawn length (`max(1, drawn - n)`); the non-range-ratio path subtracts in `_get_turn_sequence_lengths`. | Component (a) only ever has a non-zero value when the user actually has cache-bust enabled. `BenchmarkConfig.validate_cache_bust_compatibility` (in `src/aiperf/config/config.py`) refuses `--cache-bust` when a profiling phase explicitly declares a non-`agentic_replay` `timing_mode`, and refuses it outside `--endpoint-type chat` / `responses` — those checks raise as separate `ValueError`s. It does **not** fire when a `scenario` is set (scenario locks apply post-construction) or when no phase has an explicit `timing_mode` yet (effective mode is resolved later). That validator is what lets the composer assume the worker really will inject the marker once a config has passed — incompatible combinations that would silently no-op are refused when knowable at construction time, and the composer would otherwise over-subtract by `marker_tokens`. Configurations that fail the validator never reach the composer, so component (a) compensation can be unconditional once `target != NONE` and the routing in "Marker placement routing" decides which slot it lands on. @@ -38,7 +39,7 @@ wire_tokens(messages, add_generation_prompt=True) where: -- `per_request_fixed` is the BOS token plus the assistant-prompt suffix (`<|im_start|>assistant\n`, `[/INST]`, etc.). It is charged **once per request** regardless of the number of messages. +- `per_request_fixed` is the assistant-prompt suffix (`<|im_start|>assistant\n`, `[/INST]`, etc.). It is charged **once per request** regardless of the number of messages. BOS is deliberately *excluded* (`_estimate_chat_template_overheads` subtracts `bos_tokens` when computing it) because special tokens are compensated separately — see "Special tokens" below. - `per_msg_wrap` is the role header plus the end-of-turn marker (`<|im_start|>user\n` and `<|im_end|>\n`, or equivalent). It is charged **once per message**. - `content_tokens(m)` is `len(tokenizer.encode(m["content"]))` — the bare content tokens, which we already know how to compute via the same tokenizer. @@ -118,11 +119,11 @@ Once `(per_request_fixed, per_msg_wrap)` are known, the composer subtracts: | First user turn | `per_request_fixed + per_msg_wrap + first_turn_marker_tokens` | | Subsequent user turns | `per_msg_wrap` | -The first turn pays the per-request fixed cost because that's the turn that "owns" the BOS and generation-prompt tokens — even though those tokens are emitted once per request, they have to be subtracted from one specific turn's bare-prompt budget, and the first turn is the natural choice. +The first turn pays the per-request fixed cost because that's the turn that "owns" the generation-prompt tokens — even though those tokens are emitted once per request, they have to be subtracted from one specific turn's bare-prompt budget, and the first turn is the natural choice. The cache-bust marker is also charged to the first turn (when it lands there), for the same reason: it's a request-level cost that needs to come out of one turn's budget. -Subsequent turns only pay the per-message wrap because they don't own any request-level overhead — the BOS and gen-prompt are already accounted for, and the marker (if any) is on the first turn, not them. +Subsequent turns only pay the per-message wrap because they don't own any request-level overhead — the gen-prompt is already accounted for, and the marker (if any) is on the first turn, not them. Floor at 1 so prompt generation stays valid for very small `--isl` values: `isl_after = max(1, isl - adjustment)`. The synthetic generator can always produce a one-token prompt; it cannot produce a zero-token prompt. diff --git a/docs/reference/isl-distribution-examples.html b/docs/reference/isl-distribution-examples.html new file mode 100644 index 0000000000..da6566dfab --- /dev/null +++ b/docs/reference/isl-distribution-examples.html @@ -0,0 +1,124 @@ + + + + + +ISL Distribution Validation Examples + + + +

ISL Distribution Validation Examples

+
+ Companion to docs/reference/validating-isl-distribution.md. + All runs: ISL=128, OSL=128, range_ratio=0.3, corpus=random, style=vllm, seed=0, n=20,000. + Mock server applies chat template; ISL is template-inclusive. +
+ +

GPT-2 tokenizer — aiperf vs vLLM Docker

+
+
GPT-2 ISL distribution (2-token bins)
+
+ Blue = aiperf  |  Red = vLLM Docker  |  + Overlapping bars appear purple — both distributions are identical. + GPT-2 has no native chat template; ChatML fallback adds +27 tokens. +
+ +
+ aiperf: mean=154.9, std=22.86, min=116, max=194  |  + vLLM Docker: mean=154.9, std=22.86, min=116, max=194  |  + delta = 0.0 +
+
+ +

Llama 3.1 8B Instruct tokenizer — aiperf vs vLLM Docker

+
+
Llama 3.1 ISL distribution (2-token bins)
+
+ Blue = aiperf  |  Red = vLLM Docker  |  + Distributions match after mock server multimodal content normalisation fix. + Native Llama chat template adds ~39 tokens (BOS + system message + role headers). +
+ +
+ aiperf: mean=162.3, std=22.87, min=123, max=203  |  + vLLM Docker: mean=162.3, std=22.87, min=123, max=203  |  + delta = 0.0 +
+
+ + + + diff --git a/docs/reference/isl-example-gpt2.svg b/docs/reference/isl-example-gpt2.svg new file mode 100644 index 0000000000..a6f856d596 --- /dev/null +++ b/docs/reference/isl-example-gpt2.svg @@ -0,0 +1,11 @@ + + GPT-2: aiperf vs vLLM Docker — ISL distribution (n=20,000, ISL=128, range_ratio=0.3) + ChatML fallback template (+27 tokens). Both distributions identical. + aiperfvLLM Docker + 3.1%2.3%1.5%0.8%0.0% + + 110118126134142150158166174182190198 + % of requests + ISL (tokens, template-inclusive) + aiperf mean=154.9 std=22.86 | vLLM Docker mean=154.9 std=22.86 | delta = 0.0 + \ No newline at end of file diff --git a/docs/reference/isl-example-llama.svg b/docs/reference/isl-example-llama.svg new file mode 100644 index 0000000000..c7e1038c4b --- /dev/null +++ b/docs/reference/isl-example-llama.svg @@ -0,0 +1,11 @@ + + Llama 3.1 8B Instruct: aiperf vs vLLM Docker — ISL distribution (n=20,000, ISL=128, range_ratio=0.3) + Native chat template (~39 tokens overhead). Both distributions identical after multimodal content fix. + aiperfvLLM Docker + 3.0%2.3%1.5%0.8%0.0% + + 118128138148158168178188198208 + % of requests + ISL (tokens, template-inclusive) + aiperf mean=162.3 std=22.87 | vLLM Docker mean=162.3 std=22.87 | delta = 0.0 + \ No newline at end of file diff --git a/docs/reference/prompt-corpus.md b/docs/reference/prompt-corpus.md index 1b4998bfef..37dca8b0fb 100644 --- a/docs/reference/prompt-corpus.md +++ b/docs/reference/prompt-corpus.md @@ -15,6 +15,21 @@ or pass ``--prompt-corpus`` on the CLI. |-------|---------| | `sonnet` | Shakespeare sonnets (default for synthetic and most loaders) | | `coding` | Procedural coding / tool-use content | +| `random` | Synthetic prompts from random vocabulary token IDs — no text file required. Matches the token-generation algorithm used by `vllm bench serve` and by `sglang.benchmark.serving` under `--dataset-name random-ids` (see note below). Use with `--random-range-ratio` for ISL/OSL variance. | + +> **Which SGLang algorithm this matches.** The vocab-offset arithmetic implemented +> here is SGLang's `random_sample=False` branch, reached via +> `--dataset-name random-ids`. SGLang's *default* `--dataset-name random` is a +> different algorithm — it repeats/truncates ShareGPT token ids to hit the target +> length. Behavior is pinned against SGLang HEAD; the OSL lower bound was +> un-clamped before v0.5.x (`int(output_len * range_ratio)` with no `max(..., 1)`), +> so older SGLang releases will not reproduce these lengths. +> +> Byte-exactness also stops at the prompt round trip: AIPerf runs vLLM's +> decode → encode → trim/top-up loop, and those top-up draws consume the shared +> preseed stream. SGLang decodes once and reports `prompt_len = input_lens[i]`, +> consuming no such state — so `sglang`-style prompts diverge from SGLang +> whenever that round trip drifts. ## When it applies @@ -49,8 +64,179 @@ datasets: corpus: coding ``` +## How each corpus generates content + +### sonnet + +At startup `PromptGenerator` reads `assets/shakespeare.txt`, strips blank +lines, and splits the text into fixed 10,000-character chunks (deterministic +regardless of CPU count). Each chunk is tokenized in parallel and the results +are concatenated into a single flat token array. + +At request time `_sample_tokens` picks a random start position in that array +via a derived RNG and returns a contiguous window of the requested length, +wrapping at the end of the corpus. + +The BPE fixup loop in `generate_prompt` then validates the window: + +1. Decode the token window to text. +2. Re-encode the text and compare the actual token count to the target. +3. If too long, trim tokens from the end. If too short, draw additional tokens + from the corpus and extend. +4. Repeat up to 10 times until the re-encoded length matches the target. + +A BPE-stable terminator token is probed at init time and appended to segment +boundaries so that concatenating multiple windows does not cause merge/split +drift at the join points. + +### coding + +`CodingContentGenerator` builds a single shuffled `_tool_pool` from roughly +22 template generators covering Python, Go, Rust, TypeScript, ML +training/inference code, bash output, JSON payloads, error tracebacks, CUDA +errors, SQL queries, git diffs, CI/CD logs, config files, markdown docs, test +output, multi-turn coding conversations, and user prompts. The approximate +weighted breakdown is: ~28% general code, ~11% ML code, ~20% bash/training +logs, ~11% JSON, ~9% errors, ~3% SQL, ~10% miscellaneous, ~8% user prompts. + +All template blocks are shuffled with a seeded RNG at init time, joined with +`"\n\n"`, and tokenized once to produce the pool. At request time sampling is +identical to `sonnet` — a contiguous window with wraparound. + +Unlike `sonnet`, the `coding` path does **not** run a BPE fixup loop. +`generate_prompt` decodes the sampled window and returns it directly. This +means the actual ISL may differ slightly from the target for corpora with +high BPE merge/split rates, but keeps latency low for large ISLs. + +### random + +No text file is loaded. For each token `j` in the request sequence: + +``` +token_id = allowed_tokens[(offset + request_index + j) % n] +``` + +where `offset` is a per-request value drawn from the RNG, `request_index` +increments across requests so successive prompts do not overlap, and `n` is +the size of the allowed token pool. The token IDs are decoded to text and +then passed through the same BPE fixup loop as `sonnet` (up to 10 retries). + +See [## Random corpus](#random-corpus) below for the full explanation of +corpus style, token pool composition, and RNG alignment. + +## Corpus selection resolution + +`resolve_prompt_generator` in `dataset/generator/corpus.py` selects the +generator in this order: + +1. Explicit `prompts.corpus` / `--prompt-corpus` value +2. Loader's `default_prompt_corpus` from the plugin registry +3. `sonnet` as the global fallback + +`coding` returns a `CodingContentGenerator`; `sonnet` and `random` both +return a `PromptGenerator` with the appropriate `PromptCorpus` enum value. + ## Prefix prompts ``coding`` uses the same synthetic prefix / shared-system / user-context surface as ``sonnet``: those features sample from the selected corpus rather than requiring a separate generator type. + +``random`` also supports prefix prompt pools and shared system prompts. When +a prefix pool is configured, prefix prompts are generated using the same +arithmetic token-ID sequence as body prompts. + +### Token-level prefix concatenation + +The prefix is joined to the body at the **token-ID level**, and the combined +sequence goes through a single decode/re-encode correction targeting exactly +`prefix_len + body_len` — the same contract as vLLM's `generate_token_sequence`. +A prompt therefore contains exactly the configured number of tokens. + +Joining the two decoded segments as strings instead (`f"{prefix} {body}"`) would +let BPE charge for the separator: measured on GPT-2 that cost one extra token on +~81% of requests, +0.84 on average, and it perturbed the per-request token index. +See AIP-1118. + +For the RANDOM corpus the pool is drawn from the shared preseed stream *after* +the ISL/OSL/offset draws, bounded by `len(allowed_tokens)`, mirroring vLLM's +`RandomDataset.get_prefix`. Byte-exact prompt parity with vLLM requires +`--prompt-prefix-pool-size 1`, since vLLM has exactly one prefix; larger pools +consume additional draws from the shared stream and so diverge in the +(rarely-exercised) top-up token values. + +### Prefix semantics vs vLLM / SGLang + +The prefix is **additive** in every corpus, matching the reference benchmarkers: + +| Tool | Prefix behaviour | Wire ISL | +|------|-----------------|----------| +| aiperf (`--prompt-prefix-pool-size` / `--prompt-prefix-length`) | Prefix is **additive** — body tokens = configured ISL, prefix prepended on top | configured ISL + prefix_len | +| vLLM (`--random-prefix-len`) | Same — additive | configured ISL + prefix_len | +| SGLang (`prefix_len`) | Same — additive | configured ISL + prefix_len | + +So `--prompt-input-tokens-mean 128 --prompt-prefix-length 20` produces 148-token +prompts, not 128. `--prompt-input-tokens-mean` describes the *body*; the prefix +represents cached shared context riding on top of it, which is what makes +prefix pools useful for KV-cache benchmarking. + +Note this differs from the special-token and chat-template compensation, which +*are* subtracted from the ISL target so the wire length lands on the configured +value. Only the prefix is additive. + +The two compensations are applied by different mechanisms, and every corpus +style applies both. Special tokens come off the window bounds under `vllm` +style and off each drawn length under `sglang` style; chat-template wrapping +always comes off per-request in the composer. See +[ISL budget compensation](./isl-budget-compensation.md). + +To keep a fixed total instead, subtract the prefix length yourself: +`--prompt-input-tokens-mean 108 --prompt-prefix-length 20` gives 128 on the wire. + +## Random corpus + +``random`` generates prompts entirely from vocabulary token IDs using the +formula `(offset + request_index + j) % len(allowed_tokens)` for each token +`j` in the sequence, then decodes to text and re-encodes to verify the +round-trip token count (BPE fixup, up to 10 retries). No text file is loaded. + +### Corpus style + +``random`` is paired with `--random-corpus-style` (or `random_corpus_style` +in YAML) to select which benchmarking tool's behavior to replicate: + +| Style | `--random-corpus-style` | Token pool | BOS adjustment | Range formula | +|-------|------------------------|-----------|---------------|---------------| +| vLLM (default) | `vllm` | Non-special tokens only | Subtract BOS from ISL mean | Symmetric: `[floor(mean*(1-r)), ceil(mean*(1+r))]` | +| SGLang | `sglang` | Full `vocab_size` range | Subtract per-request after sampling | Lower-bounded: `[max(1, int(mean*r)), mean]` | + +### RNG alignment (vLLM style) + +When `--random-corpus-style vllm` and `--random-seed` are set, aiperf +pre-generates all ISL values, then all OSL values, then all per-request +offsets from a single `numpy.random.default_rng(seed)` — matching vLLM's +`get_sampling_params` draw order. This ensures identical token sequences for +the same seed when comparing aiperf against `vllm bench serve`. + +### Example + +```yaml +datasets: + - type: synthetic + prompts: + isl: 128 + osl: 128 + corpus: random + random_range_ratio: "0.3" + random_corpus_style: vllm +``` + +```bash +aiperf profile \ + --prompt-corpus random \ + --random-range-ratio 0.3 \ + --random-corpus-style vllm \ + --prompt-input-tokens-mean 128 \ + --prompt-output-tokens-mean 128 \ + --random-seed 0 +``` diff --git a/docs/reference/validating-isl-distribution.md b/docs/reference/validating-isl-distribution.md new file mode 100644 index 0000000000..c00eff0324 --- /dev/null +++ b/docs/reference/validating-isl-distribution.md @@ -0,0 +1,269 @@ + + +# Validating ISL/OSL distributions with the mock server + +The in-repo mock server records the token length of every request it receives. +Use this to verify that aiperf is generating prompts at the ISL and OSL you +configured — and to compare distributions against other benchmarking tools such +as `vllm bench serve` or `sglang bench_serving`. + +The histograms below show what a correctly-matched comparison looks like — aiperf +and vLLM Docker producing identical ISL distributions for the same seed and +parameters. Blue = aiperf, red = vLLM Docker; overlapping bars appear purple. + +![GPT-2 ISL distribution — aiperf vs vLLM Docker](isl-example-gpt2.svg) + +![Llama 3.1 8B Instruct ISL distribution — aiperf vs vLLM Docker](isl-example-llama.svg) + +An interactive version with additional tokenizer comparisons is available in +[isl-distribution-examples.html](isl-distribution-examples.html). + +## Quick start + +```bash +# 1. Start the mock server with a tokenizer and a recording file +aiperf-mock-server \ + --port 18000 \ + --tokenizer gpt2 \ + --default-model my-model \ + --record-requests /tmp/recording.jsonl + +# 2. Run your benchmark against it +aiperf profile \ + --url http://localhost:18000 \ + --model-names my-model \ + --tokenizer gpt2 \ + --prompt-corpus random \ + --random-range-ratio 0.3 \ + --random-corpus-style vllm \ + --prompt-input-tokens-mean 128 \ + --prompt-output-tokens-mean 128 \ + --concurrency 10 \ + --conversation-num 5000 \ + --random-seed 0 +``` + +When the benchmark finishes the mock server writes two files: + +| File | Contents | +|------|----------| +| `recording.jsonl` | One JSON line per request: `isl`, `requested_osl`, `tokenization_mode`, and OSL-shaping fields | +| `recording.jsonl.summary.json` | Per-endpoint statistics: ISL/OSL histograms, vocab distribution, unique-value counts | + +## Recording JSONL schema + +Each line is a self-contained record: + +```json +{ + "ts": 1785526379.59, + "request_id": "chatcmpl-6348ad24-…", + "endpoint": "/v1/chat/completions", + "model": "my-model", + "isl": 148, + "requested_osl": 133, + "max_tokens": null, + "max_completion_tokens": 133, + "min_tokens": null, + "ignore_eos": true, + "stream": true, + "tokenization_mode": "chat_template" +} +``` + +`isl` is the **template-inclusive** input token count — the full tokenized +representation the server receives, including any chat template markup. For +`PromptCorpus.RANDOM` without `--apply-chat-template` the `isl` value matches +the configured `--prompt-input-tokens-mean` target directly. + +`tokenization_mode` indicates how ISL was measured: + +| Mode | Meaning | +|------|---------| +| `chat_template` | `apply_chat_template` called; full template overhead included | +| `chat_template_fallback` | No native template; ChatML fallback applied (+27 tokens for GPT-2) | +| `prompt_token_ids` | Request carried pre-tokenized `prompt_token_ids`; used directly | +| `tokenizer_call` | Completion endpoint; raw text encoded directly | + +## Inspecting the distribution + +```python +import json, statistics, collections + +isls = [json.loads(l)["isl"] for l in open("/tmp/recording.jsonl")] +s = sorted(isls) +n = len(s) + +print(f"n={n}") +print(f"min={s[0]}, p5={s[n//20]}, mean={statistics.mean(isls):.1f}, " + f"p95={s[int(n*0.95)]}, max={s[-1]}, std={statistics.stdev(isls):.2f}") +``` + +## Comparing two recordings + +The `tools/compare_recordings.py` utility produces a self-contained HTML +report from any two JSONL recording files: + +```bash +python tools/compare_recordings.py \ + --a /tmp/aiperf_recording.jsonl --label-a "aiperf" \ + --b /tmp/vllm_recording.jsonl --label-b "vllm bench serve" \ + --out comparison.html +open comparison.html +``` + +### What the report contains + +- **Stat cards** for ISL and OSL — mean, std, p5/p95, min/max, and delta + between the two runs +- **Overlapping histograms** for ISL and OSL (2-token bins, blue/red + transparent bars so overlapping regions are visually distinct) +- **Tokenization-mode breakdown table** — count and percentage for each + `tokenization_mode` value per run +- **Vocabulary top-N diff table** — the token IDs with the largest count + difference between runs (loaded automatically from the companion + `.summary.json` files if present) + +### CLI reference + +``` +usage: compare_recordings.py --a FILE --b FILE [--label-a LABEL] + [--label-b LABEL] [--out FILE] + + --a FILE First recording JSONL (A) + --b FILE Second recording JSONL (B) + --label-a LABEL Display label for file A (default: A) + --label-b LABEL Display label for file B (default: B) + --out FILE Output HTML file (default: comparison.html) +``` + +The `.summary.json` companion file is detected automatically at +`.summary.json` — no extra flag needed. + +## Validating against vLLM Docker + +Run both tools against separate mock server instances in parallel, then +compare the recordings: + +```bash +# Mock server for aiperf (port 18000) +aiperf-mock-server --port 18000 --tokenizer meta-llama/Llama-3.1-8B-Instruct \ + --default-model mock-model \ + --record-requests /tmp/aiperf.jsonl & + +# Mock server for vLLM (port 18001) +aiperf-mock-server --port 18001 --tokenizer meta-llama/Llama-3.1-8B-Instruct \ + --default-model mock-model \ + --record-requests /tmp/vllm.jsonl & + +# aiperf run +aiperf profile --url http://localhost:18000 --model-names mock-model \ + --tokenizer meta-llama/Llama-3.1-8B-Instruct \ + --prompt-corpus random --random-range-ratio 0.3 --random-corpus-style vllm \ + --prompt-input-tokens-mean 128 --prompt-output-tokens-mean 128 \ + --concurrency 10 --conversation-num 20000 --random-seed 0 & + +# vLLM Docker run +docker run --rm --platform linux/arm64 \ + -v ~/.cache/huggingface:/root/.cache/huggingface:ro \ + --entrypoint vllm vllm/vllm-openai-cpu:latest-arm64 \ + bench serve --backend openai-chat \ + --base-url http://host.docker.internal:18001 \ + --endpoint /v1/chat/completions --model mock-model \ + --tokenizer meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random --random-input-len 128 --random-output-len 128 \ + --random-range-ratio 0.3 --num-prompts 20000 \ + --request-rate inf --max-concurrency 30 \ + --ready-check-timeout-sec 0 --seed 0 + +# Compare +python tools/compare_recordings.py \ + --a /tmp/aiperf.jsonl --label-a "aiperf" \ + --b /tmp/vllm.jsonl --label-b "vLLM Docker" \ + --out comparison.html +open comparison.html +``` + +## Capturing raw prompts + +To capture the exact prompt text sent by a benchmark client — useful for +diagnosing content differences between tools — use `tools/capture_server.py`. +It records every request's `messages` array byte-for-byte without tokenizing +and returns a minimal synthetic streaming response so the client does not +error out. + +```bash +python tools/capture_server.py \ + --out /tmp/captured.jsonl \ + --port 18000 \ + --host 127.0.0.1 # optional, default +``` + +Then point your benchmark client at `http://localhost:18000`. Each line in the +output JSONL contains: + +```json +{ + "i": 0, + "messages": [{"role": "user", "content": "..."}], + "model": "mock-model", + "max_completion_tokens": 128, + "stream": true, + "ignore_eos": true +} +``` + +### CLI reference + +``` +usage: capture_server.py --out FILE [--port PORT] [--host HOST] + + --out FILE Output JSONL file — one captured request per line + --port PORT Listen port (default: 18000) + --host HOST Listen host (default: 127.0.0.1) +``` + +### Diffing captured prompts + +After capturing from two tools with the same seed, compare content at the +Python level: + +```python +import json + +a = [json.loads(l)["messages"][0]["content"] + for l in open("/tmp/captured_aiperf.jsonl")] +b = [json.loads(l)["messages"][0]["content"] + for l in open("/tmp/captured_vllm.jsonl")] + +# Sort both by content since concurrent sends may arrive out of order +a.sort(); b.sort() + +diffs = [(i, ai, bi) for i, (ai, bi) in enumerate(zip(a, b)) if ai != bi] +print(f"{len(diffs)} / {min(len(a), len(b))} prompts differ") +for i, ai, bi in diffs[:3]: + pos = next((j for j in range(min(len(ai),len(bi))) if ai[j]!=bi[j]), None) + print(f" [{i}] first diff at char {pos}") +``` + +Note that with `--max-concurrency > 1` requests arrive at the capture server +out of order. Sort by content (not by the sequential `"i"` field) before +comparing, or use `--max-concurrency 1` to force serial delivery. + +## Notes on template-inclusive ISL + +The mock server applies the tokenizer's chat template before measuring ISL +when a native template is available. This means the recorded ISL includes +template overhead (role markers, system prompt, generation prompt suffix) +on top of the prompt content. For a model like Llama 3.1 8B Instruct the +template adds approximately 35 tokens per request. + +Clients that send message content as a multimodal list +`[{"type":"text","text":"..."}]` rather than a plain string — including +`vllm bench serve --backend openai-chat` — are handled transparently: the +mock server normalises list content via `_content_to_text` before passing +to `apply_chat_template`, so ISL measurement is consistent regardless of +how the client formats the content field. diff --git a/src/aiperf/common/enums/__init__.py b/src/aiperf/common/enums/__init__.py index 1a41f45fa9..bbd8fa3e3e 100644 --- a/src/aiperf/common/enums/__init__.py +++ b/src/aiperf/common/enums/__init__.py @@ -45,6 +45,7 @@ PrometheusMetricType, PromptCorpus, PromptSource, + RandomCorpusStyle, RequestContentType, ServerMetricsDiscoveryMode, ServerMetricsFormat, @@ -157,6 +158,7 @@ "PrometheusMetricType", "PromptCorpus", "PromptSource", + "RandomCorpusStyle", "RequestContentType", "SSEEventType", "SSEFieldType", diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index 8e17af4db4..52f2238589 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -269,6 +269,10 @@ class PromptCorpus(CaseInsensitiveStrEnum): CODING = "coding" """Realistic coding content: code, bash output, JSON, error tracebacks, git diffs.""" + RANDOM = "random" + """Random tokens drawn from the full tokenizer vocabulary minus special tokens. + Matches vLLM bench's RandomDataset token-generation strategy.""" + class MemoryMapFormat(CaseInsensitiveStrEnum): """Storage format for memory-mapped dataset files.""" @@ -787,3 +791,72 @@ class ExportFormat(CaseInsensitiveStrEnum): JSON = "json" JSONL = "jsonl" CSV = "csv" + + +class RandomCorpusStyle(CaseInsensitiveStrEnum): + """Benchmark style for RANDOM corpus generation. + + Controls the full set of behaviors that vary between tools. Each style + is a bundle of decisions that together reproduce the statistical + distribution of a specific benchmarking tool: + + .. list-table:: + :header-rows: 1 + :widths: 30 35 35 + + * - Concern + - VLLM + - SGLANG + * - Token pool + - ``valid_token_ids`` (special tokens excluded via ``all_special_ids``) + - ``all_token_ids`` (full ``range(vocab_size)``, no exclusion) + * - BOS / special-token adjustment + - ``max(0, mean - num_special_tokens)``, applied to the mean before + the window bounds are computed + - ``max(1, drawn - num_special_tokens)``, applied per-request after + sampling so the window keeps its raw-mean shape + * - ISL/OSL range formula + - Symmetric: ``[floor(mean*(1-r)), ceil(mean*(1+r))]`` + - Lower-bounded: ``[max(1, int(mean*r)), mean]`` + * - RNG algorithm (preseed) + - PCG64 via ``numpy.random.default_rng(seed)`` + - MT19937 via a private ``numpy.random.RandomState(seed)`` + * - RNG draw order (with preseed) + - All ISLs → all OSLs → all offsets + - All ISLs → all OSLs → all offsets (same order, different algorithm) + * - Top-up RNG + - Continues from the same ``default_rng`` stream (``_preseed_rng``) + - Continues from the same ``RandomState`` stream (``_preseed_rng``) + + Both styles are preseeded whenever the corpus is + :attr:`PromptCorpus.RANDOM` — the composer gates on + ``isinstance(dist, RangeRatioDistribution)``, which the SGLANG subclass + satisfies. ``_corpus_rng`` is the fallback for the non-preseeded paths + (non-RANDOM corpora, and draws past cache exhaustion), not a per-style + difference. + + .. note:: + The two styles subtract special tokens at different points on purpose, + each following its own upstream. VLLM folds the count into the bounds; + SGLANG shifts each drawn length, mirroring ``sample_random_requests`` + under ``return_text`` (the default, and what aiperf sends):: + + input_lens[i] = max(1, input_lens[i] - num_special_tokens) + + Both land the same wire ISL on the configured value; only the resulting + window shape differs. + """ + + VLLM = "vllm" + """vllm bench serve semantics: symmetric window ``[floor(mean*(1-r)), ceil(mean*(1+r))]``. + r=0 is fixed at mean; larger r widens the window on both sides. r must be in [0, 1). + Special tokens excluded from the sampling pool. BOS subtracted from ISL mean.""" + + SGLANG = "sglang" + """sglang ``benchmark.serving`` semantics under ``--dataset-name random-ids``: + lower-bounded window ``[max(1, int(mean*r)), mean]``. + r=0 allows full variability [1, mean]; r=1 fixes length at mean. Full vocab_size range + used for token sampling (no special-token exclusion). No BOS adjustment. + + The default ``--dataset-name random`` is a different algorithm upstream + (repeat/truncate ShareGPT token ids) and is not what this style mirrors.""" diff --git a/src/aiperf/common/models/sequence_distribution.py b/src/aiperf/common/models/sequence_distribution.py index cf4d87fd16..04be16c585 100644 --- a/src/aiperf/common/models/sequence_distribution.py +++ b/src/aiperf/common/models/sequence_distribution.py @@ -32,16 +32,28 @@ from __future__ import annotations +import math import re from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import ( + TYPE_CHECKING, + Annotated, + Any, + ClassVar, + Protocol, + Self, + runtime_checkable, +) import numpy as np import orjson +from pydantic import ConfigDict, Field, field_validator, model_validator from aiperf.common import random_generator as rng from aiperf.common.aiperf_logger import AIPerfLogger +from aiperf.common.enums import RandomCorpusStyle from aiperf.common.utils import load_json_str +from aiperf.config.base import BaseConfig if TYPE_CHECKING: from aiperf.common.random_generator import RandomGenerator @@ -49,6 +61,13 @@ logger = AIPerfLogger(__name__) +@runtime_checkable +class SequenceLengthSampler(Protocol): + """Anything that yields an (ISL, OSL) pair per call.""" + + def sample(self) -> tuple[int, int]: ... + + def _validate_probability_sum(pairs: list[SequenceLengthPair]) -> None: """ Validate that probabilities sum to approximately 100.0. @@ -517,3 +536,574 @@ def create_balanced_distribution( seq_pairs = [SequenceLengthPair(isl, osl, prob_per_pair) for isl, osl in pairs] return SequenceLengthDistribution(seq_pairs) + + +def _parse_vllm_ratio_string(value: str) -> tuple[float, float]: + """Parse a vLLM-style CLI ratio string. + + Accepts a plain float string (``"0.3"``) applied to both dimensions, or a + JSON object (``'{"input": 0.3, "output": 0.5}'``) for independent values. + """ + value = value.strip() + if not value: + raise ValueError("--random-range-ratio value cannot be empty") + try: + ratio = float(value) + return ratio, ratio + except ValueError: + pass + try: + data = orjson.loads(value) + except orjson.JSONDecodeError as e: + raise ValueError( + f"--random-range-ratio must be a float or a JSON object with " + f"'input' and 'output' keys, got: {value!r} ({e})" + ) from e + if not isinstance(data, dict): + raise ValueError( + f"--random-range-ratio must be a float or a JSON object with " + f"'input' and 'output' keys, got: {value!r}" + ) + missing = {"input", "output"} - data.keys() + if missing: + raise ValueError( + f"--random-range-ratio JSON object missing keys: {sorted(missing)}" + ) + extra = data.keys() - {"input", "output"} + if extra: + raise ValueError( + f"--random-range-ratio JSON object has unexpected keys: {sorted(extra)}" + ) + return float(data["input"]), float(data["output"]) + + +def _parse_sglang_ratio_string(value: str) -> tuple[float, float]: + """Parse an SGLang-style CLI ratio string. + + Accepts only a plain float string (``"0.3"``); independent input/output + values via a JSON dict are rejected because SGLang applies a single ratio + to both dimensions. + """ + value = value.strip() + if not value: + raise ValueError("--random-range-ratio value cannot be empty") + try: + ratio = float(value) + return ratio, ratio + except ValueError: + pass + try: + data = orjson.loads(value) + if isinstance(data, dict): + raise ValueError( + "SGLang corpus style applies a single ratio to both ISL and OSL. " + "Independent input/output values are not supported; " + "provide a plain float instead (e.g. 0.3)." + ) + except orjson.JSONDecodeError: + pass + raise ValueError( + f"--random-range-ratio must be a plain float for SGLang corpus style, got: {value!r}" + ) + + +def _coerce_ratio_input(v: Any, parser: Any) -> tuple[float, float]: + """Coerce ``range_ratio`` field input to a ``(isl_ratio, osl_ratio)`` tuple. + + Handles float/int (symmetric), string (via ``parser``), and 2-element + list/tuple. + + Raises ``ValueError`` on anything else rather than returning the input + unchanged. Both callers destructure the result immediately + (``ir, or_ = _coerce_ratio_input(...)``), so a pass-through never reached + Pydantic's type check -- it surfaced as a bare + ``TypeError: cannot unpack non-iterable NoneType object``, which names + neither the field nor the accepted forms. + """ + # bool is an int subclass, so `range_ratio: true` would otherwise coerce to + # 1.0 -- accepted by SGLang (whose range is [0, 1]) as "pin at the mean". + # Rejected explicitly: a YAML bool here is a typo, not a ratio. + if isinstance(v, bool): + raise ValueError(f"range_ratio must be a number, got bool: {v!r}") + if isinstance(v, (int, float)): + return float(v), float(v) + if isinstance(v, str): + return parser(v) + if isinstance(v, (list, tuple)) and len(v) == 2: + return float(v[0]), float(v[1]) + raise ValueError( + f"range_ratio must be a float, a string, or a 2-element sequence, " + f"got {type(v).__name__}: {v!r}" + ) + + +class VLLMRatioConfig(BaseConfig): + """Config for vLLM-style range-ratio sampling. + + ``range_ratio`` accepts: + + - a plain float (``0.3``) — applied to both ISL and OSL + - a CLI string (``"0.3"`` or ``'{"input": 0.3, "output": 0.5}'``) + - a 2-tuple ``(isl_ratio, osl_ratio)`` + + Ratios must be in ``[0.0, 1.0)``. + """ + + model_config = ConfigDict(extra="forbid") + + isl_mean: Annotated[int, Field(ge=1, description="Mean input sequence length.")] + osl_mean: Annotated[int, Field(ge=1, description="Mean output sequence length.")] + isl_stddev: Annotated[ + float, + Field( + default=0.0, + ge=0.0, + description="Must be 0; asserts caller resolved a fixed ISL mean.", + ), + ] = 0.0 + osl_stddev: Annotated[ + float, + Field( + default=0.0, + ge=0.0, + description="Must be 0; asserts caller resolved a fixed OSL mean.", + ), + ] = 0.0 + num_special_tokens: Annotated[ + int, + Field( + default=0, + ge=0, + description="Special tokens subtracted from isl_mean before bounds.", + ), + ] = 0 + range_ratio: Annotated[ + tuple[float, float], + Field( + description="(isl_ratio, osl_ratio); symmetric window [floor(mean*(1-r)), ceil(mean*(1+r))]." + ), + ] + + @model_validator(mode="after") + def _no_stddev(self) -> Self: + if self.isl_stddev != 0.0: + raise ValueError( + "--isl-stddev cannot be combined with --random-range-ratio; " + "the ratio window already controls ISL variance." + ) + if self.osl_stddev != 0.0: + raise ValueError( + "--osl-stddev cannot be combined with --random-range-ratio; " + "the ratio window already controls OSL variance." + ) + return self + + @field_validator("range_ratio", mode="before") + @classmethod + def _coerce_and_validate(cls, v: Any) -> tuple[float, float]: + ir, or_ = _coerce_ratio_input(v, _parse_vllm_ratio_string) + if not math.isfinite(ir) or not (0.0 <= ir < 1.0): + raise ValueError(f"ISL range_ratio must be in [0.0, 1.0), got {ir}") + if not math.isfinite(or_) or not (0.0 <= or_ < 1.0): + raise ValueError(f"OSL range_ratio must be in [0.0, 1.0), got {or_}") + return ir, or_ + + def compute_input_bounds(self) -> tuple[int, int]: + adjusted = max(0, self.isl_mean - self.num_special_tokens) + r = self.range_ratio[0] + return max(0, math.floor(adjusted * (1 - r))), math.ceil(adjusted * (1 + r)) + + def compute_output_bounds(self) -> tuple[int, int]: + r = self.range_ratio[1] + return ( + max(1, math.floor(self.osl_mean * (1 - r))), + max(1, math.ceil(self.osl_mean * (1 + r))), + ) + + +class SGLangRatioConfig(BaseConfig): + """Config for SGLang-style range-ratio sampling. + + ``range_ratio`` accepts: + + - a plain float (``0.3``) — applied to both ISL and OSL + - a plain-float CLI string (``"0.3"``); JSON dict form is rejected + - a 2-tuple ``(r, r)`` where both elements must be equal + + Ratios must be in ``[0.0, 1.0]``. + """ + + model_config = ConfigDict(extra="forbid") + + isl_mean: Annotated[int, Field(ge=1, description="Mean input sequence length.")] + osl_mean: Annotated[int, Field(ge=1, description="Mean output sequence length.")] + isl_stddev: Annotated[ + float, + Field( + default=0.0, + ge=0.0, + description="Must be 0; asserts caller resolved a fixed ISL mean.", + ), + ] = 0.0 + osl_stddev: Annotated[ + float, + Field( + default=0.0, + ge=0.0, + description="Must be 0; asserts caller resolved a fixed OSL mean.", + ), + ] = 0.0 + num_special_tokens: Annotated[ + int, + Field( + default=0, + ge=0, + description="Server-added special tokens, subtracted per-request " + "AFTER sampling (not from isl_mean, which would rescale the window's " + "lower bound). Mirrors SGLang's post-sampling input_lens[i] shift.", + ), + ] = 0 + range_ratio: Annotated[ + tuple[float, float], + Field( + description="(isl_ratio, osl_ratio); lower-bounded window [max(1, int(mean*r)), mean]." + ), + ] + + @model_validator(mode="after") + def _no_stddev(self) -> Self: + if self.isl_stddev != 0.0: + raise ValueError( + "--isl-stddev cannot be combined with --random-range-ratio; " + "the ratio window already controls ISL variance." + ) + if self.osl_stddev != 0.0: + raise ValueError( + "--osl-stddev cannot be combined with --random-range-ratio; " + "the ratio window already controls OSL variance." + ) + return self + + @field_validator("range_ratio", mode="before") + @classmethod + def _coerce_and_validate(cls, v: Any) -> tuple[float, float]: + ir, or_ = _coerce_ratio_input(v, _parse_sglang_ratio_string) + if not math.isfinite(ir) or not (0.0 <= ir <= 1.0): + raise ValueError(f"range_ratio must be in [0.0, 1.0], got {ir}") + if ir != or_: + raise ValueError( + f"SGLang corpus style requires equal ISL and OSL ratios, got ({ir}, {or_})" + ) + return ir, or_ + + def compute_input_bounds(self) -> tuple[int, int]: + r = self.range_ratio[0] + return max(1, int(self.isl_mean * r)), self.isl_mean + + def compute_output_bounds(self) -> tuple[int, int]: + r = self.range_ratio[1] + return max(1, int(self.osl_mean * r)), self.osl_mean + + +class RangeRatioDistribution: + """Uniform ISL/OSL sampling in a ratio-defined integer window around configured means. + + Instantiate via the registry for mode-driven dispatch:: + + DistClass = _CLASS_FOR_MODE[mode] + config = DistClass.get_config_class()(isl_mean=512, osl_mean=128, range_ratio="0.3") + dist = DistClass(config) + + Two concrete styles: + + - :class:`RangeRatioDistribution` (VLLM): symmetric window + ``[floor(mean*(1-r)), ceil(mean*(1+r))]``, ``r ∈ [0.0, 1.0)``. + - :class:`SGLangRangeRatioDistribution` (SGLANG): lower-bounded window + ``[max(1, int(mean*r)), mean]``, ``r ∈ [0.0, 1.0]``. + """ + + _style: ClassVar[RandomCorpusStyle] = RandomCorpusStyle.VLLM + + @classmethod + def get_config_class(cls) -> type[VLLMRatioConfig]: + return VLLMRatioConfig + + def __init__(self, config: VLLMRatioConfig) -> None: + self._rng = rng.derive("models.range_ratio.distribution") + self._isl_mean = config.isl_mean + self._osl_mean = config.osl_mean + self._config = config + self._warned_cache_exhausted = False + + self._input_low, self._input_high = config.compute_input_bounds() + self._output_low, self._output_high = config.compute_output_bounds() + + def preseed(self, n: int, seed: int | None) -> None: + """Pre-generate all ISL then all OSL values using vLLM's PCG64 draw order. + + Creates ``numpy.random.default_rng(seed)`` internally so the RNG + algorithm and seeding are encapsulated here rather than in the caller. + Stores the generator after ISL/OSL draws as ``_preseed_rng`` so that + :meth:`PromptGenerator.preseed` can continue drawing offsets from the + same stream without the caller needing to manage generator state. + + Prefix prompts do not participate: they are additive and prepended to the + body after generation, so the cached ISLs describe the body alone. + + Subclasses override this method to use a different RNG algorithm + (e.g. :class:`SGLangRangeRatioDistribution` uses MT19937 to match + SGLang's ``benchmark_serving.py``). + """ + g = np.random.default_rng(seed) + self._isl_cache = g.integers( + self._input_low, self._input_high + 1, size=n + ).tolist() + self._osl_cache = g.integers( + self._output_low, self._output_high + 1, size=n + ).tolist() + self._cache_idx = 0 + self._preseed_rng: object = g + + def sample(self) -> tuple[int, int]: + """Sample a single (ISL, OSL) pair with independent uniform integers. + + Reads the preseed cache when one is active, falling back to live draws + once it is exhausted. The cache is sized by conversation count but read + once per turn, so any multi-turn run outruns it; degrading keeps the run + alive instead of raising ``IndexError`` from the dataset composer. + Alignment with the reference implementation is already lost past that + point -- vLLM has no notion of turns -- so there is nothing left to + preserve by failing hard. + """ + cache = getattr(self, "_isl_cache", None) + if cache is not None and self._cache_idx < len(cache): + idx = self._cache_idx + self._cache_idx = idx + 1 + return self._isl_cache[idx], self._osl_cache[idx] + if cache is not None and not self._warned_cache_exhausted: + self._warned_cache_exhausted = True + logger.warning( + f"Preseeded ISL/OSL cache exhausted after {len(cache)} draws " + "(sized by conversation count, consumed once per turn). " + "Falling back to live sampling; prompts past this point no " + f"longer match {type(self)._style} for the same seed." + ) + isl = self.adjust_sampled_isl( + int(self._rng.integers(self._input_low, self._input_high + 1)) + ) + osl = int(self._rng.integers(self._output_low, self._output_high + 1)) + return isl, osl + + def adjust_sampled_isl(self, isl: int) -> int: + """Per-request ISL adjustment applied AFTER a length is drawn. + + No-op for VLLM style, which folds ``num_special_tokens`` into the window + bounds instead (see :meth:`VLLMRatioConfig.compute_input_bounds`). + Overridden by :class:`SGLangRangeRatioDistribution`, whose upstream + shifts drawn lengths rather than rescaling the window. + + Preseeded values are stored already-adjusted, so :meth:`sample` applies + this only on the live-draw path -- applying it to both would subtract + twice. + """ + return isl + + @property + def input_bounds(self) -> tuple[int, int]: + """Inclusive [low, high] integer bounds for ISL sampling.""" + return self._input_low, self._input_high + + @property + def output_bounds(self) -> tuple[int, int]: + """Inclusive [low, high] integer bounds for OSL sampling.""" + return self._output_low, self._output_high + + @property + def mode(self) -> RandomCorpusStyle: + return type(self)._style + + def __repr__(self) -> str: + return ( + f"RangeRatioDistribution(mode={type(self)._style}, isl_mean={self._isl_mean}, " + f"osl_mean={self._osl_mean}, range_ratio={self._config.range_ratio})" + ) + + +_MT19937_SEED_LIMIT = 2**32 + + +class _FoldedSeedWarnings: + """Tracks which folded seeds have been reported. + + Keyed by seed rather than a single flag so a sweep re-seeding per variation + flags each aliasing seed, instead of only the first one it happens to hit. + """ + + _seen: ClassVar[set[int]] = set() + + @classmethod + def claim(cls, seed: int) -> bool: + """Return True the first time ``seed`` is seen, False afterwards.""" + if seed in cls._seen: + return False + cls._seen.add(seed) + return True + + @classmethod + def reset(cls) -> None: + """Clear reported seeds. For tests.""" + cls._seen.clear() + + +def _warn_if_seed_is_folded(seed: int) -> None: + """Warn when a seed is too wide for MT19937 and must be folded. + + numpy's legacy seeder caps at ``2**32 - 1``, so wider seeds are XOR-folded + (see :func:`~aiperf.common.random_generator.fold_seed_to_uint32`). The fold + is deterministic and reproducible, but not injective: for + ``seed = hi * 2**32 + lo`` it reduces to ``hi ^ lo``, so ``--random-seed 5`` + and ``--random-seed 4294967300`` drive the same SGLANG stream. Two seeds the + user believes are independent are not. + + Only this style is affected. ``derive()`` hashes the full seed, and the + VLLM/PCG64 path takes it verbatim, so those stay distinct for the same pair. + Warned rather than rejected because the seed is perfectly valid everywhere + else; the aliasing is a property of MT19937's seeding API, not of the value. + """ + if seed < _MT19937_SEED_LIMIT or not _FoldedSeedWarnings.claim(seed): + return + logger.warning( + f"--random-seed {seed} exceeds MT19937's {_MT19937_SEED_LIMIT - 1} limit " + f"and was folded to {rng.fold_seed_to_uint32(seed)} for " + f"--random-corpus-style sglang. Draws stay reproducible for this seed, " + "but other seeds fold to the same value and produce an identical SGLang " + "workload. Use a seed below 2**32 if you need seeds to be distinguishable. " + "Other RNG streams (including --random-corpus-style vllm) use the full seed." + ) + + +class _LegacyRNG: + """Thin wrapper over a legacy ``numpy.random.RandomState`` (MT19937) + exposing the same ``.integers()`` interface as ``numpy.random.Generator`` + (PCG64). + + Used by :class:`SGLangRangeRatioDistribution` so that preseed callers + (including :meth:`PromptGenerator.preseed`) can treat both RNG backends + identically without branching on algorithm. + + Holds its own ``RandomState`` rather than driving module-level + ``numpy.random``: ``RandomState(s)`` yields draws identical to + ``numpy.random.seed(s)`` followed by module-level calls, so SGLang MT19937 + parity is preserved, but the stream is private. That keeps a second + distribution (or any future global ``numpy.random`` user) from perturbing + an in-flight preseed stream, which matters because the same instance is + handed to :meth:`PromptGenerator.preseed` and then read again for BPE + top-up draws long after ``preseed`` returns. + """ + + def __init__(self, seed: int | None = None) -> None: + """Create a private MT19937 stream. + + Args: + seed: Seed in ``[0, 2**32 - 1]``, or None to draw from OS entropy. + Callers with a wider seed must fold it first (see + :func:`~aiperf.common.random_generator.fold_seed_to_uint32`). + """ + self._state = np.random.RandomState(seed) + + def integers( + self, + low: int, + high: int | None = None, + size: int | None = None, + ) -> np.ndarray: + if high is None: + return self._state.randint(0, low, size=size) + return self._state.randint(low, high, size=size) + + +class SGLangRangeRatioDistribution(RangeRatioDistribution): + """RangeRatioDistribution with SGLang-compatible MT19937 preseed. + + The sampling window derives from the raw configured ``isl_mean``. Both + overheads are then subtracted per-request *after* sampling, matching + SGLang, which shifts drawn lengths rather than rescaling the window: + + - **Special tokens** — :meth:`adjust_sampled_isl` on this class, mirroring + ``input_lens[i] = max(1, input_lens[i] - num_special_tokens)``. + - **Chat-template wrapping** — the composer's + ``first_turn_isl_adjustment`` / ``subsequent_turn_isl_adjustment``, + which carry the template and cache-bust terms only. + + Those are two separate mechanisms; neither covers the other. Subtracting + from the mean beforehand would rescale the lower bound too + (``int((mean-c)*r)`` instead of ``int(mean*r)``), producing a different + distribution rather than a shifted one. + + Overrides :meth:`preseed` to use MT19937 (via a private + ``numpy.random.RandomState``) instead of ``numpy.random.default_rng`` + (PCG64), matching the draw order in SGLang's ``benchmark_serving.py``:: + + input_lens = np.random.randint(lower, upper + 1, size=n) + output_lens = np.random.randint(lower, upper + 1, size=n) + offsets = np.random.randint(0, vocab_size, size=n) + + When ``seed`` is provided, the MT19937 stream is seeded before the draws + so that aiperf runs are reproducible even though SGLang itself never seeds + before sampling. The seed is folded through + :func:`~aiperf.common.random_generator.fold_seed_to_uint32` first, since + the legacy seeder rejects the 64-bit seeds that adaptive sweeps and + ``multi_run.vary_seed_per_trial`` produce. + """ + + _style: ClassVar[RandomCorpusStyle] = RandomCorpusStyle.SGLANG + + @classmethod + def get_config_class(cls) -> type[SGLangRatioConfig]: + return SGLangRatioConfig + + def __init__(self, config: SGLangRatioConfig) -> None: + super().__init__(config) + + def adjust_sampled_isl(self, isl: int) -> int: + """Subtract server-added special tokens from a drawn length. + + Mirrors ``sample_random_requests`` (``benchmark/datasets/random.py``), + which shifts each drawn length after sampling:: + + if return_text: # default, and what aiperf sends + num_special_tokens = int(tokenizer.num_special_tokens_to_add()) + input_lens[i] = max(1, input_lens[i] - num_special_tokens) + + Applied here rather than to ``isl_mean`` because subtracting before + :meth:`SGLangRatioConfig.compute_input_bounds` would rescale the lower + bound too -- ``int((mean-c)*r)`` instead of ``int(mean*r)`` -- producing + a different distribution rather than a shifted one. + """ + return max(1, isl - self._config.num_special_tokens) + + def preseed(self, n: int, seed: int | None) -> None: + # numpy's legacy MT19937 seeder caps at 2**32-1, but run seeds are + # 64-bit on the adaptive-sweep and vary_seed_per_trial paths (and + # --random-seed is only bounded ge=0), so fold before seeding. + # The PCG64 parent path takes 64-bit seeds directly and needs no fold. + if seed is not None: + _warn_if_seed_is_folded(seed) + g = _LegacyRNG(rng.fold_seed_to_uint32(seed) if seed is not None else None) + self._isl_cache = [ + self.adjust_sampled_isl(isl) + for isl in g.integers( + self._input_low, self._input_high + 1, size=n + ).tolist() + ] + self._osl_cache = g.integers( + self._output_low, self._output_high + 1, size=n + ).tolist() + self._cache_idx = 0 + self._preseed_rng: object = g + + +_CLASS_FOR_MODE: dict[RandomCorpusStyle, type[RangeRatioDistribution]] = { + RandomCorpusStyle.VLLM: RangeRatioDistribution, + RandomCorpusStyle.SGLANG: SGLangRangeRatioDistribution, +} diff --git a/src/aiperf/common/random_generator.py b/src/aiperf/common/random_generator.py index 2132e7e389..bb92873c08 100644 --- a/src/aiperf/common/random_generator.py +++ b/src/aiperf/common/random_generator.py @@ -57,6 +57,7 @@ __all__ = [ "RandomGenerator", "derive", + "fold_seed_to_uint32", "init", "reset", ] @@ -424,6 +425,28 @@ def derive(self, identifier: str) -> RandomGenerator: _manager: _RNGManager | None = None +def fold_seed_to_uint32(seed: int) -> int: + """Fold a seed of any width into numpy's legacy 32-bit seeding range. + + ``numpy.random.seed`` (MT19937 global state) rejects anything outside + ``[0, 2**32 - 1]``, but AIPerf seeds are 64-bit: ``derive_variation_seed`` + returns ``int.from_bytes(sha256(...)[:8])`` for adaptive-sweep variations + and for ``multi_run.vary_seed_per_trial``, and ``--random-seed`` itself is + only bounded ``ge=0``. XOR-folding the high half into the low half keeps + every input bit contributing to the result while landing in range. + + Seeds already below ``2**32`` are returned unchanged, so folding never + perturbs an existing reproducible run. + + Args: + seed: Non-negative seed of any width. + + Returns: + Equivalent seed in ``[0, 2**32 - 1]``, suitable for ``numpy.random.seed``. + """ + return (seed ^ (seed >> 32)) & 0xFFFFFFFF + + def init(seed: int | None) -> None: """Initialize global RNG manager. Called once at startup (bootstrap.py). @@ -450,9 +473,7 @@ def init(seed: int | None) -> None: # This protects against third-party code or future changes that might use global state if seed is not None: random.seed(seed) - # Normalize seed to numpy's 32-bit range by folding high and low bits - np_seed = (seed ^ (seed >> 32)) & 0xFFFFFFFF - np.random.seed(np_seed) + np.random.seed(fold_seed_to_uint32(seed)) _manager = _RNGManager(seed) diff --git a/src/aiperf/common/tokenizer.py b/src/aiperf/common/tokenizer.py index 462b0f575b..03d4d8f893 100644 --- a/src/aiperf/common/tokenizer.py +++ b/src/aiperf/common/tokenizer.py @@ -38,6 +38,32 @@ ) +def _tiktoken_internal(encoding: "tiktoken.Encoding", attr: str) -> object: + """Read a private ``tiktoken.Encoding`` attribute, or fail with a clear error. + + tiktoken exposes no public API for enumerating its token IDs, and its ID + space is sparse -- gaps between mergeable ranks and special tokens are not + decodable -- so ``range(n_vocab)`` is not a usable substitute. These + privates are the only source of a correct token pool. + + Deliberately raises instead of degrading. Unlike + :meth:`Tokenizer.num_prompt_special_tokens`, where falling back to 0 is + conservative in a known direction, there is no safe default here: a wrong + token pool silently produces prompts drawn from invalid IDs, which either + raise ``KeyError`` deep inside a decode or corrupt the benchmark's ISL + accounting. Failing at setup with the attribute name beats either. + """ + value = getattr(encoding, attr, None) + if value is None: + raise TokenizerError( + f"tiktoken.Encoding has no {attr!r}, which AIPerf needs to enumerate " + "token IDs for --prompt-corpus random. This usually means the " + "installed tiktoken renamed or removed the attribute. Pin a " + "compatible tiktoken, or use a HuggingFace tokenizer instead." + ) + return value + + class _TiktokenAdapter: """Adapts tiktoken.Encoding to the interface expected by Tokenizer._tokenizer.""" @@ -800,6 +826,83 @@ def block_separation_token_id(self) -> int | None: return self.eos_token_id return None + @property + def vocab_size(self) -> int: + """Total vocabulary size of the underlying tokenizer.""" + self._require_init() + if isinstance(self._tokenizer, _TiktokenAdapter): + return self._tokenizer._encoding.n_vocab + return self._tokenizer.vocab_size + + @property + def all_special_ids(self) -> set[int]: + """Set of special token IDs that should be excluded from random sampling.""" + self._require_init() + if isinstance(self._tokenizer, _TiktokenAdapter): + return set( + _tiktoken_internal(self._tokenizer._encoding, "_special_token_values") + ) + return set(self._tokenizer.all_special_ids) + + @property + def valid_token_ids(self) -> list[int]: + """Sorted list of all decodable, non-special token IDs. + + Tiktoken encodings have sparse ID spaces (gaps between mergeable ranks + and special tokens), so range(vocab_size) includes invalid IDs that + raise KeyError on decode. This property returns only IDs that are both + decodable and non-special. + """ + self._require_init() + if isinstance(self._tokenizer, _TiktokenAdapter): + enc = self._tokenizer._encoding + special = _tiktoken_internal(enc, "_special_token_values") + ranks = _tiktoken_internal(enc, "_mergeable_ranks") + return sorted(id_ for id_ in ranks.values() if id_ not in special) + special = self.all_special_ids + return [i for i in range(self._tokenizer.vocab_size) if i not in special] + + @property + def all_token_ids(self) -> list[int]: + """Sorted list of all decodable token IDs, including special tokens. + + For tiktoken encodings gap IDs are excluded because they cannot be + decoded. Use valid_token_ids to additionally exclude special tokens. + """ + self._require_init() + if isinstance(self._tokenizer, _TiktokenAdapter): + enc = self._tokenizer._encoding + ranks = _tiktoken_internal(enc, "_mergeable_ranks") + special = _tiktoken_internal(enc, "_special_token_values") + return sorted(set(ranks.values()) | set(special)) + return list(range(self._tokenizer.vocab_size)) + + def num_prompt_special_tokens(self) -> int: + """Number of special tokens the server adds to each prompt. + + Mirrors vllm's ``tokenizer.num_special_tokens_to_add(pair=False)``, + which returns the count from ``build_inputs_with_special_tokens([])``: + e.g. 1 for Llama/Mistral (BOS), 0 for GPT-2 (no auto-BOS despite + having ``bos_token_id``), 2 for BERT (CLS+SEP), 1 for T5 (EOS). + Used to adjust the ISL mean in RangeRatioDistribution so that + ``--isl`` represents the total server-side token count rather than + the aiperf-side content count. + + Falls back to 0 for tokenizers that don't expose + ``num_special_tokens_to_add`` (tiktoken adapter, some custom + tokenizers) — conservative rather than wrong-directionally-confident, + since ``bos_token_id`` being defined does not mean the tokenizer + prepends it (e.g. GPT-2). + """ + self._require_init() + method = getattr(self._tokenizer, "num_special_tokens_to_add", None) + if callable(method): + try: + return int(method(pair=False)) + except (TypeError, NotImplementedError): + pass + return 0 + def __repr__(self) -> str: """ Return a string representation of the underlying tokenizer. diff --git a/src/aiperf/config/dataset/config.py b/src/aiperf/config/dataset/config.py index 382bfdcd5c..d72305eef1 100644 --- a/src/aiperf/config/dataset/config.py +++ b/src/aiperf/config/dataset/config.py @@ -15,7 +15,7 @@ from __future__ import annotations from pathlib import Path -from typing import Annotated, Any, Literal +from typing import Annotated, Any, ClassVar, Literal from pydantic import ( ConfigDict, @@ -28,6 +28,7 @@ from aiperf.common.enums import ( DatasetFormat, DatasetType, + PromptCorpus, ) from aiperf.config.dataset.content import ( AudioConfig, @@ -657,6 +658,31 @@ def _validate_source_xor(self) -> FileDataset: return self + _HASH_ID_TRACE_FORMATS: ClassVar[frozenset[DatasetFormat]] = frozenset( + { + DatasetFormat.MOONCAKE_TRACE, + DatasetFormat.BAILIAN_TRACE, + DatasetFormat.BURST_GPT_TRACE, + DatasetFormat.SAGEMAKER_DATA_CAPTURE, + DatasetFormat.WEKA_TRACE, + } + ) + + @model_validator(mode="after") + def _validate_random_corpus_incompatible_with_trace(self) -> FileDataset: + if ( + self.prompts is not None + and self.prompts.corpus == PromptCorpus.RANDOM + and self.format in self._HASH_ID_TRACE_FORMATS + ): + raise ValueError( + f"prompt_corpus 'random' is not compatible with format '{self.format}'; " + "trace loaders synthesize prompt content from a text corpus keyed by " + "hash_id — there is no corpus to draw from when corpus='random'. " + "Use corpus='sonnet' or 'coding', or switch to a synthetic dataset format." + ) + return self + @model_validator(mode="after") def _validate_open_loop_strict_requires_open_loop(self) -> FileDataset: # open_loop_strict is an open-loop-only modifier; the loader would diff --git a/src/aiperf/config/dataset/content.py b/src/aiperf/config/dataset/content.py index b90ae4130e..dd4e5f1e9f 100644 --- a/src/aiperf/config/dataset/content.py +++ b/src/aiperf/config/dataset/content.py @@ -12,8 +12,9 @@ from __future__ import annotations from pathlib import Path -from typing import Annotated, Self +from typing import TYPE_CHECKING, Annotated, Self +import orjson from pydantic import ( BeforeValidator, ConfigDict, @@ -29,8 +30,13 @@ ImageSource, ImageSourceSamplingStrategy, PromptCorpus, + RandomCorpusStyle, ) from aiperf.config.base import BaseConfig + +if TYPE_CHECKING: + from aiperf.common.models.sequence_distribution import SequenceLengthSampler + from aiperf.config.types import ( FixedDistribution, SamplingDistribution, @@ -109,6 +115,21 @@ def _record_explicit_set_flags(self) -> Self: return self +def _coerce_range_ratio(v: object) -> object: + """Normalize native float/dict inputs to the string form the ratio configs expect. + + YAML parses ``random_range_ratio: 0.3`` as float and ``{input: 0.3, output: 0.5}`` + as dict. Both are valid user intent but the ratio field validators require a string. + """ + if isinstance(v, bool): + return v # let Pydantic reject booleans via type validation + if isinstance(v, (int, float)): + return str(v) + if isinstance(v, dict): + return orjson.dumps(v).decode() + return v + + class PromptConfig(BaseConfig): """ Configuration for prompt/token specifications in synthetic datasets. @@ -200,6 +221,30 @@ class PromptConfig(BaseConfig): ), ] + random_range_ratio: Annotated[ + str | None, + BeforeValidator(_coerce_range_ratio), + Field( + default=None, + description="Sample ISL and OSL uniformly from a ratio-defined integer window around the configured means. " + "Accepts a single float (applied to both ISL and OSL) or a JSON object " + '{"input": 0.3, "output": 0.5} for independent values. ' + "Mutually exclusive with sequence_distribution. " + "When a tokenizer is configured, the ISL mean is automatically reduced by " + "tokenizer.num_special_tokens_to_add(pair=False) so --isl represents total server-side input tokens.", + ), + ] + + random_corpus_style: Annotated[ + RandomCorpusStyle, + Field( + default=RandomCorpusStyle.VLLM, + description="Sampling formula for random_range_ratio. " + "vllm (default): symmetric window [floor(mean*(1-r)), ceil(mean*(1+r))], ratio in [0, 1). " + "sglang: lower-bounded window [max(1, int(mean*r)), mean], ratio in [0, 1].", + ), + ] + @field_validator("sequence_distribution") @classmethod def validate_sequence_probabilities( @@ -209,6 +254,104 @@ def validate_sequence_probabilities( validate_probability_distribution(v) return v + @model_validator(mode="after") + def _validate_random_range_ratio(self) -> Self: + if self.random_range_ratio is None: + return self + from aiperf.common.models.sequence_distribution import _CLASS_FOR_MODE + + if self.sequence_distribution is not None: + raise ValueError( + "random_range_ratio cannot be combined with sequence_distribution; " + "use one or the other." + ) + + if self.isl is None: + raise ValueError( + "--random-range-ratio requires --isl to be set explicitly. " + "There is no safe default when a ratio window is requested." + ) + if self.osl is None: + raise ValueError( + "--random-range-ratio requires --osl to be set explicitly. " + "There is no safe default when a ratio window is requested." + ) + + # Checked here rather than left to the ratio config's `ge=1` bound: both + # means are declared `ge=0`, so `--isl 0` is a legal value that only + # fails once the sampler is constructed, surfacing a nested pydantic + # error about `isl_mean` that names neither flag. The tokenizer-aware + # half of this check (vLLM's min_total_input guard, which needs + # num_special_tokens and the prefix length) lives in + # BaseDatasetComposer._build_sequence_distribution. + if int(self.isl.expected_value) < 1: + raise ValueError( + "--random-range-ratio requires --isl >= 1, got " + f"{int(self.isl.expected_value)}. A zero-token input window has " + "no prompt to scale." + ) + if int(self.osl.expected_value) < 1: + raise ValueError( + "--random-range-ratio requires --osl >= 1, got " + f"{int(self.osl.expected_value)}. A zero-token output window " + "would request no generation." + ) + + try: + _CLASS_FOR_MODE[self.random_corpus_style].get_config_class()( + isl_mean=int(self.isl.expected_value), + osl_mean=int(self.osl.expected_value), + isl_stddev=float(getattr(self.isl, "stddev", 0.0) or 0.0), + osl_stddev=float(getattr(self.osl, "stddev", 0.0) or 0.0), + range_ratio=self.random_range_ratio, + ) + except Exception as e: + raise ValueError(f"Invalid random_range_ratio value: {e}") from e + + return self + + def get_sequence_distribution( + self, num_special_tokens: int = 0 + ) -> SequenceLengthSampler | None: + """Return a sampler for (ISL, OSL) pairs, or None if not configured.""" + if self.sequence_distribution is not None: + from aiperf.common.models.sequence_distribution import ( + SequenceLengthDistribution, + SequenceLengthPair, + ) + + pairs = [ + SequenceLengthPair( + input_seq_len=int(entry.isl.expected_value), + output_seq_len=int(entry.osl.expected_value), + probability=float(entry.probability), + input_seq_len_stddev=float( + getattr(entry.isl, "stddev", 0.0) or 0.0 + ), + output_seq_len_stddev=float( + getattr(entry.osl, "stddev", 0.0) or 0.0 + ), + ) + for entry in self.sequence_distribution + ] + return SequenceLengthDistribution(pairs) + + if self.random_range_ratio is not None: + from aiperf.common.models.sequence_distribution import _CLASS_FOR_MODE + + DistClass = _CLASS_FOR_MODE[self.random_corpus_style] + config = DistClass.get_config_class()( + isl_mean=int(self.isl.expected_value), + osl_mean=int(self.osl.expected_value), + isl_stddev=0.0, + osl_stddev=0.0, + range_ratio=self.random_range_ratio, + num_special_tokens=num_special_tokens, + ) + return DistClass(config) + + return None + class PromptSelectionConfig(BaseConfig): """Slim prompts block for file/public datasets (corpus selection only).""" @@ -262,8 +405,8 @@ class PrefixPromptConfig(BaseConfig): default=None, description="Token length for each prefix prompt in the pool. " "Only used when pool_size is set. " - "Note: due to prefix and user prompts being concatenated, " - "the final prompt token count may be off by one. " + "The prefix is concatenated to the body at the token level, so the " + "final prompt contains exactly prefix length + input length tokens. " "Mutually exclusive with shared_system_length/user_context_length.", ), ] diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index ea7c4342ee..99a086693c 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -426,6 +426,45 @@ def _apply_sequence_distribution(d: dict[str, Any], cli: CLIConfig) -> None: ] +def _apply_random_corpus_style_and_range_ratio( + d: dict[str, Any], cli: CLIConfig +) -> None: + """Write ``random_corpus_style`` and ``random_range_ratio`` onto the prompts dict. + + The style is written FIRST, before the ratio gate. It is not merely a + modifier of the ratio: with no ratio set it still selects the token pool + (``valid_token_ids`` vs ``all_token_ids``), which is then the only thing it + selects. Writing it after an early return meant ``--random-corpus-style + sglang`` alone was silently discarded and the user got vLLM's pool. + + Only the sglang request was observably lost, since vllm is the default and + a dropped write lands there anyway -- which is why this survived so long. + """ + if "prompt_random_corpus_style" in cli.model_fields_set: + d.setdefault("prompts", {})["random_corpus_style"] = ( + cli.prompt_random_corpus_style + ) + + if not cli.prompt_random_range_ratio: + return + from aiperf.common.enums import RandomCorpusStyle + from aiperf.common.models.sequence_distribution import ( + _parse_sglang_ratio_string, + _parse_vllm_ratio_string, + ) + + parser = ( + _parse_sglang_ratio_string + if cli.prompt_random_corpus_style == RandomCorpusStyle.SGLANG + else _parse_vllm_ratio_string + ) + try: + parser(cli.prompt_random_range_ratio) + except ValueError as e: + raise ValueError(f"Invalid --random-range-ratio value: {e}") from e + d.setdefault("prompts", {})["random_range_ratio"] = cli.prompt_random_range_ratio + + def _apply_turns(d: dict[str, Any], cli: CLIConfig) -> None: fields_set = cli.model_fields_set if ( @@ -551,6 +590,7 @@ def _apply_implicit_media_batch(d: dict[str, Any], cli: CLIConfig) -> None: ), ("prompt_batch_size", "--prompt-batch-size/--batch-size-text"), ("prompt_sequence_distribution", "--seq-dist/--sequence-distribution"), + ("prompt_random_range_ratio", "--random-range-ratio"), ("image_batch_size", "--image-batch-size"), ("image_source", "--image-source"), ("image_source_sampling", "--image-source-sampling"), @@ -995,6 +1035,7 @@ def _apply_block_size(d: dict[str, Any], cli: CLIConfig) -> None: ), ("prompt_batch_size", "--prompt-batch-size/--batch-size-text"), ("prompt_sequence_distribution", "--seq-dist/--sequence-distribution"), + ("prompt_random_range_ratio", "--random-range-ratio"), ) # Tokenizer options are also rejected for non-tokenizing endpoints @@ -1073,6 +1114,7 @@ def build_dataset(cli: CLIConfig) -> dict[str, Any]: _attach_subtables(d, cli) _apply_dataset_type(d, cli, needs_text) _apply_sequence_distribution(d, cli) + _apply_random_corpus_style_and_range_ratio(d, cli) _apply_turns(d, cli) _apply_synthesis(d, cli) _apply_implicit_media_batch(d, cli) diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 936a82b84c..01e7d10d68 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -49,6 +49,7 @@ ImageSourceSamplingStrategy, ModelSelectionStrategy, PromptCorpus, + RandomCorpusStyle, RequestContentType, ServerMetricsFormat, SweepMode, @@ -1366,6 +1367,54 @@ def url(self) -> str: ), ] = None + prompt_random_range_ratio: Annotated[ + str | None, + Field( + default=None, + description="Sample ISL and OSL uniformly from a ratio-defined integer window around the configured means. " + "The window is computed from `--random-corpus-style` (defaults to `vllm`): " + "vllm style → `[floor(mean*(1-r)), ceil(mean*(1+r))]` (symmetric); " + "sglang style → `[max(1, int(mean*r)), mean]` (lower-bounded). " + "Accepts a single float applied to both ISL and OSL. The JSON object form " + '`{"input": 0.3, "output": 0.5}` for independent values is accepted only under ' + "vllm style; sglang style applies one ratio to both and requires a plain float. " + "Requires both `--isl` and `--osl` to be set explicitly — there is no default " + "mean for either. " + "Mutually exclusive with `--seq-dist`. " + "Under vllm style with a tokenizer configured, the ISL mean is automatically reduced by " + "`tokenizer.num_special_tokens_to_add(pair=False)` so `--isl` represents total " + "server-side input tokens, matching `vllm bench serve` semantics; sglang style " + "applies no such adjustment.", + ), + CLIParameter( + name=("--random-range-ratio",), + group=Groups.ISL, + ), + ] = None + + prompt_random_corpus_style: Annotated[ + RandomCorpusStyle, + Field( + default=RandomCorpusStyle.VLLM, + description="Benchmark style for RANDOM corpus generation. " + "Controls range ratio formula, token pool composition, and other " + "per-tool behaviors. " + "`vllm` (default) mirrors `vllm bench serve`: symmetric range window, " + "non-special token pool (special tokens excluded). " + "`sglang` mirrors `sglang.benchmark.serving` run with `--dataset-name random-ids`: " + "lower-bounded range window, full `range(vocab_size)` token pool (no exclusion). " + "Applies in two independent places: token-pool selection whenever " + "`--prompt-corpus random` is set (including with no `--random-range-ratio`, " + "where the pool is the only thing the style selects), and the range-window " + "formula, special-token accounting and RNG algorithm whenever " + "`--random-range-ratio` is set — the latter regardless of corpus.", + ), + CLIParameter( + name=("--random-corpus-style",), + group=Groups.ISL, + ), + ] = RandomCorpusStyle.VLLM + ############################################################################## # Output Sequence Length (OSL) ############################################################################## diff --git a/src/aiperf/dataset/composer/base.py b/src/aiperf/dataset/composer/base.py index 25d41ea27e..9fd81f841d 100644 --- a/src/aiperf/dataset/composer/base.py +++ b/src/aiperf/dataset/composer/base.py @@ -8,21 +8,21 @@ from aiperf.common import random_generator as rng from aiperf.common.constants import SYSTEM_PROMPT_JOIN_SEP -from aiperf.common.enums import ConversationContextMode, ModelSelectionStrategy +from aiperf.common.enums import ( + ConversationContextMode, + ModelSelectionStrategy, + RandomCorpusStyle, +) from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.common.models import Conversation, Turn -from aiperf.common.models.sequence_distribution import ( - SequenceLengthDistribution, - SequenceLengthPair, -) +from aiperf.common.models.sequence_distribution import SequenceLengthSampler from aiperf.common.tokenizer import Tokenizer from aiperf.config.dataset import SyntheticDataset from aiperf.dataset.generator.audio import AudioGenerator -from aiperf.dataset.generator.coding_content import CodingContentGenerator from aiperf.dataset.generator.corpus import resolve_prompt_generator from aiperf.dataset.generator.image import ImageGenerator -from aiperf.dataset.generator.prompt import PromptGenerator from aiperf.dataset.generator.video import VideoGenerator +from aiperf.dataset.protocols import CorpusGeneratorProtocol if TYPE_CHECKING: from aiperf.config.dataset import VideoConfig @@ -54,11 +54,17 @@ def _estimate_chat_template_overheads( wire_tokens = per_request_fixed + Σ_{m in messages} (per_msg_wrap + content_tokens(m)) - where ``per_request_fixed`` ≈ BOS + generation-prompt suffix and + where ``per_request_fixed`` ≈ generation-prompt suffix and ``per_msg_wrap`` ≈ role header + end-of-turn marker (averaged over - user/assistant). Measuring the two separately lets callers apply the - fixed cost only to the first user turn and the per-message wrap to - every turn. + user/assistant). Measuring the two separately lets callers apply the fixed + cost only to the first user turn and the per-message wrap to every turn. + + BOS is deliberately excluded from ``per_request_fixed`` because every ISL + path already subtracts special tokens on its own, and double-counting would + shorten prompts: VLLM style folds them into the window bounds + (``VLLMRatioConfig.compute_input_bounds``), SGLANG style shifts each drawn + length (``SGLangRangeRatioDistribution.adjust_sampled_isl``), and the + non-range-ratio path subtracts in ``_get_turn_sequence_lengths``. Returns ``(0, 0)`` when the tokenizer is ``None``/has no underlying HF tokenizer, has no ``apply_chat_template`` (e.g. tiktoken), the model has @@ -72,29 +78,34 @@ def _estimate_chat_template_overheads( if apply is None: return 0, 0 + bos_tokens = tokenizer.num_prompt_special_tokens() fixed_costs: list[float] = [] wrap_costs: list[float] = [] for sample in _CHAT_TEMPLATE_PROBE_SAMPLES: try: - single = apply( + single_text = apply( [{"role": "user", "content": sample}], - tokenize=True, + tokenize=False, add_generation_prompt=True, ) - triple = apply( + triple_text = apply( [ {"role": "user", "content": sample}, {"role": "assistant", "content": sample}, {"role": "user", "content": sample}, ], - tokenize=True, + tokenize=False, add_generation_prompt=True, ) except Exception: return 0, 0 + if not isinstance(single_text, str) or not isinstance(triple_text, str): + return 0, 0 + single_len = len(tokenizer.encode(single_text)) + triple_len = len(tokenizer.encode(triple_text)) bare_len = len(tokenizer.encode(sample)) - avg_wrap = (len(triple) - len(single) - 2 * bare_len) / 2 - per_request_fixed = len(single) - bare_len - avg_wrap + avg_wrap = (triple_len - single_len - 2 * bare_len) / 2 + per_request_fixed = single_len - bare_len - avg_wrap - bos_tokens if avg_wrap < 0 or per_request_fixed < 0: return 0, 0 wrap_costs.append(avg_wrap) @@ -148,7 +159,7 @@ def __init__( compensated_prefix_prompts = self._init_isl_compensation(tokenizer) # Create generators (prompt generator requires a tokenizer) - self.prompt_generator: PromptGenerator | CodingContentGenerator | None = ( + self.prompt_generator: CorpusGeneratorProtocol | None = ( resolve_prompt_generator( corpus=self.run.cfg.get_prompt_corpus(), default_corpus=None, @@ -168,13 +179,12 @@ def __init__( self.turn_count = 0 - # ``PromptConfig.sequence_distribution`` is a - # ``list[SequenceDistributionEntry]`` of typed ``SamplingDistribution`` - # objects. Convert each entry directly to a ``SequenceLengthPair`` - # (extracting mean + stddev from the underlying distribution) and - # build the runtime distribution without re-serializing through - # ``DistributionParser.parse``, which only accepts strings. - self._seq_distribution = self._build_sequence_distribution() + self._num_special_tokens = ( + tokenizer.num_prompt_special_tokens() if tokenizer is not None else 0 + ) + self._seq_distribution: SequenceLengthSampler | None = ( + self._build_sequence_distribution(self._num_special_tokens) + ) # Cache for turn-level sequence lengths to ensure ISL/OSL pairing consistency self._turn_sequence_cache: dict[int, tuple[int, int]] = {} @@ -274,15 +284,23 @@ def _init_isl_compensation( return prefix_prompts.model_copy( update={"shared_system_length": compensated_shared_sys_len} ) + + # Prefix prompts are additive: total wire ISL is body + prefix, matching + # vLLM's --random-prefix-len and SGLang's prefix_len. The prefix is NOT + # subtracted from the ISL budget, so --prompt-input-tokens-mean describes + # the body and the prefix rides on top. return prefix_prompts @property def first_turn_isl_adjustment(self) -> int: """Total tokens to subtract from the FIRST user turn's synthetic ISL. - Composed of the per-request chat-template fixed cost (BOS + gen-prompt + Composed of the per-request chat-template fixed cost (gen-prompt suffix), the per-message chat-template wrap (role header + EOT), and the cache-bust marker when it lands on the first user turn. + + Prefix prompts are deliberately absent: they are additive, so the body + keeps the full configured ISL and the prefix is prepended on top. """ return ( self._chat_template_per_request_fixed_tokens @@ -311,33 +329,77 @@ def create_dataset(self) -> list[Conversation]: """ ... - def _build_sequence_distribution(self) -> SequenceLengthDistribution | None: - """Build a runtime sequence-length distribution from config entries. + def _build_sequence_distribution( + self, num_special_tokens: int = 0 + ) -> SequenceLengthSampler | None: + """Build a runtime sequence-length sampler from config. - ``PromptConfig.sequence_distribution`` is a list of - ``SequenceDistributionEntry`` carrying typed ``SamplingDistribution`` - ISL/OSL fields (Fixed/Normal/LogNormal/...). Pull the mean and the - normal-distribution stddev (0 for non-normal types) off each entry to - construct ``SequenceLengthPair`` directly. ``DistributionParser.parse`` - only accepts strings and would reject this list shape. + Delegates to ``PromptConfig.get_sequence_distribution()`` which handles + both ``sequence_distribution`` (probabilistic ISL/OSL pairs) and + ``random_range_ratio`` (uniform window sampling). + + Chat-template overhead is deliberately NOT passed down. It is already + subtracted per-request by ``first_turn_isl_adjustment`` / + ``subsequent_turn_isl_adjustment``; feeding it to the window as well + charged it twice (AIP-873 review) and shrank wire ISL below the + configured value under ``--apply-chat-template``. """ if self._synthetic_prompts is None: return None - entries = self._synthetic_prompts.sequence_distribution - if not entries: - return None + self._reject_degenerate_range_ratio(num_special_tokens) + return self._synthetic_prompts.get_sequence_distribution( + num_special_tokens=num_special_tokens, + ) - pairs = [ - SequenceLengthPair( - input_seq_len=int(entry.isl.expected_value), - output_seq_len=int(entry.osl.expected_value), - probability=float(entry.probability), - input_seq_len_stddev=float(getattr(entry.isl, "stddev", 0.0) or 0.0), - output_seq_len_stddev=float(getattr(entry.osl, "stddev", 0.0) or 0.0), - ) - for entry in entries - ] - return SequenceLengthDistribution(pairs) + def _reject_degenerate_range_ratio(self, num_special_tokens: int) -> None: + """Fail when the ratio window cannot yield a usable prompt. + + Mirrors vLLM's ``RandomDataset.sample`` guard:: + + min_total_input = prefix_len + floor(max(0, isl - num_special) * (1 - r)) + if min_total_input < 1: raise ValueError(...) + + Without it, ``--isl 2 --random-range-ratio 0.9`` yields bounds ``(0, 4)`` + and ``calculate_num_tokens`` silently clamps draws of 0 up to 1, so the + run completes and reports numbers for one-token prompts the user never + asked for. vLLM refuses the config instead. + + Lives here rather than in ``PromptConfig`` because the predicate needs + ``num_special_tokens`` (tokenizer-dependent, unknown at config-parse + time) and the prefix length (a sibling config the prompt validator + cannot reach). + + Only VLLM-style windows can bottom out: the SGLANG window is + ``[max(1, int(mean*r)), mean]``, whose lower bound is >= 1 by + construction. + """ + prompts = self._synthetic_prompts + if prompts is None or prompts.random_range_ratio is None: + return + if prompts.random_corpus_style != RandomCorpusStyle.VLLM: + return + + distribution = prompts.get_sequence_distribution( + num_special_tokens=num_special_tokens + ) + low = getattr(distribution, "input_bounds", (1, 1))[0] + prefix_len = ( + getattr(self._synthetic_prefix_prompts, "length", None) or 0 + if self._synthetic_prefix_prompts is not None + else 0 + ) + if prefix_len + low >= 1: + return + + isl_mean = int(prompts.isl.expected_value) if prompts.isl is not None else 0 + raise ValueError( + f"--random-range-ratio {prompts.random_range_ratio} with --isl " + f"{isl_mean} produces a minimum input of {prefix_len + low} tokens " + f"(tokenizer adds {num_special_tokens} special token(s), prefix " + f"contributes {prefix_len}). Increase --isl, add " + "--prompt-prefix-length, or lower --random-range-ratio so the " + "minimum is at least 1 token." + ) def _osl_distribution(self) -> SamplingDistribution | None: """Resolve the OSL distribution to use as a fallback for max_tokens. @@ -395,6 +457,17 @@ def _get_turn_sequence_lengths(self, turn_id: int) -> tuple[int, int]: and self._synthetic_prompts.isl is not None else 0 ) + # Subtract the tokens the server prepends (BOS et al.), mirroring + # what VLLMRatioConfig.compute_input_bounds does for the + # range-ratio path. Without it this path was the only one not + # accounting for them, so --isl 128 on a BOS tokenizer put 129 on + # the wire. _estimate_chat_template_overheads deliberately excludes + # BOS from per_request_fixed on the assumption the mean was already + # adjusted -- an assumption only the range-ratio path was meeting. + # max(0, ...) matches vLLM's real_input_len; the downstream + # sample_positive_normal_integer floors the draw at 1. + if isl_mean > 0: + isl_mean = max(0, isl_mean - self._num_special_tokens) osl_mean = ( int(self._synthetic_prompts.osl.expected_value) if self._synthetic_prompts is not None diff --git a/src/aiperf/dataset/composer/synthetic.py b/src/aiperf/dataset/composer/synthetic.py index e540f3468d..3cca81a12b 100644 --- a/src/aiperf/dataset/composer/synthetic.py +++ b/src/aiperf/dataset/composer/synthetic.py @@ -5,7 +5,9 @@ from typing import TYPE_CHECKING from aiperf.common import random_generator as rng +from aiperf.common.enums import PromptCorpus from aiperf.common.models import Audio, Conversation, Image, Text, Turn, Video +from aiperf.common.models.sequence_distribution import RangeRatioDistribution from aiperf.common.session_id_generator import SessionIDGenerator from aiperf.common.tokenizer import Tokenizer from aiperf.config.dataset import SyntheticDataset @@ -98,6 +100,28 @@ def __init__(self, *, run: BenchmarkRun, tokenizer: Tokenizer | None, **kwargs): "setting the mean to a positive value." ) + # RNG-aligned preseed: pre-generate ISLs, OSLs, and offsets upfront so + # the draw order matches the target tool's benchmark implementation. + # Each RangeRatioDistribution subclass owns its own RNG algorithm: + # - RangeRatioDistribution (VLLM): numpy.random.default_rng (PCG64) + # - SGLangRangeRatioDistribution (SGLANG): numpy.random (MT19937) + # The preseed() call stores _preseed_rng for PromptGenerator to continue + # from the same stream for offset draws. No style branching needed here. + if ( + self.prompt_generator is not None + and getattr(self.prompt_generator, "_corpus", None) == PromptCorpus.RANDOM + and isinstance(self._seq_distribution, RangeRatioDistribution) + ): + self._seq_distribution.preseed(self._num_entries, run.random_seed) + self.prompt_generator.preseed( + self._num_entries, self._seq_distribution._preseed_rng + ) + + # Must follow preseed: the RANDOM corpus draws its prefixes from the shared + # stream at vLLM's position (after the offset draws). No-op without a pool. + if self.prompt_generator is not None: + self.prompt_generator.initialize_prefix_pool() + def create_dataset(self) -> list[Conversation]: """Create a synthetic conversation dataset from the given configuration. @@ -196,31 +220,44 @@ def _generate_text_payloads(self, turn: Turn, is_first: bool) -> Text: turn_id = id(turn) isl, _ = self._get_turn_sequence_lengths(turn_id) + # The prefix is prepended inside the generator at the token level so that + # prefix + body tokenizes to exactly prefix_len + isl (AIP-1118). + with_prefix = is_first and self.prefix_prompt_enabled + # ISL budget compensation. See ``base.first_turn_isl_adjustment`` and - # ``base.subsequent_turn_isl_adjustment`` for the model. Floored at 1 so - # prompt generation stays valid for very small ISLs (a one-token prompt - # rather than a crash or empty content). + # ``base.subsequent_turn_isl_adjustment`` for the model. adjustment = ( self.first_turn_isl_adjustment if is_first else self.subsequent_turn_isl_adjustment ) + # A range-ratio window can legitimately bottom out at 0 (bounds (0, N)), + # and the degenerate-config guard deliberately accepts that when a + # prefix covers the request -- vLLM does too, emitting prefix-only. So + # only floor at 1 when there is no prefix to carry it; flooring + # unconditionally emitted one more token than the reference. + floor = 0 if with_prefix else 1 if adjustment > 0: - isl = max(1, isl - adjustment) + isl = max(floor, isl - adjustment) + isl = max(floor, isl) + + # An active sequence distribution has already produced an exact length; + # resampling it through calculate_num_tokens would floor a sampled 0 to 1. + exact_length = self._seq_distribution is not None # Preserve original variance unless sequence distribution is active stddev = 0 if self._seq_distribution is not None else self._isl_stddev for _ in range(self._prompt_batch_size): # Generate prompt content using the sampled input sequence length - content = self.prompt_generator.generate(mean=isl, stddev=stddev) - - # Add prefix prompt if this is the first turn and prefix is enabled - if is_first and self.prefix_prompt_enabled: - prefix = self.prompt_generator.get_random_prefix_prompt() - content = f"{prefix} {content}" - - text.contents.append(content) + text.contents.append( + self.prompt_generator.generate( + mean=isl, + stddev=stddev, + with_prefix=with_prefix, + exact_length=exact_length, + ) + ) return text diff --git a/src/aiperf/dataset/generator/coding_content.py b/src/aiperf/dataset/generator/coding_content.py index 73780f2a5d..a43e59ce87 100644 --- a/src/aiperf/dataset/generator/coding_content.py +++ b/src/aiperf/dataset/generator/coding_content.py @@ -725,6 +725,7 @@ def __init__( self._bpe_stable_terminator_tokens: list[int] = [] self._prefix_prompts: list[str] = [] + self._prefix_prompt_tokens: list[list[int]] = [] self._shared_system_prompt: str | None = None self._user_context_prompts: list[str] = [] @@ -748,23 +749,45 @@ def __init__( if shared_system_length is not None: self._generate_shared_system_prompt() + def preseed(self, n: int, generator: object) -> None: + """No-op — CodingContentGenerator draws on demand, no preseed needed.""" + + def initialize_prefix_pool(self) -> None: + """No-op — the coding prefix pool is built during construction. + + Present so callers can invoke it uniformly across generators; only + PromptGenerator needs the post-preseed ordering. + """ + def generate( self, mean: int | None = None, stddev: int | None = None, hash_ids: list[int] | None = None, block_size: int | None = None, + *, + with_prefix: bool = False, + exact_length: bool = False, ) -> str: if hash_ids: if mean is None: raise ValueError("mean must be provided when hash_ids is set.") bs = block_size or self.config.block_size return self._generate_cached_prompt(mean, hash_ids, bs) - num_tokens = self.calculate_num_tokens(mean, stddev) - return self.generate_prompt(num_tokens) + # An active sequence distribution already produced an exact length; + # resampling would floor a legitimately-sampled 0 up to 1. + num_tokens = ( + int(mean or 0) if exact_length else self.calculate_num_tokens(mean, stddev) + ) + prefix_tokens = self.get_random_prefix_prompt_tokens() if with_prefix else None + return self.generate_prompt(num_tokens, prefix_tokens=prefix_tokens) - def generate_prompt(self, num_tokens: int) -> str: + def generate_prompt( + self, num_tokens: int, prefix_tokens: list[int] | None = None + ) -> str: tokens = self._sample_tokens(num_tokens, self._tool_pool) + if prefix_tokens: + tokens = list(prefix_tokens) + list(tokens) return self.tokenizer.decode(tokens) def calculate_num_tokens( @@ -780,19 +803,38 @@ def _create_prefix_prompt_pool(self) -> None: return length = self.prefix_prompts.length or 0 pool_size = self.prefix_prompts.pool_size or 0 - self._prefix_prompts = [self.generate_prompt(length) for _ in range(pool_size)] + self._prefix_prompt_tokens = [ + list(self._sample_tokens(length, self._tool_pool)) for _ in range(pool_size) + ] + self._prefix_prompts = [ + self.tokenizer.decode(tokens) for tokens in self._prefix_prompt_tokens + ] self.debug( - lambda: f"Initialized coding prefix prompts pool with {len(self._prefix_prompts)} prompts" + lambda: ( + f"Initialized coding prefix prompts pool with {len(self._prefix_prompts)} prompts" + ) ) - def get_random_prefix_prompt(self) -> str: - """Fetch a random prefix prompt from the coding corpus pool.""" - if not self._prefix_prompts: + def _random_prefix_index(self) -> int: + """Pick a pool slot, raising when the pool was never initialized.""" + if not self._prefix_prompt_tokens: raise InvalidStateError( "Attempted to sample a prefix prompt but the prefix prompts pool is empty. " "Please ensure that the prefix prompts pool is initialized." ) - return self._prefix_rng.choice(self._prefix_prompts) + return self._prefix_rng.choice(range(len(self._prefix_prompt_tokens))) + + def get_random_prefix_prompt(self) -> str: + """Fetch a random prefix prompt from the coding corpus pool.""" + return self._prefix_prompts[self._random_prefix_index()] + + def get_random_prefix_prompt_tokens(self) -> list[int]: + """Fetch the token IDs of a random prefix prompt from the coding pool. + + Concatenating token IDs rather than strings avoids the BPE seam cost + described in :meth:`PromptGenerator.get_random_prefix_prompt_tokens`. + """ + return self._prefix_prompt_tokens[self._random_prefix_index()] def _generate_shared_system_prompt(self) -> None: """Generate the shared system prompt once from the coding corpus.""" @@ -832,8 +874,10 @@ def generate_user_context_prompt(self, session_index: int) -> str: while session_index >= len(self._user_context_prompts): self._user_context_prompts.append(self.generate_prompt(length)) self.debug( - lambda: f"Generated coding user context prompt " - f"#{len(self._user_context_prompts) - 1}" + lambda: ( + f"Generated coding user context prompt " + f"#{len(self._user_context_prompts) - 1}" + ) ) return self._user_context_prompts[session_index] @@ -864,8 +908,10 @@ def _build_tool_pool(self) -> None: text = "\n\n".join(blocks) self._tool_pool = self.tokenizer.encode(text) self.debug( - lambda: f"Built tool pool with {len(self._tool_pool)} tokens " - f"from {len(blocks)} blocks" + lambda: ( + f"Built tool pool with {len(self._tool_pool)} tokens " + f"from {len(blocks)} blocks" + ) ) def _sample_tokens(self, num_tokens: int, pool: list[int]) -> list[int]: diff --git a/src/aiperf/dataset/generator/corpus.py b/src/aiperf/dataset/generator/corpus.py index 429bc66f00..11c5fda885 100644 --- a/src/aiperf/dataset/generator/corpus.py +++ b/src/aiperf/dataset/generator/corpus.py @@ -10,6 +10,7 @@ from aiperf.config.dataset.content import PrefixPromptConfig, PromptConfig from aiperf.dataset.generator.coding_content import CodingContentGenerator from aiperf.dataset.generator.prompt import PromptGenerator +from aiperf.dataset.protocols import CorpusGeneratorProtocol def resolve_prompt_generator( @@ -19,7 +20,7 @@ def resolve_prompt_generator( tokenizer: Tokenizer, prompts: PromptConfig | None = None, prefix_prompts: PrefixPromptConfig | None = None, -) -> PromptGenerator | CodingContentGenerator: +) -> CorpusGeneratorProtocol: """Pick sonnet vs coding generator from authored corpus + loader default. Resolution: explicit ``corpus`` -> ``default_corpus`` -> ``PromptCorpus.SONNET``. @@ -31,6 +32,21 @@ def resolve_prompt_generator( tokenizer=tokenizer, prefix_prompts=prefix_prompts, ) + if resolved == PromptCorpus.RANDOM or resolved == "random": + from aiperf.common.enums import RandomCorpusStyle + + style = ( + prompts.random_corpus_style + if prompts is not None + else RandomCorpusStyle.VLLM + ) + return PromptGenerator( + prompts=prompts, + prefix_prompts=prefix_prompts, + tokenizer=tokenizer, + corpus=PromptCorpus.RANDOM, + corpus_style=style, + ) return PromptGenerator( prompts=prompts, prefix_prompts=prefix_prompts, diff --git a/src/aiperf/dataset/generator/prompt.py b/src/aiperf/dataset/generator/prompt.py index 1f3cf9ba99..6feaad6eee 100644 --- a/src/aiperf/dataset/generator/prompt.py +++ b/src/aiperf/dataset/generator/prompt.py @@ -6,6 +6,7 @@ from pathlib import Path from aiperf.common import random_generator as rng +from aiperf.common.enums import PromptCorpus, RandomCorpusStyle from aiperf.common.exceptions import ( ConfigurationError, InvalidStateError, @@ -80,14 +81,23 @@ def __init__( prompts: PromptConfig | None, prefix_prompts: PrefixPromptConfig | None, tokenizer: Tokenizer, + corpus: PromptCorpus = PromptCorpus.SONNET, + corpus_style: RandomCorpusStyle = RandomCorpusStyle.VLLM, **kwargs, ): self.prompts = prompts self.prefix_prompts = prefix_prompts self.tokenizer = tokenizer + self._corpus = corpus + self._corpus_style = corpus_style self._tokenized_corpus = None self._corpus_size = 0 + self._allowed_tokens: list[int] = [] + self._random_request_index: int = 0 self._prefix_prompts: list[str] = [] + self._topup_tokens: list[int] = [] + self._warned_offsets_exhausted = False + self._prefix_prompt_tokens: list[list[int]] = [] # Conversation context prompts self._shared_system_prompt: str | None = None @@ -117,6 +127,9 @@ def __init__( if self._tokenized_corpus is None: self._initialize_corpus() + if self._corpus == PromptCorpus.RANDOM: + self._build_allowed_tokens() + # Probe the tokenizer for a BPE-stable terminator we can append to # every reconstructed segment so aiperf's join-with-" " ISL formula # equals the sum of per-segment token counts (eliminates segment-join @@ -125,12 +138,11 @@ def __init__( self._determine_bpe_stable_terminator() ) - # Initialize prefix prompts pool if the pool size > 0 - pool_size = ( - self.prefix_prompts.pool_size if self.prefix_prompts is not None else None - ) or 0 - if pool_size > 0: - self._create_prefix_prompt_pool() + # The prefix prompt pool is NOT built here. vLLM's RandomDataset draws its + # prefix from the shared generator immediately after get_sampling_params has + # drawn ISLs, OSLs and offsets, so the pool has to observe that same stream + # position — which only exists after preseed(). Callers invoke + # initialize_prefix_pool() once the generator is seeded. # Initialize shared context prompts if configured shared_system_length = ( @@ -142,9 +154,106 @@ def __init__( self._generate_shared_system_prompt() # Note: User context prompts are generated on-demand in generate_user_context_prompt() + def _build_allowed_tokens(self) -> None: + """Build the list of token IDs for random vocab sampling. + + Token pool composition is controlled by corpus_style: + - VLLM: exclude special tokens via valid_token_ids, matching vLLM's + ``all_special_ids`` exclusion in RandomDataset. + - SGLANG: use all_token_ids (full vocab range without special-token + filtering), matching SGLang's ``% vocab_size`` arithmetic which uses + the raw vocab_size as modulus and does not exclude special tokens. + """ + if self._corpus_style == RandomCorpusStyle.VLLM: + self._allowed_tokens = self.tokenizer.valid_token_ids + else: + self._allowed_tokens = self.tokenizer.all_token_ids + + # Separate, wider pool for BPE top-up draws. vLLM's + # gen_prompt_decode_to_target_len tops up from the full + # [0, vocab_size) range without excluding special tokens, so the + # narrower VLLM-style _allowed_tokens would be the wrong pool here. + # + # Byte parity: for HF tokenizers all_token_ids IS list(range(vocab_size)), + # so indexing into it is bit-for-bit identical to drawing raw + # [0, vocab_size) -- same bound, same value. For tiktoken the two differ + # (cl100k_base: 100261 decodable vs vocab_size 100277) and the raw draw + # could return a gap ID that raises KeyError on decode. + self._topup_tokens = self.tokenizer.all_token_ids + self.debug( + lambda: ( + f"Built random vocab corpus with {len(self._allowed_tokens)} allowed tokens " + f"({len(self._topup_tokens)} in the top-up pool)" + ) + ) + + def _sample_topup_tokens(self, num_tokens: int) -> list[int]: + """Sample tokens for BPE top-up without consuming the preseed offset cache. + + Top-up draws happen inside the retry loop of generate_prompt when + re-encoding produces fewer tokens than the target. They must not + advance _random_request_index (which would shift the arithmetic-sequence + index for all subsequent requests) and must not consume preseed cache + entries (one cache entry per request, not per retry). + + When preseed is active, draws from the same generator stream used for + ISL/OSL/offset draws (matching vLLM's gen_prompt_decode_to_target_len + which uses its single shared rng for top-up). Falls back to + _corpus_rng when preseed is not in use. + """ + if self._corpus == PromptCorpus.RANDOM: + # vLLM bench passes rng=self._rng (the shared seeded generator) to + # gen_prompt_decode_to_target_len, drawing top-up tokens from the + # same stream as ISL/OSL/offsets over the full vocab. + # + # Indexed into _topup_tokens rather than emitted as raw draws so + # sparse-vocabulary tokenizers cannot yield an undecodable ID. This + # is a no-op for HF tokenizers, where the pool is exactly + # range(vocab_size) -- same bound, same values, byte-identical + # prompts. See _build_allowed_tokens. + pool = self._topup_tokens or self._allowed_tokens + generator = getattr(self, "_preseed_rng", None) or self._corpus_rng + return [ + pool[int(i)] + for i in generator.integers(0, len(pool), size=num_tokens) # type: ignore[union-attr] + ] + return self._sample_tokens(num_tokens) + + def preseed(self, n: int, generator: object) -> None: + """Pre-generate all offsets for the RANDOM corpus using vLLM's draw order. + + Must be called after :meth:`_build_allowed_tokens`. ``generator`` should + be the same numpy Generator passed to + :meth:`RangeRatioDistribution.preseed`, advanced past its ISL/OSL draws, + so that offsets are drawn from the correct position in the shared stream. + + The generator is retained as ``_preseed_rng`` so that BPE top-up draws + in :meth:`_sample_topup_tokens` continue from the same stream, matching + vLLM's ``gen_prompt_decode_to_target_len`` which uses its single shared + rng for both the initial sequence and any top-up tokens. + + """ + # vLLM bench draws offsets from [0, tokenizer.vocab_size), not from + # [0, len(allowed_tokens)). Using vocab_size as the draw bound matches + # vLLM bench's get_sampling_params and produces the same offset values + # for the same seed, giving identical prompts. + vocab_size = ( + self.tokenizer.vocab_size + if self.tokenizer is not None + else len(self._allowed_tokens) + ) + self._offset_cache: list[int] = generator.integers( + 0, vocab_size, size=n + ).tolist() # type: ignore[union-attr] + self._offset_idx: int = 0 + self._preseed_rng = generator + def _initialize_corpus(self) -> None: """Load and tokenize the corpus once, storing it for reuse. + No-op when corpus is RANDOM — token IDs are drawn directly from the + vocabulary without loading a text file. + Uses character-based chunking for reproducibility across different machines. The chunk size is fixed (not CPU-dependent) to ensure the same tokenization boundaries regardless of hardware, which guarantees identical prompts with @@ -156,6 +265,9 @@ def _initialize_corpus(self) -> None: Thread count doesn't affect reproducibility since chunks have deterministic boundaries based on character count. """ + if self._corpus == PromptCorpus.RANDOM: + return + corpus_path = Path(__file__).parent / DEFAULT_CORPUS_FILE with open(corpus_path, encoding="utf-8") as f: @@ -204,8 +316,10 @@ def tokenize_chunk(chunk): ] self._corpus_size = len(self._tokenized_corpus) self.debug( - lambda: f"Initialized corpus with {self._corpus_size} tokens " - f"from {len(chunks)} chunks using {num_threads} thread(s)" + lambda: ( + f"Initialized corpus with {self._corpus_size} tokens " + f"from {len(chunks)} chunks using {num_threads} thread(s)" + ) ) def _determine_bpe_stable_terminator(self) -> list[int]: @@ -253,26 +367,72 @@ def _determine_bpe_stable_terminator(self) -> list[int]: self.debug("No BPE-stable terminator found; segment-join drift will be unfixed") return [] - def _create_prefix_prompt_pool(self) -> None: - """Generate a pool of prefix prompts to sample from.""" - if self._tokenized_corpus is None: - raise NotInitializedError("Tokenized corpus is not initialized.") + def initialize_prefix_pool(self) -> None: + """Build the pool of prefix prompts, as token IDs and decoded text. + Must be called after :meth:`preseed` for the RANDOM corpus so the prefix + draws land at vLLM's stream position (after ISLs, OSLs and offsets). A + no-op when no pool is configured, so callers can invoke it unconditionally. + """ if self.prefix_prompts is None: return length = self.prefix_prompts.length or 0 pool_size = self.prefix_prompts.pool_size or 0 - self._prefix_prompts = [self.generate_prompt(length) for _ in range(pool_size)] + if pool_size <= 0: + return + + if self._tokenized_corpus is None and self._corpus != PromptCorpus.RANDOM: + raise NotInitializedError("Tokenized corpus is not initialized.") + + if length <= 0: + self._prefix_prompt_tokens = [[] for _ in range(pool_size)] + self._prefix_prompts = [""] * pool_size + return + + self._prefix_prompt_tokens = [ + self._create_prefix_tokens(length) for _ in range(pool_size) + ] + self._prefix_prompts = [ + self.tokenizer.decode(tokens) for tokens in self._prefix_prompt_tokens + ] self.debug( - lambda: f"Initialized prefix prompts pool with {len(self._prefix_prompts)} prompts" + lambda: ( + f"Initialized prefix prompts pool with {len(self._prefix_prompts)} prompts" + ) ) + def _create_prefix_tokens(self, length: int) -> list[int]: + """Draw and length-correct a single prefix, returning its token IDs. + + Mirrors vLLM's ``RandomDataset.get_prefix``: the draw is bounded by + ``len(allowed_tokens)`` (NOT vocab_size, which bounds the offset draws) + and comes from the shared preseed stream. Deliberately avoids + :meth:`_sample_tokens` for the RANDOM corpus — that would advance + ``_random_request_index`` and shift every subsequent request body away + from vLLM's ``index=i`` sequence. + """ + if self._corpus == PromptCorpus.RANDOM: + generator = getattr(self, "_preseed_rng", None) or self._corpus_rng + n = len(self._allowed_tokens) + tokens = [ + self._allowed_tokens[int(i)] + for i in generator.integers(0, n, size=length) + ] + else: + tokens = self._sample_tokens(length) + + text = self._decode_to_exact_len(tokens, length) + return self.tokenizer.encode(text, add_special_tokens=False) + def generate( self, mean: int | None = None, stddev: int | None = None, hash_ids: list[int] | None = None, + *, + with_prefix: bool = False, + exact_length: bool = False, ) -> str: """Generate a synthetic prompt with the configuration parameters. Serves as a wrapper around other internal methods to provide a unified interface. @@ -281,6 +441,14 @@ def generate( mean: The mean of the normal distribution. stddev: The standard deviation of the normal distribution. hash_ids: A list of hash indices used for token reuse. + with_prefix: Prepend a pooled prefix prompt at the token level. Ignored + when hash_ids is set, since that path composes its own blocks. + exact_length: Treat ``mean`` as an already-decided length and use it + verbatim instead of resampling. Set by callers driving a + sequence distribution, which has already produced an exact + value; routing that through ``calculate_num_tokens`` would floor + a legitimately-sampled 0 up to 1 and emit one more token than + the reference implementation. Returns: A synthetic prompt as a string. @@ -293,8 +461,11 @@ def generate( ) or InputTokensDefaults.BLOCK_SIZE return self._generate_cached_prompt(mean, hash_ids, block_size) - num_tokens = self.calculate_num_tokens(mean, stddev) - return self.generate_prompt(num_tokens) + num_tokens = ( + int(mean or 0) if exact_length else self.calculate_num_tokens(mean, stddev) + ) + prefix_tokens = self.get_random_prefix_prompt_tokens() if with_prefix else None + return self.generate_prompt(num_tokens, prefix_tokens=prefix_tokens) def calculate_num_tokens( self, @@ -310,16 +481,71 @@ def calculate_num_tokens( return self._length_rng.sample_positive_normal_integer(mean, stddev) - def generate_prompt(self, num_tokens: int) -> str: - """Generate a prompt containing exactly `num_tokens` number of tokens. + def _decode_to_exact_len( + self, tokens: list[int], target: int, *, _max_retries: int = 10 + ) -> str: + """Decode `tokens` to text whose re-encoded length is exactly `target`. + + Decodes, re-encodes to verify the round-trip token count, and trims or + tops up until the count matches, up to `_max_retries` times. Matches + vLLM bench's ``gen_prompt_decode_to_target_len`` contract. Args: - num_tokens: Number of tokens required in the prompt. + tokens: Initial token sequence to decode. + target: Required token count of the re-encoded result. + _max_retries: Maximum adjustment iterations before accepting mismatch. + + Returns: + The decoded prompt text. + """ + for _ in range(_max_retries): + text = self.tokenizer.decode(tokens) + # Re-assign tokens from the re-encoded result, matching vLLM bench's + # gen_prompt_decode_to_target_len which reassigns token_sequence after + # encode before trimming or extending. Trimming the re-encoded sequence + # (not the original) is what produces identical prompts for the same seed. + tokens = self.tokenizer.encode(text, add_special_tokens=False) + if len(tokens) == target: + return text + if len(tokens) > target: + tokens = tokens[:target] + else: + tokens = tokens + self._sample_topup_tokens(target - len(tokens)) + return self.tokenizer.decode(tokens) + + def generate_prompt( + self, + num_tokens: int, + *, + _max_retries: int = 10, + prefix_tokens: list[int] | None = None, + ) -> str: + """Generate a prompt whose re-encoded length matches `num_tokens`. + + When `prefix_tokens` is supplied it is concatenated at the TOKEN-ID level + and the combined sequence is corrected once to ``len(prefix_tokens) + + num_tokens``, matching vLLM's ``generate_token_sequence``. Joining the + decoded segments as strings instead would let BPE charge for the seam, + which cost ~0.84 extra tokens per request (AIP-1118). + + Args: + num_tokens: Target number of body tokens in the final prompt. + _max_retries: Maximum adjustment iterations before accepting mismatch. + prefix_tokens: Prefix token IDs to prepend, or None for no prefix. Returns: A synthetic prompt as a string. """ - return self.tokenizer.decode(self._sample_tokens(num_tokens)) + if num_tokens < 0 or (num_tokens == 0 and not prefix_tokens): + raise ValueError( + f"num_tokens must be > 0 without a prefix, got {num_tokens}" + ) + tokens = self._sample_tokens(num_tokens) + target = num_tokens + if prefix_tokens: + tokens = list(prefix_tokens) + tokens + target += len(prefix_tokens) + return self._decode_to_exact_len(tokens, target, _max_retries=_max_retries) def _generate_cached_prompt( self, @@ -453,8 +679,14 @@ def _build_token_sequence( return final_prompt def _sample_tokens(self, num_tokens: int) -> list[int]: - """Generate a list of token IDs containing exactly `num_tokens` number of tokens - using the preloaded tokenized corpus. + """Generate a list of token IDs containing exactly `num_tokens` number of tokens. + + For SONNET corpus: samples a contiguous window from the preloaded Shakespeare + corpus starting at a random offset (wraps at end). + + For RANDOM corpus: samples a contiguous window from the sorted allowed-token + list using (offset + request_index + j) % n, matching vLLM bench's + RandomDataset strategy including the per-request index shift. Args: num_tokens: Number of tokens required in the prompt. @@ -463,8 +695,38 @@ def _sample_tokens(self, num_tokens: int) -> list[int]: A list of token IDs. Raises: - NotInitializedError: If the tokenized corpus is not initialized + NotInitializedError: If the relevant corpus is not initialized. """ + if self._corpus == PromptCorpus.RANDOM: + if not self._allowed_tokens: + raise NotInitializedError("Random vocab corpus is not initialized.") + n = len(self._allowed_tokens) + offset_cache = getattr(self, "_offset_cache", None) + if offset_cache is not None and self._offset_idx < len(offset_cache): + offset = offset_cache[self._offset_idx] + self._offset_idx += 1 + else: + # The cache holds one offset per conversation but is read once + # per generate_prompt call, so multi-turn, --prompt-batch-size + # > 1, prefix pools and user-context prompts all outrun it. + # Degrade to a live draw rather than raising IndexError out of + # the dataset composer; reference alignment is already lost. + if offset_cache is not None and not self._warned_offsets_exhausted: + self._warned_offsets_exhausted = True + self.warning( + f"Preseeded offset cache exhausted after " + f"{len(offset_cache)} draws (sized by conversation " + "count, consumed once per prompt). Falling back to " + "live sampling; prompts past this point no longer " + "match the reference implementation for the same seed." + ) + offset = int(self._corpus_rng.integers(0, n)) + idx = self._random_request_index + self._random_request_index += 1 + return [ + self._allowed_tokens[(offset + idx + j) % n] for j in range(num_tokens) + ] + if not self._tokenized_corpus: raise NotInitializedError("Tokenized corpus is not initialized.") if num_tokens > self._corpus_size: @@ -483,6 +745,15 @@ def _sample_tokens(self, num_tokens: int) -> list[int]: self.trace(lambda: f"Sampled {len(prompt_tokens)} tokens from corpus") return prompt_tokens + def _random_prefix_index(self) -> int: + """Pick a pool slot, raising when the pool was never initialized.""" + if not self._prefix_prompt_tokens: + raise InvalidStateError( + "Attempted to sample a prefix prompt but the prefix prompts pool is empty. " + "Please ensure that initialize_prefix_pool() has been called." + ) + return self._prefix_rng.choice(range(len(self._prefix_prompt_tokens))) + def get_random_prefix_prompt(self) -> str: """ Fetch a random prefix prompt from the pool. @@ -493,12 +764,23 @@ def get_random_prefix_prompt(self) -> str: Raises: InvalidStateError: If the prefix prompts pool is empty. """ - if not self._prefix_prompts: - raise InvalidStateError( - "Attempted to sample a prefix prompt but the prefix prompts pool is empty. " - "Please ensure that the prefix prompts pool is initialized." - ) - return self._prefix_rng.choice(self._prefix_prompts) + return self._prefix_prompts[self._random_prefix_index()] + + def get_random_prefix_prompt_tokens(self) -> list[int]: + """ + Fetch the token IDs of a random prefix prompt from the pool. + + Preferred over :meth:`get_random_prefix_prompt` when the prefix is about + to be joined to a body: concatenating token IDs avoids the BPE seam cost + that string concatenation incurs. + + Returns: + The token IDs of a random prefix prompt. + + Raises: + InvalidStateError: If the prefix prompts pool is empty. + """ + return self._prefix_prompt_tokens[self._random_prefix_index()] def _generate_shared_system_prompt(self) -> None: """Generate the shared system prompt. @@ -506,7 +788,7 @@ def _generate_shared_system_prompt(self) -> None: This prompt is generated once and is identical across all sessions. It appears as a system message in turn 0 of every conversation. """ - if self._tokenized_corpus is None: + if self._tokenized_corpus is None and self._corpus != PromptCorpus.RANDOM: raise NotInitializedError("Tokenized corpus is not initialized.") length = ( @@ -552,7 +834,7 @@ def generate_user_context_prompt(self, session_index: int) -> str: NotInitializedError: If tokenized corpus is not initialized. InvalidStateError: If user context prompt length is not configured. """ - if self._tokenized_corpus is None: + if self._tokenized_corpus is None and self._corpus != PromptCorpus.RANDOM: raise NotInitializedError("Tokenized corpus is not initialized.") length = ( @@ -571,8 +853,10 @@ def generate_user_context_prompt(self, session_index: int) -> str: new_prompt = self.generate_prompt(length) self._user_context_prompts.append(new_prompt) self.debug( - lambda: f"Generated user context prompt #{len(self._user_context_prompts) - 1} " - f"for session {len(self._user_context_prompts) - 1}" + lambda: ( + f"Generated user context prompt #{len(self._user_context_prompts) - 1} " + f"for session {len(self._user_context_prompts) - 1}" + ) ) return self._user_context_prompts[session_index] diff --git a/src/aiperf/dataset/protocols.py b/src/aiperf/dataset/protocols.py index 94c3c99a62..cc540dadff 100644 --- a/src/aiperf/dataset/protocols.py +++ b/src/aiperf/dataset/protocols.py @@ -15,6 +15,71 @@ from aiperf.plugin.enums import DatasetSamplingStrategy +@runtime_checkable +class CorpusGeneratorProtocol(Protocol): + """Protocol for corpus-based prompt generators. + + Both ``PromptGenerator`` (SONNET / RANDOM) and ``CodingContentGenerator`` + satisfy this protocol structurally. Callers should annotate against + ``CorpusGeneratorProtocol`` rather than the concrete union + ``PromptGenerator | CodingContentGenerator`` so that new corpus styles + (e.g. SGLang-native) can be introduced without touching every call site. + + ``preseed`` is part of the protocol with a no-op default so that callers + can always call it — only generators that support upfront vectorised draws + (currently ``PromptGenerator`` with ``RandomCorpusStyle.VLLM``) populate + the cache; others silently do nothing. + """ + + def generate( + self, + mean: int, + stddev: int = 0, + *, + with_prefix: bool = False, + exact_length: bool = False, + ) -> str: + """Generate a single prompt of approximately ``mean`` tokens. + + When ``with_prefix`` is set the prefix is prepended at the token level so + the result tokenizes to exactly ``prefix_len + mean``. + + When ``exact_length`` is set, ``mean`` is an already-decided length and + is used verbatim rather than resampled. Callers driving a sequence + distribution set this: that distribution has already produced an exact + value, and resampling floors a legitimately-sampled 0 up to 1. + """ + ... + + def get_random_prefix_prompt(self) -> str: + """Return a randomly sampled prefix from the prefix pool.""" + ... + + def get_random_prefix_prompt_tokens(self) -> list[int]: + """Return the token IDs of a randomly sampled prefix from the pool.""" + ... + + def initialize_prefix_pool(self) -> None: + """Build the prefix pool, after ``preseed``. No-op without a pool.""" + ... + + def get_shared_system_prompt(self) -> str | None: + """Return the shared system prompt, or ``None`` if not configured.""" + ... + + def generate_user_context_prompt(self, session_id: str) -> str | None: + """Return a per-conversation user-context prompt, or ``None``.""" + ... + + def preseed(self, n: int, generator: object) -> None: + """Pre-generate per-request values from a shared RNG (optional). + + Implementations that do not support preseeding should leave this as a + no-op so callers can always invoke it unconditionally. + """ + ... + + @runtime_checkable class CustomDatasetLoaderProtocol(Protocol): """Protocol for custom dataset loaders that load dataset from a file and convert it to a list of Conversation objects.""" diff --git a/tests/aiperf_mock_server/request_recorder.py b/tests/aiperf_mock_server/request_recorder.py index 88c12b16da..2e815a16ee 100644 --- a/tests/aiperf_mock_server/request_recorder.py +++ b/tests/aiperf_mock_server/request_recorder.py @@ -313,7 +313,18 @@ def _encode_chat_prompt_ids( tokenizer: Any, request: ChatCompletionRequest ) -> tuple[list[int], str]: """Mirror trtllm-serve chat prompt tokenization when a chat template exists.""" - messages = [_message_to_dict(msg) for msg in request.messages] + # Normalize multimodal list content to plain strings so apply_chat_template + # receives the actual text rather than a Python dict repr. Clients such as + # vllm bench serve send content as [{"type":"text","text":"..."}] even for + # text-only prompts; without normalization the template renders the list + # literal instead of the text, inflating the ISL measurement. + raw_messages = [_message_to_dict(msg) for msg in request.messages] + messages = [ + {**m, "content": _content_to_text(m["content"])} + if not isinstance(m.get("content"), str) + else m + for m in raw_messages + ] add_generation_prompt = bool(getattr(request, "add_generation_prompt", True)) inner = _unwrap_tokenizer(tokenizer) apply_chat_template = getattr(inner, "apply_chat_template", None) diff --git a/tests/unit/common/models/test_sequence_distribution.py b/tests/unit/common/models/test_sequence_distribution.py index 5f0a2dec01..84fb45f20d 100644 --- a/tests/unit/common/models/test_sequence_distribution.py +++ b/tests/unit/common/models/test_sequence_distribution.py @@ -12,14 +12,25 @@ - Edge cases, error handling, and boundary conditions """ +import math + import numpy as np import pytest +from pydantic import ValidationError +from pytest import param from aiperf.common import random_generator as rng +from aiperf.common.enums import RandomCorpusStyle from aiperf.common.models.sequence_distribution import ( + _CLASS_FOR_MODE, DistributionParser, + RangeRatioDistribution, SequenceLengthDistribution, SequenceLengthPair, + SequenceLengthSampler, + SGLangRangeRatioDistribution, + SGLangRatioConfig, + VLLMRatioConfig, create_balanced_distribution, create_uniform_distribution, ) @@ -746,3 +757,658 @@ def create_dataset(self): composer._clear_turn_cache(turn1_id) assert turn1_id not in composer._turn_sequence_cache assert turn2_id in composer._turn_sequence_cache + + +def _vllm( + isl_mean: int, osl_mean: int, ratio: float | tuple[float, float], **kw: int +) -> RangeRatioDistribution: + return RangeRatioDistribution( + VLLMRatioConfig(isl_mean=isl_mean, osl_mean=osl_mean, range_ratio=ratio, **kw) + ) + + +def _sglang( + isl_mean: int, osl_mean: int, ratio: float, **kw: int +) -> SGLangRangeRatioDistribution: + return SGLangRangeRatioDistribution( + SGLangRatioConfig(isl_mean=isl_mean, osl_mean=osl_mean, range_ratio=ratio, **kw) + ) + + +class TestRangeRatioDistribution: + """Tests for vllm-style uniform ISL/OSL sampling around configured means.""" + + def test_implements_sequence_length_sampler_protocol(self): + assert isinstance(_vllm(100, 50, 0.2), SequenceLengthSampler) + + def test_ratio_zero_is_fixed(self): + dist = _vllm(1024, 128, 0.0) + for _ in range(50): + assert dist.sample() == (1024, 128) + + def test_bounds_are_computed_per_vllm_formula(self): + dist = _vllm(1024, 128, 0.3) + assert dist.input_bounds == (math.floor(1024 * 0.7), math.ceil(1024 * 1.3)) + assert dist.output_bounds == (math.floor(128 * 0.7), math.ceil(128 * 1.3)) + + def test_samples_are_inclusive_on_both_ends(self): + dist = _vllm(100, 40, (0.5, 0.25)) + in_low, in_high = dist.input_bounds + out_low, out_high = dist.output_bounds + + samples = [dist.sample() for _ in range(2000)] + isls = [isl for isl, _ in samples] + osls = [osl for _, osl in samples] + + assert min(isls) >= in_low + assert max(isls) <= in_high + assert min(osls) >= out_low + assert max(osls) <= out_high + assert in_low in isls and in_high in isls + assert out_low in osls and out_high in osls + + def test_sample_values_are_python_ints(self): + isl, osl = _vllm(100, 40, 0.1).sample() + assert type(isl) is int + assert type(osl) is int + + def test_independent_input_and_output_ratios(self): + dist = _vllm(1024, 128, (0.1, 0.5)) + assert dist.input_bounds == (math.floor(1024 * 0.9), math.ceil(1024 * 1.1)) + assert dist.output_bounds == (math.floor(128 * 0.5), math.ceil(128 * 1.5)) + + def test_vllm_input_allows_zero_matching_upstream(self): + # vllm bench allows 0-length input ranges; floor(1 * (1 - 0.9)) = 0 is valid. + assert _vllm(1, 1, 0.9).input_bounds[0] == 0 + + def test_output_always_floors_at_one(self): + dist = _vllm(1, 1, 0.9) + assert dist.output_bounds[0] >= 1 + for _ in range(20): + _, osl = dist.sample() + assert osl >= 1 + + def test_sglang_input_floors_at_one(self): + dist = _sglang(1, 1, 0.9) + assert dist.input_bounds[0] >= 1 + for _ in range(20): + isl, _ = dist.sample() + assert isl >= 1 + + @pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5, float("inf"), float("nan")]) + def test_rejects_isl_ratio_out_of_range(self, bad_ratio): + with pytest.raises(ValueError, match="ISL range_ratio"): + _vllm(100, 50, (bad_ratio, 0.1)) + + @pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5, float("inf"), float("nan")]) + def test_rejects_osl_ratio_out_of_range(self, bad_ratio): + with pytest.raises(ValueError, match="OSL range_ratio"): + _vllm(100, 50, (0.1, bad_ratio)) + + def test_rejects_non_positive_isl_mean(self): + with pytest.raises(ValueError, match="isl_mean"): + _vllm(0, 50, 0.1) + + def test_rejects_non_positive_osl_mean(self): + with pytest.raises(ValueError, match="osl_mean"): + _vllm(100, 0, 0.1) + + def test_sampling_is_deterministic_with_fixed_seed(self): + dist1 = _vllm(1024, 128, 0.3) + first_run = [dist1.sample() for _ in range(10)] + + rng.reset() + rng.init(42) + dist2 = _vllm(1024, 128, 0.3) + second_run = [dist2.sample() for _ in range(10)] + + assert first_run == second_run + + def test_preseed_draws_all_isls_then_osls(self): + """preseed(seed) uses PCG64; sample reads from the cache.""" + dist = _vllm(128, 128, 0.3) + n = 5 + g = np.random.default_rng(0) + expected_isls = g.integers( + dist.input_bounds[0], dist.input_bounds[1] + 1, size=n + ).tolist() + expected_osls = g.integers( + dist.output_bounds[0], dist.output_bounds[1] + 1, size=n + ).tolist() + + dist.preseed(n, 0) + + for i in range(n): + isl, osl = dist.sample() + assert isl == expected_isls[i] + assert osl == expected_osls[i] + + def test_preseed_isl_cache_is_body_only(self): + """Cached ISLs describe the body alone; prefixes are additive and added later. + + preseed() takes no prefix_len — the prefix is prepended to the generated + body at request time, so the sampled distribution must be unaffected by + prefix configuration. + """ + n = 5 + dist = _vllm(128, 128, 0.3) + dist.preseed(n, 0) + low, high = dist._config.compute_input_bounds() + + for _ in range(n): + isl, _ = dist.sample() + assert low <= isl <= high + + def test_sample_past_cache_falls_back_to_live_draw(self): + """The ISL/OSL cache is sized per conversation but read once per turn, + so any multi-turn run outruns it. Exhaustion must degrade to a live + draw rather than raising `IndexError` out of the dataset composer. + """ + dist = _vllm(64, 16, 0.3) + dist.preseed(2, 42) + + cached = [dist.sample() for _ in range(2)] + assert len(cached) == 2 + + for _ in range(4): + isl, osl = dist.sample() + assert dist.input_bounds[0] <= isl <= dist.input_bounds[1] + assert dist.output_bounds[0] <= osl <= dist.output_bounds[1] + + def test_sample_past_cache_warns_once(self, caplog): + """Degradation is otherwise invisible, so it warns exactly once.""" + import logging + + dist = _vllm(64, 16, 0.3) + dist.preseed(1, 42) + dist.sample() + + with caplog.at_level(logging.WARNING): + for _ in range(5): + dist.sample() + + exhausted = [ + r for r in caplog.records if "ISL/OSL cache exhausted" in r.getMessage() + ] + assert len(exhausted) == 1 + assert "consumed once per turn" in exhausted[0].getMessage() + + def test_sample_within_cache_is_unaffected(self): + """Draws inside the cache still come from the preseed stream in order.""" + dist = _vllm(64, 16, 0.3) + dist.preseed(4, 42) + expected = list(zip(dist._isl_cache, dist._osl_cache, strict=True)) + + assert [dist.sample() for _ in range(4)] == expected + + def test_preseed_exposes_rng_for_offset_draws(self): + """_preseed_rng after preseed is advanced past ISL+OSL draws.""" + dist = _vllm(128, 128, 0.3) + dist.preseed(3, 42) + g_ref = np.random.default_rng(42) + g_ref.integers(dist.input_bounds[0], dist.input_bounds[1] + 1, size=3) + g_ref.integers(dist.output_bounds[0], dist.output_bounds[1] + 1, size=3) + assert int(dist._preseed_rng.integers(0, 1000)) == int(g_ref.integers(0, 1000)) + + def test_num_special_tokens_adjusts_isl_mean_only(self): + """Style's adjust_mean subtracts special tokens from isl_mean; osl is unchanged.""" + dist_plain = _vllm(512, 128, 0.3) + dist_adjusted = _vllm(512, 128, 0.3, num_special_tokens=1) + assert dist_adjusted.input_bounds == ( + math.floor(511 * 0.7), + math.ceil(511 * 1.3), + ) + assert dist_adjusted.output_bounds == dist_plain.output_bounds + + def test_num_special_tokens_vllm_allows_zero_adjusted_mean(self): + """vllm adjust_mean: max(0, isl_mean - n) — allows zero.""" + assert _vllm(1, 128, 0.0, num_special_tokens=5).input_bounds == (0, 0) + + def test_num_special_tokens_zero_is_identity(self): + """num_special_tokens=0 (default) leaves bounds unchanged.""" + assert ( + _vllm(512, 128, 0.3).input_bounds + == _vllm(512, 128, 0.3, num_special_tokens=0).input_bounds + ) + + def test_num_special_tokens_negative_raises(self): + """Negative num_special_tokens is invalid.""" + with pytest.raises(ValueError, match="num_special_tokens"): + _vllm(512, 128, 0.3, num_special_tokens=-1) + + +class TestRangeRatioDistributionSglangMode: + """sglang-mode semantics: lower-bounded window [max(1, int(mean*r)), mean].""" + + def test_ratio_zero_gives_full_variability(self): + dist = _sglang(1024, 128, 0.0) + assert dist.input_bounds == (1, 1024) + assert dist.output_bounds == (1, 128) + + def test_ratio_one_is_fixed_at_mean(self): + dist = _sglang(1024, 128, 1.0) + assert dist.input_bounds == (1024, 1024) + assert dist.output_bounds == (128, 128) + for _ in range(20): + assert dist.sample() == (1024, 128) + + def test_ratio_half_gives_lower_bounded_window(self): + dist = _sglang(1024, 128, 0.5) + assert dist.input_bounds == (512, 1024) + assert dist.output_bounds == (64, 128) + + def test_samples_never_exceed_mean(self): + dist = _sglang(200, 50, 0.3) + for _ in range(2000): + isl, osl = dist.sample() + assert 1 <= isl <= 200 + assert 1 <= osl <= 50 + + def test_rejects_ratio_below_zero(self): + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"): + _sglang(100, 50, -0.1) + + def test_rejects_ratio_above_one(self): + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"): + _sglang(100, 50, 1.5) + + def test_rejects_json_dict_form(self): + """SGLang only allows a single ratio; independent input/output values are rejected.""" + with pytest.raises(ValueError, match="single ratio"): + SGLangRatioConfig( + isl_mean=100, osl_mean=50, range_ratio='{"input": 0.3, "output": 0.5}' + ) + + def test_num_special_tokens_sglang_does_not_adjust_mean(self): + """sglang adjust_mean is a no-op — SGLang uses raw input_len without BOS subtraction.""" + assert ( + _sglang(128, 128, 0.3).input_bounds + == _sglang(128, 128, 0.3, num_special_tokens=5).input_bounds + ) + + def test_chat_template_len_is_not_a_config_field(self): + """The window must derive from the raw mean. + + `chat_template_len` used to shrink `isl_mean` before bounds while the + composer ALSO subtracted the same overhead from each sampled ISL via + `first_turn_isl_adjustment` — charging it twice and landing wire ISL + ~13 tokens under the configured value under `--apply-chat-template`. + The plumbing is gone; `extra="forbid"` makes reintroducing it by + accident a hard error rather than a silent regression. + """ + with pytest.raises(ValidationError, match="chat_template_len"): + SGLangRatioConfig( + isl_mean=128, osl_mean=16, range_ratio=1.0, chat_template_len=13 + ) + with pytest.raises(ValidationError, match="chat_template_len"): + VLLMRatioConfig( + isl_mean=128, osl_mean=16, range_ratio=0.3, chat_template_len=13 + ) + + def test_window_matches_sglang_upstream_shape(self): + """Bounds come from the raw mean, matching SGLang's compute_random_lens. + + Upstream computes `[max(int(full_len * ratio), 1), full_len]` and then + shifts each drawn sample by the overhead. Subtracting the overhead from + the mean first would rescale the LOWER bound too -- `int((mean-c)*r)` + instead of `int(mean*r)` -- yielding a different distribution rather + than a shifted one. At mean=100, r=0.5, overhead=10 that was (45, 90) + where upstream gives (50, 100). + """ + assert _sglang(100, 16, 0.5).input_bounds == (50, 100) + assert _sglang(128, 16, 1.0).input_bounds == (128, 128) + + def test_preseed_accepts_uint64_run_seed(self): + """A 64-bit run seed must not blow up the MT19937 global seeder. + + `resolve_run_seed` returns `sha256(...)[:8]` as a uint64 for adaptive + sweeps and `multi_run.vary_seed_per_trial`, and `--random-seed` is only + bounded `ge=0`. Passing that straight to `numpy.random.seed` raises + "Seed must be between 0 and 2**32 - 1" and kills the DatasetManager. + """ + from aiperf.common.random_generator import derive_variation_seed + + seed = derive_variation_seed(42, "concurrency_10:trial:1") + assert seed >= 2**32 + + dist = _sglang(1024, 128, 0.5) + dist.preseed(4, seed) + + assert len(dist._isl_cache) == 4 + assert all(512 <= isl <= 1024 for isl in dist._isl_cache) + + def test_preseed_uint64_seed_is_reproducible(self): + """Same uint64 seed => same draws, so folding stays deterministic.""" + from aiperf.common.random_generator import derive_variation_seed + + seed = derive_variation_seed(42, "concurrency_10") + a, b = _sglang(1024, 128, 0.5), _sglang(1024, 128, 0.5) + a.preseed(8, seed) + b.preseed(8, seed) + assert a._isl_cache == b._isl_cache + assert a._osl_cache == b._osl_cache + + def test_preseed_small_seed_draws_are_unchanged_by_folding(self): + """Seeds under 2**32 fold to themselves, pinning pre-fix draw values.""" + dist = _sglang(1024, 128, 0.5) + dist.preseed(4, 42) + + np.random.seed(42) + expected_isl = np.random.randint(512, 1025, size=4).tolist() + assert dist._isl_cache == expected_isl + + def test_preseed_does_not_mutate_global_numpy_state(self): + """MT19937 parity is achieved with a private RandomState, not by + seeding module-level `numpy.random`, so an unrelated global consumer + sees the same stream before and after a preseed.""" + np.random.seed(7) + baseline = np.random.randint(0, 1000, size=3).tolist() + + np.random.seed(7) + _sglang(1024, 128, 0.5).preseed(64, 12345) + after = np.random.randint(0, 1000, size=3).tolist() + + assert after == baseline + + def test_concurrent_distributions_have_independent_streams(self): + """Two preseeded distributions must not consume each other's draws. + + With module-level `numpy.random` the second preseed reseeded the shared + state, so `a`'s deferred `_preseed_rng` reads (offset + BPE top-up draws + in PromptGenerator) silently continued from `b`'s stream. + """ + a = _sglang(1024, 128, 0.5) + a.preseed(4, 42) + solo = a._preseed_rng.integers(0, 50000, size=3).tolist() + + a2, b2 = _sglang(1024, 128, 0.5), _sglang(1024, 128, 0.5) + a2.preseed(4, 42) + b2.preseed(4, 99) + interleaved = a2._preseed_rng.integers(0, 50000, size=3).tolist() + + assert interleaved == solo + + +class TestParseRandomRangeRatio: + """Tests for --random-range-ratio CLI string parsing via VLLMRatioConfig.""" + + def _parse(self, s: str) -> tuple[float, float]: + return VLLMRatioConfig(isl_mean=1, osl_mean=1, range_ratio=s).range_ratio + + def test_float_applies_to_both_dimensions(self): + assert self._parse("0.3") == (0.3, 0.3) + + def test_zero_is_valid(self): + assert self._parse("0") == (0.0, 0.0) + assert self._parse("0.0") == (0.0, 0.0) + + def test_whitespace_is_tolerated(self): + assert self._parse(" 0.25 ") == (0.25, 0.25) + + def test_json_dict_sets_input_and_output_independently(self): + assert self._parse('{"input": 0.3, "output": 0.5}') == (0.3, 0.5) + + def test_json_dict_accepts_integer_values(self): + assert self._parse('{"input": 0, "output": 0}') == (0.0, 0.0) + + @pytest.mark.parametrize("bad", [-0.1, 1.0, 1.5]) + def test_float_out_of_range_rejected(self, bad): + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\)"): + self._parse(str(bad)) + + def test_json_out_of_range_rejected(self): + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\)"): + self._parse('{"input": 0.2, "output": 1.2}') + + def test_json_missing_keys_rejected(self): + with pytest.raises(ValueError, match="missing keys"): + self._parse('{"input": 0.3}') + + def test_json_extra_keys_rejected(self): + with pytest.raises(ValueError, match="unexpected keys"): + self._parse('{"input": 0.3, "output": 0.5, "extra": 1}') + + def test_empty_value_rejected(self): + with pytest.raises(ValueError, match="cannot be empty"): + self._parse("") + + def test_whitespace_only_rejected(self): + with pytest.raises(ValueError, match="cannot be empty"): + self._parse(" ") + + def test_invalid_json_rejected(self): + with pytest.raises(ValueError, match="must be a float or a JSON object"): + self._parse("{not valid json") + + def test_non_object_json_rejected(self): + # A bare JSON number string parses as a float via the first-try path, + # but a JSON array or string must error in the dict-parse branch. + with pytest.raises(ValueError, match="must be a float or a JSON object"): + self._parse("[0.3, 0.5]") + + def test_registry_returns_correct_class_for_mode(self): + assert _CLASS_FOR_MODE[RandomCorpusStyle.VLLM] is RangeRatioDistribution + assert _CLASS_FOR_MODE[RandomCorpusStyle.SGLANG] is SGLangRangeRatioDistribution + + +class TestRatioInputCoercion: + """`_coerce_ratio_input` must reject unhandled types, not pass them through. + + Both callers destructure the result immediately, so a pass-through never + reached Pydantic's type check: `range_ratio=None` surfaced as a bare + `TypeError: cannot unpack non-iterable NoneType object` naming neither the + field nor the accepted forms. + """ + + @pytest.mark.parametrize( + "value", + [ + param(None, id="none"), + param({"input": 0.3}, id="dict"), + param(["a", "b"], id="list-of-str"), + param({0.3}, id="set"), + param(object(), id="arbitrary-object"), + ], + ) # fmt: skip + @pytest.mark.parametrize( + "config_cls", + [param(VLLMRatioConfig, id="vllm"), param(SGLangRatioConfig, id="sglang")], + ) # fmt: skip + def test_unhandled_types_raise_validation_error(self, config_cls, value): + with pytest.raises(ValidationError): + config_cls(isl_mean=10, osl_mean=10, range_ratio=value) + + @pytest.mark.parametrize( + "config_cls", + [param(VLLMRatioConfig, id="vllm"), param(SGLangRatioConfig, id="sglang")], + ) # fmt: skip + def test_bool_is_rejected(self, config_cls): + """bool is an int subclass, so `range_ratio: true` would coerce to 1.0 -- + silently accepted by SGLang (range [0, 1]) as "pin at the mean". A YAML + bool here is a typo, not a ratio. + """ + with pytest.raises(ValidationError, match="bool"): + config_cls(isl_mean=10, osl_mean=10, range_ratio=True) + + def test_accepted_forms_still_work(self): + """Guard against over-rejecting: the documented forms must still parse.""" + assert VLLMRatioConfig( + isl_mean=10, osl_mean=10, range_ratio=0.3 + ).range_ratio == (0.3, 0.3) + assert VLLMRatioConfig( + isl_mean=10, osl_mean=10, range_ratio="0.3" + ).range_ratio == (0.3, 0.3) + assert VLLMRatioConfig( + isl_mean=10, osl_mean=10, range_ratio=(0.2, 0.4) + ).range_ratio == (0.2, 0.4) + assert VLLMRatioConfig( + isl_mean=10, osl_mean=10, range_ratio='{"input": 0.2, "output": 0.4}' + ).range_ratio == (0.2, 0.4) + assert SGLangRatioConfig( + isl_mean=10, osl_mean=10, range_ratio=1.0 + ).range_ratio == (1.0, 1.0) + + +class TestFoldedSeedWarning: + """Seeds wider than MT19937's 32-bit limit are folded, which is lossy. + + `fold_seed_to_uint32` reduces to `hi ^ lo` for `seed = hi * 2**32 + lo`, so + distinct seeds collide onto one SGLANG stream. The seed stays valid + everywhere else -- `derive()` hashes the full value and the VLLM/PCG64 path + takes it verbatim -- so this warns rather than rejects. + """ + + @pytest.fixture(autouse=True) + def _clear_warned(self): + from aiperf.common.models.sequence_distribution import _FoldedSeedWarnings + + _FoldedSeedWarnings.reset() + yield + _FoldedSeedWarnings.reset() + + def _preseed(self, seed): + _sglang(128, 16, 0.5).preseed(4, seed) + + def test_seed_below_limit_does_not_warn(self, caplog): + import logging + + with caplog.at_level(logging.WARNING): + self._preseed(42) + self._preseed(2**32 - 1) + assert not [r for r in caplog.records if "MT19937" in r.getMessage()] + + def test_seed_above_limit_warns_with_folded_value(self, caplog): + import logging + + with caplog.at_level(logging.WARNING): + self._preseed((1 << 32) | 4) # folds to 1 ^ 4 == 5 + warned = [r for r in caplog.records if "MT19937" in r.getMessage()] + assert len(warned) == 1 + message = warned[0].getMessage() + assert "4294967300" in message, "names the seed the user passed" + assert "folded to 5" in message, "names the value actually used" + assert "sglang" in message, "names the affected style" + + def test_warns_once_per_seed(self, caplog): + import logging + + with caplog.at_level(logging.WARNING): + for _ in range(4): + self._preseed((1 << 32) | 4) + assert len([r for r in caplog.records if "MT19937" in r.getMessage()]) == 1 + + def test_distinct_large_seeds_each_warn(self, caplog): + """A sweep re-seeding per variation should flag each aliasing seed.""" + import logging + + with caplog.at_level(logging.WARNING): + self._preseed((1 << 32) | 4) + self._preseed((1 << 33) | 9) + assert len([r for r in caplog.records if "MT19937" in r.getMessage()]) == 2 + + def test_aliasing_pair_produces_identical_sglang_draws(self): + """The warning exists because this is true: hi ^ lo collides.""" + a, b = _sglang(128, 16, 0.5), _sglang(128, 16, 0.5) + a.preseed(6, 5) + b.preseed(6, (1 << 32) | 4) + assert a._isl_cache == b._isl_cache + + def test_vllm_path_keeps_the_full_seed(self): + """Same pair must stay distinct on PCG64, so the warning is correctly + scoped to SGLANG rather than being a global seed limitation.""" + a, b = _vllm(128, 16, 0.5), _vllm(128, 16, 0.5) + a.preseed(6, 5) + b.preseed(6, (1 << 32) | 4) + assert a._isl_cache != b._isl_cache + + +class TestSGLangSpecialTokenAdjustment: + """SGLANG subtracts special tokens per-request AFTER sampling. + + Mirrors upstream `sample_random_requests`, which shifts drawn lengths:: + + input_lens[i] = max(1, input_lens[i] - num_special_tokens) + + Subtracting from `isl_mean` instead would rescale the window's lower bound + (`int((mean-c)*r)` vs `int(mean*r)`), changing the distribution's shape + rather than shifting it. Before this, the field was ignored entirely and + wire ISL landed at `--isl + num_special` on BOS tokenizers. + """ + + def test_window_bounds_keep_the_raw_mean(self): + """The adjustment must not leak into the bounds.""" + assert _sglang(128, 16, 0.5, num_special_tokens=1).input_bounds == (64, 128) + assert _sglang(128, 16, 0.5).input_bounds == (64, 128) + + def test_sampled_isl_is_shifted_down(self): + with_special = _sglang(128, 16, 1.0, num_special_tokens=1) + without = _sglang(128, 16, 1.0) + # ratio 1.0 pins the window to [128, 128], so this is exact, not noise. + assert without.sample()[0] == 128 + assert with_special.sample()[0] == 127 + + def test_preseeded_values_are_adjusted_exactly_once(self): + """The cache stores adjusted values and `sample` returns them verbatim; + applying the hook in both places would subtract twice.""" + dist = _sglang(128, 16, 1.0, num_special_tokens=1) + dist.preseed(4, 42) + assert dist._isl_cache == [127, 127, 127, 127] + assert [dist.sample()[0] for _ in range(4)] == [127, 127, 127, 127] + + def test_multi_special_token_tokenizer(self): + assert _sglang(128, 16, 1.0, num_special_tokens=2).sample()[0] == 126 + + def test_floors_at_one(self): + """Mirrors upstream's `max(1, ...)`: a tiny mean cannot go to zero.""" + assert _sglang(1, 16, 1.0, num_special_tokens=5).sample()[0] == 1 + + def test_osl_is_not_adjusted(self): + """Upstream shifts `input_lens` only, never `output_lens`.""" + dist = _sglang(128, 64, 1.0, num_special_tokens=1) + assert dist.sample()[1] == 64 + + def test_vllm_style_is_unaffected(self): + """VLLM folds specials into the bounds, so the post-sample hook is a + no-op there -- otherwise it would double-subtract.""" + dist = _vllm(128, 16, 0.0, num_special_tokens=1) + assert dist.input_bounds == (127, 127) + assert dist.sample()[0] == 127 + + +class TestZeroLengthBodyIsEmittedVerbatim: + """A range-ratio window that bottoms out at 0 must not be floored to 1. + + `calculate_num_tokens` routes through `sample_positive_normal_integer`, + whose `max(1, round(mean))` is correct for its original callers (a normal + draw around a positive mean cannot be 0) but wrong for an already-decided + length. The degenerate-config guard deliberately accepts `isl=1, r=0.0` + when a prefix covers the request -- vLLM does too, emitting prefix-only -- + so flooring produced one token more than the reference. + """ + + def test_window_can_legitimately_bottom_out_at_zero(self): + cfg = VLLMRatioConfig( + isl_mean=1, osl_mean=16, num_special_tokens=1, range_ratio=0.0 + ) + assert cfg.compute_input_bounds() == (0, 0) + + @pytest.mark.parametrize( + "isl_mean,ratio", + [ + param(2, 0.5, id="isl2-r0.5"), + param(10, 0.9, id="isl10-r0.9"), + param(100, 0.99, id="isl100-r0.99"), + ], + ) # fmt: skip + def test_other_configs_reaching_a_zero_lower_bound(self, isl_mean, ratio): + """Not unique to --isl 1: any sufficiently high ratio gets there.""" + low, _ = VLLMRatioConfig( + isl_mean=isl_mean, osl_mean=16, num_special_tokens=1, range_ratio=ratio + ).compute_input_bounds() + assert low == 0 + + def test_sampler_returns_zero_rather_than_flooring(self): + """The distribution itself must report the zero; the floor lived + downstream in the generator, not here.""" + dist = _vllm(1, 16, 0.0, num_special_tokens=1) + assert dist.sample()[0] == 0 diff --git a/tests/unit/common/test_random_generator.py b/tests/unit/common/test_random_generator.py index add6cc402f..5a5f718307 100644 --- a/tests/unit/common/test_random_generator.py +++ b/tests/unit/common/test_random_generator.py @@ -415,5 +415,53 @@ def test_output_fits_in_uint64(self): assert 0 <= seed < 2**64 +class TestFoldSeedToUint32: + """`fold_seed_to_uint32` bridges 64-bit AIPerf seeds to numpy's legacy seeder. + + `numpy.random.seed` (MT19937 global state) rejects anything outside + [0, 2**32 - 1], but `derive_variation_seed` returns a full uint64 for + adaptive-sweep variations and `vary_seed_per_trial` runs. + """ + + def test_small_seed_is_unchanged(self): + """Seeds already in range must round-trip identically, so folding + never perturbs an existing reproducible run.""" + from aiperf.common.random_generator import fold_seed_to_uint32 + + for seed in (0, 1, 42, 2**32 - 1): + assert fold_seed_to_uint32(seed) == seed + + def test_uint64_seed_folds_into_range(self): + from aiperf.common.random_generator import ( + derive_variation_seed, + fold_seed_to_uint32, + ) + + seed = derive_variation_seed(42, "concurrency_10:trial:1") + assert seed >= 2**32 + assert 0 <= fold_seed_to_uint32(seed) < 2**32 + + def test_folded_seed_is_accepted_by_numpy_legacy_seeder(self): + import numpy as np + + from aiperf.common.random_generator import ( + derive_variation_seed, + fold_seed_to_uint32, + ) + + seed = derive_variation_seed(42, "concurrency_10") + with pytest.raises(ValueError, match=r"2\*\*32"): + np.random.seed(seed) + np.random.seed(fold_seed_to_uint32(seed)) + + def test_high_bits_contribute_to_result(self): + """XOR-fold, not truncation: two seeds sharing a low half must not collide.""" + from aiperf.common.random_generator import fold_seed_to_uint32 + + low = 0x0000_0000_DEAD_BEEF + high = 0x1234_5678_DEAD_BEEF + assert fold_seed_to_uint32(low) != fold_seed_to_uint32(high) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/unit/common/test_tokenizer.py b/tests/unit/common/test_tokenizer.py index 972c25dea3..99ac19bc63 100644 --- a/tests/unit/common/test_tokenizer.py +++ b/tests/unit/common/test_tokenizer.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest +from pytest import param from aiperf.common.exceptions import NotInitializedError, TokenizerError from aiperf.common.tokenizer import ( @@ -66,6 +67,84 @@ def test_block_separation_token_id_falls_back_to_eos( ) -> None: assert tokenizer.block_separation_token_id == tokenizer.eos_token_id + def test_num_prompt_special_tokens_returns_zero_when_no_bos( + self, tokenizer: Tokenizer + ) -> None: + """Builtin (tiktoken) tokenizer has no BOS — zero special tokens to add.""" + assert tokenizer.num_prompt_special_tokens() == 0 + + def test_num_prompt_special_tokens_delegates_to_hf_method(self) -> None: + """Returns whatever HF's num_special_tokens_to_add(pair=False) reports. + + For BERT-like tokenizers this is 2 (CLS+SEP); the BOS heuristic alone + would under-count it. + """ + from types import SimpleNamespace + + mock_inner = SimpleNamespace( + bos_token_id=101, + eos_token_id=102, + num_special_tokens_to_add=lambda pair=False: 2 if not pair else 3, + ) + tok = Tokenizer() + with patch.object(tok, "_tokenizer", mock_inner): + assert tok.num_prompt_special_tokens() == 2 + + def test_num_prompt_special_tokens_returns_zero_when_hf_method_says_zero( + self, + ) -> None: + """GPT-2 has bos_token_id but doesn't auto-prepend it; HF method returns 0.""" + from types import SimpleNamespace + + mock_inner = SimpleNamespace( + bos_token_id=50256, + eos_token_id=50256, + num_special_tokens_to_add=lambda pair=False: 0, + ) + tok = Tokenizer() + with patch.object(tok, "_tokenizer", mock_inner): + assert tok.num_prompt_special_tokens() == 0 + + def test_num_prompt_special_tokens_falls_back_to_zero(self) -> None: + """Tokenizers without num_special_tokens_to_add always return 0 — bos_token_id + being defined does not mean the tokenizer prepends it (e.g. GPT-2).""" + from types import SimpleNamespace + + with_bos = SimpleNamespace(bos_token_id=1, eos_token_id=2) + without_bos = SimpleNamespace(bos_token_id=None, eos_token_id=2) + + tok = Tokenizer() + with patch.object(tok, "_tokenizer", with_bos): + assert tok.num_prompt_special_tokens() == 0 + with patch.object(tok, "_tokenizer", without_bos): + assert tok.num_prompt_special_tokens() == 0 + + def test_num_prompt_special_tokens_falls_back_when_hf_method_raises(self) -> None: + """Stubbed (NotImplementedError) or signature-mismatched (TypeError) methods + fall back to 0; other exceptions propagate as real bugs.""" + from types import SimpleNamespace + + def _not_impl(pair: bool = False) -> int: + raise NotImplementedError + + def _bad_sig() -> int: # rejects pair kwarg → TypeError + return 0 + + bos = {"bos_token_id": 1, "eos_token_id": 2} + tok = Tokenizer() + with patch.object( + tok, + "_tokenizer", + SimpleNamespace(**bos, num_special_tokens_to_add=_not_impl), + ): + assert tok.num_prompt_special_tokens() == 0 + with patch.object( + tok, + "_tokenizer", + SimpleNamespace(**bos, num_special_tokens_to_add=_bad_sig), + ): + assert tok.num_prompt_special_tokens() == 0 + def test_call_returns_input_ids(self, tokenizer: Tokenizer) -> None: result = tokenizer("hello") assert "input_ids" in result @@ -285,3 +364,72 @@ def test_each_encoding_url_appears_in_tiktoken_source(self) -> None: f"tiktoken_ext.openai_public.{target} — tiktoken changed the URL; " f"update _TIKTOKEN_ENCODING_URLS in src/aiperf/common/tokenizer.py" ) + + +class TestTiktokenInternalAttributeGuard: + """tiktoken privates must fail loudly, not as AttributeError/KeyError. + + `valid_token_ids` / `all_token_ids` / `all_special_ids` read + `_mergeable_ranks` and `_special_token_values` off `tiktoken.Encoding`. + tiktoken exposes no public equivalent, and its sparse ID space means + `range(n_vocab)` is not a usable substitute -- gap IDs are not decodable. + A rename in tiktoken previously broke `--prompt-corpus random` at runtime, + with a traceback pointing nowhere useful. + """ + + @pytest.fixture + def tiktoken_tokenizer(self): + return Tokenizer.from_pretrained("o200k_base") + + @pytest.mark.parametrize( + "missing", + [ + param("_mergeable_ranks", id="mergeable-ranks"), + param("_special_token_values", id="special-token-values"), + ], + ) # fmt: skip + @pytest.mark.parametrize( + "accessor", + [ + param("valid_token_ids", id="valid-token-ids"), + param("all_token_ids", id="all-token-ids"), + ], + ) # fmt: skip + def test_missing_internal_raises_tokenizer_error( + self, tiktoken_tokenizer, accessor, missing + ): + encoding = tiktoken_tokenizer._tokenizer._encoding + + class _Stripped: + """Encoding proxy with one attribute removed, mimicking a rename.""" + + def __getattr__(self, name): + if name == missing: + raise AttributeError(name) + return getattr(encoding, name) + + tiktoken_tokenizer._tokenizer._encoding = _Stripped() + with pytest.raises(TokenizerError, match=missing): + getattr(tiktoken_tokenizer, accessor) + + def test_error_names_the_affected_flag(self, tiktoken_tokenizer): + """The message has to be actionable from a benchmark run, so it names + the flag that breaks rather than only the attribute.""" + encoding = tiktoken_tokenizer._tokenizer._encoding + + class _Stripped: + def __getattr__(self, name): + if name == "_mergeable_ranks": + raise AttributeError(name) + return getattr(encoding, name) + + tiktoken_tokenizer._tokenizer._encoding = _Stripped() + with pytest.raises(TokenizerError) as exc: + _ = tiktoken_tokenizer.valid_token_ids + assert "--prompt-corpus random" in str(exc.value) + + def test_intact_encoding_is_unaffected(self, tiktoken_tokenizer): + """Guard must be transparent when the attributes are present.""" + assert len(tiktoken_tokenizer.valid_token_ids) == 199998 + assert len(tiktoken_tokenizer.all_token_ids) == 200000 + assert len(tiktoken_tokenizer.all_special_ids) == 2 diff --git a/tests/unit/config/test_converter_random_corpus_style.py b/tests/unit/config/test_converter_random_corpus_style.py new file mode 100644 index 0000000000..6aa72bca84 --- /dev/null +++ b/tests/unit/config/test_converter_random_corpus_style.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""`--random-corpus-style` must survive the CLI converter without a range ratio. + +The style write used to sit after `_apply_random_range_ratio`'s early return, +so `--random-corpus-style sglang` alone was silently discarded and the user got +vLLM's token pool. Only the sglang request was observably lost -- vllm is the +default, so a dropped write lands there anyway -- which is why one of the four +cells below is the entire bug. + +With no ratio, the pool is the *only* thing the style selects, so this is +precisely the case where someone reaches for the flag on its own. +""" + +import pytest +from pytest import param + +from aiperf.common.enums import RandomCorpusStyle +from aiperf.config.flags.cli_config import CLIConfig +from tests.unit.conftest import make_run_from_cli + +_BASE = dict( + model_names=["m"], + conversation_num_dataset_entries=2, + prompt_corpus="random", + prompt_input_tokens_mean=128, + prompt_output_tokens_mean=16, +) + + +def _resolved_style(**overrides) -> RandomCorpusStyle: + run = make_run_from_cli(CLIConfig(**{**_BASE, **overrides})) + return run.cfg.get_default_dataset().prompts.random_corpus_style + + +@pytest.mark.parametrize( + "style", + [ + param(RandomCorpusStyle.VLLM, id="vllm"), + param(RandomCorpusStyle.SGLANG, id="sglang"), + ], +) # fmt: skip +@pytest.mark.parametrize( + "ratio", + [param(None, id="no-ratio"), param("0.3", id="with-ratio")], +) # fmt: skip +def test_style_survives_with_and_without_a_ratio(style, ratio): + overrides = {"prompt_random_corpus_style": style} + if ratio is not None: + overrides["prompt_random_range_ratio"] = ratio + assert _resolved_style(**overrides) == style + + +def test_sglang_without_ratio_is_the_regression_case(): + """Pinned separately: this is the one cell that was broken, and it is the + one a user hits when they want SGLang's full-vocab pool and nothing else.""" + assert ( + _resolved_style(prompt_random_corpus_style=RandomCorpusStyle.SGLANG) + == RandomCorpusStyle.SGLANG + ) + + +def test_default_is_vllm_when_flag_is_absent(): + """Guard against over-correcting: an unset flag must not start writing.""" + assert _resolved_style() == RandomCorpusStyle.VLLM diff --git a/tests/unit/config/test_prompt_config_range_ratio.py b/tests/unit/config/test_prompt_config_range_ratio.py new file mode 100644 index 0000000000..3302b3bcbb --- /dev/null +++ b/tests/unit/config/test_prompt_config_range_ratio.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for random_range_ratio fields on the v2 PromptConfig.""" + +import math + +import pytest + +from aiperf.config.dataset.content import PromptConfig +from aiperf.config.types import FixedDistribution + + +def test_prompt_config_random_range_ratio_defaults_to_none(): + """Test that random_range_ratio defaults to None.""" + config = PromptConfig() + assert config.random_range_ratio is None + + +def test_prompt_config_random_corpus_style_defaults_to_vllm(): + from aiperf.common.enums import RandomCorpusStyle + + config = PromptConfig() + assert config.random_corpus_style == RandomCorpusStyle.VLLM + + +def test_prompt_config_random_range_ratio_float_builds_distribution(): + """A plain float string produces a RangeRatioDistribution applied to both dims.""" + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio="0.3", + isl=FixedDistribution(value=1024), + osl=FixedDistribution(value=128), + ) + dist = config.get_sequence_distribution() + assert isinstance(dist, RangeRatioDistribution) + assert dist.input_bounds == (math.floor(1024 * 0.7), math.ceil(1024 * 1.3)) + assert dist.output_bounds == (math.floor(128 * 0.7), math.ceil(128 * 1.3)) + + +def test_prompt_config_random_range_ratio_json_dict_builds_distribution(): + """A JSON dict sets input and output ratios independently.""" + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio='{"input": 0.2, "output": 0.5}', + isl=FixedDistribution(value=1000), + osl=FixedDistribution(value=100), + ) + dist = config.get_sequence_distribution() + assert isinstance(dist, RangeRatioDistribution) + assert dist.input_bounds == (800, 1200) + assert dist.output_bounds == (50, 150) + + +def test_prompt_config_random_range_ratio_requires_osl(): + """--random-range-ratio without --osl raises at parse time.""" + with pytest.raises(ValueError, match="--osl"): + PromptConfig( + random_range_ratio="0.0", + isl=FixedDistribution(value=1024), + ) + + +def test_prompt_config_random_range_ratio_requires_isl(): + """--random-range-ratio without --isl raises at parse time.""" + with pytest.raises(ValueError, match="--isl"): + PromptConfig( + random_range_ratio="0.0", + osl=FixedDistribution(value=128), + ) + + +def test_prompt_config_random_range_ratio_rejects_isl_stddev(): + """--isl-stddev with --random-range-ratio is rejected at validation time.""" + from aiperf.config.distributions import NormalDistribution + + with pytest.raises(ValueError, match="--isl-stddev"): + PromptConfig( + random_range_ratio="0.3", + isl=NormalDistribution(mean=128, stddev=20), + osl=FixedDistribution(value=128), + ) + + +def test_prompt_config_random_range_ratio_rejects_osl_stddev(): + """--osl-stddev with --random-range-ratio is rejected at validation time.""" + from aiperf.config.distributions import NormalDistribution + + with pytest.raises(ValueError, match="--osl-stddev"): + PromptConfig( + random_range_ratio="0.3", + isl=FixedDistribution(value=128), + osl=NormalDistribution(mean=128, stddev=20), + ) + + +def test_prompt_config_random_range_ratio_invalid_value_rejected(): + """Bad ratio value is rejected at validation time, not on first use.""" + with pytest.raises(ValueError, match="Invalid random_range_ratio value"): + PromptConfig( + random_range_ratio="1.5", + isl=FixedDistribution(value=128), + osl=FixedDistribution(value=128), + ) + + +def test_prompt_config_random_range_ratio_conflicts_with_sequence_distribution(): + """Setting both random_range_ratio and sequence_distribution is rejected.""" + from aiperf.config.types import SequenceDistributionEntry + + with pytest.raises( + ValueError, match="cannot be combined with sequence_distribution" + ): + PromptConfig( + random_range_ratio="0.3", + sequence_distribution=[ + SequenceDistributionEntry( + isl=FixedDistribution(value=256), + osl=FixedDistribution(value=128), + probability=100.0, + ) + ], + ) + + +def test_prompt_config_random_range_ratio_none_returns_none(): + """When random_range_ratio is None, get_sequence_distribution returns None.""" + config = PromptConfig() + assert config.get_sequence_distribution() is None + + +def test_prompt_config_random_range_ratio_zero_is_valid(): + """Ratio of 0.0 is valid (fixed at mean).""" + config = PromptConfig( + random_range_ratio="0.0", + isl=FixedDistribution(value=512), + osl=FixedDistribution(value=128), + ) + assert config.random_range_ratio == "0.0" + dist = config.get_sequence_distribution() + assert dist is not None + assert dist.input_bounds == (512, 512) + assert dist.output_bounds == (128, 128) + + +def test_prompt_config_random_range_ratio_sglang_mode_builds_sglang_distribution(): + from aiperf.common.enums import RandomCorpusStyle + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio="0.5", + random_corpus_style=RandomCorpusStyle.SGLANG, + isl=FixedDistribution(value=1024), + osl=FixedDistribution(value=128), + ) + dist = config.get_sequence_distribution() + assert isinstance(dist, RangeRatioDistribution) + assert dist.mode == RandomCorpusStyle.SGLANG + assert dist.input_bounds == (512, 1024) + assert dist.output_bounds == (64, 128) + + +def test_prompt_config_random_range_ratio_sglang_mode_allows_ratio_one(): + """sglang mode accepts r=1.0 (fixed at mean); vllm mode rejects it.""" + from aiperf.common.enums import RandomCorpusStyle + + config = PromptConfig( + random_range_ratio="1.0", + random_corpus_style=RandomCorpusStyle.SGLANG, + isl=FixedDistribution(value=1024), + osl=FixedDistribution(value=128), + ) + dist = config.get_sequence_distribution() + assert dist.input_bounds == (1024, 1024) + assert dist.output_bounds == (128, 128) + + +def test_prompt_config_random_range_ratio_vllm_mode_rejects_ratio_one(): + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\)"): + PromptConfig( + random_range_ratio="1.0", + isl=FixedDistribution(value=1024), + osl=FixedDistribution(value=128), + ) + + +def test_prompt_config_get_sequence_distribution_passes_num_special_tokens(): + """num_special_tokens is forwarded to RangeRatioDistribution via the style's adjust_mean.""" + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio="0.3", + isl=FixedDistribution(value=512), + osl=FixedDistribution(value=128), + ) + dist = config.get_sequence_distribution(num_special_tokens=1) + assert isinstance(dist, RangeRatioDistribution) + assert dist.input_bounds == (math.floor(511 * 0.7), math.ceil(511 * 1.3)) + + +def test_prompt_config_get_sequence_distribution_default_num_special_tokens_zero(): + """Default num_special_tokens=0 leaves bounds at the configured isl_mean.""" + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio="0.3", + isl=FixedDistribution(value=512), + osl=FixedDistribution(value=128), + ) + dist_default = config.get_sequence_distribution() + dist_explicit = config.get_sequence_distribution(num_special_tokens=0) + assert isinstance(dist_default, RangeRatioDistribution) + assert dist_default.input_bounds == dist_explicit.input_bounds + + +def test_prompt_config_random_range_ratio_accepts_native_float(): + """A native Python float (e.g. from YAML) is coerced to string before validation.""" + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio=0.3, + isl=FixedDistribution(value=1024), + osl=FixedDistribution(value=128), + ) + assert config.random_range_ratio == "0.3" + dist = config.get_sequence_distribution() + assert isinstance(dist, RangeRatioDistribution) + assert dist.input_bounds == (math.floor(1024 * 0.7), math.ceil(1024 * 1.3)) + + +def test_prompt_config_random_range_ratio_accepts_native_dict(): + """A native Python dict (e.g. from YAML) is JSON-serialized before validation.""" + from aiperf.common.models.sequence_distribution import RangeRatioDistribution + + config = PromptConfig( + random_range_ratio={"input": 0.2, "output": 0.5}, + isl=FixedDistribution(value=1000), + osl=FixedDistribution(value=100), + ) + dist = config.get_sequence_distribution() + assert isinstance(dist, RangeRatioDistribution) + assert dist.input_bounds == (800, 1200) + assert dist.output_bounds == (50, 150) diff --git a/tests/unit/dataset/composer/test_base_composer.py b/tests/unit/dataset/composer/test_base_composer.py index c97d2444c7..7cd31a8d74 100644 --- a/tests/unit/dataset/composer/test_base_composer.py +++ b/tests/unit/dataset/composer/test_base_composer.py @@ -60,8 +60,15 @@ def sequence_dist_config(self): @pytest.fixture def mock_tokenizer(self): - """Create a mock tokenizer.""" - return MagicMock() + """Create a mock tokenizer. + + ``num_prompt_special_tokens`` returns a real int because the composer + subtracts it from the ISL target; 0 (the gpt2 case) keeps these tests + asserting the configured ISL directly. + """ + tokenizer = MagicMock() + tokenizer.num_prompt_special_tokens = MagicMock(return_value=0) + return tokenizer def test_initialization_with_sequence_distribution( self, sequence_dist_config, mock_tokenizer diff --git a/tests/unit/dataset/composer/test_degenerate_range_ratio.py b/tests/unit/dataset/composer/test_degenerate_range_ratio.py new file mode 100644 index 0000000000..07a6aa7bc5 --- /dev/null +++ b/tests/unit/dataset/composer/test_degenerate_range_ratio.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Degenerate range-ratio configs must be rejected, mirroring vLLM's guard. + +vLLM's ``RandomDataset.sample`` raises when the minimum possible input is +unusable:: + + min_total_input = prefix_len + floor(max(0, isl - num_special) * (1 - r)) + if min_total_input < 1: raise ValueError(...) + +Without the equivalent guard, ``--isl 2 --random-range-ratio 0.9`` yields +bounds ``(0, 4)`` and draws of 0 are silently clamped to 1, so the run +completes and reports numbers for one-token prompts nobody asked for. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from aiperf.common.enums import RandomCorpusStyle +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.dataset.composer.synthetic import SyntheticDatasetComposer +from tests.unit.conftest import make_run_from_cli + +_MARKER_COST_PATCH = "aiperf.timing.strategies.cache_bust.estimate_marker_token_cost" + + +def _tokenizer(num_special: int = 0): + tok = MagicMock() + tok.encode = MagicMock(return_value=list(range(10))) + tok._tokenizer = MagicMock(spec=[]) + tok.num_prompt_special_tokens = MagicMock(return_value=num_special) + return tok + + +def _build( + *, + isl: int, + ratio: float, + prefix_len: int | None = None, + style: RandomCorpusStyle = RandomCorpusStyle.VLLM, + num_special: int = 0, +): + overrides: dict = { + "model_names": ["test-model"], + "conversation_num_dataset_entries": 1, + "prompt_input_tokens_mean": isl, + "prompt_output_tokens_mean": 16, + "prompt_random_range_ratio": str(ratio), + "prompt_random_corpus_style": style, + "prompt_corpus": "random", + } + if prefix_len is not None: + overrides["prompt_prefix_length"] = prefix_len + overrides["prompt_prefix_pool_size"] = 1 + run = make_run_from_cli(CLIConfig(**overrides)) + # Patch the generator factory rather than PromptGenerator itself: the real + # factory builds a real generator, and prefix-pool construction then samples + # against the mock tokenizer's vocab and dies on unrelated plumbing. The + # guard runs before any prompt is generated, so stubbing the factory keeps + # this test on the guard alone. Prefix composition against a real tokenizer + # is covered end-to-end via the CLI. + with ( + patch(_MARKER_COST_PATCH, return_value=0), + patch( + "aiperf.dataset.composer.base._estimate_chat_template_overheads", + return_value=(0, 0), + ), + patch("aiperf.dataset.composer.base.resolve_prompt_generator"), + ): + return SyntheticDatasetComposer(run=run, tokenizer=_tokenizer(num_special)) + + +class TestDegenerateRangeRatioGuard: + def test_degenerate_window_is_rejected(self): + """isl=2, r=0.9 -> bounds (0, 4); vLLM raises, so we must too.""" + with pytest.raises(ValueError, match="minimum input of"): + _build(isl=2, ratio=0.9) + + def test_prefix_rescues_a_tiny_body(self): + """Mirrors vLLM: min_total_input counts prefix_len, so a large prefix + makes an otherwise-degenerate body acceptable.""" + composer = _build(isl=2, ratio=0.9, prefix_len=20) + assert composer is not None + + def test_normal_config_unaffected(self): + composer = _build(isl=128, ratio=0.3) + assert composer is not None + + def test_sglang_style_cannot_bottom_out(self): + """SGLANG's window is [max(1, int(mean*r)), mean] -- lower bound >= 1 by + construction, so the guard is VLLM-specific and must not fire here.""" + composer = _build(isl=2, ratio=0.9, style=RandomCorpusStyle.SGLANG) + assert composer is not None + + def test_special_tokens_counted_against_the_budget(self): + """num_special is subtracted before the window is computed, so a + tokenizer with a BOS can push a marginal config over the edge.""" + with pytest.raises(ValueError, match="special token"): + _build(isl=1, ratio=0.5, num_special=1) + + def test_error_names_all_three_levers(self): + with pytest.raises(ValueError) as exc: + _build(isl=2, ratio=0.9) + message = str(exc.value) + assert "--isl" in message + assert "--prompt-prefix-length" in message + assert "--random-range-ratio" in message diff --git a/tests/unit/dataset/composer/test_isl_budget_compensation.py b/tests/unit/dataset/composer/test_isl_budget_compensation.py index cde0adbb34..24bc3b3417 100644 --- a/tests/unit/dataset/composer/test_isl_budget_compensation.py +++ b/tests/unit/dataset/composer/test_isl_budget_compensation.py @@ -22,6 +22,8 @@ def _make_config( *, cache_bust_target: CacheBustTarget = CacheBustTarget.NONE, shared_system_prompt_length: int | None = None, + prefix_pool_size: int | None = None, + prefix_length: int | None = None, isl_mean: int = 100, apply_chat_template: bool = True, ): @@ -36,14 +38,25 @@ def _make_config( } if shared_system_prompt_length is not None: overrides["prompt_prefix_shared_system_length"] = shared_system_prompt_length + if prefix_pool_size is not None: + overrides["prompt_prefix_pool_size"] = prefix_pool_size + if prefix_length is not None: + overrides["prompt_prefix_length"] = prefix_length return make_run_from_cli(CLIConfig(**overrides)) def _make_tokenizer_no_chat_template(): - """A tokenizer mock with no apply_chat_template — overheads collapse to 0.""" + """A tokenizer mock with no apply_chat_template — overheads collapse to 0. + + ``num_prompt_special_tokens`` is stubbed to 0 (the gpt2 case) because the + composer now subtracts it from the ISL target; an unstubbed MagicMock would + reach real arithmetic. 0 keeps these budget assertions about chat-template + and cache-bust costs alone, with no special-token contribution. + """ tokenizer = MagicMock() tokenizer.encode = MagicMock(return_value=list(range(10))) tokenizer._tokenizer = MagicMock(spec=[]) # spec=[] -> no attributes + tokenizer.num_prompt_special_tokens = MagicMock(return_value=0) return tokenizer @@ -250,19 +263,29 @@ def test_returns_zeros_when_apply_chat_template_raises(self): assert _estimate_chat_template_overheads(tokenizer) == (0, 0) def test_decomposes_fixed_and_wrap(self): - """Synthetic Llama-3-like template: BOS=1, gen_prompt=3, wrap=5/msg.""" + """Synthetic Llama-3-like template: BOS=1, gen_prompt=3, wrap=5/msg. + + BOS is excluded from per_request_fixed because RangeRatioDistribution + already accounts for it via num_special_tokens. The function returns + only the template-structural overhead (gen_prompt suffix, not BOS). + """ from aiperf.dataset.composer.base import ( _CHAT_TEMPLATE_PROBE_SAMPLES, _estimate_chat_template_overheads, ) per_msg_wrap = 5 - per_request_fixed = 4 # BOS(1) + gen_prompt(3) + bos = 1 + gen_prompt = 3 + template_total_fixed = bos + gen_prompt # what apply_chat_template adds - def fake_apply(messages, **_kwargs): + def fake_apply(messages, tokenize=True, add_generation_prompt=True, **_kwargs): + # tokenizer.encode counts by whitespace-delimited words, so return a + # string with exactly the right number of space-separated words. content_tokens = sum(len(m["content"].split()) for m in messages) wrapping = per_msg_wrap * len(messages) - return list(range(per_request_fixed + wrapping + content_tokens)) + total = template_total_fixed + wrapping + content_tokens + return " ".join(["x"] * total) inner = MagicMock() inner.apply_chat_template = MagicMock(side_effect=fake_apply) @@ -271,9 +294,10 @@ def fake_apply(messages, **_kwargs): tokenizer.encode = MagicMock( side_effect=lambda text: list(range(len(text.split()))) ) + tokenizer.num_prompt_special_tokens = MagicMock(return_value=bos) fixed, wrap = _estimate_chat_template_overheads(tokenizer) - assert fixed == per_request_fixed + assert fixed == gen_prompt # BOS excluded — handled by RangeRatioDistribution assert wrap == per_msg_wrap # 2 templates per sample -> 2 * len(samples) apply calls. assert inner.apply_chat_template.call_count == 2 * len( @@ -301,15 +325,20 @@ class TestAdjustmentProperties: """The two public properties must compose the components correctly.""" def test_first_turn_adjustment_composes_all_three(self): - config = _make_config(cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX) + config = _make_config( + cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX, + prefix_pool_size=3, + prefix_length=20, + ) composer = _build_composer(config, marker_cost=10, chat_fixed=4, chat_wrap=5) - # 4 (fixed) + 5 (wrap) + 10 (marker) = 19 + # 4 (fixed) + 5 (wrap) + 10 (marker) = 19. The 20-token prefix is NOT + # subtracted: prefixes are additive. assert composer.first_turn_isl_adjustment == 19 def test_subsequent_turn_adjustment_only_per_msg_wrap(self): config = _make_config(cache_bust_target=CacheBustTarget.FIRST_TURN_PREFIX) composer = _build_composer(config, marker_cost=10, chat_fixed=4, chat_wrap=5) - # Only 5 (wrap), no fixed and no marker. + # Only 5 (wrap): fixed, marker, and prefix all apply only to the first turn. assert composer.subsequent_turn_isl_adjustment == 5 def test_no_adjustment_when_everything_zero(self): @@ -318,6 +347,23 @@ def test_no_adjustment_when_everything_zero(self): assert composer.first_turn_isl_adjustment == 0 assert composer.subsequent_turn_isl_adjustment == 0 + def test_prefix_length_excluded_from_first_turn_adjustment(self): + """Prefixes are additive: the body keeps the full ISL and the prefix rides on top. + + Matches vLLM's --random-prefix-len and SGLang's prefix_len, so + --prompt-input-tokens-mean describes the body rather than the wire total. + """ + config = _make_config(prefix_pool_size=3, prefix_length=30) + composer = _build_composer(config) + assert composer.first_turn_isl_adjustment == 0 + assert composer.subsequent_turn_isl_adjustment == 0 + + def test_prefix_length_without_pool_size_also_excluded(self): + """prefix_length without pool_size likewise has no effect on adjustment.""" + config = _make_config(prefix_length=30) + composer = _build_composer(config) + assert composer.first_turn_isl_adjustment == 0 + class TestSyntheticPromptBudgetSubtraction: """End-to-end: synthetic composer reduces ISL passed to prompt generator.""" @@ -485,3 +531,54 @@ def test_marker_estimator_is_invoked_when_compensation_is_needed( SyntheticDatasetComposer(run=config, tokenizer=tokenizer) assert mock_estimate.call_count == expected_marker_estimator_calls + + +class TestSpecialTokenBudgetOnNonRangeRatioPath: + """Server-added special tokens must be budgeted on every path. + + `_estimate_chat_template_overheads` deliberately excludes BOS from + `per_request_fixed`, documenting that the mean is adjusted elsewhere -- but + only `VLLMRatioConfig.compute_input_bounds` was doing that adjustment. The + non-range-ratio path used `--isl` raw, so `--isl 128` on a BOS tokenizer put + 129 tokens on the wire (measured on TinyLlama, before and after this fix). + """ + + def _composer(self, *, num_special: int, isl_mean: int = 128): + run = _make_config(isl_mean=isl_mean, apply_chat_template=False) + tokenizer = _make_tokenizer_no_chat_template() + tokenizer.num_prompt_special_tokens = MagicMock(return_value=num_special) + with ( + patch(_MARKER_COST_PATCH, return_value=0), + patch( + "aiperf.dataset.composer.base._estimate_chat_template_overheads", + return_value=(0, 0), + ), + patch("aiperf.dataset.composer.base.resolve_prompt_generator"), + ): + return SyntheticDatasetComposer(run=run, tokenizer=tokenizer) + + def test_special_tokens_subtracted_from_isl_target(self): + composer = self._composer(num_special=1) + isl, _ = composer._get_turn_sequence_lengths(turn_id=1) + assert isl == 127, ( + "BOS must come out of the body budget so wire ISL lands on 128" + ) + + def test_no_special_tokens_leaves_target_untouched(self): + """gpt2-style tokenizers (no auto-BOS) must see no change.""" + composer = self._composer(num_special=0) + isl, _ = composer._get_turn_sequence_lengths(turn_id=1) + assert isl == 128 + + def test_multi_special_token_tokenizer(self): + """BERT-style (CLS+SEP) subtracts both.""" + composer = self._composer(num_special=2) + isl, _ = composer._get_turn_sequence_lengths(turn_id=1) + assert isl == 126 + + def test_target_floors_at_zero_not_negative(self): + """Mirrors vLLM's max(0, input_len - num_special); the downstream + sample_positive_normal_integer floors the actual draw at 1.""" + composer = self._composer(num_special=5, isl_mean=2) + isl, _ = composer._get_turn_sequence_lengths(turn_id=1) + assert isl == 0 diff --git a/tests/unit/dataset/composer/test_synthetic_composer.py b/tests/unit/dataset/composer/test_synthetic_composer.py index 9e16d26ac8..dff0441268 100644 --- a/tests/unit/dataset/composer/test_synthetic_composer.py +++ b/tests/unit/dataset/composer/test_synthetic_composer.py @@ -408,27 +408,28 @@ def test_generate_text_payloads_basic( assert text_payload.name == "text" assert text_payload.contents == ["Generated text content"] - @patch("aiperf.dataset.generator.prompt.PromptGenerator.get_random_prefix_prompt") @patch("aiperf.dataset.generator.prompt.PromptGenerator.generate") def test_generate_text_payloads_first_turn_with_prefix( - self, mock_generate, mock_prefix, prefix_prompt_config, mock_tokenizer + self, mock_generate, prefix_prompt_config, mock_tokenizer ): - """Test _generate_text_payloads for first turn with prefix prompts.""" - mock_generate.return_value = "User message" - mock_prefix.return_value = "Prefix prompt:" + """First turn delegates the prefix to the generator via with_prefix. + + The composer no longer joins prefix and body as strings — that seam cost + ~0.84 extra tokens per request (AIP-1118). The generator concatenates the + token IDs instead, so the composer only signals intent. + """ + mock_generate.return_value = "Prefixed user message" composer = SyntheticDatasetComposer( run=make_run(prefix_prompt_config), tokenizer=mock_tokenizer ) - # Test prefix prompt is added to first turn turn = Turn() text = composer._generate_text_payloads(turn, is_first=True) turn.texts.append(text) - text_payload = turn.texts[0] - # Test prefix prompt format ("prefix prompt") - assert text_payload.contents == ["Prefix prompt: User message"] + assert turn.texts[0].contents == ["Prefixed user message"] + assert mock_generate.call_args.kwargs["with_prefix"] is True @patch("aiperf.dataset.generator.prompt.PromptGenerator.generate") def test_generate_text_payloads_subsequent_turn_no_prefix( diff --git a/tests/unit/dataset/generator/test_prompt_generator.py b/tests/unit/dataset/generator/test_prompt_generator.py index f9ef8a740f..5736530947 100644 --- a/tests/unit/dataset/generator/test_prompt_generator.py +++ b/tests/unit/dataset/generator/test_prompt_generator.py @@ -8,15 +8,18 @@ including edge cases, error conditions, and integration scenarios. """ -from unittest.mock import mock_open, patch +from unittest.mock import MagicMock, mock_open, patch +import numpy as np import pytest +from aiperf.common.enums import PromptCorpus from aiperf.common.exceptions import ( ConfigurationError, InvalidStateError, NotInitializedError, ) +from aiperf.common.tokenizer import Tokenizer from aiperf.config.dataset.content import PrefixPromptConfig, PromptConfig from aiperf.dataset.generator.prompt import PromptGenerator @@ -29,12 +32,18 @@ def _make_generator( prompts: PromptConfig | None = None, prefix_prompts: PrefixPromptConfig | None = None, ) -> PromptGenerator: - """Construct a PromptGenerator with the v2 keyword-only signature.""" - return PromptGenerator( + """Construct a fully initialized PromptGenerator. + + Mirrors the composer, which calls initialize_prefix_pool() once the generator + is seeded; the pool is no longer built during __init__. + """ + generator = PromptGenerator( prompts=prompts, prefix_prompts=prefix_prompts, tokenizer=tokenizer, ) + generator.initialize_prefix_pool() + return generator @patch("builtins.open", mock_open(read_data=MOCK_CORPUS_CONTENT)) @@ -161,13 +170,14 @@ def testgenerate_prompt_normal_case(self, basic_config): assert result.startswith("token_") def testgenerate_prompt_zero_tokens(self, basic_config): - """Test generate_prompt with zero tokens.""" + """generate_prompt raises on num_tokens <= 0 rather than silently returning empty.""" tokenizer, prompts, prefix_prompts = basic_config generator = _make_generator( tokenizer, prompts=prompts, prefix_prompts=prefix_prompts ) - generator.generate_prompt(0) + with pytest.raises(ValueError, match="num_tokens must be > 0"): + generator.generate_prompt(0) def testgenerate_prompt_large_number(self, basic_config): """Test generate_prompt with large number of tokens.""" @@ -178,6 +188,48 @@ def testgenerate_prompt_large_number(self, basic_config): generator.generate_prompt(1000) + def test_generate_prompt_retries_when_encode_drifts_long(self, basic_config): + """generate_prompt trims and retries when re-encode returns too many tokens.""" + tokenizer, prompts, prefix_prompts = basic_config + + # First encode call returns 2 extra tokens; subsequent calls are accurate. + real_encode = tokenizer._mock_encode + call_count = {"n": 0} + + def drifting_encode(text, **kwargs): + call_count["n"] += 1 + ids = real_encode(text, **kwargs) + if call_count["n"] == 1: + return ids + [998, 999] + return ids + + tokenizer.encode = drifting_encode + generator = _make_generator( + tokenizer, prompts=prompts, prefix_prompts=prefix_prompts + ) + + result = generator.generate_prompt(3) + assert isinstance(result, str) + assert call_count["n"] >= 2 + + def test_generate_prompt_accepts_mismatch_after_budget(self, basic_config): + """generate_prompt accepts result after exhausting _max_retries.""" + tokenizer, prompts, prefix_prompts = basic_config + + # encode always returns one extra token — convergence is impossible. + real_encode = tokenizer._mock_encode + + def always_drifting_encode(text, **kwargs): + return real_encode(text, **kwargs) + [999] + + tokenizer.encode = always_drifting_encode + generator = _make_generator( + tokenizer, prompts=prompts, prefix_prompts=prefix_prompts + ) + + result = generator.generate_prompt(3, _max_retries=3) + assert isinstance(result, str) + # ============================================================================ # _generate_cached_prompt Method Tests # ============================================================================ @@ -521,11 +573,11 @@ def test_initialize_corpus_success(self, mock_cpu_count, basic_config): assert all(isinstance(token, int) for token in generator._tokenized_corpus) # ============================================================================ - # _create_prefix_prompt_pool Method Tests + # initialize_prefix_pool Method Tests # ============================================================================ - def test_create_prefix_prompt_pool_success(self, prefix_config): - """Test _create_prefix_prompt_pool successful creation.""" + def test_initialize_prefix_pool_success(self, prefix_config): + """Test initialize_prefix_pool successful creation.""" tokenizer, prompts, prefix_prompts = prefix_config generator = _make_generator( tokenizer, prompts=prompts, prefix_prompts=prefix_prompts @@ -534,8 +586,8 @@ def test_create_prefix_prompt_pool_success(self, prefix_config): assert len(generator._prefix_prompts) == 5 assert all(isinstance(prompt, str) for prompt in generator._prefix_prompts) - def test_create_prefix_prompt_pool_no_corpus(self, prefix_config): - """Test _create_prefix_prompt_pool when corpus is not initialized.""" + def test_initialize_prefix_pool_no_corpus(self, prefix_config): + """Test initialize_prefix_pool when corpus is not initialized.""" tokenizer, prompts, prefix_prompts = prefix_config generator = _make_generator( tokenizer, prompts=prompts, prefix_prompts=prefix_prompts @@ -543,10 +595,10 @@ def test_create_prefix_prompt_pool_no_corpus(self, prefix_config): generator._tokenized_corpus = None with pytest.raises(NotInitializedError): - generator._create_prefix_prompt_pool() + generator.initialize_prefix_pool() - def test_create_prefix_prompt_pool_zero_length(self, mock_tokenizer): - """Test _create_prefix_prompt_pool with zero length prompts. + def test_initialize_prefix_pool_zero_length(self, mock_tokenizer): + """Test initialize_prefix_pool with zero length prompts. v2 PrefixPromptConfig requires length >= 1, so we mutate the value post-init via Pydantic's allow-attribute-assignment behavior; if the @@ -564,7 +616,7 @@ def test_create_prefix_prompt_pool_zero_length(self, mock_tokenizer): pool_size=5, length=0 ) generator._prefix_prompts = [] - generator._create_prefix_prompt_pool() + generator.initialize_prefix_pool() assert len(generator._prefix_prompts) == 5 assert all(prompt == "" for prompt in generator._prefix_prompts) @@ -795,3 +847,429 @@ def test_build_token_sequence_same_validation_as_generate_cached( # This should raise same error as _generate_cached_prompt with pytest.raises(ConfigurationError): generator._build_token_sequence(10, [1, 2, 3], 5) # final_block_size = 0 + + +class TestPromptGeneratorRandomCorpus: + """Tests for PromptCorpus.RANDOM generate_prompt top-up behavior.""" + + @pytest.fixture + def random_tokenizer(self, mock_tokenizer_cls): + tok = mock_tokenizer_cls.from_pretrained("gpt2") + tok._tokenizer.vocab_size = 100 + return tok + + @pytest.fixture + def random_generator(self, random_tokenizer): + return PromptGenerator( + prompts=None, + prefix_prompts=None, + tokenizer=random_tokenizer, + corpus=PromptCorpus.RANDOM, + ) + + def test_generate_prompt_random_retries_when_encode_is_short( + self, random_tokenizer, random_generator + ): + """RANDOM generate_prompt re-enters the loop when BPE re-encode yields fewer tokens.""" + real_encode = random_tokenizer._mock_encode + call_count = {"n": 0} + + def shrink_first(text, **kwargs): + call_count["n"] += 1 + ids = real_encode(text, **kwargs) + if call_count["n"] == 1: + return ids[:-1] + return ids + + random_tokenizer.encode = shrink_first + + result = random_generator.generate_prompt(3) + + assert isinstance(result, str) + assert call_count["n"] >= 2 + + def test_generate_prompt_random_converges_when_exact_on_first_try( + self, random_tokenizer, random_generator + ): + """RANDOM generate_prompt exits after the first iteration when re-encode is exact.""" + real_encode = random_tokenizer._mock_encode + call_count = {"n": 0} + + def counting_encode(text, **kwargs): + call_count["n"] += 1 + return real_encode(text, **kwargs) + + random_tokenizer.encode = counting_encode + + random_generator.generate_prompt(3) + + assert call_count["n"] == 1 + + def test_generate_prompt_random_trims_when_encode_is_long( + self, random_tokenizer, random_generator + ): + """RANDOM generate_prompt trims and retries when BPE re-encode yields extra tokens.""" + real_encode = random_tokenizer._mock_encode + call_count = {"n": 0} + + def grow_first(text, **kwargs): + call_count["n"] += 1 + ids = real_encode(text, **kwargs) + if call_count["n"] == 1: + return ids + [998] + return ids + + random_tokenizer.encode = grow_first + + result = random_generator.generate_prompt(3) + + assert isinstance(result, str) + assert call_count["n"] >= 2 + + def test_random_corpus_prefix_pool_does_not_raise(self, random_tokenizer): + """RANDOM corpus can generate a prefix prompt pool without a text corpus.""" + from aiperf.config.dataset.content import PrefixPromptConfig + + generator = PromptGenerator( + prompts=None, + prefix_prompts=PrefixPromptConfig(pool_size=3, length=5), + tokenizer=random_tokenizer, + corpus=PromptCorpus.RANDOM, + ) + generator.initialize_prefix_pool() + assert len(generator._prefix_prompts) == 3 + assert all(isinstance(p, str) for p in generator._prefix_prompts) + + def test_random_corpus_shared_system_prompt_does_not_raise(self, random_tokenizer): + """RANDOM corpus can generate a shared system prompt without a text corpus.""" + from aiperf.config.dataset.content import PrefixPromptConfig + + generator = PromptGenerator( + prompts=None, + prefix_prompts=PrefixPromptConfig(shared_system_length=10), + tokenizer=random_tokenizer, + corpus=PromptCorpus.RANDOM, + ) + assert generator._shared_system_prompt is not None + assert isinstance(generator._shared_system_prompt, str) + + def test_random_corpus_excludes_special_tokens_from_sampling_pool( + self, mock_tokenizer_cls + ): + """_allowed_tokens must not contain any special token IDs.""" + from aiperf.common.enums import RandomCorpusStyle + + tok = mock_tokenizer_cls.from_pretrained("gpt2") + tok._tokenizer.vocab_size = 100 + tok._tokenizer.all_special_ids = [1, 2] + + generator = PromptGenerator( + prompts=None, + prefix_prompts=None, + tokenizer=tok, + corpus=PromptCorpus.RANDOM, + corpus_style=RandomCorpusStyle.VLLM, + ) + + allowed = set(generator._allowed_tokens) + assert 1 not in allowed + assert 2 not in allowed + assert len(generator._allowed_tokens) == 98 + + def test_random_corpus_sglang_uses_full_vocab(self, mock_tokenizer_cls): + """SGLANG style uses all_token_ids (full vocab_size range, no special-token exclusion).""" + from aiperf.common.enums import RandomCorpusStyle + + tok = mock_tokenizer_cls.from_pretrained("gpt2") + tok._tokenizer.vocab_size = 100 + tok._tokenizer.all_special_ids = [1, 2] + + generator = PromptGenerator( + prompts=None, + prefix_prompts=None, + tokenizer=tok, + corpus=PromptCorpus.RANDOM, + corpus_style=RandomCorpusStyle.SGLANG, + ) + + # SGLANG does not exclude special tokens — full range(vocab_size) + assert len(generator._allowed_tokens) == 100 + + def test_offset_cache_exhaustion_falls_back_to_live_draw(self, random_generator): + """Preseed sizes the offset cache per conversation but it is read once + per prompt, so multi-turn / batched / prefix-pool runs outrun it. + + Exhaustion must degrade to a live `_corpus_rng` draw rather than raising + `IndexError` out of the dataset composer. + """ + import numpy as np + + random_generator.preseed(2, np.random.default_rng(42)) + assert len(random_generator._offset_cache) == 2 + + for _ in range(2): + random_generator._sample_tokens(4) + assert random_generator._offset_idx == 2 + + # Third read is past the end of the cache. + tokens = random_generator._sample_tokens(4) + + assert len(tokens) == 4 + assert all(t in random_generator._allowed_tokens for t in tokens) + + def test_offset_cache_exhaustion_warns_once(self, random_generator, caplog): + """The fallback is silent-by-default degradation otherwise, so it warns + exactly once naming the cause rather than per prompt.""" + import logging + + import numpy as np + + random_generator.preseed(1, np.random.default_rng(42)) + random_generator._sample_tokens(4) + + with caplog.at_level(logging.WARNING): + for _ in range(5): + random_generator._sample_tokens(4) + + exhausted = [ + r for r in caplog.records if "offset cache exhausted" in r.getMessage() + ] + assert len(exhausted) == 1 + assert "consumed once per prompt" in exhausted[0].getMessage() + + def test_offset_cache_within_bounds_is_unaffected(self, random_generator): + """Reads inside the cache must still come from the preseed stream.""" + import numpy as np + + random_generator.preseed(4, np.random.default_rng(42)) + expected = list(random_generator._offset_cache) + + random_generator._sample_tokens(4) + random_generator._sample_tokens(4) + + assert random_generator._offset_idx == 2 + assert random_generator._offset_cache == expected + + def test_topup_draws_stay_inside_the_decodable_pool(self, random_generator): + """Top-up must never emit an ID outside the tokenizer's decodable set. + + vLLM tops up from raw `[0, vocab_size)`, which is safe only because HF + vocabularies are dense. tiktoken's ID space is sparse -- cl100k_base has + 16 undecodable gap IDs -- and those raise `KeyError` inside + `_decode_to_exact_len`, aborting dataset generation. + """ + import numpy as np + + random_generator.preseed(4, np.random.default_rng(42)) + allowed = set(random_generator._topup_tokens) + tokens = random_generator._sample_topup_tokens(5000) + + assert len(tokens) == 5000 + assert not [t for t in tokens if t not in allowed] + + def test_topup_pool_is_the_full_vocab_not_the_vllm_subset(self, random_generator): + """vLLM's top-up does not exclude special tokens, so the narrower + VLLM-style `_allowed_tokens` would be the wrong pool and would change + the draw bound, breaking byte parity.""" + tok = random_generator.tokenizer + assert random_generator._topup_tokens == tok.all_token_ids + assert len(random_generator._topup_tokens) == tok.vocab_size + + def test_indexed_draw_equals_raw_draw_on_dense_vocab(self, random_generator): + """The parity guarantee: for a dense vocabulary `all_token_ids` is + `range(vocab_size)`, so indexing into it is bit-for-bit the same + sequence as drawing raw `[0, vocab_size)` from the same generator. + """ + import numpy as np + + tok = random_generator.tokenizer + pool = random_generator._topup_tokens + raw = np.random.default_rng(7).integers(0, tok.vocab_size, size=500).tolist() + indexed = [ + pool[int(i)] + for i in np.random.default_rng(7).integers(0, len(pool), size=500) + ] + assert indexed == raw + + def test_generate_with_exact_length_bypasses_the_floor(self, random_generator): + """`exact_length` uses the value verbatim instead of resampling. + + Without it, an already-decided 0 is floored to 1 by + `sample_positive_normal_integer`, emitting one more token than vLLM. + """ + import numpy as np + + random_generator.preseed(4, np.random.default_rng(42)) + random_generator._prefix_prompt_tokens = [[7]] + random_generator._prefix_prompts = ["x"] + # body 0 + a 1-token prefix -> prefix-only, which is what vLLM emits + text = random_generator.generate( + mean=0, stddev=0, with_prefix=True, exact_length=True + ) + assert isinstance(text, str) + + def test_generate_prompt_rejects_zero_without_a_prefix(self, random_generator): + """The relaxation is scoped: with no prefix there is nothing to carry + the request, and the degenerate-config guard already rejects that + combination upstream.""" + with pytest.raises(ValueError, match="without a prefix"): + random_generator.generate_prompt(0) + + def test_generate_prompt_still_rejects_negative(self, random_generator): + with pytest.raises(ValueError): + random_generator.generate_prompt(-1) + + def test_exact_length_off_still_floors(self, random_generator): + """Default path is untouched: a mean of 0 without `exact_length` + continues to floor at 1, which is correct for a sampled length.""" + assert random_generator.calculate_num_tokens(0, 0) == 1 + + +class _SeamTokenizer(Tokenizer): + """Deterministic BPE-like tokenizer that charges a token for a joined space. + + Token id ``t`` in ``[0, VOCAB_SIZE)`` maps to the unique two-letter string + ``chr(97 + t // 10) + chr(97 + t % 10)``. Decoding concatenates those pairs + with no separator, so a decoded run of N tokens is exactly 2N letters. + Encoding walks the text two letters at a time and emits ``SPACE_ID`` for any + literal space. + + This models the one property AIP-1118 turns on: joining two independently + decoded segments with ``" "`` costs one extra token, while concatenating + their token IDs costs nothing. The shared ``mock_tokenizer_cls`` fixture + splits on whitespace and therefore cannot express a seam at all. + """ + + VOCAB_SIZE = 100 + SPACE_ID = 100 + + def __init__(self) -> None: + super().__init__() + self._tokenizer = MagicMock() + self._tokenizer.vocab_size = self.VOCAB_SIZE + self._tokenizer.all_special_ids = [] + + @property + def vocab_size(self) -> int: + return self.VOCAB_SIZE + + @property + def all_special_ids(self) -> set[int]: + return set() + + @property + def valid_token_ids(self) -> list[int]: + return list(range(self.VOCAB_SIZE)) + + @property + def all_token_ids(self) -> list[int]: + return list(range(self.VOCAB_SIZE)) + + def decode(self, token_ids, **kwargs) -> str: + return "".join( + " " if t == self.SPACE_ID else chr(97 + t // 10) + chr(97 + t % 10) + for t in token_ids + ) + + def encode(self, text: str, **kwargs) -> list[int]: + out: list[int] = [] + i = 0 + while i < len(text): + if text[i] == " ": + out.append(self.SPACE_ID) + i += 1 + else: + pair = text[i : i + 2] + out.append((ord(pair[0]) - 97) * 10 + (ord(pair[1]) - 97)) + i += 2 + return out + + +class TestPromptGeneratorPrefixVLLMParity: + """AIP-1118: prefix prompts must match vLLM's RandomDataset token accounting. + + vLLM concatenates the prefix and body at the TOKEN-ID level and runs a single + decode/re-encode correction over the combined sequence targeting exactly + ``prefix_len + input_len``. aiperf historically decoded each segment + independently and joined the strings with a space, which cost ~0.84 extra + tokens per request and shifted the per-request token index. + """ + + PREFIX_LEN = 5 + BODY_LEN = 20 + POOL_SIZE = 4 + SEED = 0 + + def _make(self, pool_size: int | None, length: int | None) -> PromptGenerator: + from aiperf.common.enums import RandomCorpusStyle + + return PromptGenerator( + prompts=None, + prefix_prompts=PrefixPromptConfig(pool_size=pool_size, length=length), + tokenizer=_SeamTokenizer(), + corpus=PromptCorpus.RANDOM, + corpus_style=RandomCorpusStyle.VLLM, + ) + + def test_generate_with_prefix_yields_exact_total_token_count(self): + """prefix + body must tokenize to exactly prefix_len + body_len. + + The string join ``f"{prefix} {content}"`` charged one extra token for the + separator on ~81% of requests (mean +0.84). + """ + gen = self._make(pool_size=1, length=self.PREFIX_LEN) + gen.preseed(8, np.random.default_rng(self.SEED)) + gen.initialize_prefix_pool() + + content = gen.generate(mean=self.BODY_LEN, stddev=0, with_prefix=True) + + assert ( + len(gen.tokenizer.encode(content, add_special_tokens=False)) + == self.PREFIX_LEN + self.BODY_LEN + ) + + def test_prefix_pool_build_does_not_advance_request_index(self): + """Pool construction must not consume per-request token indices. + + vLLM builds its prefix from a standalone draw; the per-request index + starts at 0 for request 0 regardless of prefix configuration. + """ + gen = self._make(pool_size=self.POOL_SIZE, length=self.PREFIX_LEN) + gen.preseed(8, np.random.default_rng(self.SEED)) + gen.initialize_prefix_pool() + + assert gen._random_request_index == 0 + + def test_prefix_pool_does_not_shift_body_token_sequence(self): + """Body content must be byte-identical with and without a prefix pool.""" + with_pool = self._make(pool_size=self.POOL_SIZE, length=self.PREFIX_LEN) + with_pool.preseed(8, np.random.default_rng(self.SEED)) + with_pool.initialize_prefix_pool() + + without_pool = self._make(pool_size=None, length=None) + without_pool.preseed(8, np.random.default_rng(self.SEED)) + without_pool.initialize_prefix_pool() + + assert with_pool.generate( + mean=self.BODY_LEN, stddev=0 + ) == without_pool.generate(mean=self.BODY_LEN, stddev=0) + + def test_prefix_pool_matches_vllm_get_prefix_draw_order(self): + """Prefix tokens must come from the shared stream, after the offset draws. + + vLLM's ``RandomDataset.get_prefix`` draws with + ``rng.integers(0, len(allowed_tokens), size=prefix_len)`` — bounded by + the allowed-token count, not vocab_size — immediately after + ``get_sampling_params`` has drawn ISLs, OSLs and offsets. + """ + n = 8 + gen = self._make(pool_size=1, length=self.PREFIX_LEN) + gen.preseed(n, np.random.default_rng(self.SEED)) + gen.initialize_prefix_pool() + + ref = np.random.default_rng(self.SEED) + ref.integers(0, _SeamTokenizer.VOCAB_SIZE, size=n) # offsets + allowed = np.array(gen._allowed_tokens) + expected = allowed[ref.integers(0, len(allowed), size=self.PREFIX_LEN)].tolist() + + assert gen.get_random_prefix_prompt_tokens() == expected diff --git a/tests/unit/property/test_cli_help_references.py b/tests/unit/property/test_cli_help_references.py new file mode 100644 index 0000000000..b927d74d4e --- /dev/null +++ b/tests/unit/property/test_cli_help_references.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Mechanical invariant: CLI help text may only reference flags that exist. + +`CLIConfig` field descriptions are the source of `docs/cli-options.md`, so a +description naming a flag that was renamed (or never existed) ships a wrong +instruction to users in two places at once and no existing check catches it. +This pins the reference graph closed. +""" + +import re + +from aiperf.config.cli_parameter import CLIParameter +from aiperf.config.flags.cli_config import CLIConfig + +# Matches an inline-code flag reference such as `--random-corpus-style`. +_FLAG_REFERENCE = re.compile(r"`(--[a-z0-9][a-z0-9-]*)`") + + +def _as_tuple(value: str | tuple[str, ...] | bool | None) -> tuple[str, ...]: + """Normalize a CLIParameter name/negative to a tuple of flag strings. + + cyclopts accepts a bare string and normalizes it to a tuple, and `negative` + may additionally be a bool (auto-derive) or None (disabled). + """ + if isinstance(value, str): + return (value,) + if isinstance(value, tuple): + return value + return () + + +def _declared_flags() -> set[str]: + """Every flag CLIConfig actually exposes, including negative forms.""" + declared: set[str] = set() + for field in CLIConfig.model_fields.values(): + for meta in field.metadata: + if not isinstance(meta, CLIParameter): + continue + names = _as_tuple(meta.name) + declared.update(names) + declared.update(_as_tuple(meta.negative)) + # `negative=True` means cyclopts derives `--no-` itself. + if meta.negative is True: + declared.update(f"--no-{n.lstrip('-')}" for n in names) + return declared + + +def _referenced_flags() -> dict[str, set[str]]: + """Flags cited in field descriptions, mapped to the citing field names.""" + referenced: dict[str, set[str]] = {} + for name, field in CLIConfig.model_fields.items(): + if not field.description: + continue + for flag in _FLAG_REFERENCE.findall(field.description): + referenced.setdefault(flag, set()).add(name) + return referenced + + +def test_help_text_references_only_existing_flags(): + """A description citing a nonexistent flag is a user-facing lie. + + This caught `--random-range-ratio-mode` in the `--random-range-ratio` + description, which had never existed under that name (the flag is + `--random-corpus-style`) and was mirrored into docs/cli-options.md twice. + """ + declared = _declared_flags() + dangling = { + flag: sorted(fields) + for flag, fields in _referenced_flags().items() + if flag not in declared + } + + assert not dangling, ( + "CLI help text references flags that do not exist:\n" + + "\n".join( + f" {flag} <- cited by {fields}" for flag, fields in dangling.items() + ) + ) + + +def test_invariant_has_coverage(): + """Guards the test above against silently passing on an empty scan.""" + assert len(_declared_flags()) > 100 + assert len(_referenced_flags()) > 20 diff --git a/tests/unit/property/test_finite_invariants.py b/tests/unit/property/test_finite_invariants.py index 5841bcec3d..ec7dd4c234 100644 --- a/tests/unit/property/test_finite_invariants.py +++ b/tests/unit/property/test_finite_invariants.py @@ -379,6 +379,11 @@ def _load_or_init_metric_baseline(current: list[str]) -> list[str]: # _Environment.ENDPOINT: nested _EndpointSettings object, not a numeric # leaf. The heuristic fires because "int" appears in "_EndpointSettings". "_Environment.ENDPOINT", + # VLLMRatioConfig.range_ratio / SGLangRatioConfig.range_ratio: tuple[float, float] + # container — a field-level ge/lt cannot apply to the tuple; bounds [0.0, 1.0) + # are enforced by the field_validator on each component. + "VLLMRatioConfig.range_ratio", + "SGLangRatioConfig.range_ratio", } From b4e36b6cdef27f6e35ae982a2750c6d582a64566 Mon Sep 17 00:00:00 2001 From: Nate Mailhot Date: Fri, 21 Aug 2026 18:49:30 -0700 Subject: [PATCH 034/119] chore: bump aiperf version to 0.13.0 (#1317) Signed-off-by: Nate Mailhot (cherry picked from commit e10d53b1d30b5845f56cbea63d0560f10ff5aa4e) --- docs/plugins/creating-your-first-plugin.md | 2 +- docs/server-metrics/server-metrics-json-schema.md | 6 +++--- pyproject.toml | 2 +- tests/aiperf_mock_server/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/plugins/creating-your-first-plugin.md b/docs/plugins/creating-your-first-plugin.md index 0c65200ebd..5c70047cb7 100644 --- a/docs/plugins/creating-your-first-plugin.md +++ b/docs/plugins/creating-your-first-plugin.md @@ -234,7 +234,7 @@ You should see both packages listed in the same environment: ```text Name: aiperf -Version: 0.12.0 +Version: 0.13.0 Location: ... Requires: ... Required-by: my-aiperf-plugins diff --git a/docs/server-metrics/server-metrics-json-schema.md b/docs/server-metrics/server-metrics-json-schema.md index 95018eb05b..1b041660e4 100644 --- a/docs/server-metrics/server-metrics-json-schema.md +++ b/docs/server-metrics/server-metrics-json-schema.md @@ -60,7 +60,7 @@ data["metrics"]["metric_name"]["series"][0]["stats"]["p99"] ```json { "schema_version": "1.1", - "aiperf_version": "0.12.0", + "aiperf_version": "0.13.0", "benchmark_id": "550e8400-e29b-41d4-a716-446655440000", "summary": { ... }, "metrics_phase": "profiling", @@ -73,7 +73,7 @@ data["metrics"]["metric_name"]["series"][0]["stats"]["p99"] | Field | Type | Description | |-------|------|-------------| | `schema_version` | string | Schema version for this export format (e.g., `"1.1"`) | -| `aiperf_version` | string or null | AIPerf version that generated this export (e.g., `"0.12.0"`). `null` if version unavailable. | +| `aiperf_version` | string or null | AIPerf version that generated this export (e.g., `"0.13.0"`). `null` if version unavailable. | | `benchmark_id` | string or null | Unique UUID identifying this benchmark run. `null` if not available. | | [`summary`](#summary-section) | object | Collection metadata and endpoint information. `phase_time_ranges` is present with at least a `profiling` entry when the profiling window is non-degenerate; `warmup` is the optional second key. | | `metrics_phase` | string | Benchmark phase represented by the top-level `metrics` field. Always `"profiling"` (kept for backward compatibility). | @@ -883,7 +883,7 @@ for series in metric["series"]: ```json { "schema_version": "1.1", - "aiperf_version": "0.12.0", + "aiperf_version": "0.13.0", "benchmark_id": "550e8400-e29b-41d4-a716-446655440000", "metrics_phase": "profiling", "summary": { diff --git a/pyproject.toml b/pyproject.toml index 51ccc5585e..5dd141a64a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ allow-direct-references = true [project] name = "aiperf" -version = "0.12.0" +version = "0.13.0" description = "AIPerf is a package for performance testing of AI models" readme = "README.md" license = {text = "Apache-2.0"} diff --git a/tests/aiperf_mock_server/pyproject.toml b/tests/aiperf_mock_server/pyproject.toml index c7ac3a4344..6b9d595c47 100644 --- a/tests/aiperf_mock_server/pyproject.toml +++ b/tests/aiperf_mock_server/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "aiperf-mock-server" -version = "0.12.0" +version = "0.13.0" description = "AIPerf FastAPI-based OpenAI mock server for AI performance testing" authors = [] dependencies = [ From 457d00b8d87b36771971b2c2d10e81dbbe039c41 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:35:47 -0700 Subject: [PATCH 035/119] fix(test-harness): implement num_prompt_special_tokens on FakeTokenizer (#1322) Signed-off-by: Elias Bermudez (cherry picked from commit 9b60a3d479fd808373c077686c0bbe367d431bec) --- tests/harness/fake_tokenizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/harness/fake_tokenizer.py b/tests/harness/fake_tokenizer.py index 931c3ee9b7..74f0d0323f 100644 --- a/tests/harness/fake_tokenizer.py +++ b/tests/harness/fake_tokenizer.py @@ -66,3 +66,6 @@ def resolved_name(self) -> str | None: @property def block_separation_token_id(self) -> int: return 1 + + def num_prompt_special_tokens(self) -> int: + return 0 From f26b27ed7ae783414845d561aa5e539b72efe48b Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Mon, 24 Aug 2026 16:35:47 -0700 Subject: [PATCH 036/119] fix(mmap): make get_conversation thread-safe, prefault pages, drop executor hop (#1245) Signed-off-by: Anthony Casagrande Co-authored-by: Xianjie Qiao <5410381+qiaoxj07@users.noreply.github.com> (cherry picked from commit c9288da6c107c145a310dfe23f64c6ab35328feb) --- docs/environment-variables.md | 1 + src/aiperf/common/environment.py | 10 + src/aiperf/dataset/memory_map_utils.py | 66 ++++--- .../test_mmap_conversation_concurrency.py | 172 ++++++++++++++++++ 4 files changed, 226 insertions(+), 23 deletions(-) create mode 100644 tests/unit/dataset/test_mmap_conversation_concurrency.py diff --git a/docs/environment-variables.md b/docs/environment-variables.md index ac5070cd94..5afae4d928 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -100,6 +100,7 @@ Dataset loading and configuration. Controls timeouts and behavior for dataset lo | `AIPERF_DATASET_MMAP_BASE_PATH` | `None` | — | Base path for memory-mapped dataset files. If None, uses system temp directory. Set to a shared filesystem path for Kubernetes mounted volumes. Example: AIPERF_DATASET_MMAP_BASE_PATH=/mnt/shared-pvc creates files at /mnt/shared-pvc/aiperf_mmap_{benchmark_id}/ | | `AIPERF_DATASET_MMAP_CACHE_ENABLED` | `True` | — | If True, AIPerf reuses memory-mapped dataset files across runs whose input bytes, tokenizer identity, and prompt/input settings are byte-identical. Set to False to force every run to re-tokenize and re-write its mmap files. Cache misses still produce byte-identical mmap files to a non-cached run. | | `AIPERF_DATASET_MMAP_CACHE_DIR` | `None` | — | Directory holding the content-addressed mmap cache. If None, defaults to ~/.cache/aiperf/dataset_mmap. Each cache entry lives under a `dir/key` subpath and contains dataset.dat, index.dat, manifest.json, and (when produced) inputs.json. No automatic eviction is implemented yet -- delete the directory to reclaim disk. | +| `AIPERF_DATASET_MMAP_PREFAULT` | `True` | — | If True, each memory-mapped dataset client walks every page of the data file at open time (after madvise(MADV_WILLNEED)) to force-populate the OS page cache. Reads afterwards are served warm, so no request pays a major page fault mid-benchmark -- which would otherwise land in the measured latency. Workers share the kernel page cache, so the disk read happens once regardless of worker count. Costs a one-time startup pass proportional to dataset size; set to False to trade predictable tail latency for faster startup on very large datasets. | | `AIPERF_DATASET_PREFORMAT_PAYLOADS` | `False` | — | If True, pre-encode single-turn / self-contained synthetic conversations to the PAYLOAD_BYTES mmap fast path at dataset-build time so workers stream the bytes verbatim and skip per-request encoding. This is a throughput optimization that DROPS input-tokenization metrics (input_sequence_length, image counts) because the structured prompt is discarded. Default False keeps the structured-turns (CONVERSATION) path so those metrics are computed. Datasets that natively ship raw payloads (raw_payload / inputs_json / mooncake-with-payload) always use PAYLOAD_BYTES regardless of this flag; cache-bust runs always use CONVERSATION regardless. | | `AIPERF_DATASET_PUBLIC_DATASET_TIMEOUT` | `300.0` | ≥ 1.0, ≤ 100000.0 | Timeout in seconds for public dataset loading operations | | `AIPERF_DATASET_MEDIA_DOWNLOAD_TIMEOUT` | `60.0` | ≥ 1.0, ≤ 100000.0 | Timeout in seconds per media URL download when inline encoding is required | diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 3d19d7d835..cce590d5de 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -335,6 +335,16 @@ class _DatasetSettings(BaseSettings): "dataset.dat, index.dat, manifest.json, and (when produced) inputs.json. " "No automatic eviction is implemented yet -- delete the directory to reclaim disk.", ) + MMAP_PREFAULT: bool = Field( + default=True, + description="If True, each memory-mapped dataset client walks every page of the " + "data file at open time (after madvise(MADV_WILLNEED)) to force-populate the OS " + "page cache. Reads afterwards are served warm, so no request pays a major page " + "fault mid-benchmark -- which would otherwise land in the measured latency. " + "Workers share the kernel page cache, so the disk read happens once regardless of " + "worker count. Costs a one-time startup pass proportional to dataset size; set to " + "False to trade predictable tail latency for faster startup on very large datasets.", + ) PREFORMAT_PAYLOADS: bool = Field( default=False, description="If True, pre-encode single-turn / self-contained synthetic " diff --git a/src/aiperf/dataset/memory_map_utils.py b/src/aiperf/dataset/memory_map_utils.py index de4612fb63..53b667980b 100644 --- a/src/aiperf/dataset/memory_map_utils.py +++ b/src/aiperf/dataset/memory_map_utils.py @@ -403,12 +403,10 @@ def __init__(self, client_metadata: MemoryMapClientMetadata, **kwargs) -> None: self._data_path: Path = client_metadata.data_file_path self._index_path: Path = client_metadata.index_file_path self._client: MemoryMapDatasetClient | None = None - self._loop: asyncio.AbstractEventLoop | None = None @on_init async def _setup(self) -> None: """Open memory-mapped files (read-only).""" - self._loop = asyncio.get_running_loop() self.debug( lambda: f"Opening memory-mapped files: data={self._data_path}, index={self._index_path}" ) @@ -421,7 +419,14 @@ async def _setup(self) -> None: async def get_conversation(self, conversation_id: str) -> Conversation: """Retrieve conversation from memory-mapped file. - Runs in executor since mmap reads can block on page faults. + No executor hop: pages are prefaulted at open (when MMAP_PREFAULT is + enabled) so reads don't trigger major faults. Remains async-callable + for API parity with other dataset clients. + + When MMAP_PREFAULT is disabled, reads may still take major faults, and + those now land on the event loop thread rather than an executor thread. + That is deliberate: CPython's mmap slice holds the GIL while copying, + so an executor relocates the stall instead of removing it. Args: conversation_id: Session ID of the conversation @@ -432,11 +437,9 @@ async def get_conversation(self, conversation_id: str) -> Conversation: Raises: KeyError: If conversation_id not found """ - if self._client is None or self._loop is None: + if self._client is None: raise RuntimeError("Client store not initialized. Call initialize() first.") - return await self._loop.run_in_executor( - None, self._client.get_conversation, conversation_id - ) + return self._client.get_conversation(conversation_id) async def get_payload_bytes( self, conversation_id: str, turn_index: int @@ -451,11 +454,9 @@ async def get_payload_bytes( Pre-encoded JSON bytes, or None when the dataset is not in PAYLOAD_BYTES format or the turn has no payload. """ - if self._client is None or self._loop is None: + if self._client is None: raise RuntimeError("Client store not initialized. Call initialize() first.") - return await self._loop.run_in_executor( - None, self._client.get_payload_bytes, conversation_id, turn_index - ) + return self._client.get_payload_bytes(conversation_id, turn_index) async def get_payload_turn( self, conversation_id: str, turn_index: int @@ -470,11 +471,9 @@ async def get_payload_turn( ``PayloadTurnData`` or None when the dataset is not in PAYLOAD_BYTES format or the turn has no payload. """ - if self._client is None or self._loop is None: + if self._client is None: raise RuntimeError("Client store not initialized. Call initialize() first.") - return await self._loop.run_in_executor( - None, self._client.get_payload_turn, conversation_id, turn_index - ) + return self._client.get_payload_turn(conversation_id, turn_index) @on_stop async def _cleanup(self) -> None: @@ -673,6 +672,15 @@ def __init__(self, data_file_path: Path | str, index_file_path: Path | str) -> N index_data = self.index_mmap.read() self.index = MemoryMapDatasetIndex.model_validate_json(index_data) + # Flatten to a plain dict[str, tuple[int, int]] to avoid per-lookup + # Pydantic attribute overhead on the hot path. Wire format unchanged; + # self.index stays for callers that iterate index.conversation_ids. + self._offsets: dict[str, tuple[int, int]] = { + cid: (o.offset, o.size) for cid, o in self.index.offsets.items() + } + + if Environment.DATASET.MMAP_PREFAULT: + self._prefault_data_mmap() except OSError as e: self._cleanup_resources() @@ -735,6 +743,19 @@ def _cleanup_resources(self) -> None: with suppress(Exception): obj.close() + def _prefault_data_mmap(self) -> None: + """Advise WILLNEED and touch every page to populate the page cache. + + Without this, the first read of each page takes a major fault whose + latency lands inside a measured request. Workers share the kernel page + cache, so N workers cost one disk read rather than N serialized ones. + """ + # madvise is unavailable on some platforms; the page walk still works. + with suppress(OSError, AttributeError): + self.data_mmap.madvise(mmap.MADV_WILLNEED) + for offset in range(0, len(self.data_mmap), mmap.PAGESIZE): + _ = self.data_mmap[offset] + def _deserialize_conversation(self, data: bytes) -> Conversation: """Deserialize a single conversation from bytes. @@ -774,20 +795,19 @@ def get_conversation(self, conversation_id: str) -> Conversation: "format. Use get_payload_bytes() instead." ) - if conversation_id not in self.index.offsets: + if conversation_id not in self._offsets: raise KeyError(f"Conversation '{conversation_id}' not found in dataset") - offset_info = self.index.offsets[conversation_id] + offset, size = self._offsets[conversation_id] try: - conv_bytes = bytes( - self.data_mmap[ - offset_info.offset : offset_info.offset + offset_info.size - ] - ) + # Slice rather than seek()+read(): the mmap file position is shared + # state, so two concurrent readers can interleave a seek between + # another's seek and read and serve foreign bytes. + conv_bytes = self.data_mmap[offset : offset + size] _logger.debug( - lambda: f"Loading conversation '{conversation_id}': offset={offset_info.offset}, size={offset_info.size} bytes" + lambda: f"Loading conversation '{conversation_id}': offset={offset}, size={size} bytes" ) return self._deserialize_conversation(conv_bytes) diff --git a/tests/unit/dataset/test_mmap_conversation_concurrency.py b/tests/unit/dataset/test_mmap_conversation_concurrency.py new file mode 100644 index 0000000000..7339b962ae --- /dev/null +++ b/tests/unit/dataset/test_mmap_conversation_concurrency.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Concurrency and resource-lifetime tests for MemoryMapDatasetClient. + +``get_conversation`` used to read through the mmap's shared file position +(``seek()`` then ``read()``). Any two readers sharing one client could +interleave those calls and serve each other's bytes, surfacing either as a +spurious ``MemoryMapSerializationError`` on a random conversation or, when the +foreign slice happened to parse, as silently wrong data. + +The reads are now position-free slices, so no reader depends on the shared +position. +""" + +import mmap +from collections.abc import AsyncIterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import pytest + +from aiperf.common.enums import MemoryMapFormat +from aiperf.common.models import Conversation, Text, Turn +from aiperf.dataset.memory_map_utils import ( + MemoryMapDatasetBackingStore, + MemoryMapDatasetClient, +) + +# Wildly varying sizes so a cross-read lands mid-record and is caught either as +# a decode error or as a mismatched session_id. +_CONVERSATION_COUNT = 24 +_THREADS = 8 +_LOOKUPS_PER_THREAD = 150 + + +def _make_conversation(index: int) -> Conversation: + """Conversation whose size grows with ``index`` and whose text encodes it.""" + text = f"conv-{index}:" + ("x" * (64 * (index + 1))) + return Conversation( + session_id=f"conv-{index}", + turns=[Turn(role="user", texts=[Text(contents=[text])])], + ) + + +@asynccontextmanager +async def _open_client( + tmp_path: Path, benchmark_id: str, count: int +) -> AsyncIterator[MemoryMapDatasetClient]: + """Write ``count`` conversations and open a client over them. + + The backing store is stopped on exit, so its ``@on_stop`` cleanup unlinks + the mmap files instead of leaking them for the run's lifetime. The client + is closed first: the store's cleanup deletes the files it maps. + """ + store = MemoryMapDatasetBackingStore( + benchmark_id=benchmark_id, format=MemoryMapFormat.CONVERSATION + ) + await store.initialize() + for i in range(count): + await store.add_conversation(f"conv-{i}", _make_conversation(i)) + await store.finalize() + + metadata = store.get_client_metadata() + client: MemoryMapDatasetClient | None = None + try: + # Constructed inside the guard: if opening the mmap raises, the store + # still gets stopped and its files still get unlinked. + client = MemoryMapDatasetClient( + metadata.data_file_path, + metadata.index_file_path, + ) + yield client + finally: + if client is not None: + client.close() + await store.stop() + + +@pytest.mark.asyncio +async def test_get_conversation_concurrent_readers_returns_own_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Concurrent get_conversation() calls must not read each other's bytes.""" + monkeypatch.setenv("AIPERF_DATASET_MMAP_BASE_PATH", str(tmp_path)) + async with _open_client( + tmp_path, "test_conv_concurrency", _CONVERSATION_COUNT + ) as client: + + def _read_many(thread_index: int) -> None: + for n in range(_LOOKUPS_PER_THREAD): + # Stagger the starting id per thread so the interleaving covers + # many (offset, size) pairs rather than repeatedly hitting one + # record. + i = (thread_index + n) % _CONVERSATION_COUNT + conv = client.get_conversation(f"conv-{i}") + assert conv.session_id == f"conv-{i}" + + with ThreadPoolExecutor(max_workers=_THREADS) as pool: + # list() resolves every future so exceptions propagate. + list(pool.map(_read_many, range(_THREADS))) + + +class _InterleavingMmap: + """mmap wrapper that moves the shared position between seek() and read(). + + This is the race, made deterministic. Merely leaving a stale position + behind proves nothing: a ``seek()`` + ``read()`` reader overwrites it on + the way in and still returns the right record. The bug needs a competing + seek to land *after* this reader seeks and *before* it reads. + + A position-free reader goes through ``__getitem__`` and is unaffected. + """ + + def __init__(self, real: mmap.mmap, foreign_offset: int) -> None: + self._real = real + self._foreign_offset = foreign_offset + + def seek(self, pos: int) -> None: + self._real.seek(pos) + + def read(self, size: int) -> bytes: + # The competing reader lands here, between our seek and our read. + self._real.seek(self._foreign_offset) + return self._real.read(size) + + def __getitem__(self, item: slice | int) -> bytes | int: + return self._real[item] + + def __len__(self) -> int: + return len(self._real) + + def __getattr__(self, name: str) -> Any: + return getattr(self._real, name) + + +@pytest.mark.asyncio +async def test_get_conversation_ignores_competing_seek_between_seek_and_read( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An interleaved foreign seek must not change what get_conversation reads. + + Fails against a ``seek()`` + ``read()`` implementation, which serves the + foreign record's bytes. + """ + monkeypatch.setenv("AIPERF_DATASET_MMAP_BASE_PATH", str(tmp_path)) + async with _open_client(tmp_path, "test_conv_position", 3) as client: + foreign = client.index.offsets["conv-2"].offset + client.data_mmap = _InterleavingMmap(client.data_mmap, foreign) + + conversation = client.get_conversation("conv-0") + + assert conversation.session_id == "conv-0" + assert conversation.turns[0].texts[0].contents[0].startswith("conv-0:") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefault", [True, False]) # fmt: skip +async def test_get_conversation_matches_written_data_under_prefault_setting( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, prefault: bool +) -> None: + """Reads are identical whether or not pages were prefaulted at open.""" + monkeypatch.setenv("AIPERF_DATASET_MMAP_BASE_PATH", str(tmp_path)) + monkeypatch.setattr( + "aiperf.common.environment.Environment.DATASET.MMAP_PREFAULT", prefault + ) + async with _open_client(tmp_path, f"test_conv_prefault_{prefault}", 5) as client: + for i in range(5): + conv = client.get_conversation(f"conv-{i}") + assert conv.session_id == f"conv-{i}" + assert conv.turns[0].texts[0].contents[0].startswith(f"conv-{i}:") From 29b1449bb5a508f7494a9fc8dfef540461af142b Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:41:37 -0700 Subject: [PATCH 037/119] fix(dataset): restore random_pool batch-size support with --input-file (#1274) Signed-off-by: Elias Bermudez Signed-off-by: Anthony Casagrande Co-authored-by: Anthony Casagrande (cherry picked from commit c2889280a66fc85b44e9456fd7020874c73a44fc) --- docs/tutorials/custom-dataset.md | 42 ++ src/aiperf/config/dataset/config.py | 66 +++ src/aiperf/config/flags/_converter_dataset.py | 96 +++- src/aiperf/config/flags/resolver.py | 134 ++++- src/aiperf/dataset/loader/random_pool.py | 167 +++++- ...st_random_pool_batch_size_yaml_override.py | 353 ++++++++++++ .../config/test_v1_file_dataset_rejections.py | 244 ++++++++ tests/unit/dataset/loader/test_random_pool.py | 527 ++++++++++++++++-- 8 files changed, 1559 insertions(+), 70 deletions(-) create mode 100644 tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py diff --git a/docs/tutorials/custom-dataset.md b/docs/tutorials/custom-dataset.md index acb876d018..9bd1351b7f 100644 --- a/docs/tutorials/custom-dataset.md +++ b/docs/tutorials/custom-dataset.md @@ -439,6 +439,48 @@ Log File: artifacts/Qwen_Qwen3-0.6B-openai-chat-concurrency4/logs/aiperf.log - Sampling with replacement (entries can repeat) - Use `--random-seed` for reproducibility +### Multimodal batch sizes + +For multimodal workloads, `random_pool` supports per-modality batch-size flags that control how many items are packed into each request: + +| Flag | Field | Default | Notes | +|---|---|---|---| +| `--prompt-batch-size N` | `prompt_batch_size` | 1 | Text items per request. Set to `0` to disable text inputs entirely. | +| `--image-batch-size N` | `image_batch_size` | 1 | Images per request. Set to `0` to disable image inputs. | +| `--audio-batch-size N` | `audio_batch_size` | 1 | Audio items per request. Set to `0` to disable audio inputs. | +| `--video-batch-size N` | `video_batch_size` | 1 | Video items per request. Set to `0` to disable video inputs. | + +Setting a batch size to `0` for a modality that is absent from the pool is a no-op: it neither suppresses anything nor counts as batching, so `--image-batch-size 0` against a single-file text-only pool leaves that pool sampled normally. A configuration where *every* modality is either absent from the pool or set to `0` produces empty requests and is rejected, since there is nothing left to send. Directory input rejects any batch size other than `1` regardless of which modalities are present, because pool contents are not known at config time -- see below. + +These flags are only valid with `format: random_pool` (set via `--custom-dataset-type random_pool` on the CLI, or `format: random_pool` directly in a YAML dataset config -- either selects the same format). Using them with other file dataset formats (e.g. `mooncake_trace`) is an error. + +Batch sizes other than 1 are also rejected outright whenever batching would discard pool identity: (1) multiple separately named pools -- directory input (multiple files, e.g. `queries.jsonl` / `passages.jsonl`) or inline YAML `records:` with multiple top-level keys (e.g. `records: {queries: [...], passages: [...]}`) both flatten into one anonymous pool per modality, discarding the names that name-sensitive endpoints (e.g. rankings) depend on; (2) a single pool whose entries embed named `Text`/`Image`/`Audio`/`Video` objects, or `Image` objects carrying `uuids` (vLLM cache-reuse IDs) -- reachable from a single file or a single-key inline `records:` list, not just directory input. Batch sizes only apply to a single unnamed pool with unnamed entries. + +The directory case is caught up front, before the benchmark starts. The remaining shapes are only visible once the pool has been parsed, so they surface when the dataset is built. + +Batching does not preserve per-entry associations across modalities. If a pool entry pairs a specific text with a specific image, a batch size > 1 flattens each modality into an independent pool and samples from them separately, so the original text-image pairing is not preserved in the resulting request. Use `single_turn` instead of `random_pool` if exact pairing must be preserved. + +The `pool.jsonl` file above is text-only, so `--image-batch-size` would have no images to sample. A multimodal `random_pool` needs an `image` field in the pool entries: + +```bash +cat > multimodal_pool.jsonl << 'EOF' +{"text": "Describe what you see in this image.", "image": "https://example.com/img1.png"} +{"text": "What objects are visible here?", "image": "https://example.com/img2.png"} +{"text": "Summarize the scene.", "image": "https://example.com/img3.png"} +EOF + +aiperf profile \ + --model Qwen/Qwen3-VL-7B \ + --endpoint-type chat \ + --input-file multimodal_pool.jsonl \ + --custom-dataset-type random_pool \ + --image-batch-size 2 \ + --num-conversations 50 \ + --streaming \ + --concurrency 4 \ + --url localhost:8000 +``` + ### Inline alternative (multi-pool) ```yaml diff --git a/src/aiperf/config/dataset/config.py b/src/aiperf/config/dataset/config.py index d72305eef1..ccb73fff88 100644 --- a/src/aiperf/config/dataset/config.py +++ b/src/aiperf/config/dataset/config.py @@ -493,6 +493,50 @@ class FileDataset(SystemPromptMixin): ), ] + prompt_batch_size: Annotated[ + int | None, + Field( + default=None, + ge=0, + description="Number of text items per request. " + "Only applies to format: random_pool; rejected on other formats. " + "Set to 0 to disable text inputs entirely (e.g. image/audio/video-only workloads).", + ), + ] + + image_batch_size: Annotated[ + int | None, + Field( + default=None, + ge=0, + description="Number of images per request. " + "Only applies to format: random_pool; rejected on other formats. " + "Set to 0 to disable image inputs entirely.", + ), + ] + + audio_batch_size: Annotated[ + int | None, + Field( + default=None, + ge=0, + description="Number of audio items per request. " + "Only applies to format: random_pool; rejected on other formats. " + "Set to 0 to disable audio inputs entirely.", + ), + ] + + video_batch_size: Annotated[ + int | None, + Field( + default=None, + ge=0, + description="Number of video items per request. " + "Only applies to format: random_pool; rejected on other formats. " + "Set to 0 to disable video inputs entirely.", + ), + ] + block_size: Annotated[ int | None, Field( @@ -604,6 +648,28 @@ def _validate_trace_delay_exclusivity(self) -> FileDataset: ) return self + @model_validator(mode="after") + def _validate_batch_sizes_random_pool_only(self) -> FileDataset: + """Reject per-modality batch sizes on non-random_pool formats. + + These fields are only consumed by RandomPoolDatasetLoader. Setting them + on other formats is an error, not a silent no-op. + """ + batch_fields = { + "prompt_batch_size": self.prompt_batch_size, + "image_batch_size": self.image_batch_size, + "audio_batch_size": self.audio_batch_size, + "video_batch_size": self.video_batch_size, + } + set_fields = {k: v for k, v in batch_fields.items() if v is not None} + if set_fields and self.format != DatasetFormat.RANDOM_POOL: + names = ", ".join(set_fields) + raise ValueError( + f"{names} are rejected on formats other than random_pool; " + f"got format: {self.format}." + ) + return self + @model_validator(mode="after") def _validate_max_context_length_weka_only(self) -> FileDataset: """Reject max_context_length on provably non-Weka file formats. diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index 99a086693c..2ad16b3476 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -13,6 +13,7 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING, Any from aiperf.config.flags._section_fields import ( @@ -607,6 +608,13 @@ def _apply_implicit_media_batch(d: dict[str, Any], cli: CLIConfig) -> None: ) +# Batch-size flags are synthetic-only for trace/single-turn datasets, but +# random_pool supports them to control how many items are packed per request. +_RANDOM_POOL_BATCH_SIZE_FLAGS: frozenset[str] = frozenset( + {"prompt_batch_size", "image_batch_size", "audio_batch_size", "video_batch_size"} +) + + def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: """Reject synthetic-only flags on FILE or PUBLIC (trace) datasets. @@ -619,6 +627,11 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: AIPerfConfig validation with ``extra_forbidden``). Surface a clear message instead. + Exception: batch-size flags (--prompt-batch-size, --image-batch-size, + --audio-batch-size, --video-batch-size) are allowed when + ``--custom-dataset-type random_pool`` is set; they control per-modality + packing in ``RandomPoolDatasetLoader``. + --osl / --osl-stddev are NOT rejected — they're routed onto ``FileDataset.osl`` / ``PublicDataset.osl`` by ``_apply_file_osl`` as a per-record fallback. @@ -626,16 +639,87 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: if not cli.input_file and not _implies_public_dataset(cli): return s = cli.model_fields_set - violations = [ - flag for attr, flag in _FILE_DATASET_INCOMPATIBLE_TRIGGERS if attr in s + from aiperf.plugin.enums import CustomDatasetType + + is_random_pool = ( + cli.input_file is not None + and cli.custom_dataset_type == CustomDatasetType.RANDOM_POOL + ) + batch_size_attrs = _RANDOM_POOL_BATCH_SIZE_FLAGS + non_batch_violations = [ + flag + for attr, flag in _FILE_DATASET_INCOMPATIBLE_TRIGGERS + if attr in s and attr not in batch_size_attrs ] - if violations: + batch_violations = [ + flag + for attr, flag in _FILE_DATASET_INCOMPATIBLE_TRIGGERS + if attr in s and attr in batch_size_attrs and not is_random_pool + ] + if non_batch_violations: raise ValueError( - f"{', '.join(violations)} is only supported with synthetic datasets; " + f"{', '.join(non_batch_violations)} is only supported with synthetic datasets; " "remove --input-file / --public-dataset (use a synthetic dataset) to " "apply synthetic-only prompt shaping (ISL, prefix prompts, multimodal " "generation, multi-turn conversation, etc)." ) + # Directory input is decidable here, before any service starts: each file forms + # a separately named pool and batching flattens them all into one anonymous pool + # per modality. Checked ahead of the random_pool gating below so a directory is + # not first told to add --custom-dataset-type random_pool and only then rejected + # at load time. The loader repeats an equivalent check for the pool shapes that + # are only visible once the files are parsed (inline records, named entries). + set_batch_flags = [ + flag + for attr, flag in _FILE_DATASET_INCOMPATIBLE_TRIGGERS + if attr in s and attr in batch_size_attrs and getattr(cli, attr) != 1 + ] + if set_batch_flags and cli.input_file is not None and Path(cli.input_file).is_dir(): + raise ValueError( + f"{', '.join(set_batch_flags)} is not supported when --input-file is a " + "directory: each file forms a separately named pool, and batching " + "flattens them into one anonymous pool per modality. Point --input-file " + "at a single random_pool file, or drop the batch-size flags." + ) + if batch_violations and cli.input_file is not None: + remedy = ( + "Either add --custom-dataset-type random_pool to keep --input-file" + if cli.custom_dataset_type is None + else f"Either change --custom-dataset-type from {cli.custom_dataset_type} " + "to random_pool to keep --input-file" + ) + raise ValueError( + f"{', '.join(batch_violations)} requires --custom-dataset-type random_pool " + f"when used with --input-file. {remedy}, or remove --input-file to use a " + "synthetic dataset." + ) + if batch_violations: + raise ValueError( + f"{', '.join(batch_violations)} is only supported with synthetic datasets " + "or --input-file --custom-dataset-type random_pool; remove --public-dataset " + "to use a synthetic dataset." + ) + + +def _apply_random_pool_batch_sizes(d: dict[str, Any], cli: CLIConfig) -> None: + """Route batch-size CLI flags onto FileDataset fields when format is random_pool. + + FileDataset has no prompts/images/audio/video sub-configs, so batch sizes + live as flat fields and are threaded directly to RandomPoolDatasetLoader. + """ + from aiperf.plugin.enums import CustomDatasetType + + if not cli.input_file or cli.custom_dataset_type != CustomDatasetType.RANDOM_POOL: + return + s = cli.model_fields_set + if "prompt_batch_size" in s: + d["prompt_batch_size"] = cli.prompt_batch_size + if "image_batch_size" in s: + d["image_batch_size"] = cli.image_batch_size + if "audio_batch_size" in s: + d["audio_batch_size"] = cli.audio_batch_size + if "video_batch_size" in s: + d["video_batch_size"] = cli.video_batch_size _BASETEN_ONLY_TRACE_FLAGS: tuple[tuple[str, str], ...] = ( @@ -1097,7 +1181,8 @@ def build_dataset(cli: CLIConfig) -> dict[str, Any]: flat input fields and sub-config holders on ``cli``, then assembles the sub-fields into the correct dataset shape. Rejects synthetic-only flags (prefix, ISL shaping, batch_size, seq-dist, multimodal batch_size) - when --input-file is set. + when --input-file is set, except batch-size flags when + ``--custom-dataset-type random_pool`` is the custom dataset type. Returns: A dict suitable for ``DatasetConfig.model_validate({"name": "main", **out})``. @@ -1119,6 +1204,7 @@ def build_dataset(cli: CLIConfig) -> dict[str, Any]: _apply_synthesis(d, cli) _apply_implicit_media_batch(d, cli) _apply_file_osl(d, cli) + _apply_random_pool_batch_sizes(d, cli) # block_size for FILE hash-id traces is owned by _apply_block_size (which # also rejects weka / non-hash-id formats). Do not also call # _apply_file_block_size — that helper is broader and redundant here. diff --git a/src/aiperf/config/flags/resolver.py b/src/aiperf/config/flags/resolver.py index 62000823a3..decb5c3546 100644 --- a/src/aiperf/config/flags/resolver.py +++ b/src/aiperf/config/flags/resolver.py @@ -18,6 +18,8 @@ import logging from typing import TYPE_CHECKING, Any +from pydantic.alias_generators import to_camel + from aiperf.common.enums import DatasetType from aiperf.common.phase import infer_legacy_phase_kind from aiperf.config.flags._resolver_gpu_telemetry import ( @@ -36,7 +38,7 @@ OUTPUT_FIELDS, SWEEPING_FIELDS, ) -from aiperf.plugin.enums import ArrivalPattern, PhaseType +from aiperf.plugin.enums import ArrivalPattern, DatasetFormat, PhaseType if TYPE_CHECKING: from pathlib import Path @@ -111,6 +113,7 @@ def resolve_config( merged = deep_merge(yaml_dict, overrides) if overrides else yaml_dict _apply_dataset_synthesis_overrides(merged, cli_config) _apply_dataset_filter_overrides(merged, cli_config) + _apply_random_pool_batch_size_overrides(merged, cli_config) _apply_phase_loadgen_overrides(merged, cli_config) promote_benchmark_magic_lists( merged, @@ -401,6 +404,135 @@ def _apply_dataset_filter_overrides(merged: dict[str, Any], cli: CLIConfig) -> N filters.update(_parse_dataset_filters(cli.dataset_filters)) +def _first_yaml_dataset( + benchmark: dict[str, Any], *, warn_context: str +) -> dict[str, Any] | None: + """Resolve the singular ``dataset`` or first entry of ``datasets`` from a + merged YAML ``benchmark`` mapping. Returns ``None`` if neither is present. + + ``warn_context`` names the flag/feature in the "multiple datasets" warning + (e.g. ``"Batch-size flags"``), consistent with the convention shared by + ``_apply_dataset_filter_overrides`` and ``_apply_dataset_synthesis_overrides``. + """ + dataset = benchmark.get("dataset") + if isinstance(dataset, dict): + return dataset + + datasets = benchmark.get("datasets") + if not isinstance(datasets, list) or not datasets: + return None + if len(datasets) > 1: + logger.warning( + "%s with multiple YAML datasets apply only to the first dataset", + warn_context, + ) + dataset = datasets[0] + return dataset if isinstance(dataset, dict) else None + + +# Maps CLIConfig attribute name -> (FileDataset field name, CLI flag display name). +# Used by _apply_random_pool_batch_size_overrides for gating and error messages. +_RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP: tuple[tuple[str, str, str], ...] = ( + ("prompt_batch_size", "prompt_batch_size", "--prompt-batch-size"), + ("image_batch_size", "image_batch_size", "--image-batch-size"), + ("audio_batch_size", "audio_batch_size", "--audio-batch-size"), + ("video_batch_size", "video_batch_size", "--video-batch-size"), +) + + +def _apply_random_pool_batch_size_overrides( + merged: dict[str, Any], cli: CLIConfig +) -> None: + """Overlay explicit batch-size CLI flags onto a YAML-supplied random_pool dataset. + + In the YAML+CLI path ``_apply_input_overrides`` only routes ``headers`` and + ``extra_inputs``; every other ``INPUT_FIELDS`` member (including the four + batch-size fields added by this PR) was silently discarded. This function + closes that gap for the four fields that ``RandomPoolDatasetLoader`` consumes. + + Gating is on ``cli.model_fields_set`` — not truthiness, not ``is not None`` + against the field value — so an unset flag never clobbers a YAML-supplied value. + Zero is a valid value (``image/audio/video_batch_size=0`` disables that modality). + + Only applies to ``type: file`` datasets. Synthetic and public datasets have no + ``format`` field at all, so this function must not touch or reject them here. + Note this does NOT mean the flags take effect there: nothing in the YAML+CLI + path currently routes batch-size flags onto ``SyntheticDataset.prompts/ + images/audio/video.batch_size`` (``_apply_input_overrides`` only handles + ``headers``/``extra_inputs``), so a batch-size flag against a synthetic YAML + dataset has no effect -- a pre-existing gap this function does not close and + is out of scope to fix here. It logs a warning rather than dropping the flag + silently, since the neighbouring wrong-format case raises loudly. The CLI-only + path (no ``--config``) applies these flags correctly; only the YAML+CLI overlay + drops them. + + For a ``type: file`` dataset that isn't ``format: random_pool``, a ``ValueError`` + is raised with a message that names the flag and the format, matching the + friendly error the CLI-only path produces instead of letting the ``FileDataset`` + model validator fire a raw Pydantic trace. + + With multiple YAML datasets the override applies to the first dataset only, + consistent with the convention in ``_apply_dataset_synthesis_overrides`` and + ``_apply_dataset_filter_overrides``. + """ + set_fields = cli.model_fields_set & { + cli_attr for cli_attr, _, _ in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP + } + if not set_fields: + return + + benchmark = merged.get("benchmark") + if not isinstance(benchmark, dict): + return + + dataset = _first_yaml_dataset(benchmark, warn_context="Batch-size flags") + if dataset is None: + return + if dataset.get("type") != DatasetType.FILE: + # Adjacent to a loud ValueError for a file dataset of the wrong format, so + # do not drop this one in silence: the flag genuinely has no effect here. + logger.warning( + "%s ignored: batch-size flags are only applied to a YAML dataset with " + "type: file and format: random_pool (got type: %s). The CLI-only path " + "(no --config) applies them normally.", + ", ".join( + flag + for cli_attr, _, flag in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP + if cli_attr in set_fields + ), + dataset.get("type"), + ) + return + + # dataset.get("format") reads the raw pre-validation YAML dict, so an omitted + # `format:` key reads back as None here even though FileDataset.format defaults + # to DatasetFormat.SINGLE_TURN -- fall back to that default so the error message + # below reports the actual effective format instead of a misleading "None". + fmt = dataset.get("format") or DatasetFormat.SINGLE_TURN + if fmt != DatasetFormat.RANDOM_POOL: + flag_names = ", ".join( + flag + for cli_attr, _, flag in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP + if cli_attr in set_fields + ) + raise ValueError( + f"{flag_names} requires format: random_pool on the YAML dataset " + f"(got format: {fmt}). Either set format: random_pool in the dataset " + "config, or remove these flags." + ) + + for cli_attr, dataset_field, _ in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP: + if cli_attr in set_fields: + # FileDataset uses alias_generator=to_camel with extra="forbid": if the + # YAML already supplied this field under its camelCase alias (e.g. + # promptBatchSize, the shipped template idiom), writing the snake_case + # key here leaves both present and Pydantic rejects the snake_case one + # as extra. Drop whichever spelling is already there before writing. + dataset.pop(to_camel(dataset_field), None) + dataset.pop(dataset_field, None) + dataset[dataset_field] = getattr(cli, cli_attr) + + def _apply_dataset_synthesis_overrides(merged: dict[str, Any], cli: CLIConfig) -> None: """Overlay explicit synthesis flags onto a YAML-supplied dataset.""" if not any( diff --git a/src/aiperf/dataset/loader/random_pool.py b/src/aiperf/dataset/loader/random_pool.py index 15a5ac6c3e..aeac6ddfb7 100644 --- a/src/aiperf/dataset/loader/random_pool.py +++ b/src/aiperf/dataset/loader/random_pool.py @@ -12,7 +12,8 @@ from aiperf.common import random_generator as rng from aiperf.common.enums import MediaType -from aiperf.common.models import Audio, Conversation, Image, Text, Turn, Video +from aiperf.common.models import Audio, Conversation, Image, Media, Text, Turn, Video +from aiperf.config.dataset.config import FileDataset from aiperf.dataset.loader.base_loader import BaseFileLoader from aiperf.dataset.loader.mixins import MediaConversionMixin from aiperf.dataset.loader.models import RandomPool @@ -42,6 +43,15 @@ class RandomPoolDatasetLoader(BaseFileLoader, MediaConversionMixin): - supports named fields for each modality (e.g. text_field_a, text_field_b, etc.) - DOES NOT support multi-turn or its features (e.g. delay, sessions, etc.) + Batching and named pools are mutually exclusive: batch sizes other than 1 are + rejected outright for directory input (multiple named pools), since batching + flattens every pool into one anonymous pool per modality. Directory input is + caught at config time by ``_reject_file_dataset_incompatible``; the pool shapes + only visible after parsing (inline multi-key ``records:``, entries embedding + named media objects) are caught here by ``_reject_batching_with_named_pools``. + A batch size on a modality absent from the pool does not count as batching -- + see ``_batching_requested``. + Note on batching and associations: When entries have paired data across modalities (e.g. {"image": "cat.png", "text": "describe this cat"}), enabling batch sizes > 1 flattens each modality into an independent pool and @@ -103,18 +113,35 @@ def __init__( self.num_conversations = ( num_conversations if num_conversations is not None else 100 ) - # Per-modality batch_size only exists on SyntheticDataset (and even then - # only when the modality config block is present). FileDataset has none - # of these, so default to 1 for missing fields rather than crashing. + # Per-modality batch sizes come from different places depending on the + # dataset type. SyntheticDataset carries them nested under + # prompts/images/audio/video sub-configs. FileDataset (the --input-file + # path) stores them as flat fields populated by the CLI converter. dataset = self.run.cfg.get_default_dataset() - prompts = getattr(dataset, "prompts", None) - images = getattr(dataset, "images", None) - audio = getattr(dataset, "audio", None) - video = getattr(dataset, "video", None) - self.batch_size_image = getattr(images, "batch_size", 1) if images else 1 - self.batch_size_text = getattr(prompts, "batch_size", 1) if prompts else 1 - self.batch_size_audio = getattr(audio, "batch_size", 1) if audio else 1 - self.batch_size_video = getattr(video, "batch_size", 1) if video else 1 + if isinstance(dataset, FileDataset): + self.batch_size_text = ( + dataset.prompt_batch_size + if dataset.prompt_batch_size is not None + else 1 + ) + self.batch_size_image = ( + dataset.image_batch_size if dataset.image_batch_size is not None else 1 + ) + self.batch_size_audio = ( + dataset.audio_batch_size if dataset.audio_batch_size is not None else 1 + ) + self.batch_size_video = ( + dataset.video_batch_size if dataset.video_batch_size is not None else 1 + ) + else: + prompts = getattr(dataset, "prompts", None) + images = getattr(dataset, "images", None) + audio = getattr(dataset, "audio", None) + video = getattr(dataset, "video", None) + self.batch_size_image = getattr(images, "batch_size", 1) if images else 1 + self.batch_size_text = getattr(prompts, "batch_size", 1) if prompts else 1 + self.batch_size_audio = getattr(audio, "batch_size", 1) if audio else 1 + self.batch_size_video = getattr(video, "batch_size", 1) if video else 1 @staticmethod def _validate_path(path: Path) -> int: @@ -286,12 +313,8 @@ def convert_to_conversations( "Sampling random_pool dataset entries with replacement. " "Duplicates within a single request are possible when batch_size exceeds pool size." ) - if ( - self.batch_size_image != 1 - or self.batch_size_text != 1 - or self.batch_size_audio != 1 - or self.batch_size_video != 1 - ): + if self._batching_requested(data): + self._reject_batching_with_named_pools(data) return self._convert_to_conversations_batched(data) conversations = [ @@ -325,6 +348,95 @@ def convert_to_conversations( return conversations + def _batching_requested(self, data: dict[Filename, list[RandomPool]]) -> bool: + """Return True if a modality actually present in the pool has batch size != 1. + + A batch size on a modality absent from the pool produces the same (empty) + output on either path, so it must not select the flattened one: + ``--image-batch-size 0`` against a text-only pool means "disable image + inputs entirely" (as the flag's own description says), not "flatten every + named text pool into one anonymous pool". + """ + for batch_size, singular, plural in ( + (self.batch_size_text, "text", "texts"), + (self.batch_size_image, "image", "images"), + (self.batch_size_audio, "audio", "audios"), + (self.batch_size_video, "video", "videos"), + ): + if batch_size == 1: + continue + if any( + getattr(entry, singular) is not None or getattr(entry, plural) + for pool in data.values() + for entry in pool + ): + return True + return False + + @staticmethod + def _reject_batching_with_named_pools( + data: dict[Filename, list[RandomPool]], + ) -> None: + """Reject batch sizes other than 1 when batching would discard pool identity. + + Two distinct ways this happens, both from the same root cause: + ``_build_flat_pool`` unwraps embedded ``Text``/``Image``/``Audio``/``Video`` + objects down to bare content strings, and ``_convert_to_conversations_batched`` + rebuilds them as ``X(name="", contents=...)`` -- discarding any authored + ``name`` and, for images, any authored ``uuids`` (vLLM cache-reuse IDs). + + 1. Multiple named pools in ``data`` -- one per key. Directory input (multiple + files, e.g. ``queries.jsonl`` -> ``query``, ``passages.jsonl`` -> ``passage``) + and inline YAML ``records:`` with multiple top-level keys (e.g. + ``records: {queries: [...], passages: [...]}``) both land here; either + way, each key is a separately named pool that name-sensitive endpoints + (e.g. rankings, which routes on the ``query``/``queries`` and ``passages`` + field names) depend on. Flattening across pools merges them into one + anonymous pool per modality. + + 2. A single pool whose entries embed named ``Text``/``Image``/``Audio``/ + ``Video`` objects, or ``Image`` objects carrying ``uuids``. This is + reachable from a single file (or single-key inline YAML ``records``) + and is not caught by pool count alone. + + Raises: + ValueError: If either condition applies. + """ + if len(data) > 1: + names = ", ".join(sorted(data)) + raise ValueError( + f"random_pool batch sizes other than 1 are not supported for named " + f"pools (found {len(data)}: {names}). Drop the batch-size flags, or " + "use a single unnamed pool -- one file, or a flat inline records: list." + ) + if RandomPoolDatasetLoader._pool_entries_carry_metadata(data): + raise ValueError( + "random_pool batch sizes other than 1 are not supported when pool " + "entries carry named Text/Image/Audio/Video objects or image cache " + "uuids. Drop the batch-size flags, or strip name/uuids from the " + "entries if only their contents matter." + ) + + @staticmethod + def _pool_entries_carry_metadata(data: dict[Filename, list[RandomPool]]) -> bool: + """Return True if any pool entry authors a Media name or (image) uuids. + + Checks the plural list fields (``texts``/``images``/``audios``/``videos`` -- + the only ones that can hold ``Media`` objects rather than bare strings) + across every ``RandomPool`` entry in every pool. + """ + for pool in data.values(): + for entry in pool: + for items in (entry.texts, entry.images, entry.audios, entry.videos): + if not items: + continue + for item in items: + if isinstance(item, Media) and ( + item.name or getattr(item, "uuids", None) + ): + return True + return False + def _build_flat_pool( self, data: dict[Filename, list[RandomPool]], @@ -382,6 +494,25 @@ def _convert_to_conversations_batched( audio_pool = self._build_flat_pool(data, "audio", "audios") video_pool = self._build_flat_pool(data, "video", "videos") + if not ( + (image_pool and self.batch_size_image > 0) + or (text_pool and self.batch_size_text > 0) + or (audio_pool and self.batch_size_audio > 0) + or (video_pool and self.batch_size_video > 0) + ): + # Raise rather than warn: this loader runs inside the DatasetManager + # subprocess, whose stdlib-logger output only reaches the log file, so a + # warning here never surfaces on the console. The run would then fail + # with a generic "check the server URL/endpoint/response format" error + # for what is purely a local config mistake. + raise ValueError( + f"random_pool batch sizes produce turns with no content: every " + f"modality is absent from the pool or has batch size 0 (text=" + f"{self.batch_size_text}, image={self.batch_size_image}, audio=" + f"{self.batch_size_audio}, video={self.batch_size_video}). Set a " + "batch size above 0 for a modality that is present in the pool." + ) + conversations = [] for _ in range(self.num_conversations): images: list[Image] = [] diff --git a/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py b/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py new file mode 100644 index 0000000000..108bf77693 --- /dev/null +++ b/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py @@ -0,0 +1,353 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Regression: batch-size CLI flags must override YAML-supplied random_pool datasets. + +In the YAML+CLI path, ``_apply_input_overrides`` only routed ``headers`` and +``extra_inputs``; all other INPUT_FIELDS members were silently discarded. For +the four batch-size fields this was newly reachable after this PR added them to +``FileDataset``. ``_apply_random_pool_batch_size_overrides`` closes the gap. + +All tests drive ``resolve_config`` with a real YAML file, not ``convert_cli_to_aiperf``. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from aiperf.config import AIPerfConfig +from aiperf.config.dataset.config import FileDataset +from aiperf.config.flags import CLIConfig +from aiperf.config.flags.resolver import resolve_config + + +def _write_random_pool_yaml( + tmp_path: Path, pool_path: Path, **extra_dataset_fields: object +) -> Path: + """Write a minimal YAML config with a random_pool file dataset. + + Extra keyword arguments are serialised as YAML dataset fields (one per line, + two-space indent) so individual tests can pre-set batch sizes in the YAML. + """ + extra_lines = "".join( + f" {key}: {value}\n" for key, value in extra_dataset_fields.items() + ) + yaml_content = f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: random_pool + path: {pool_path} +{extra_lines}\ + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + cfg_path = tmp_path / "random_pool.yaml" + cfg_path.write_text(yaml_content) + return cfg_path + + +def _cli(**kwargs: object) -> CLIConfig: + """Build a CLIConfig with only the supplied fields in model_fields_set.""" + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +def _dataset(cfg: AIPerfConfig) -> FileDataset: + return cfg.benchmark.datasets[0] + + +# --------------------------------------------------------------------------- +# Core override semantics +# --------------------------------------------------------------------------- + + +def test_prompt_batch_size_cli_overrides_yaml_silence(tmp_path: Path) -> None: + """CLI --prompt-batch-size wins when YAML has no batch-size set.""" + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_random_pool_yaml(tmp_path, pool) + cli = _cli(prompt_batch_size=7) + cfg = resolve_config(cli, yaml_path) + assert _dataset(cfg).prompt_batch_size == 7 + + +def test_prompt_batch_size_cli_wins_over_yaml(tmp_path: Path) -> None: + """CLI --prompt-batch-size overrides a value already set in the YAML.""" + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_random_pool_yaml(tmp_path, pool, prompt_batch_size=10) + cli = _cli(prompt_batch_size=3) + cfg = resolve_config(cli, yaml_path) + assert _dataset(cfg).prompt_batch_size == 3 + + +def test_yaml_batch_size_survives_when_cli_not_set(tmp_path: Path) -> None: + """YAML batch-size must survive when the CLI flag is absent. + + This is the regression guard for the 'gate on model_fields_set' requirement: + if the check were truthiness-based the YAML value would be clobbered on + every run that omits the flag. + """ + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_random_pool_yaml(tmp_path, pool, prompt_batch_size=10) + cli = CLIConfig() # no batch-size flags set at all + cfg = resolve_config(cli, yaml_path) + assert _dataset(cfg).prompt_batch_size == 10 + + +def test_image_batch_size_zero_resolves_to_zero(tmp_path: Path) -> None: + """image_batch_size=0 (disable images) must not be treated as unset or clamped to 1.""" + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_random_pool_yaml(tmp_path, pool) + cli = _cli(image_batch_size=0) + cfg = resolve_config(cli, yaml_path) + assert _dataset(cfg).image_batch_size == 0 + + +def test_all_four_modalities_distinct_values(tmp_path: Path) -> None: + """All four batch-size fields are written independently; a field-order swap fails.""" + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_random_pool_yaml(tmp_path, pool) + cli = _cli( + prompt_batch_size=2, image_batch_size=3, audio_batch_size=5, video_batch_size=7 + ) + cfg = resolve_config(cli, yaml_path) + ds = _dataset(cfg) + assert ds.prompt_batch_size == 2 + assert ds.image_batch_size == 3 + assert ds.audio_batch_size == 5 + assert ds.video_batch_size == 7 + + +def test_camel_case_yaml_field_overridden_by_cli_flag_does_not_crash( + tmp_path: Path, +) -> None: + """CLI --prompt-batch-size against a YAML that already spells the field in + its camelCase alias (promptBatchSize -- the shipped template idiom, see + e.g. templates/embeddings.yaml's batchSize) must override cleanly, not + crash with extra_forbidden. + + Regression test: the override previously always wrote the snake_case key + (dataset["prompt_batch_size"] = ...) without removing a pre-existing + camelCase key. FileDataset has alias_generator=to_camel, extra="forbid", + so both keys ending up present made Pydantic bind from the alias and + reject the snake_case key as extra. + """ + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_content = f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: random_pool + path: {pool} + promptBatchSize: 10 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + yaml_path = tmp_path / "camel.yaml" + yaml_path.write_text(yaml_content) + cli = _cli(prompt_batch_size=3) + cfg = resolve_config(cli, yaml_path) + assert _dataset(cfg).prompt_batch_size == 3 + + +# --------------------------------------------------------------------------- +# Non-random_pool dataset: friendly error, not raw Pydantic trace +# --------------------------------------------------------------------------- + + +def test_batch_size_on_non_random_pool_yaml_raises_friendly_error( + tmp_path: Path, +) -> None: + """Batch-size CLI flag on a mooncake_trace YAML dataset must raise a clear ValueError. + + The FileDataset model validator would fire anyway, but its message does not + name the CLI flag. The override helper must intercept first and raise a + message that tells the user what to do. + """ + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_content = f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: mooncake_trace + path: {pool} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + yaml_path = tmp_path / "trace.yaml" + yaml_path.write_text(yaml_content) + cli = _cli(prompt_batch_size=4) + with pytest.raises(ValueError, match="random_pool"): + resolve_config(cli, yaml_path) + + +def test_batch_size_on_yaml_with_omitted_format_reports_effective_default( + tmp_path: Path, +) -> None: + """The rejection message must report the effective default format + ('single_turn'), not a bare 'None', when the YAML omits `format:` entirely. + + Regression test: dataset.get("format") reads the raw pre-validation YAML + dict, so an omitted format key read back as None even though + FileDataset.format defaults to DatasetFormat.SINGLE_TURN -- "got format: + None" reads like a parse failure rather than "you didn't set format, so + it's using the default". + """ + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_content = f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + path: {pool} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + yaml_path = tmp_path / "no_format.yaml" + yaml_path.write_text(yaml_content) + cli = _cli(prompt_batch_size=4) + with pytest.raises(ValueError, match=re.escape("got format: single_turn")): + resolve_config(cli, yaml_path) + + +# --------------------------------------------------------------------------- +# Synthetic / public datasets: batch-size flags are legitimate elsewhere and +# must not be rejected by the random_pool-specific override helper. +# --------------------------------------------------------------------------- + + +def _write_synthetic_yaml(tmp_path: Path, **extra_prompts_fields: object) -> Path: + """Write a minimal YAML config with a synthetic dataset.""" + extra_lines = "".join( + f" {key}: {value}\n" for key, value in extra_prompts_fields.items() + ) + yaml_content = f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + prompts: +{extra_lines}\ + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + cfg_path = tmp_path / "synthetic.yaml" + cfg_path.write_text(yaml_content) + return cfg_path + + +@pytest.mark.parametrize( + "batch_kwarg", + [ + pytest.param({"prompt_batch_size": 4}, id="text"), + pytest.param({"image_batch_size": 2}, id="image"), + pytest.param({"audio_batch_size": 2}, id="audio"), + pytest.param({"video_batch_size": 2}, id="video"), + ], +) +def test_batch_size_flag_on_synthetic_yaml_does_not_raise( + tmp_path: Path, batch_kwarg: dict +) -> None: + """Batch-size CLI flags must not raise on a synthetic YAML dataset. + + Regression test: synthetic datasets have no ``format`` field at all, so + ``dataset.get("format")`` returned ``None`` and the ``!= RANDOM_POOL`` check + incorrectly raised for perfectly legitimate synthetic/embeddings batch-size + usage. + """ + yaml_path = _write_synthetic_yaml(tmp_path, batch_size=1) + cli = _cli(**batch_kwarg) + cfg = resolve_config(cli, yaml_path) + assert cfg.benchmark.datasets[0].type == "synthetic" + + +def test_prompt_batch_size_flag_on_synthetic_yaml_is_dropped_with_warning( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Pins the known (pre-existing, out-of-scope-to-fix-here) gap: nothing in + the YAML+CLI path routes --prompt-batch-size onto a synthetic dataset's + prompts.batch_size, unlike the CLI-only path (no --config) where the same + flag applies correctly. The YAML value survives untouched and the explicit + CLI flag has no effect, inverting normal CLI-overrides-YAML precedence -- + so it must at least warn, since the adjacent wrong-format file-dataset case + raises loudly for the same flag. If the value assertion starts failing, the + gap has been closed -- update this test to assert the override took effect. + """ + yaml_path = _write_synthetic_yaml(tmp_path, batch_size=1) + cli = _cli(prompt_batch_size=4) + with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): + cfg = resolve_config(cli, yaml_path) + assert cfg.benchmark.datasets[0].prompts.batch_size == 1 + assert "--prompt-batch-size ignored" in caplog.text + + +def test_image_batch_size_flag_on_synthetic_yaml_is_dropped_with_warning( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Companion to the text case: --image-batch-size against a synthetic YAML + dataset with no `images:` block doesn't even create one -- the flag has + zero effect end to end, not just a value mismatch, and warns accordingly. + """ + yaml_path = _write_synthetic_yaml(tmp_path, batch_size=1) + cli = _cli(image_batch_size=2) + with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): + cfg = resolve_config(cli, yaml_path) + assert cfg.benchmark.datasets[0].images is None + assert "--image-batch-size ignored" in caplog.text + + +def test_batch_size_flag_applies_without_cli_custom_dataset_type( + tmp_path: Path, +) -> None: + """Batch-size flags must resolve via YAML `format: random_pool` alone, with + no `--custom-dataset-type` CLI flag set at all. + + Regression test for a doc claim ("only valid with --custom-dataset-type + random_pool") that was false: the resolver keys off `dataset["format"]` + in the merged YAML dict, never `cli.custom_dataset_type` -- the YAML + `format:` key is the same knob, reachable without touching the CLI flag. + """ + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_random_pool_yaml(tmp_path, pool) + cli = _cli(prompt_batch_size=4) + assert "custom_dataset_type" not in cli.model_fields_set + cfg = resolve_config(cli, yaml_path) + assert _dataset(cfg).prompt_batch_size == 4 diff --git a/tests/unit/config/test_v1_file_dataset_rejections.py b/tests/unit/config/test_v1_file_dataset_rejections.py index 703fd4423e..8377abc713 100644 --- a/tests/unit/config/test_v1_file_dataset_rejections.py +++ b/tests/unit/config/test_v1_file_dataset_rejections.py @@ -20,14 +20,17 @@ from __future__ import annotations +import re from pathlib import Path import pytest from pytest import param +from aiperf.config.dataset.config import FileDataset from aiperf.config.flags._converter_dataset import build_dataset from aiperf.config.flags.cli_config import CLIConfig from aiperf.config.flags.converter import convert_cli_to_aiperf +from aiperf.plugin.enums import DatasetFormat @pytest.fixture @@ -173,3 +176,244 @@ def test_synthetic_only_flag_rejected_on_public_dataset( ) with pytest.raises(ValueError, match=expected_flag_fragment): build_dataset(user) + + +@pytest.mark.parametrize( + "extra, expected_flag_fragment", + [ + param({"prompt_batch_size": 3}, "--prompt-batch-size", id="text"), + param({"image_batch_size": 2}, "--image-batch-size", id="image"), + ], +) # fmt: skip +def test_batch_size_flag_rejected_on_public_dataset_names_public_dataset( + extra: dict, expected_flag_fragment: str +) -> None: + """Batch-size flags on a PUBLIC dataset must raise a message that names + --public-dataset, not --input-file. + + Regression test: batch_violations previously always used the --input-file + wording ("requires --custom-dataset-type random_pool when used with + --input-file"), even when the user passed --public-dataset and never + touched --input-file at all -- every suggestion in that message was a + dead end since --custom-dataset-type random_pool is mutually exclusive + with --public-dataset. + """ + from aiperf.plugin.enums import PublicDatasetType + + user = CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + **CLIConfig(request_count=5, concurrency=1).model_dump(exclude_unset=True), + public_dataset=PublicDatasetType.SEMIANALYSIS_CC_TRACES_WEKA_WITH_SUBAGENTS, + **extra, + ) + with pytest.raises(ValueError, match=expected_flag_fragment) as exc_info: + build_dataset(user) + message = str(exc_info.value) + assert "remove --input-file" not in message, ( + "the user never set --input-file; telling them to remove it is a dead end" + ) + assert "--public-dataset" in message + + +class TestRandomPoolBatchSizeCarveOut: + """random_pool must be exempt from the batch-size flag rejection.""" + + @pytest.fixture + def pool_jsonl(self, tmp_path: Path) -> Path: + p = tmp_path / "pool.jsonl" + p.touch() + return p + + @pytest.mark.parametrize( + "batch_kwarg, expected_field", + [ + param({"prompt_batch_size": 5}, "prompt_batch_size", id="text"), + param( + {"prompt_batch_size": 0}, "prompt_batch_size", id="text-zero-disables" + ), + param({"image_batch_size": 3}, "image_batch_size", id="image"), + param({"audio_batch_size": 2}, "audio_batch_size", id="audio"), + param({"video_batch_size": 4}, "video_batch_size", id="video"), + ], + ) # fmt: skip + def test_batch_size_allowed_with_random_pool( + self, pool_jsonl: Path, batch_kwarg: dict, expected_field: str + ) -> None: + """Batch-size flags must not raise when custom_dataset_type is random_pool. + + prompt_batch_size=0 must be accepted (not just >=1): it's the documented way + to disable text for image/audio/video-only random_pool workloads, matching + image/audio/video_batch_size's existing 0-disables convention. + """ + cli = CLIConfig( + model_names=["test-model"], + input_file=str(pool_jsonl), + custom_dataset_type="random_pool", + **batch_kwarg, + ) + cfg = convert_cli_to_aiperf(cli) + dataset = cfg.benchmark.datasets[0] + assert getattr(dataset, expected_field) == next(iter(batch_kwarg.values())) + + @pytest.mark.parametrize( + "batch_kwarg, expected_flag_fragment", + [ + param({"prompt_batch_size": 5}, "--prompt-batch-size", id="text"), + param({"image_batch_size": 3}, "--image-batch-size", id="image"), + ], + ) # fmt: skip + def test_batch_size_still_rejected_for_trace_formats( + self, mc_jsonl: Path, batch_kwarg: dict, expected_flag_fragment: str + ) -> None: + """Batch-size flags must still be rejected for non-random-pool file datasets.""" + cli = CLIConfig( + model_names=["test-model"], + input_file=str(mc_jsonl), + custom_dataset_type="mooncake_trace", + **batch_kwarg, + ) + with pytest.raises(ValueError, match=re.escape(expected_flag_fragment)): + convert_cli_to_aiperf(cli) + + +def test_batch_size_fields_rejected_on_non_random_pool_format(tmp_path: Path) -> None: + """FileDataset model validator rejects batch-size fields when format != random_pool.""" + p = tmp_path / "f.jsonl" + p.touch() + with pytest.raises( + ValueError, match="are rejected on formats other than random_pool" + ): + FileDataset( + name="d", + type="file", + path=p, + format=DatasetFormat.MOONCAKE_TRACE, + prompt_batch_size=8, + ) + + +class TestBatchSizeDirectoryAndTypeMessages: + """Config-time coverage for the batch-size rejection messages. + + Both cases used to be discovered late or worded wrong: a directory input was + only rejected once the DatasetManager parsed the files (after SystemController + and the workers had already started, and truncated in the console panel), and + the "add --custom-dataset-type random_pool" remedy was printed even when the + user had already set that flag to something else. + """ + + @pytest.fixture + def pool_dir(self, tmp_path: Path) -> Path: + d = tmp_path / "pools" + d.mkdir() + (d / "queries.jsonl").touch() + (d / "passages.jsonl").touch() + return d + + @pytest.mark.parametrize( + "batch_kwarg, expected_flag_fragment", + [ + param({"prompt_batch_size": 4}, "--prompt-batch-size", id="text"), + param({"image_batch_size": 2}, "--image-batch-size", id="image"), + param({"audio_batch_size": 2}, "--audio-batch-size", id="audio"), + param({"video_batch_size": 2}, "--video-batch-size", id="video"), + ], + ) # fmt: skip + def test_batch_size_on_directory_input_rejected_at_config_time( + self, pool_dir: Path, batch_kwarg: dict, expected_flag_fragment: str + ) -> None: + """A directory --input-file is fully decidable here, so it must be rejected + before any service starts rather than at dataset-load time.""" + user = CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + **CLIConfig(request_count=5, concurrency=1).model_dump(exclude_unset=True), + input_file=str(pool_dir), + custom_dataset_type="random_pool", + **batch_kwarg, + ) + with pytest.raises(ValueError, match=expected_flag_fragment) as exc_info: + build_dataset(user) + assert "directory" in str(exc_info.value) + + @pytest.mark.parametrize( + "batch_kwarg", + [ + param({"prompt_batch_size": 1}, id="text"), + param({"image_batch_size": 1}, id="image"), + param({"audio_batch_size": 1}, id="audio"), + param({"video_batch_size": 1}, id="video"), + ], + ) # fmt: skip + def test_batch_size_of_one_on_directory_input_is_not_rejected( + self, pool_dir: Path, batch_kwarg: dict + ) -> None: + """Batch size 1 means "not batching" everywhere else in this PR + (``_batching_requested`` in the loader treats it as a no-op); the + directory guard must agree instead of rejecting the documented + default just because the flag was explicitly set.""" + user = CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + **CLIConfig(request_count=5, concurrency=1).model_dump(exclude_unset=True), + input_file=str(pool_dir), + custom_dataset_type="random_pool", + **batch_kwarg, + ) + build_dataset(user) + + def test_directory_batch_size_message_does_not_demand_custom_dataset_type( + self, pool_dir: Path + ) -> None: + """Directory input auto-detects as random_pool, so a directory + batch flag + with no --custom-dataset-type must not send the user through a first failed + run adding the flag only to hit the directory rejection on the second.""" + user = CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + **CLIConfig(request_count=5, concurrency=1).model_dump(exclude_unset=True), + input_file=str(pool_dir), + prompt_batch_size=4, + ) + with pytest.raises(ValueError) as exc_info: + build_dataset(user) + message = str(exc_info.value) + assert "directory" in message + assert "add --custom-dataset-type random_pool" not in message + + def test_wrong_custom_dataset_type_message_says_change_not_add( + self, mc_jsonl: Path + ) -> None: + """With --custom-dataset-type already set to a non-random_pool value there + is nothing to "add" -- the user has to change it.""" + user = CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + **CLIConfig(request_count=5, concurrency=1).model_dump(exclude_unset=True), + input_file=str(mc_jsonl), + custom_dataset_type="mooncake_trace", + prompt_batch_size=4, + ) + with pytest.raises(ValueError, match="--prompt-batch-size") as exc_info: + build_dataset(user) + message = str(exc_info.value) + assert ( + "change --custom-dataset-type from mooncake_trace to random_pool" in message + ) + assert "add --custom-dataset-type" not in message + + def test_unset_custom_dataset_type_message_still_says_add( + self, mc_jsonl: Path + ) -> None: + """The "add" wording remains correct when the flag was never set.""" + user = CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + **CLIConfig(request_count=5, concurrency=1).model_dump(exclude_unset=True), + input_file=str(mc_jsonl), + prompt_batch_size=4, + ) + with pytest.raises(ValueError, match="--prompt-batch-size") as exc_info: + build_dataset(user) + assert "add --custom-dataset-type random_pool" in str(exc_info.value) diff --git a/tests/unit/dataset/loader/test_random_pool.py b/tests/unit/dataset/loader/test_random_pool.py index d4a6be48aa..08cc062cea 100644 --- a/tests/unit/dataset/loader/test_random_pool.py +++ b/tests/unit/dataset/loader/test_random_pool.py @@ -7,6 +7,8 @@ import pytest from aiperf.common.models import Audio, Image, Text, Video +from aiperf.config import BenchmarkRun +from aiperf.config.flags.cli_config import CLIConfig from aiperf.dataset.loader.models import RandomPool from aiperf.dataset.loader.random_pool import RandomPoolDatasetLoader from aiperf.plugin.enums import CustomDatasetType @@ -659,8 +661,11 @@ def test_batch_mode_preserves_video(self, default_cfg): assert len(turn.videos) == 1 assert turn.videos[0].contents[0] in video_urls - def test_batch_mode_named_image_objects_flattened(self, default_cfg): - """Images specified as named Image objects should have their contents added to the pool.""" + def test_batch_mode_named_image_objects_rejected(self, default_cfg): + """Batching (batch_size != 1) against a pool with named Image objects must be + rejected: _build_flat_pool/_convert_to_conversations_batched would discard + the authored name (and any uuids) by rebuilding as Image(name="", ...). + """ config = self._make_config(batch_size_image=2) data = { "f.jsonl": [ @@ -680,15 +685,14 @@ def test_batch_mode_named_image_objects_flattened(self, default_cfg): loader = RandomPoolDatasetLoader( filename="dummy.jsonl", run=make_run_from_cli(config), num_conversations=2 ) - conversations = loader.convert_to_conversations(data) - - expected = {"https://example.com/a.png", "https://example.com/b.png"} - for conv in conversations: - for img_content in conv.turns[0].images[0].contents: - assert img_content in expected - - def test_batch_mode_named_text_objects_flattened(self, default_cfg): - """Texts specified as named Text objects should have their contents added to the pool.""" + with pytest.raises(ValueError, match="named"): + loader.convert_to_conversations(data) + + def test_batch_mode_named_text_objects_rejected(self, default_cfg): + """Batching against a pool with named Text objects must be rejected: it + would discard the authored name (e.g. 'query'), breaking name-sensitive + endpoints like rankings. + """ config = self._make_config(batch_size_text=2) data = { "f.jsonl": [ @@ -700,15 +704,11 @@ def test_batch_mode_named_text_objects_flattened(self, default_cfg): loader = RandomPoolDatasetLoader( filename="dummy.jsonl", run=make_run_from_cli(config), num_conversations=2 ) - conversations = loader.convert_to_conversations(data) - - expected = {"alpha", "beta", "gamma"} - for conv in conversations: - for txt_content in conv.turns[0].texts[0].contents: - assert txt_content in expected + with pytest.raises(ValueError, match="named"): + loader.convert_to_conversations(data) - def test_batch_mode_named_audio_objects_flattened(self, default_cfg): - """Audios specified as named Audio objects should have their contents added to the pool.""" + def test_batch_mode_named_audio_objects_rejected(self, default_cfg): + """Batching against a pool with named Audio objects must be rejected.""" config = self._make_config(batch_size_image=2) data = { "f.jsonl": [ @@ -733,20 +733,11 @@ def test_batch_mode_named_audio_objects_flattened(self, default_cfg): loader = RandomPoolDatasetLoader( filename="dummy.jsonl", run=make_run_from_cli(config), num_conversations=2 ) - conversations = loader.convert_to_conversations(data) - - expected = { - "https://example.com/a1.wav", - "https://example.com/a2.wav", - "https://example.com/a3.wav", - } - for conv in conversations: - turn = conv.turns[0] - assert len(turn.audios) == 1 - assert turn.audios[0].contents[0] in expected + with pytest.raises(ValueError, match="named"): + loader.convert_to_conversations(data) - def test_batch_mode_named_video_objects_flattened(self, default_cfg): - """Videos specified as named Video objects should have their contents added to the pool.""" + def test_batch_mode_named_video_objects_rejected(self, default_cfg): + """Batching against a pool with named Video objects must be rejected.""" config = self._make_config(batch_size_image=2) data = { "f.jsonl": [ @@ -771,17 +762,8 @@ def test_batch_mode_named_video_objects_flattened(self, default_cfg): loader = RandomPoolDatasetLoader( filename="dummy.jsonl", run=make_run_from_cli(config), num_conversations=2 ) - conversations = loader.convert_to_conversations(data) - - expected = { - "https://example.com/v1.mp4", - "https://example.com/v2.mp4", - "https://example.com/v3.mp4", - } - for conv in conversations: - turn = conv.turns[0] - assert len(turn.videos) == 1 - assert turn.videos[0].contents[0] in expected + with pytest.raises(ValueError, match="named"): + loader.convert_to_conversations(data) def test_batch_mode_plain_string_videos_flattened(self, default_cfg): """Videos specified as plain strings should be included in the flat pool.""" @@ -847,9 +829,13 @@ def test_image_zero_text_one_disables_images_via_legacy_path(self, default_cfg): # Removed: test_batch_size_text_zero_disables_texts and # test_image_one_text_zero_disables_texts_via_legacy_path. v2 PromptConfig - # rejects batch_size=0 (ge=1); the "disable text via batch_size_text=0" - # path is unreachable from a valid v2 config so the assertion has no path - # to exercise. Image/audio/video batch_size=0 paths remain covered. + # (SyntheticDataset.prompts) rejects batch_size=0 (ge=1); the "disable text + # via batch_size_text=0" path is unreachable from a valid v2 config so the + # assertion has no path to exercise. Image/audio/video batch_size=0 paths + # are covered here for the SyntheticDataset branch. FileDataset.prompt_batch_size + # allows 0 (image/audio/video-only random_pool workloads); that zero-value + # coverage, including text=0 disabling text output end-to-end, lives in + # TestRandomPoolBatchSizeWithFileDataset. def test_num_conversations_none_defaults_to_100(self, default_cfg): """When num_conversations=None is passed, the loader should default to 100.""" @@ -1039,3 +1025,452 @@ def test_audio_video_batch_sizes_read_from_config(self, default_cfg): ) assert loader.batch_size_audio == 2 assert loader.batch_size_video == 3 + + +class TestRandomPoolNamedPoolBatchingGuard: + """Batching (any batch size != 1) must be rejected outright for directory + input, since it flattens every named pool into one anonymous pool per + modality -- discarding the field names (e.g. 'query'/'passage') that + directory mode exists to preserve. See _reject_batching_with_named_pools. + """ + + def _named_pool_data(self): + return { + "queries.jsonl": [ + RandomPool(texts=[Text(name="query", contents=["Who are you?"])]) + ], + "passages.jsonl": [ + RandomPool(texts=[Text(name="passage", contents=["I am a cat."])]) + ], + } + + def test_batching_with_multi_file_data_raises(self, default_cfg): + """Any batch size != 1 must raise when data spans more than one named pool.""" + config = TestRandomPoolBatchSize()._make_config(batch_size_text=2) + loader = RandomPoolDatasetLoader( + filename="dummy_dir", run=make_run_from_cli(config), num_conversations=2 + ) + with pytest.raises(ValueError, match="named pools"): + loader.convert_to_conversations(self._named_pool_data()) + + def test_batching_with_single_file_data_does_not_raise(self, default_cfg): + """A single-file (single named pool) input must still batch normally.""" + config = TestRandomPoolBatchSize()._make_config(batch_size_text=2) + loader = RandomPoolDatasetLoader( + filename="dummy.jsonl", run=make_run_from_cli(config), num_conversations=2 + ) + data = { + "pool.jsonl": [ + RandomPool(text="query1"), + RandomPool(text="query2"), + ] + } + conversations = loader.convert_to_conversations(data) + assert len(conversations) == 2 + for conv in conversations: + assert len(conv.turns[0].texts[0].contents) == 2 + + def test_default_batch_sizes_with_multi_file_data_does_not_raise(self, default_cfg): + """All batch sizes at their default of 1 must not trigger the named-pool guard.""" + loader = RandomPoolDatasetLoader( + filename="dummy_dir", + run=make_run_from_cli(default_cfg), + num_conversations=2, + ) + conversations = loader.convert_to_conversations(self._named_pool_data()) + assert len(conversations) == 2 + for conv in conversations: + names = {text.name for text in conv.turns[0].texts} + assert names == {"query", "passage"} + + def test_batching_with_single_file_named_text_objects_raises(self, default_cfg): + """A single file whose entries embed named Text objects must still be + rejected: file count alone (len(data) == 1) doesn't catch this, but + batching would flatten the embedded names away exactly as directory + mode's file-level names would be. + """ + config = TestRandomPoolBatchSize()._make_config(batch_size_text=2) + loader = RandomPoolDatasetLoader( + filename="pool.jsonl", run=make_run_from_cli(config), num_conversations=2 + ) + data = { + "pool.jsonl": [ + RandomPool(texts=[Text(name="query", contents=["q1"])]), + RandomPool(texts=[Text(name="passage", contents=["p1"])]), + ] + } + with pytest.raises(ValueError, match="named"): + loader.convert_to_conversations(data) + + def test_batching_with_single_file_image_uuids_raises(self, default_cfg): + """A single file whose entries embed Image objects with authored uuids + (vLLM cache-reuse IDs) must be rejected: batching rebuilds images as + Image(name="", contents=...) with no uuids field at all, silently + dropping the cache-reuse behavior instead of erroring loudly. + """ + config = TestRandomPoolBatchSize()._make_config(batch_size_image=2) + loader = RandomPoolDatasetLoader( + filename="pool.jsonl", run=make_run_from_cli(config), num_conversations=1 + ) + data = { + "pool.jsonl": [ + RandomPool( + images=[ + Image( + name="img", + contents=["https://example.com/a.png"], + uuids=["uuid-a"], + ) + ] + ), + ] + } + with pytest.raises(ValueError, match="named"): + loader.convert_to_conversations(data) + + def test_multi_pool_rejection_message_does_not_claim_files(self, default_cfg): + """The rejection message must not call inline-records pool keys 'files' -- + the same guard fires for inline YAML `records: {queries: [...], passages: + [...]}`, which has no directory and no files on disk at all. + """ + config = TestRandomPoolBatchSize()._make_config(batch_size_text=2) + loader = RandomPoolDatasetLoader( + filename="dummy_inline", run=make_run_from_cli(config), num_conversations=2 + ) + data = { + "queries": [RandomPool(text="What is your refund policy?")], + "passages": [RandomPool(text="Refunds take 5 business days.")], + } + with pytest.raises(ValueError, match="named pools") as exc_info: + loader.convert_to_conversations(data) + message = str(exc_info.value) + assert "directory of" not in message + assert "unnamed pool file" not in message + + def test_batching_with_single_file_plain_strings_does_not_raise(self, default_cfg): + """A single file with plain-string (unnamed) entries must still batch + normally -- the metadata guard must not over-reject content with no + name or uuids to lose. + """ + config = TestRandomPoolBatchSize()._make_config(batch_size_image=2) + loader = RandomPoolDatasetLoader( + filename="pool.jsonl", run=make_run_from_cli(config), num_conversations=1 + ) + data = { + "pool.jsonl": [ + RandomPool(images=["https://example.com/a.png"]), + RandomPool(images=["https://example.com/b.png"]), + ] + } + conversations = loader.convert_to_conversations(data) + assert len(conversations[0].turns[0].images[0].contents) == 2 + + def test_batching_with_inline_records_single_pool_does_not_raise(self, default_cfg): + """Batch sizes must apply to a single-pool inline `records:` list (not a + file at all), not just single-file input. + + Regression test for a doc claim ("batch sizes only apply to single-file + random_pool input") that was false: the guard gates on pool count and + per-entry metadata, not on file-ness, so a flat inline records: list + (list-form, one unnamed pool) batches exactly like a single file. + """ + config = TestRandomPoolBatchSize()._make_config(batch_size_text=2) + loader = RandomPoolDatasetLoader( + inline_records=[{"text": "alpha"}, {"text": "beta"}], + run=make_run_from_cli(config), + num_conversations=1, + ) + data = loader.load_dataset() + conversations = loader.convert_to_conversations(data) + assert len(conversations[0].turns[0].texts[0].contents) == 2 + + +class TestRandomPoolBatchSizeWithFileDataset: + """Batch sizes must work when the dataset is a FileDataset (--input-file path).""" + + def _make_file_run( + self, + tmp_path: Path, + *, + prompt_batch_size: int | None = None, + image_batch_size: int | None = None, + audio_batch_size: int | None = None, + video_batch_size: int | None = None, + ) -> tuple[BenchmarkRun, str]: + pool = tmp_path / "pool.jsonl" + pool.touch() + kwargs = {} + if prompt_batch_size is not None: + kwargs["prompt_batch_size"] = prompt_batch_size + if image_batch_size is not None: + kwargs["image_batch_size"] = image_batch_size + if audio_batch_size is not None: + kwargs["audio_batch_size"] = audio_batch_size + if video_batch_size is not None: + kwargs["video_batch_size"] = video_batch_size + cli = CLIConfig( + model_names=["test-model"], + input_file=str(pool), + custom_dataset_type="random_pool", + **kwargs, + ) + return make_run_from_cli(cli), str(pool) + + def test_text_batch_size_via_file_dataset(self, tmp_path: Path) -> None: + """prompt_batch_size from FileDataset produces multi-text conversations.""" + run, pool_path = self._make_file_run(tmp_path, prompt_batch_size=4) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_text == 4 + + data = { + "pool.jsonl": [ + RandomPool(text="q1"), + RandomPool(text="q2"), + RandomPool(text="q3"), + ] + } + conversations = loader.convert_to_conversations(data) + assert len(conversations) == 2 + for conv in conversations: + assert len(conv.turns) == 1 + assert len(conv.turns[0].texts) == 1 + assert len(conv.turns[0].texts[0].contents) == 4 + + def test_image_batch_size_via_file_dataset(self, tmp_path: Path) -> None: + """image_batch_size from FileDataset produces multi-image conversations.""" + run, pool_path = self._make_file_run(tmp_path, image_batch_size=3) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_image == 3 + + data = { + "pool.jsonl": [ + RandomPool(image="https://example.com/img1.png"), + RandomPool(image="https://example.com/img2.png"), + RandomPool(image="https://example.com/img3.png"), + ] + } + conversations = loader.convert_to_conversations(data) + assert len(conversations) == 2 + for conv in conversations: + assert len(conv.turns[0].images[0].contents) == 3 + + def test_default_batch_size_1_when_not_set(self, tmp_path: Path) -> None: + """When no batch-size is set, FileDataset path defaults to 1 (existing behavior).""" + run, pool_path = self._make_file_run(tmp_path) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_text == 1 + assert loader.batch_size_image == 1 + assert loader.batch_size_audio == 1 + assert loader.batch_size_video == 1 + + def test_audio_batch_size_via_file_dataset(self, tmp_path: Path) -> None: + """audio_batch_size from FileDataset is stored correctly.""" + run, pool_path = self._make_file_run(tmp_path, audio_batch_size=5) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_audio == 5 + + def test_video_batch_size_via_file_dataset(self, tmp_path: Path) -> None: + """video_batch_size from FileDataset is stored correctly.""" + run, pool_path = self._make_file_run(tmp_path, video_batch_size=7) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_video == 7 + + def test_all_four_modality_batch_sizes_distinct(self, tmp_path: Path) -> None: + """All four batch-size fields are read independently; a field-order swap fails.""" + run, pool_path = self._make_file_run( + tmp_path, + prompt_batch_size=2, + image_batch_size=3, + audio_batch_size=5, + video_batch_size=7, + ) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_text == 2 + assert loader.batch_size_image == 3 + assert loader.batch_size_audio == 5 + assert loader.batch_size_video == 7 + + def test_image_batch_size_zero_via_file_dataset(self, tmp_path: Path) -> None: + """image_batch_size=0 through --input-file must not collapse to 1.""" + run, pool_path = self._make_file_run(tmp_path, image_batch_size=0) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_image == 0 + + def test_audio_batch_size_zero_via_file_dataset(self, tmp_path: Path) -> None: + """audio_batch_size=0 through --input-file must not collapse to 1.""" + run, pool_path = self._make_file_run(tmp_path, audio_batch_size=0) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_audio == 0 + + def test_video_batch_size_zero_via_file_dataset(self, tmp_path: Path) -> None: + """video_batch_size=0 through --input-file must not collapse to 1.""" + run, pool_path = self._make_file_run(tmp_path, video_batch_size=0) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_video == 0 + + def test_prompt_batch_size_zero_via_file_dataset(self, tmp_path: Path) -> None: + """prompt_batch_size=0 through --input-file must be accepted, not raise. + + FileDataset.prompt_batch_size uses ge=0 (not ge=1) so image/audio/video-only + random_pool workloads can disable text via --prompt-batch-size 0, matching + the loader's `if text_pool and self.batch_size_text > 0` guard. + """ + run, pool_path = self._make_file_run( + tmp_path, prompt_batch_size=0, image_batch_size=2 + ) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + assert loader.batch_size_text == 0 + + def test_prompt_batch_size_zero_disables_text_end_to_end( + self, tmp_path: Path + ) -> None: + """prompt_batch_size=0 must suppress text output even when text is in the pool.""" + run, pool_path = self._make_file_run( + tmp_path, prompt_batch_size=0, image_batch_size=1 + ) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + data = { + "pool.jsonl": [ + RandomPool(image="https://example.com/img1.png", text="query1"), + RandomPool(image="https://example.com/img2.png", text="query2"), + ] + } + conversations = loader.convert_to_conversations(data) + for conv in conversations: + turn = conv.turns[0] + assert turn.texts == [] + assert len(turn.images) == 1 + + def test_prompt_batch_size_zero_on_text_only_pool_raises( + self, tmp_path: Path + ) -> None: + """prompt_batch_size=0 against a text-only pool must raise: every modality + is suppressed or absent, so every turn would be empty and every request + would fail downstream with empty content. + + A warning is not enough -- this loader runs inside the DatasetManager + subprocess, so its logger output only reaches the log file and never the + console the user is watching. + """ + run, pool_path = self._make_file_run(tmp_path, prompt_batch_size=0) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + data = {"pool.jsonl": [RandomPool(text="query1"), RandomPool(text="query2")]} + with pytest.raises(ValueError, match="turns with no content"): + loader.convert_to_conversations(data) + + def test_prompt_batch_size_zero_with_images_present_does_not_raise( + self, tmp_path: Path + ) -> None: + """prompt_batch_size=0 must not raise when another modality still produces + content -- this is the legitimate image/audio/video-only workload case. + """ + run, pool_path = self._make_file_run( + tmp_path, prompt_batch_size=0, image_batch_size=1 + ) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + data = { + "pool.jsonl": [ + RandomPool(image="https://example.com/img1.png", text="query1"), + ] + } + conversations = loader.convert_to_conversations(data) + + for conv in conversations: + turn = conv.turns[0] + assert turn.texts == [] + assert len(turn.images) == 1 + + def test_zero_batch_size_for_absent_modality_does_not_select_batched_path( + self, tmp_path: Path + ) -> None: + """`--image-batch-size 0` means "disable image inputs entirely". Against a + text-only named-pool directory it must be a no-op, not a trip into the + flattened path and its named-pool rejection. + """ + run, pool_path = self._make_file_run(tmp_path, image_batch_size=0) + loader = RandomPoolDatasetLoader( + filename=pool_path, run=run, num_conversations=2 + ) + data = { + "queries.jsonl": [RandomPool(texts=[Text(name="query", contents=["q1"])])], + "passages.jsonl": [ + RandomPool(texts=[Text(name="passage", contents=["p1"])]) + ], + } + conversations = loader.convert_to_conversations(data) + + assert len(conversations) == 2 + for conv in conversations: + names = sorted(t.name for t in conv.turns[0].texts) + assert names == ["passage", "query"] + assert conv.turns[0].images == [] + + +def _extract_heredoc_lines(doc: str, heredoc_target: str) -> list[str]: + """Return the body lines of a `cat > << 'EOF' ... EOF` block.""" + marker = f"cat > {heredoc_target} << 'EOF'\n" + body = doc.split(marker, 1)[1].split("\nEOF", 1)[0] + return [line for line in body.splitlines() if line.strip()] + + +def test_custom_dataset_docs_multimodal_example_pool_has_images() -> None: + """The "Multimodal batch sizes" tutorial example must use a pool that actually + contains images -- otherwise `--image-batch-size 2` silently samples nothing. + + Regression test for a doc bug where the example reused the text-only + `pool.jsonl` from the "Basic Single-File Sampling" section above it. + """ + doc = Path("docs/tutorials/custom-dataset.md").read_text() + lines = _extract_heredoc_lines(doc, "multimodal_pool.jsonl") + assert lines, "expected a multimodal_pool.jsonl heredoc in the tutorial" + + records = [RandomPool.model_validate_json(line) for line in lines] + assert all(record.image is not None for record in records), ( + "every entry in the multimodal random_pool example must carry an image " + "field, or --image-batch-size in the example command samples nothing" + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + pool_path = Path(tmp_dir) / "multimodal_pool.jsonl" + pool_path.write_text("\n".join(lines) + "\n") + cli = CLIConfig( + model_names=["test-model"], + input_file=str(pool_path), + custom_dataset_type="random_pool", + image_batch_size=2, + ) + run = make_run_from_cli(cli) + loader = RandomPoolDatasetLoader( + filename=str(pool_path), run=run, num_conversations=3 + ) + conversations = loader.convert_to_conversations( + {"multimodal_pool.jsonl": records} + ) + for conv in conversations: + assert len(conv.turns[0].images[0].contents) == 2 From e19fb157e1905e2cdf1b09b13ea853c62695cd4c Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Tue, 25 Aug 2026 17:00:42 -0700 Subject: [PATCH 038/119] fix(search): batch the DSP kernel to match a constrained multi-output GP (#1321) Signed-off-by: Anthony Casagrande (cherry picked from commit 4edbbf39d3d3fdd805494511d4e72ba4ff3d2435) --- .github/workflows/run-unit-tests.yml | 40 ++++++++++++++ .../search_planner/_botorch_kernel.py | 17 +++++- .../search_planner/_optuna_helpers.py | 12 +++- .../search_planner/test_botorch_kernel.py | 55 +++++++++++++++++++ .../search_planner/test_optuna_dsp_kernel.py | 55 ++++++++++++++++--- 5 files changed, 170 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 71f5d6e8cb..561845153e 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -143,3 +143,43 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} slug: ai-dynamo/aiperf + + botorch-regression: + name: BoTorch Regression Tests (CPU) + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + PYTEST_XDIST_AUTO_NUM_WORKERS: "4" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Set up uv with cache + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + - name: Install CPU-only BoTorch dependencies + run: | + make ci-install PYTHON_VERSION=3.13 + uv pip install --torch-backend cpu -e ".[botorch]" + # Split across two steps rather than two lines of one ``run:``. ``run:`` + # executes under ``bash -e``, so a failure in the first file would abort + # the step and the slow suite would never run. ``!cancelled()`` on the + # second overrides the implicit ``success()`` so both suites always + # report -- matching the Windows tiers above. + - name: Run BoTorch GP kernel regression tests + run: | + uv run pytest -n auto --dist=worksteal \ + tests/unit/orchestrator/search_planner/test_botorch_kernel.py + - name: Run BoTorch GP candidates_func regression tests (slow) + if: ${{ !cancelled() }} + run: | + uv run pytest -n auto --dist=worksteal -m slow \ + tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py diff --git a/src/aiperf/orchestrator/search_planner/_botorch_kernel.py b/src/aiperf/orchestrator/search_planner/_botorch_kernel.py index 47a0d5571d..a1f197fae0 100644 --- a/src/aiperf/orchestrator/search_planner/_botorch_kernel.py +++ b/src/aiperf/orchestrator/search_planner/_botorch_kernel.py @@ -23,10 +23,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + import torch from gpytorch.kernels import ScaleKernel -def make_dsp_kernel(d: int) -> ScaleKernel: +def make_dsp_kernel(d: int, batch_shape: torch.Size | None = None) -> ScaleKernel: """Return a Hvarfner-DSP-scaled Matern 5/2 kernel for ``d`` input dims. The output ``ScaleKernel`` wraps an ARD ``MaternKernel(nu=2.5)`` with a @@ -34,6 +35,18 @@ def make_dsp_kernel(d: int) -> ScaleKernel: lengthscale -- the form verified by Hvarfner et al. 2024 section 3.3. The output-scale prior is the BoTorch default ``Gamma(2, 0.15)``, which is well-behaved for objectives normalized via ``Standardize``. + + ``batch_shape`` must match the batch shape of the model this kernel is + handed to. A ``SingleTaskGP`` built over an ``m``-column train_y is a + batched multi-output model with ``batch_shape == Size([m])``, and every + kernel parameter has to carry that leading dimension. Omitting it + produces a kernel whose parameters are size 1, and the fit dies inside + BoTorch's scipy path with ``shape '[m, 1]' is invalid for input of size + 1`` -- which is where constrained qLogNEI lands, because the constrained + path cat-stacks the objective with one column per SLA filter and one per + outcome constraint. ``None`` (the default) is equivalent to ``Size([])`` + and leaves the kernel unbatched; gpytorch's ``Kernel.__init__`` does that + normalization itself. """ import torch from gpytorch.kernels import MaternKernel, ScaleKernel @@ -48,6 +61,8 @@ def make_dsp_kernel(d: int) -> ScaleKernel: nu=2.5, ard_num_dims=d, lengthscale_prior=LogNormalPrior(loc=loc, scale=scale), + batch_shape=batch_shape, ), outputscale_prior=GammaPrior(2.0, 0.15), + batch_shape=batch_shape, ) diff --git a/src/aiperf/orchestrator/search_planner/_optuna_helpers.py b/src/aiperf/orchestrator/search_planner/_optuna_helpers.py index c29df5c6d6..4662739eee 100644 --- a/src/aiperf/orchestrator/search_planner/_optuna_helpers.py +++ b/src/aiperf/orchestrator/search_planner/_optuna_helpers.py @@ -291,10 +291,20 @@ def qlognei_candidates_func(*args: Any) -> Any: pending_x_n = ( normalize(pending_x, bounds=bounds) if pending_x is not None else None ) + # A multi-column train_y makes SingleTaskGP a batched multi-output + # model, so the custom kernel must carry the same leading dim. Derive + # it with BoTorch's own helper rather than hand-rolling Size([m]), so + # leading input batch dims stay correct too. See make_dsp_kernel's + # docstring for why the batch shape matters. + _, aug_batch_shape = SingleTaskGP.get_batch_dimensions( + train_X=train_x_n, train_Y=train_y + ) model = SingleTaskGP( train_x_n, train_y, - covar_module=make_dsp_kernel(d=train_x_n.size(-1)), + covar_module=make_dsp_kernel( + d=train_x_n.size(-1), batch_shape=aug_batch_shape + ), outcome_transform=Standardize(m=train_y.size(-1)), ) fit_gpytorch_mll(ExactMarginalLogLikelihood(model.likelihood, model)) diff --git a/tests/unit/orchestrator/search_planner/test_botorch_kernel.py b/tests/unit/orchestrator/search_planner/test_botorch_kernel.py index dcdacba526..c99c445a84 100644 --- a/tests/unit/orchestrator/search_planner/test_botorch_kernel.py +++ b/tests/unit/orchestrator/search_planner/test_botorch_kernel.py @@ -10,6 +10,10 @@ torch = pytest.importorskip("torch") gpytorch = pytest.importorskip("gpytorch") +# Module level, not inside the test: a per-test importorskip leaves the CI job +# green (exit 0) with the regression silently skipped. Collected-nothing here +# exits 5 instead, which is loud. +botorch = pytest.importorskip("botorch") from aiperf.orchestrator.search_planner._botorch_kernel import ( # noqa: E402 make_dsp_kernel, @@ -35,3 +39,54 @@ def test_dsp_kernel_lengthscale_prior_shifts_with_sqrt_d(): expected_scale = math.sqrt(3.0) assert math.isclose(prior.loc.item(), expected_loc, rel_tol=1e-9) assert math.isclose(prior.scale.item(), expected_scale, rel_tol=1e-9) + + +class TestDspKernelBatchShape: + """Regression: the DSP kernel must batch to match a multi-output GP. + + A ``SingleTaskGP`` built over an ``m``-column ``train_y`` is a batched + multi-output model with ``batch_shape == Size([m])``. Constrained qLogNEI + reaches that case whenever SLA filters are configured, because + ``_qlognei_constraint_kwargs`` cat-stacks the objective with one column per + filter. A kernel without the matching leading dim has size-1 parameters, + and the fit dies deep inside BoTorch's scipy path with + ``shape '[m, 1]' is invalid for input of size 1``. + + This crashed cluster adaptive search on every run that set ``slaFilters``. + It went unnoticed because the runtime image shipped without the [botorch] + extra, so BayesianSearchPlanner silently fell back to Optuna's TPE sampler + and this code path never executed. + """ + + def test_batch_shape_is_applied_to_both_kernels(self) -> None: + kernel = make_dsp_kernel(d=3, batch_shape=torch.Size([2])) + assert kernel.batch_shape == torch.Size([2]) + assert kernel.base_kernel.batch_shape == torch.Size([2]) + assert kernel.base_kernel.lengthscale.shape[0] == 2 + + def test_omitting_batch_shape_stays_unbatched(self) -> None: + kernel = make_dsp_kernel(d=3) + assert kernel.batch_shape == torch.Size([]) + + @pytest.mark.parametrize("n_outputs", [1, 2, 3]) + def test_gp_fit_succeeds_for_each_output_count(self, n_outputs: int) -> None: + """m=1 is unconstrained; m>1 is one column per SLA filter.""" + from botorch.fit import fit_gpytorch_mll + from botorch.models import SingleTaskGP + from botorch.models.transforms import Standardize + from gpytorch.mlls import ExactMarginalLogLikelihood + + torch.manual_seed(0) + train_x = torch.rand(6, 1, dtype=torch.float64) + train_y = torch.rand(6, n_outputs, dtype=torch.float64) + + model = SingleTaskGP( + train_x, + train_y, + covar_module=make_dsp_kernel( + d=1, + batch_shape=torch.Size([n_outputs]) if n_outputs > 1 else None, + ), + outcome_transform=Standardize(m=n_outputs), + ) + fit_gpytorch_mll(ExactMarginalLogLikelihood(model.likelihood, model)) diff --git a/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py b/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py index 9a58d2023c..ff5bc38628 100644 --- a/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py +++ b/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py @@ -10,6 +10,7 @@ import math import pytest +from pytest import param pytestmark = pytest.mark.slow @@ -24,15 +25,21 @@ def _capture_built_models(monkeypatch, captured: list): - """Patch SingleTaskGP so every constructed model lands in `captured`.""" + """Patch SingleTaskGP so every constructed model lands in `captured`. + + A subclass rather than a wrapper function: the call sites reach through + the patched symbol for classmethods/staticmethods too (e.g. + ``SingleTaskGP.get_batch_dimensions``), which a plain function does not + carry. + """ real_cls = botorch.models.SingleTaskGP - def _spy(*args, **kwargs): - instance = real_cls(*args, **kwargs) - captured.append(instance) - return instance + class _Spy(real_cls): + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) + captured.append(self) - monkeypatch.setattr(botorch.models, "SingleTaskGP", _spy) + monkeypatch.setattr(botorch.models, "SingleTaskGP", _Spy) # Also patch the symbol bound inside _optuna_helpers' candidates_func # closure — the `from botorch.models import SingleTaskGP` binding is # captured at builder construction time inside the closure, so monkeypatch @@ -40,7 +47,7 @@ def _spy(*args, **kwargs): import aiperf.orchestrator.search_planner._optuna_helpers as helpers_mod if hasattr(helpers_mod, "SingleTaskGP"): - monkeypatch.setattr(helpers_mod, "SingleTaskGP", _spy) + monkeypatch.setattr(helpers_mod, "SingleTaskGP", _Spy) def test_qlognei_candidates_func_fits_dsp_kernel(monkeypatch): @@ -67,6 +74,40 @@ def test_qlognei_candidates_func_fits_dsp_kernel(monkeypatch): assert math.isclose(prior.loc.item(), expected_loc, rel_tol=1e-9) +@pytest.mark.parametrize( + "n_sla_filters", + [ + param(1, id="one-filter"), + param(2, id="two-filters"), + ], +) # fmt: skip +def test_qlognei_candidates_func_fits_with_sla_constraints(n_sla_filters: int) -> None: + """Regression: the constrained path cat-stacks objective + one column per + SLA filter, which makes the GP a batched multi-output model. The call site + must derive the matching kernel batch_shape or the fit raises + ``shape '[m, 1]' is invalid for input of size 1``. + + Goes through ``build_qlognei_candidates_func`` rather than reconstructing + the model inline, so it exercises the wiring that actually regressed. + """ + func = build_qlognei_candidates_func() + d = 3 + torch.manual_seed(0) + train_x = torch.rand(8, d, dtype=torch.float64) + train_obj = torch.rand(8, 1, dtype=torch.float64) + # BoTorch treats con <= 0 as feasible. Pin the signs rather than relying + # on random draws: even rows strictly feasible, odd rows strictly not. + train_con = torch.rand(8, n_sla_filters, dtype=torch.float64) + 0.5 + train_con[::2] *= -1.0 + bounds = torch.stack( + [torch.zeros(d, dtype=torch.float64), torch.ones(d, dtype=torch.float64)] + ) + + candidates = func(train_x, train_obj, train_con, bounds, None) + + assert candidates.shape == (1, d) + + def test_qnehvi_candidates_func_fits_dsp_kernel_per_objective(monkeypatch): captured: list = [] _capture_built_models(monkeypatch, captured) From 6e801853b7afd31facf5f23dba604b60384df704 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:56:12 -0700 Subject: [PATCH 039/119] fix: infer benchmark shape from --search-space keywords (#1323) Signed-off-by: Elias Bermudez (cherry picked from commit d8d49e8c2adc76072625c0789ac1029967b639a5) --- docs/sweeping/bayesian-optimization.md | 51 ++ .../config/flags/_converter_profiling.py | 310 ++++++++- .../search_planner/optuna_planner.py | 44 +- .../test_converter_profiling_phase_routes.py | 657 ++++++++++++++++++ .../search_planner/test_optuna_planner.py | 106 +++ 5 files changed, 1153 insertions(+), 15 deletions(-) diff --git a/docs/sweeping/bayesian-optimization.md b/docs/sweeping/bayesian-optimization.md index f94ddb8b1b..0a0e1c607e 100644 --- a/docs/sweeping/bayesian-optimization.md +++ b/docs/sweeping/bayesian-optimization.md @@ -85,6 +85,57 @@ This runs 30 search iterations × 3 trials each = 90 benchmarks. `--search-plann | `--optuna-acquisition ACQ` | no | BoTorch acquisition override. Only consulted with `--search-planner=optuna --optuna-sampler=botorch`. Single-objective: `qnei` (Letham 2019) or `qlognei` (Ament 2023) for noisy-EI; `logei`/`qlogei` are the explicit defaults. Multi-objective: `qehvi`, `qnehvi`, or `qlognehvi` (Daulton 2021). The cross-field validator on `AdaptiveSearchSweep` requires the choice to match `len(objectives)`: single-objective acquisitions reject `len(objectives) > 1`, multi-objective acquisitions reject `len(objectives) == 1`. The `bayesian` preset auto-selects `qlognei` (single-obj) or `qlognehvi` (multi-obj) based on `len(objectives)`. See [Multi-objective Pareto BO](#multi-objective-pareto-bo). | | `--optuna-terminator MODE` | no | Posterior-regret stopping: `regret` (Makarova 2022 `RegretBoundEvaluator`) or `emmr` (Ishibashi 2023). Only consulted with `--search-planner=optuna`. Layered on top of three-signal convergence; `convergence_reason` becomes `posterior_regret_bound` or `emmr` when it fires. | +The shape-inference, seeding, and rejection rules below are enforced when +`--search-space` is parsed on the CLI conversion path (`convert_cli_to_aiperf`). +A `search_space:` entry under a YAML `sweep:` block goes through a +different code path that doesn't apply the same validation, so a +config-file sweep can still hit the underlying crashes these rules +prevent -- prefer `--search-space` on the CLI until config-file sweeps get +equivalent validation. + +A benchmark has one "shape" before it starts: `concurrency` ("send as many +requests as possible"), `rate` ("send N requests per second"), `user` +("simulate N users"), or `gamma` ("send requests with a particular +bumpy/smooth pattern"). Searching a shape-specific field -- +`rate`/`rate_ramp` (rate shape), `smoothness` (gamma shape), or `users` +(user shape) -- automatically switches the base benchmark to a compatible +shape, the same as passing the matching CLI flag (`--request-rate`, +`--arrival-pattern gamma`, `--user-centric-rate`) would. Fields from the +same shape combine freely — e.g. searching `users` and `rate` together +fully auto-seeds a user shape, since a user-shaped benchmark has both +fields (it shares the rate machinery with the rate shape). Searching +fields from two mutually-exclusive shapes (e.g. `users` and `smoothness` +together — no shape has both) is rejected with a clear error. + +`rate_series` is a piecewise-linear rate schedule, not a single number, so +it's not a valid dimension to sweep -- `--search-space "rate_series:..."` +is rejected at config time, with or without a companion +`--request-rate-series`. Pass `--request-rate-series` as a fixed schedule +instead, or search `rate`/`rate_ramp` for a scalar rate to sweep. + +`rate_ramp` and `smoothness` modulate an existing rate rather than +supplying one, so searching either alone still requires a base rate from +somewhere -- pass `--request-rate` (or `--user-centric-rate` for a user +shape), or add a `rate` dimension to the same `--search-space`. Without +one, AIPerf errors at config time with a message explaining what's +missing, rather than searching `rate_ramp`/`smoothness` in a vacuum. +`users` is a partial exception: it self-seeds the user count from +its own search range, but a user-shaped benchmark still needs the same base +rate as the rate shapes (users share a global request rate), so searching +`users` alone also requires `--request-rate`, `--user-centric-rate`, or a +`rate` dimension -- same requirement, same error, just for a different +field. A `users` dimension must also use `:int` kind (e.g. +`users:1,50:int`) -- the omitted-kind default of `:real` is rejected, since +the number of simulated users can only ever be a whole number. + +`smoothness` auto-selects the gamma shape only when `--arrival-pattern` is +left unset. An explicit non-gamma `--arrival-pattern` (`poisson` or +`constant`) wins instead and is rejected at config time if `smoothness` is +also being searched, the same way `--arrival-smoothness` is rejected +against an explicit non-gamma pattern -- drop `--arrival-pattern` to let +`smoothness` auto-select gamma, or pass `--arrival-pattern gamma` +explicitly. + ### Search space grammar `PATH:LO,HI[:KIND]` diff --git a/src/aiperf/config/flags/_converter_profiling.py b/src/aiperf/config/flags/_converter_profiling.py index d9b8f17556..fef1c2e81c 100644 --- a/src/aiperf/config/flags/_converter_profiling.py +++ b/src/aiperf/config/flags/_converter_profiling.py @@ -6,6 +6,7 @@ from __future__ import annotations import gzip +import math import zlib from typing import TYPE_CHECKING, Any @@ -87,24 +88,153 @@ def _apply_agentic_replay_fields(phase: dict[str, Any], cli: CLIConfig) -> None: phase["agentic_warmup_grace_period"] = cli.warmup_grace_period -def _profiling_phase_type(cli: CLIConfig) -> Any: +_RATE_SHAPE_SEARCH_FIELDS: frozenset[str] = frozenset( + {"rate", "rate_ramp", "smoothness"} +) + + +def _search_space_dimensions(cli: CLIConfig) -> dict[str, tuple[float, str]]: + """Field name -> (lower bound, kind) for every dimension in ``--search-space``. + + A lightweight companion to ``parse_search_space()`` (which runs later, + in ``_build_adaptive_search``): only extracts each dimension's final + path segment (resolving bare-name aliases the same way the real parser + does), its lower bound, and its declared ``kind`` (``int``/``real``, + default ``real`` when omitted -- matches ``SearchSpaceDimension``'s own + default), so ``_profiling_phase_type``/``build_profiling`` can pick a + compatible phase shape -- and seed a self-supplying field's initial + value from its own search range -- before search-space bounds/kind are + even validated by the real parser. Malformed entries are skipped here; + the real parser reports the actual grammar error. Only dimensions that + target a direct ``phases.profiling.`` scalar are considered + (bare aliases always resolve there); a fully-qualified path targeting a + different phase (e.g. ``phases.warmup.*``) OR a nested sub-field one + level deeper (e.g. ``phases.profiling.cancellation.rate``, the request- + cancellation rate -- an unrelated field that merely happens to share + the ``rate`` leaf name) is ignored by this shape-inference helper -- + the real search-space parser still processes it normally later, just + not for the purpose of phase-shape inference. + + ``rate_series`` is rejected outright, regardless of any companion CLI + flag: it's a piecewise-linear schedule (``RateSeriesConfig``), not a + scalar the planner can sample between two bounds. Letting it through + would set a phase's ``rate_series`` field to a bare Optuna-sampled + float on the first trial and crash there -- reject it immediately and + clearly instead. + """ + if not cli.search_space: + return {} + + dims: dict[str, tuple[float, str]] = {} + for raw in cli.search_space: + parsed = _parse_shape_dimension(raw) + if parsed is not None: + field, lo, kind = parsed + dims[field] = (lo, kind) + return dims + + +def _parse_shape_dimension(raw: str) -> tuple[str, float, str] | None: + """Parse one ``--search-space`` entry into ``(field, lo, kind)``, or + ``None`` if it isn't a shape-bearing ``phases.profiling.`` scalar. + + Raises for ``rate_series`` specifically (see ``_search_space_dimensions`` + docstring); every other malformed or out-of-scope entry is skipped so + the real parser (``parse_search_space``) is what reports the actual + grammar error, rather than this lightweight helper guessing wrong. + """ + from aiperf.config.loader.dotted_path import _resolve_path_alias + + path_part, _, rest = raw.partition(":") + path = path_part + if not path: + return None + if "." not in path: + path = _resolve_path_alias(path) + if not path.startswith("phases.profiling."): + return None + remainder = path[len("phases.profiling.") :] + if "." in remainder: + # A nested sub-field (e.g. "phases.profiling.cancellation.rate") is + # not the phase's own scalar field -- matching it by leaf name alone + # would misclassify e.g. a cancellation-rate sweep as the phase's + # request rate. Not a shape-bearing dimension. + return None + field = remainder + if field == "rate_series": + raise ValueError( + "--search-space 'rate_series' is not a valid adaptive-search " + "dimension: it's a piecewise-linear rate schedule (a list of " + "time/rate points), not a single number the planner can " + "sample between two bounds. Pass --request-rate-series as a " + "fixed companion instead of searching over it, or search " + "'rate'/'rate_ramp' for a scalar rate to sweep." + ) + bounds_part, _, kind_part = rest.partition(":") + if ":" in kind_part: + # More than three ':'-separated segments (e.g. "users:1,50:int:log") + # -- malformed grammar. Skip rather than guess a kind from it. + return None + kind = kind_part or "real" + if kind not in ("int", "real"): + # Not one of the real parser's _VALID_KINDS (parsing.py) -- skip so + # the real parser's own grammar error is what the user sees. This is + # also what makes _seed_users_dimension's "not ':real'" message + # always accurate: by the time kind reaches there, it can only ever + # be "int" or "real". + return None + lo_str = bounds_part.split(",", 1)[0] + try: + return field, float(lo_str), kind + except ValueError: + return None + + +def _profiling_phase_type( + cli: CLIConfig, search_dims: dict[str, tuple[float, str]] +) -> Any: from aiperf.config.phases import PhaseType from aiperf.plugin.enums import ArrivalPattern if cli.fixed_schedule: return PhaseType.FIXED_SCHEDULE - if cli.user_centric_rate is not None: + + user_centric_needed = "users" in search_dims + user_centric_selected = cli.user_centric_rate is not None or user_centric_needed + rate_shape_needed = bool(set(search_dims) & _RATE_SHAPE_SEARCH_FIELDS) + + # UserCentricPhase has no 'smoothness' field and GammaPhase has no + # 'users' field -- no phase type can satisfy both, unlike users+rate + # (UserCentricPhase legitimately has rate/rate_ramp/rate_series too, + # inherited from RatePhaseConfig). Gated on user_centric_selected (not + # just user_centric_needed) so an explicit --user-centric-rate without + # a 'users' dimension is caught too: it still resolves to USER_CENTRIC, + # which still has no 'smoothness' field for a searched 'smoothness' + # dimension to land on. + if user_centric_selected and "smoothness" in search_dims: + raise ValueError( + "--search-space targets 'smoothness' (a gamma-shaped benchmark), " + "but --user-centric-rate (or a 'users' dimension) selects a " + "user-centric-shaped benchmark. A benchmark can only have one " + "shape at a time -- search these in separate runs." + ) + + if user_centric_selected: return PhaseType.USER_CENTRIC - if cli.request_rate is not None or cli.request_rate_series is not None: + if ( + cli.request_rate is not None + or cli.request_rate_series is not None + or rate_shape_needed + ): # v1 parity (user_config.py auto-promote): --arrival-smoothness / # --vllm-burstiness without an explicit --arrival-pattern resolves to # gamma, since smoothness is a gamma-distribution knob. Without this the # flag fell through to POISSON and then _apply_phase_specific_routes # hard-rejected it ("only supported with gamma") -- a cutover regression - # that made --vllm-burstiness unusable on its own. - if ( - "arrival_pattern" not in cli.model_fields_set - and cli.arrival_smoothness is not None + # that made --vllm-burstiness unusable on its own. A 'smoothness' + # search-space dimension is the same knob, so it auto-promotes too. + if "arrival_pattern" not in cli.model_fields_set and ( + cli.arrival_smoothness is not None or "smoothness" in search_dims ): return PhaseType.GAMMA match cli.arrival_pattern: @@ -117,6 +247,151 @@ def _profiling_phase_type(cli: CLIConfig) -> Any: return PhaseType.CONCURRENCY +def _apply_search_space_shape_seeds( + prof: dict[str, Any], search_dims: dict[str, tuple[float, str]] +) -> None: + """Seed the required scalar(s) a search-space-inferred shape still needs. + + ``_profiling_phase_type`` can auto-switch the phase *type* from + ``--search-space`` alone, but rate-controlled/user-centric phases also + require a *value* for their defining scalar (``rate``, and for + user-centric also ``users``) before Pydantic will accept the base + config -- the search planner only supplies that value once trials + start. When the searched field is the scalar itself, its own lower + bound is a natural seed (the planner immediately overrides it on trial + 0 anyway). When it isn't -- e.g. searching 'smoothness' or 'rate_ramp' + without ever searching or explicitly setting 'rate' -- there's no value + to seed from; raise a clear error instead of letting Pydantic's + "rate-controlled phases require rate or rate_series" surface as an + unexplained crash on the very first config build. + + Both the 'users' and 'rate' validation below run whenever the + respective field is in search_dims, regardless of whether an explicit + CLI flag (--num-users, --request-rate, --request-rate-series, or + --user-centric-rate) already populated prof["users"]/prof["rate"]/ + prof["rate_series"] via _PROF_FIELD_ROUTES / _apply_profiling_rate_series + before this runs -- only the *assignment* is gated on absence. + Otherwise `--request-rate 10 --search-space "rate:-5,100"` would + silently skip bound validation on a dimension the planner still samples + from every trial (crashing mid-search once a negative value is drawn, + not at config-build time), the same class of bug as skipping 'users' + validation whenever --num-users was also explicit. + """ + from aiperf.config.phases import PhaseType + + # 'smoothness' (GammaPhase.smoothness, gt=0) and 'rate_ramp' + # (RampConfig.duration, gt=0.0) aren't seeded into prof here -- both are + # optional fields the planner only ever writes once trials start -- but + # they share the same "validate the bound now, not at sample time" need + # as 'rate'/'users': a negative lower bound otherwise builds a valid + # base config and only fails once the planner happens to sample a + # negative value mid-search. + _reject_non_positive_bound( + search_dims, "smoothness", "the gamma distribution's shape parameter" + ) + _reject_non_positive_bound(search_dims, "rate_ramp", "the ramp duration (seconds)") + + phase_type = prof["type"] + + # 'smoothness' only auto-promotes the phase to GAMMA when --arrival-pattern + # is unset (see the v1-parity comment in _profiling_phase_type); an + # EXPLICIT non-gamma --arrival-pattern silently wins instead, leaving a + # searched 'smoothness' dimension with nowhere to land (GammaPhase is the + # only phase with that field). Mirrors the equivalent gate for the + # --arrival-smoothness *flag* in _apply_phase_specific_routes below. + if "smoothness" in search_dims and phase_type != PhaseType.GAMMA: + raise ValueError( + "--search-space 'smoothness' is only supported with --arrival-pattern " + "gamma. Pass --arrival-pattern gamma (or omit --arrival-pattern so " + "'smoothness' can auto-select it), or drop the 'smoothness' dimension." + ) + + if phase_type == PhaseType.USER_CENTRIC and "users" in search_dims: + _seed_users_dimension(prof, search_dims) + + if phase_type in ( + PhaseType.POISSON, + PhaseType.GAMMA, + PhaseType.CONSTANT, + PhaseType.USER_CENTRIC, + ): + _seed_rate_dimension(prof, search_dims, phase_type) + + +def _reject_non_positive_bound( + search_dims: dict[str, tuple[float, str]], field: str, subject: str +) -> None: + """Raise a clear error if `field`'s search-space lower bound isn't > 0.""" + if field not in search_dims: + return + lo, _kind = search_dims[field] + if not math.isfinite(lo) or lo <= 0: + raise ValueError( + f"--search-space '{field}' lower bound must be > 0 (got {lo!r}); " + f"{subject} must be positive." + ) + + +def _seed_users_dimension( + prof: dict[str, Any], search_dims: dict[str, tuple[float, str]] +) -> None: + """Validate and (if absent) seed prof["users"] from a 'users' dimension.""" + users_lo, users_kind = search_dims["users"] + if users_kind != "int": + raise ValueError( + "--search-space 'users' must use ':int' kind (e.g. " + "'users:1,50:int'), not ':real' -- the number of simulated " + "users must be a whole number." + ) + if not math.isfinite(users_lo) or users_lo < 1: + raise ValueError( + f"--search-space 'users' lower bound must be >= 1 (got " + f"{users_lo!r}); the number of simulated users can't be " + "less than one." + ) + if "users" not in prof: + prof["users"] = int(users_lo) + + +def _seed_rate_dimension( + prof: dict[str, Any], search_dims: dict[str, tuple[float, str]], phase_type: Any +) -> None: + """Validate and (if absent) seed prof["rate"] from a 'rate' dimension. + + Raises when the phase needs a base rate that neither an explicit CLI + flag nor a 'rate' search-space dimension can supply. + """ + has_base_rate = "rate" in prof or "rate_series" in prof + if "rate" not in search_dims: + if not has_base_rate: + raise ValueError( + f"--search-space selects a rate-shaped benchmark (phase type " + f"{phase_type}), which also requires a base rate. Pass " + "--request-rate (or --user-centric-rate for a " + "user-shaped benchmark), or add a 'rate' dimension to " + "--search-space." + ) + return + + if "rate_series" in prof: + raise ValueError( + "--search-space targets 'rate', but --request-rate-series " + "already supplies a fixed rate schedule for this phase -- " + "'rate' and 'rate_series' are mutually exclusive on a " + "rate-controlled phase. Drop --request-rate-series to " + "search 'rate', or drop the 'rate' dimension to keep the " + "fixed schedule." + ) + rate_lo, _rate_kind = search_dims["rate"] + if not math.isfinite(rate_lo) or rate_lo <= 0: + raise ValueError( + f"--search-space 'rate' lower bound must be > 0 (got " + f"{rate_lo!r}); rate must be positive." + ) + if not has_base_rate: + prof["rate"] = rate_lo + + def _apply_profiling_ramps(prof: dict[str, Any], cli: CLIConfig) -> None: fields_set = cli.model_fields_set for field, key in _RAMP_FIELDS: @@ -328,16 +603,21 @@ def _maybe_auto_promote_trace( # FixedSchedulePhase doesn't accept rate/users/smoothness. If the user # explicitly opted into a rate-controlled mode against a timestamped # trace, refuse the combo loudly rather than silently dropping their - # flag — they almost certainly want one or the other, not both. + # flag — they almost certainly want one or the other, not both. `prof` + # may hold these because of an explicit CLI flag OR because + # _apply_search_space_shape_seeds seeded them from a --search-space + # dimension -- "flags" alone would be unactionable in the latter case + # (there's no flag to drop), so name both sources. conflicts = [k for k in ("rate", "users", "smoothness") if k in prof] if conflicts: raise ValueError( "Trace dataset has per-record timestamps and would be " - "auto-promoted to fixed_schedule, but the following flags " - f"are incompatible with fixed_schedule mode: {conflicts}. " - "Either drop the conflicting flags to enable auto-fixed-" - "schedule, or pass --no-fixed-schedule to keep your " - "user-selected timing mode and ignore trace timestamps." + "auto-promoted to fixed_schedule, but the following flags or " + f"--search-space dimensions are incompatible with " + f"fixed_schedule mode: {conflicts}. Either drop the " + "conflicting flags/dimensions to enable auto-fixed-schedule, " + "or pass --no-fixed-schedule to keep your user-selected timing " + "mode and ignore trace timestamps." ) prof["type"] = PhaseType.FIXED_SCHEDULE @@ -515,7 +795,9 @@ def build_profiling(cli: CLIConfig) -> dict[str, Any]: _apply_agentic_replay_fields(prof, cli) _apply_profiling_rate_series(prof, cli) - prof["type"] = _profiling_phase_type(cli) + search_dims = _search_space_dimensions(cli) + prof["type"] = _profiling_phase_type(cli, search_dims) + _apply_search_space_shape_seeds(prof, search_dims) _reject_orphan_load_generator_flags(prof, cli) _apply_phase_specific_routes(prof, cli) diff --git a/src/aiperf/orchestrator/search_planner/optuna_planner.py b/src/aiperf/orchestrator/search_planner/optuna_planner.py index f1e97c69c9..33355dbbd1 100644 --- a/src/aiperf/orchestrator/search_planner/optuna_planner.py +++ b/src/aiperf/orchestrator/search_planner/optuna_planner.py @@ -23,6 +23,8 @@ import warnings from typing import TYPE_CHECKING, Any +from pydantic import ValidationError + from aiperf.common.finite import is_finite_value from aiperf.config.config import BenchmarkConfig from aiperf.config.sweep import AdaptiveSearchSweep, SweepVariation, _set_nested_value @@ -219,7 +221,47 @@ def ask(self) -> tuple[BenchmarkConfig, SweepVariation] | None: ) for path, val in values.items(): _set_nested_value(cfg_dict, path, val) - cfg = BenchmarkConfig.model_validate(cfg_dict) + try: + cfg = BenchmarkConfig.model_validate(cfg_dict) + except ValidationError as e: + # Only reframe genuine shape mismatches -- a field that doesn't + # exist on the PROFILING phase ("extra_forbidden": e.g. injecting + # 'rate' onto a concurrency phase) or a required field the base + # config never got a chance to seed ("missing"). Ordinary + # numeric-bounds violations (e.g. a sampled concurrency=0 against + # ge=1) are unrelated to phase shape and should surface as-is -- + # reframing them as a "shape" problem is actively misleading. + # Scoped to errors under "phases" specifically: extra_forbidden + # also fires for an unrelated malformed root-level key (e.g. a + # search-space path with no "." that never resolves under a + # phase at all), which isn't a shape mismatch either. + shape_error_types = {"extra_forbidden", "missing"} + if not any( + err.get("type") in shape_error_types + and err.get("loc", ())[:1] == ("phases",) + for err in e.errors() + ): + raise + # phases[0] is the warmup phase whenever one exists (warmup is + # appended before profiling -- see converter.py's phase-list + # assembly and BenchmarkConfig's "order is execution order" + # contract), so naming it here would blame the wrong phase's + # shape. Match the profiling phase by kind instead, the same + # selector BenchmarkConfig.validate_profiling_phase_required + # uses (config.py). + base_phase_type = next( + (p.type for p in self._base.phases if p.kind == "profiling"), + self._base.phases[0].type, + ) + raise ValueError( + f"--search-space path(s) {sorted(values)} do not fit this " + f"benchmark's shape ('{base_phase_type}'). A benchmark has one " + "shape before it starts (concurrency, rate, user, or gamma); " + "searching a field that belongs to a different shape (e.g. " + "'rate' needs --request-rate, 'users' needs " + "--user-centric-rate, 'smoothness' needs --arrival-pattern " + f"gamma) fails like this. Underlying error: {e}" + ) from e variation = SweepVariation( index=self._iter, diff --git a/tests/unit/config/test_converter_profiling_phase_routes.py b/tests/unit/config/test_converter_profiling_phase_routes.py index 1ea92f33bc..fff498b062 100644 --- a/tests/unit/config/test_converter_profiling_phase_routes.py +++ b/tests/unit/config/test_converter_profiling_phase_routes.py @@ -25,6 +25,7 @@ import pytest +from aiperf.config.config import BenchmarkConfig from aiperf.config.flags._converter_profiling import build_profiling from aiperf.config.flags.cli_config import CLIConfig from aiperf.plugin.enums import ArrivalPattern, PhaseType @@ -691,3 +692,659 @@ def test_rate_series_with_user_centric_rate_raises(self, tmp_path: Path) -> None with pytest.raises(ValueError, match="user-centric-rate"): build_profiling(user) + + +# --------------------------------------------------------------------------- +# BUG 5 (NVBugs 6656707) — --search-space keywords must auto-infer phase shape +# --------------------------------------------------------------------------- + + +class TestSearchSpacePhaseShapeInference: + def test_bare_rate_keyword_infers_poisson_phase(self) -> None: + """--search-space 'rate:...' with no --request-rate must not crash; + it should auto-switch to a rate-controlled (poisson default) phase, + seeding 'rate' from the search-space dimension's own lower bound.""" + loadgen = CLIConfig( + search_space=["rate:1,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.POISSON + assert prof["rate"] == 1.0 + + def test_dotted_rate_path_infers_poisson_phase(self) -> None: + """Full dotted path form ('phases.profiling.rate') resolves the same + as the bare alias.""" + loadgen = CLIConfig( + search_space=["phases.profiling.rate:1,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.POISSON + assert prof["rate"] == 1.0 + + def test_rate_search_space_respects_explicit_arrival_pattern(self) -> None: + """--arrival-pattern gamma + --search-space 'rate:...' should still + pick GAMMA, not the POISSON default.""" + loadgen = CLIConfig( + search_space=["rate:1,100:real"], + arrival_pattern=ArrivalPattern.GAMMA, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.GAMMA + assert prof["rate"] == 1.0 + + def test_rate_ramp_keyword_with_request_rate_stays_poisson(self) -> None: + """--request-rate supplies the base rate; 'rate_ramp' alone in + search-space just needs a companion rate source, per the bug + report's documented workaround.""" + loadgen = CLIConfig( + search_space=["rate_ramp:1,60:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.POISSON + assert prof["rate"] == 10.0 + + def test_rate_ramp_keyword_alone_raises_clear_error(self) -> None: + """'rate_ramp' layers a ramp on top of a base rate -- it cannot + supply that base rate itself. Without --request-rate or a 'rate' + search-space dimension, this must fail with a clear error, not a + raw Pydantic 'rate-controlled phases require rate or rate_series'.""" + loadgen = CLIConfig( + search_space=["rate_ramp:1,60:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="base rate"): + build_profiling(user) + + def test_rate_series_keyword_alone_raises_clear_error(self) -> None: + """'rate_series' is a piecewise-linear schedule, not a scalar the + planner can sample between two bounds -- it must never be treated + as a searchable numeric dimension, regardless of companion flags.""" + loadgen = CLIConfig( + search_space=["rate_series:1,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="not a valid adaptive-search"): + build_profiling(user) + + def test_rate_series_keyword_with_request_rate_still_raises(self) -> None: + """A --request-rate companion doesn't change anything -- 'rate_series' + itself still can't be a scalar search dimension (it was previously + treated as equivalent to 'rate', which silently masked the fact + that a sampled float would later crash writing into a + RateSeriesConfig-typed field).""" + loadgen = CLIConfig( + search_space=["rate_series:1,100:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="not a valid adaptive-search"): + build_profiling(user) + + def test_rate_series_keyword_with_request_rate_series_still_raises( + self, tmp_path: Path + ) -> None: + """Even the documented --request-rate-series companion doesn't make + the schedule itself searchable as a number -- this combo was + already reachable before this PR (via the pre-existing + request_rate_series-sets-a-rate-controlled-type logic) and already + crashed once a sampled float reached the planner; now it's rejected + immediately and clearly instead.""" + json_path = tmp_path / "rate.json" + json_path.write_text( + '{"points":[{"time_s":0,"qps":1},{"time_s":60,"qps":7}]}', + encoding="utf-8", + ) + loadgen = CLIConfig( + search_space=["rate_series:1,100:real"], + request_rate_series=json_path, + arrival_pattern=ArrivalPattern.CONSTANT, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="not a valid adaptive-search"): + build_profiling(user) + + def test_rate_keyword_with_request_rate_series_raises_mutual_exclusion_error( + self, tmp_path: Path + ) -> None: + """Searching 'rate' (a valid scalar dimension on its own) while + --request-rate-series already supplies a fixed schedule passes + config conversion cleanly if left unchecked -- prof["rate_series"] + is already set, so the seed step's has_base_rate short-circuit + skips seeding prof["rate"] without complaint. But the planner + still samples 'rate' every trial and injects it into a phase dict + that already has rate_series, violating RatePhaseConfig's mutual + exclusion at the first trial instead of failing clearly here.""" + json_path = tmp_path / "rate.json" + json_path.write_text( + '{"points":[{"time_s":0,"qps":1},{"time_s":60,"qps":7}]}', + encoding="utf-8", + ) + loadgen = CLIConfig( + search_space=["rate:1,100:real"], + request_rate_series=json_path, + arrival_pattern=ArrivalPattern.CONSTANT, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="mutually exclusive"): + build_profiling(user) + + def test_smoothness_keyword_with_request_rate_infers_gamma_phase(self) -> None: + """--search-space 'smoothness:...' auto-switches to gamma, but still + needs a base rate from somewhere -- --request-rate supplies it.""" + loadgen = CLIConfig( + search_space=["smoothness:0.5,2.0:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.GAMMA + assert prof["rate"] == 10.0 + + def test_smoothness_keyword_alone_raises_clear_error(self) -> None: + """'smoothness' alone (no --request-rate, no 'rate' in search-space) + auto-switches the *type* to gamma but has no base rate to seed -- + must fail with a clear error, not a raw Pydantic crash.""" + loadgen = CLIConfig( + search_space=["smoothness:0.5,2.0:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="base rate"): + build_profiling(user) + + def test_users_keyword_infers_user_centric_phase(self) -> None: + """--user-centric-rate supplies the shared base rate; 'users' in + search-space self-seeds the user count from its own lower bound.""" + loadgen = CLIConfig( + search_space=["users:1,50:int"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.USER_CENTRIC + assert prof["users"] == 1 + assert prof["rate"] == 10.0 + + def test_users_keyword_alone_raises_clear_error(self) -> None: + """'users' alone (no --user-centric-rate, no 'rate' in search-space) + auto-switches the *type* to user-centric and self-seeds 'users', but + has no base rate to seed -- must fail with a clear error.""" + loadgen = CLIConfig( + search_space=["users:1,50:int"], + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="base rate"): + build_profiling(user) + + def test_users_and_rate_together_infers_user_centric_with_both_seeded( + self, + ) -> None: + """'users' and 'rate' are compatible, not a conflict: UserCentricPhase + subclasses RatePhaseConfig, so it legitimately has both fields. + Searching them together fully self-seeds a user-centric shape with + no companion flags needed at all.""" + loadgen = CLIConfig( + search_space=["users:1,50:int", "rate:1,100:real"], + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.USER_CENTRIC + assert prof["users"] == 1 + assert prof["rate"] == 1.0 + + def test_users_and_smoothness_together_raises_clear_conflict_error(self) -> None: + """This IS a genuine, unresolvable conflict: no phase type has both + a 'users' field (UserCentricPhase-only) and a 'smoothness' field + (GammaPhase-only).""" + loadgen = CLIConfig( + search_space=["users:1,50:int", "smoothness:0.5,2.0:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="only have one shape"): + build_profiling(user) + + def test_explicit_user_centric_rate_and_smoothness_raises_clear_conflict_error( + self, + ) -> None: + """The same conflict as 'users'+'smoothness', but triggered via an + explicit --user-centric-rate instead of a 'users' search-space + dimension -- either way the phase resolves to USER_CENTRIC, which + still has no 'smoothness' field. Previously only the 'users'-in- + search-space trigger was checked, so this combo built a + USER_CENTRIC phase that would only fail later, at the planner.""" + loadgen = CLIConfig( + search_space=["smoothness:0.5,2.0:real"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="only have one shape"): + build_profiling(user) + + def test_rate_nan_lower_bound_raises_clear_error(self) -> None: + """A NaN lower bound bypasses 'rate_lo <= 0' (NaN comparisons are + always False), so it must be explicitly rejected with + math.isfinite() rather than silently seeding prof["rate"] = nan.""" + loadgen = CLIConfig( + search_space=["rate:nan,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be > 0"): + build_profiling(user) + + def test_users_nan_lower_bound_raises_clear_error(self) -> None: + loadgen = CLIConfig( + search_space=["users:nan,50:int"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be >= 1"): + build_profiling(user) + + def test_smoothness_negative_lower_bound_raises_clear_error(self) -> None: + """'smoothness' isn't seeded into prof at config-build time (it's + optional, only ever written by the planner once trials start), so + it needs its own bound check independent of the seeding logic -- + GammaPhase.smoothness has the same gt=0 constraint 'rate' has.""" + loadgen = CLIConfig( + search_space=["smoothness:-5,-1:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be > 0"): + build_profiling(user) + + def test_rate_ramp_negative_lower_bound_raises_clear_error(self) -> None: + """Same as smoothness: 'rate_ramp' isn't seeded at config-build + time either, but RampConfig.duration has gt=0.0.""" + loadgen = CLIConfig( + search_space=["rate_ramp:-50,-10:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be > 0"): + build_profiling(user) + + def test_smoothness_with_explicit_non_gamma_arrival_pattern_raises_clear_error( + self, + ) -> None: + """'smoothness' only auto-promotes the phase to GAMMA when + --arrival-pattern is unset; an explicit non-gamma pattern wins + instead, leaving 'smoothness' with nowhere to land (only GammaPhase + has that field). Must fail clearly at config time, not at trial 0 + with a raw Pydantic extra_forbidden.""" + loadgen = CLIConfig( + search_space=["smoothness:0.5,2.0:real"], + request_rate=10.0, + arrival_pattern=ArrivalPattern.POISSON, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="only supported with --arrival-pattern"): + build_profiling(user) + + def test_smoothness_with_explicit_constant_arrival_pattern_raises_clear_error( + self, + ) -> None: + loadgen = CLIConfig( + search_space=["smoothness:0.5,2.0:real"], + request_rate=10.0, + arrival_pattern=ArrivalPattern.CONSTANT, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="only supported with --arrival-pattern"): + build_profiling(user) + + def test_search_space_path_with_leading_whitespace_ignored_for_shape_inference( + self, + ) -> None: + """The real parser (parse_search_space) doesn't strip whitespace -- + ' rate' and 'rate' are different paths to it, and ' rate' is + accepted as a literal (broken) path rather than resolved as the + 'rate' alias. This helper must match that exactly: stripping the + path here would silently reshape the benchmark for a dimension the + real parser treats as something else entirely.""" + loadgen = CLIConfig( + search_space=[" rate:1,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.CONCURRENCY + assert "rate" not in prof + + def test_search_space_kind_with_leading_whitespace_ignored_for_shape_inference( + self, + ) -> None: + """'rate:1,100: int' (space before kind) is REJECTED by the real + parser (kind must be exactly 'int' or 'real'). This helper must + not silently normalize ' int' to 'int' and proceed as if the + dimension were well-formed -- skip it and let the real parser + report the actual grammar error.""" + loadgen = CLIConfig( + search_space=["rate:1,100: int"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.CONCURRENCY + assert "rate" not in prof + + def test_trace_auto_promote_conflict_message_mentions_search_space( + self, tmp_path: Path + ) -> None: + """When 'rate' was seeded from --search-space (not an explicit + --request-rate flag), the trace auto-promote conflict error must + not tell the user to "drop the conflicting flags" -- there is no + flag to drop, only a --search-space dimension.""" + trace_path = tmp_path / "trace.jsonl" + trace_path.write_text( + '{"timestamp": 0, "input_length": 100, "output_length": 50}\n' + '{"timestamp": 100, "input_length": 120, "output_length": 60}\n', + encoding="utf-8", + ) + # Built directly (not via _make_user's two-CLIConfig merge) since + # input_file's validator only accepts a literal str, and + # _make_user's model_dump()/reconstruct round-trip turns it back + # into a Path first. + user = CLIConfig( + url="http://localhost:8000/test", + model_names=["test-model"], + search_space=["rate:1,100:real"], + input_file=str(trace_path), + custom_dataset_type="mooncake_trace", + request_count=10, + ) + with pytest.raises(ValueError, match="--search-space dimensions"): + build_profiling(user) + + def test_explicit_request_rate_still_wins_without_search_space(self) -> None: + """Regression guard: explicit --request-rate path (no search-space) + is unaffected by the new inference logic.""" + loadgen = CLIConfig(request_rate=50.0, request_count=10) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.POISSON + assert prof["rate"] == 50.0 + + def test_concurrency_search_space_keyword_unaffected(self) -> None: + """'concurrency' is already valid on every phase type incl. the + default -- must keep resolving to PhaseType.CONCURRENCY.""" + loadgen = CLIConfig( + search_space=["concurrency:1,1000:int"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.CONCURRENCY + + def test_smoothness_search_space_with_gamma_and_explicit_smoothness_flag( + self, + ) -> None: + """--search-space 'smoothness:...' combined with an explicit + --arrival-smoothness flag under gamma still succeeds normally (the + two features are independent; this just proves no interaction bug).""" + loadgen = CLIConfig( + search_space=["smoothness:0.5,2.0:real"], + arrival_pattern=ArrivalPattern.GAMMA, + arrival_smoothness=1.0, + request_rate=50.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.GAMMA + assert prof["smoothness"] == 1.0 + + def test_bare_rate_keyword_produces_a_fully_valid_benchmark_config(self) -> None: + """End-to-end proof, not just build_profiling()'s dict: the seeded + phase actually passes full BenchmarkConfig/Pydantic validation.""" + loadgen = CLIConfig( + search_space=["rate:1,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + cfg = BenchmarkConfig.model_validate( + { + "models": ["m"], + "endpoint": {"urls": ["http://x"], "type": "chat"}, + "datasets": [{"name": "profiling", "type": "synthetic"}], + "phases": [{"name": "profiling", **prof}], + } + ) + assert cfg.phases[0].type == PhaseType.POISSON + + def test_users_and_rate_together_produces_a_fully_valid_benchmark_config( + self, + ) -> None: + loadgen = CLIConfig( + search_space=["users:1,50:int", "rate:1,100:real"], + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + cfg = BenchmarkConfig.model_validate( + { + "models": ["m"], + "endpoint": {"urls": ["http://x"], "type": "chat"}, + "datasets": [{"name": "profiling", "type": "synthetic"}], + "phases": [{"name": "profiling", **prof}], + } + ) + assert cfg.phases[0].type == PhaseType.USER_CENTRIC + + def test_rate_lower_bound_zero_raises_clear_error(self) -> None: + """A zero/negative --search-space 'rate' lower bound must not + silently seed an invalid value and crash with a raw Pydantic + 'greater than' error -- it should fail clearly at seed time.""" + loadgen = CLIConfig( + search_space=["rate:0,100:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be > 0"): + build_profiling(user) + + def test_users_lower_bound_zero_raises_clear_error(self) -> None: + loadgen = CLIConfig( + search_space=["users:0,50:int"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be >= 1"): + build_profiling(user) + + def test_users_real_kind_raises_clear_error(self) -> None: + """'users:1.5,50:real' must not silently truncate 1.5 -> 1 (a value + below the user's own declared lower bound) as the seed -- the number + of simulated users can only ever be a whole number, so a ':real' + kind for 'users' must fail clearly instead.""" + loadgen = CLIConfig( + search_space=["users:1.5,50:real"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="':int' kind"): + build_profiling(user) + + def test_users_default_kind_is_real_and_still_raises(self) -> None: + """Omitting ':kind' defaults to 'real' (matching + SearchSpaceDimension's own default) -- 'users' without an explicit + ':int' suffix must also raise, not silently pass.""" + loadgen = CLIConfig( + search_space=["users:1,50"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="':int' kind"): + build_profiling(user) + + def test_users_real_kind_with_explicit_num_users_still_raises(self) -> None: + """An explicit --num-users must not bypass ':int'-kind validation + for a 'users' search-space dimension. Before this was fixed, an + already-set prof["users"] (from --num-users) short-circuited the + seed function entirely, letting 'users:1.5,50:real' silently reach + the search planner as a real-valued dimension for a field that + requires an integer.""" + loadgen = CLIConfig( + search_space=["users:1.5,50:real"], + user_centric_rate=10.0, + num_users=5, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="':int' kind"): + build_profiling(user) + + def test_users_lower_bound_zero_with_explicit_num_users_still_raises( + self, + ) -> None: + """Same bypass check for the bound validation: an explicit + --num-users must not let an out-of-bounds 'users' search-space + lower bound go unvalidated.""" + loadgen = CLIConfig( + search_space=["users:0,50:int"], + user_centric_rate=10.0, + num_users=5, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be >= 1"): + build_profiling(user) + + def test_users_explicit_num_users_wins_over_search_space_lower_bound( + self, + ) -> None: + """When --num-users is explicit AND 'users' is validly searched + (:int kind, valid bound), the explicit --num-users value is kept + rather than overwritten by the search-space lower bound -- the + search-space dimension is still validated, just not used as the + seed when an explicit value already exists.""" + loadgen = CLIConfig( + search_space=["users:1,50:int"], + user_centric_rate=10.0, + num_users=5, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.USER_CENTRIC + assert prof["users"] == 5 + + def test_rate_negative_bound_with_explicit_request_rate_still_raises( + self, + ) -> None: + """An explicit --request-rate must not bypass bound validation for a + 'rate' search-space dimension -- mirrors the analogous --num-users + fix. Before this was fixed, `--request-rate 10 --search-space + "rate:-5,100"` skipped the rate_lo <= 0 check entirely (since + prof["rate"] was already set from --request-rate), letting a + negative rate reach the planner and crash mid-search once sampled, + instead of failing clearly at config-build time.""" + loadgen = CLIConfig( + search_space=["rate:-5,100:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + with pytest.raises(ValueError, match="must be > 0"): + build_profiling(user) + + def test_rate_explicit_request_rate_wins_over_search_space_lower_bound( + self, + ) -> None: + """When --request-rate is explicit AND 'rate' is validly searched + (positive bound), the explicit --request-rate value is kept rather + than overwritten by the search-space lower bound -- the dimension + is still validated, just not used as the seed when an explicit + value already exists.""" + loadgen = CLIConfig( + search_space=["rate:1,100:real"], + request_rate=10.0, + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.POISSON + assert prof["rate"] == 10.0 + + def test_search_space_extra_kind_segment_is_skipped_not_misreported( + self, + ) -> None: + """'users:1,50:int:log' has an extra ':'-separated segment (a + malformed grammar the real parser will reject with its own error). + This lightweight helper must skip it rather than misparse + 'int:log' as the kind and report the wrong complaint (e.g. telling + a user who wrote ':int' that they wrote ':real').""" + loadgen = CLIConfig( + search_space=["users:1,50:int:log"], + user_centric_rate=10.0, + conversation_turn_mean=4, + ) + user = _make_user(loadgen=loadgen) + # The malformed dimension is invisible to shape inference (skipped), + # so with no other search-space field this falls through to + # whatever --user-centric-rate alone would produce: USER_CENTRIC + # type, but no 'users' seed and no ':int'-kind complaint about a + # dimension this helper never parsed. + prof = build_profiling(user) + assert prof["type"] == PhaseType.USER_CENTRIC + + def test_warmup_path_search_space_dimension_ignored_for_shape_inference( + self, + ) -> None: + """A dotted path targeting a non-profiling phase (e.g. warmup) must + not be mistaken for a profiling-phase field by its trailing segment + alone -- it should be ignored by shape inference entirely.""" + loadgen = CLIConfig( + search_space=["phases.warmup.rate:1,10:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.CONCURRENCY + assert "rate" not in prof + + def test_nested_cancellation_rate_path_ignored_for_shape_inference(self) -> None: + """'phases.profiling.cancellation.rate' is the request-cancellation + rate (an unrelated CancellationConfig sub-field), not the phase's + own 'rate'. Matching by trailing path segment alone previously + misclassified this as a phase-shape-selecting 'rate' dimension, + wrongly switching the phase to POISSON and seeding prof["rate"] + from the cancellation-rate range.""" + loadgen = CLIConfig( + search_space=["phases.profiling.cancellation.rate:1,50:real"], + request_count=10, + ) + user = _make_user(loadgen=loadgen) + prof = build_profiling(user) + assert prof["type"] == PhaseType.CONCURRENCY + assert "rate" not in prof diff --git a/tests/unit/orchestrator/search_planner/test_optuna_planner.py b/tests/unit/orchestrator/search_planner/test_optuna_planner.py index 281e5d9d25..377692542d 100644 --- a/tests/unit/orchestrator/search_planner/test_optuna_planner.py +++ b/tests/unit/orchestrator/search_planner/test_optuna_planner.py @@ -15,6 +15,8 @@ # Imports below depend on optuna being importable. pytest.importorskip must # precede them so the whole module is skipped when the `optuna` extra is absent. +from pydantic import ValidationError # noqa: E402 + from aiperf.common.models.export_models import JsonMetricResult # noqa: E402 from aiperf.config.config import BenchmarkConfig # noqa: E402 from aiperf.config.sweep import ( # noqa: E402 @@ -617,3 +619,107 @@ def test_ask_preserves_url_userinfo() -> None: assert cfg.endpoint.urls == [ "http://alice:s3cret@host1.example.com/v1/chat/completions" ] + + +def test_ask_with_shape_mismatched_search_space_raises_clear_error() -> None: + """Safety net: if a search-space dimension still doesn't fit the base + phase's shape (e.g. a hand-written dotted path our lightweight CLI-time + inference can't see, since this planner is constructed directly here + with no CLI conversion in between), ask() must raise a clear, actionable + ValueError -- not let a raw pydantic extra_forbidden ValidationError + propagate to the user.""" + base = _base_config() # phases[0].type == "concurrency" + cfg = _cfg( + extra_dims=[ + SearchSpaceDimension( + path="phases.profiling.rate", lo=1, hi=100, kind="real" + ) + ] + ) + planner = OptunaSearchPlanner(base, cfg) + + with pytest.raises(ValueError, match="shape"): + planner.ask() + + +def test_ask_with_ordinary_bounds_violation_raises_original_error() -> None: + """A search-space dimension that samples an out-of-bounds value for + a field that DOES belong on the base phase (e.g. concurrency=0 on a + ConcurrencyPhase, which requires ge=1) is not a shape mismatch -- + the original pydantic ValidationError must propagate unreframed, + not get relabeled as a 'shape' problem.""" + # SearchSpaceDimension itself enforces hi > lo strictly, so lo=hi=0 isn't + # constructible; lo=-1, hi=0 is the smallest range where every integer + # in [lo, hi] still violates concurrency's ge=1, so the outcome is + # deterministic regardless of which value the sampler picks. This + # replaces (rather than extends) _cfg's default 'concurrency' dimension, + # since AdaptiveSearchSweep forbids duplicate-path dimensions. + base = _base_config() # phases[0].type == "concurrency" + cfg = _cfg( + search_space=[ + SearchSpaceDimension( + path="phases.profiling.concurrency", lo=-1, hi=0, kind="int" + ) + ] + ) + planner = OptunaSearchPlanner(base, cfg) + + with pytest.raises(ValidationError): + planner.ask() + + +def test_ask_shape_mismatch_names_profiling_phase_not_warmup() -> None: + """phases[0] is the warmup phase whenever one exists (warmup is + appended before profiling), so naming self._base.phases[0].type in + the shape-mismatch message would blame the wrong phase. The message + must name the PROFILING phase's shape, matched by kind (like + BenchmarkConfig.validate_profiling_phase_required does), not by + list position.""" + base = BenchmarkConfig.model_validate( + { + "models": ["m"], + "endpoint": {"urls": ["http://x"], "type": "chat"}, + "datasets": [{"name": "profiling", "type": "synthetic"}], + "phases": [ + { + "name": "warmup", + "type": "concurrency", + "concurrency": 1, + "requests": 5, + }, + { + "name": "profiling", + "type": "poisson", + "rate": 10.0, + "requests": 10, + }, + ], + } + ) + cfg = _cfg( + extra_dims=[ + SearchSpaceDimension(path="phases.profiling.users", lo=1, hi=50, kind="int") + ] + ) + planner = OptunaSearchPlanner(base, cfg) + + with pytest.raises(ValueError, match=r"shape \('poisson'\)"): + planner.ask() + + +def test_ask_with_root_level_extra_field_raises_original_error() -> None: + """extra_forbidden also fires for a malformed path that lands as a + phantom ROOT-level key (e.g. a bare, non-dotted path unrelated to any + phase field) -- that's not a phase-shape mismatch, and reframing it + as one would be actively wrong. Must propagate the original + ValidationError unreframed, same as an ordinary bounds violation.""" + base = _base_config() # phases[0].type == "concurrency" + cfg = _cfg( + search_space=[ + SearchSpaceDimension(path="bogus_root_field", lo=1, hi=100, kind="real") + ] + ) + planner = OptunaSearchPlanner(base, cfg) + + with pytest.raises(ValidationError): + planner.ask() From 2ad2e24e2508055ef094ec8417c43a20ba6bae79 Mon Sep 17 00:00:00 2001 From: Nate Mailhot Date: Thu, 27 Aug 2026 17:08:33 -0700 Subject: [PATCH 040/119] chore: update attributions for 0.12.0 (#1212) Signed-off-by: Nate Mailhot (cherry picked from commit 32e0a3a1f325b8ef6aa9a48adba28237df765520) --- ATTRIBUTIONS-Python.md | 62 +++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/ATTRIBUTIONS-Python.md b/ATTRIBUTIONS-Python.md index d44425f409..8d0f4856ee 100644 --- a/ATTRIBUTIONS-Python.md +++ b/ATTRIBUTIONS-Python.md @@ -801,7 +801,7 @@ Apache License - `Homepage`: https://github.com/aio-libs/aiosignal -## alembic (1.18.5) +## alembic (1.19.0) ### Licenses License: `MIT` @@ -830,7 +830,7 @@ SOFTWARE. ``` -## annotated-doc (0.0.4) +## annotated-doc (0.0.5) ### Licenses License: `MIT` @@ -868,7 +868,7 @@ THE SOFTWARE. - `Repository`: https://github.com/fastapi/annotated-doc -## annotated-types (0.7.0) +## annotated-types (0.8.0) ### Licenses License: `MIT License` @@ -904,7 +904,7 @@ SOFTWARE. - `Source`: https://github.com/annotated-types/annotated-types -## anyio (4.14.1) +## anyio (4.14.2) ### Licenses License: `MIT` @@ -1045,7 +1045,7 @@ THE SOFTWARE. - `Homepage`: https://github.com/google/brotli -## certifi (2026.6.17) +## certifi (2026.7.22) ### Licenses License: `MPL-2.0` @@ -1079,7 +1079,7 @@ one at http://mozilla.org/MPL/2.0/. - `Source`: https://github.com/certifi/python-certifi -## cffi (2.1.0) +## cffi (2.1.1) ### Licenses License: `MIT` @@ -1195,7 +1195,7 @@ OTHER DEALINGS IN THE SOFTWARE. - `Repository`: https://github.com/plotly/choreographer -## click (8.2.1) +## click (8.3.3) ### Licenses License: `BSD-3-Clause` @@ -1240,7 +1240,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Source`: https://github.com/pallets/click/ -## colorlog (6.10.1) +## colorlog (6.12.0) ### Licenses License: `MIT` @@ -1392,7 +1392,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.``` - `repository`: https://github.com/matplotlib/cycler -## cyclopts (4.20.0) +## cyclopts (4.22.5) ### Licenses License: `Apache-2.0` @@ -1861,7 +1861,7 @@ limitations under the License. - `Source`: https://github.com/dbc-team/dash-bootstrap-components/ -## datasets (5.0.0) +## datasets (5.0.1) ### Licenses License: `Apache 2.0` @@ -2165,7 +2165,7 @@ SOFTWARE. - `repository`: https://github.com/rr-/docstring_parser -## fastapi (0.139.0) +## fastapi (0.141.1) ### Licenses License: `MIT` @@ -2418,7 +2418,7 @@ License: `Apache Software License` - `Homepage`: https://github.com/kkroening/ffmpeg-python -## filelock (3.29.6) +## filelock (3.32.2) ### Licenses License: `MIT` @@ -3226,7 +3226,7 @@ THE SOFTWARE. - `Source`: https://github.com/PythonCharmers/python-future -## greenlet (3.5.3) +## greenlet (3.5.4) ### Licenses License: `MIT AND PSF-2.0` @@ -3301,7 +3301,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - `Homepage`: https://github.com/python-hyper/h11 -## hf-xet (1.5.1) +## hf-xet (1.6.0) ### Licenses License: `Apache Software License` @@ -4343,7 +4343,7 @@ SOFTWARE. - `Repository`: https://github.com/geopozo/logistro -## mako (1.3.12) +## mako (1.4.0) ### Licenses License: `MIT` @@ -4478,7 +4478,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Source`: https://github.com/pallets/markupsafe/ -## matplotlib (3.11.0) +## matplotlib (3.11.1) ### Licenses License: `Python Software Foundation License` @@ -5676,7 +5676,7 @@ SUCH DAMAGE. - `Source Code`: https://github.com/uqfoundation/multiprocess -## narwhals (2.23.0) +## narwhals (2.24.0) ### Licenses License: `MIT License` @@ -6552,7 +6552,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice - `source`: https://github.com/ijl/orjson -## packaging (26.2) +## packaging (26.3) ### Licenses License: `Apache-2.0 OR BSD-2-Clause` @@ -9583,7 +9583,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Source`: https://github.com/python-pillow/Pillow -## platformdirs (4.10.0) +## platformdirs (4.11.0) ### Licenses License: `MIT` @@ -10213,7 +10213,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Homepage`: https://github.com/giampaolo/psutil -## pyarrow (24.0.0) +## pyarrow (25.0.0) ### Licenses License: `Apache-2.0` @@ -12972,7 +12972,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Source`: https://github.com/theskumar/python-dotenv -## pytz (2026.2) +## pytz (2026.3.post1) ### Licenses License: `MIT` @@ -13695,7 +13695,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice - `Tracker`: https://github.com/zeromq/pyzmq/issues -## regex (2026.6.28) +## regex (2026.7.19) ### Licenses License: `Apache-2.0 AND CNRI-Python` @@ -14891,7 +14891,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Source`: https://github.com/mwaskom/seaborn -## sentencepiece (0.2.1) +## sentencepiece (0.2.2) ### Licenses License: `Apache-2.0` @@ -15407,7 +15407,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Source`: https://github.com/Kludex/starlette -## starlette-compress (1.7.1) +## starlette-compress (1.8.0) ### Licenses License: `0BSD` @@ -15495,7 +15495,7 @@ License: `Apache Software License` - `Source`: https://github.com/huggingface/tokenizers -## tqdm (4.68.4) +## tqdm (4.70.0) ### Licenses License: `MPL-2.0 AND MIT` @@ -15560,7 +15560,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - `wiki`: https://github.com/tqdm/tqdm/wiki -## transformers (5.13.0) +## transformers (5.14.1) ### Licenses License: `Apache 2.0 License` @@ -15776,7 +15776,7 @@ Copyright 2018- The Hugging Face team. All rights reserved. - `Homepage`: https://github.com/huggingface/transformers -## typer (0.26.8) +## typer (0.27.1) ### Licenses License: `MIT` @@ -16148,7 +16148,7 @@ SOFTWARE. - `Source`: https://github.com/pydantic/typing-inspection -## tzdata (2026.2) +## tzdata (2026.3) ### Licenses License: `Apache-2.0` @@ -16480,7 +16480,7 @@ SOFTWARE. - `Issue tracker`: https://github.com/urllib3/urllib3/issues -## uvicorn (0.50.2) +## uvicorn (0.52.1) ### Licenses License: `BSD-3-Clause` @@ -16802,7 +16802,7 @@ SOFTWARE. - `Source`: https://github.com/samuelcolvin/watchfiles -## websockets (16.0) +## websockets (17.0.1) ### Licenses License: `BSD-3-Clause` @@ -16923,7 +16923,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - `Homepage`: https://github.com/ifduyue/python-xxhash -## yarl (1.24.2) +## yarl (1.24.5) ### Licenses License: `Apache-2.0` From 108459378c8ea705dc1a44693ce732b01fc86267 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:52:04 -0700 Subject: [PATCH 041/119] fix: stop numpy.float64 from aborting search_history.json export (#1343) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 261ba8ce2d7f866acbf187e9805f3fb7751dc0bf) --- docs/dev/patterns.md | 11 ++- src/aiperf/common/finite.py | 25 +++-- .../search_planner/optuna_planner.py | 8 +- tests/unit/common/test_finite.py | 30 ++++++ .../test_search_history_numpy_guard.py | 91 +++++++++++++++++++ .../search_planner/test_optuna_planner.py | 67 ++++++++++++++ 6 files changed, 222 insertions(+), 10 deletions(-) create mode 100644 tests/unit/exporters/test_search_history_numpy_guard.py diff --git a/docs/dev/patterns.md b/docs/dev/patterns.md index 3fdf8d346e..cac76579b5 100644 --- a/docs/dev/patterns.md +++ b/docs/dev/patterns.md @@ -359,8 +359,15 @@ def export_records_json(records: list[Record], out_path: Path) -> None: `scrub_non_finite` recursively walks `dict`/`list`/`tuple` containers and rewrites non-finite numeric values to `None`. It leaves `str`/`bytes`/`bool` -alone and handles numpy scalar types correctly (`numpy.float32`, -`numpy.float64`). +alone and coerces numpy scalar types to native Python numbers +(`numpy.float32`, `numpy.float64`, `numpy.int64`, ...). + +That coercion is load-bearing, not incidental: `orjson.dumps` **raises** +`TypeError: Type is not JSON serializable: numpy.float64` rather than +degrading, so a numpy scalar anywhere in a payload aborts the export and +takes the run down with it. Any code that hands values to an exporter +(planners, scorers, analysis helpers) should still return native floats — +`scrub_non_finite` is the backstop, not the excuse. ### `is_finite_value` for the canonical finiteness check diff --git a/src/aiperf/common/finite.py b/src/aiperf/common/finite.py index 53487a2404..c9730a5c74 100644 --- a/src/aiperf/common/finite.py +++ b/src/aiperf/common/finite.py @@ -107,14 +107,17 @@ def scrub_non_finite(obj: Any) -> Any: Walks ``dict``, ``list``, and ``tuple`` containers; leaves ``str``, ``bytes``, and ``bytearray`` alone (a string literal ``"nan"`` is not a - numeric NaN and must not be rewritten). Coerces numpy floats correctly - by checking ``__float__`` and using ``float()`` directly (mirroring - :func:`is_finite_value`'s strategy) -- ``isinstance(x, float)`` would - miss ``numpy.float32``/``numpy.float64`` on some numpy versions. + numeric NaN and must not be rewritten). Numpy scalars are handled two + ways: ``numpy.float64`` subclasses ``float`` and is cast explicitly, + while every other numpy type (``numpy.float32``, ``numpy.int64``, ...) + reaches the duck-typed ``__float__``/``float()`` branch that mirrors + :func:`is_finite_value`'s strategy. Use before ``orjson.dumps`` on any payload that may contain metric - values. orjson 3.x silently coerces NaN/inf to JSON ``null`` which is - indistinguishable from explicit-None semantics in downstream tooling. + values. This is the guard for two distinct orjson behaviors: it + silently coerces NaN/inf to JSON ``null`` (indistinguishable from + explicit-None semantics downstream), and it raises outright on numpy + scalars ("Type is not JSON serializable: numpy.float64"). The returned structure preserves the input container types (dict stays dict, tuple stays tuple); numpy scalars are coerced to Python @@ -132,7 +135,15 @@ def scrub_non_finite(obj: Any) -> Any: if isinstance(obj, tuple): return tuple(scrub_non_finite(v) for v in obj) if isinstance(obj, float): - return obj if math.isfinite(obj) else None + if not math.isfinite(obj): + return None + # ``numpy.float64`` is the one numpy scalar type that subclasses + # ``float``, so it matches here instead of falling through to the + # duck-typed branch below that coerces every other numpy type. + # Returning it unconverted leaks it into orjson.dumps(), which + # rejects it outright ("Type is not JSON serializable"). float() is + # a no-op on an exact float -- CPython hands back the same object. + return float(obj) # Numpy scalar / other numeric: duck-typed ``float()`` coercion (same # strategy as is_finite_value). Non-numeric objects fall through unchanged. if hasattr(obj, "__float__") and not isinstance(obj, int): diff --git a/src/aiperf/orchestrator/search_planner/optuna_planner.py b/src/aiperf/orchestrator/search_planner/optuna_planner.py index 33355dbbd1..baea83b825 100644 --- a/src/aiperf/orchestrator/search_planner/optuna_planner.py +++ b/src/aiperf/orchestrator/search_planner/optuna_planner.py @@ -207,7 +207,13 @@ def ask(self) -> tuple[BenchmarkConfig, SweepVariation] | None: dim.path, int(dim.lo), int(dim.hi), log=log ) else: - v = trial.suggest_float(dim.path, dim.lo, dim.hi, log=log) + # BoTorch -- the default sampler -- suggests numpy.float64, and + # optuna only coerces to a native type for int distributions; + # FloatDistribution hands the sampler's scalar straight back. + # Unconverted it survives into the search history and blows up + # orjson.dumps() in write_search_history() with + # "Type is not JSON serializable: numpy.float64". + v = float(trial.suggest_float(dim.path, dim.lo, dim.hi, log=log)) values[dim.path] = v # mode="python" + context={"include_secrets": True} so neither the diff --git a/tests/unit/common/test_finite.py b/tests/unit/common/test_finite.py index 468ef15a76..461e72adb1 100644 --- a/tests/unit/common/test_finite.py +++ b/tests/unit/common/test_finite.py @@ -145,10 +145,40 @@ def test_scrub_non_finite_numpy_scalars() -> None: assert out["f64_inf"] is None # finite numpy float coerced through the same path -> python float, finite assert out["f64_finite"] == pytest.approx(3.14) + assert type(out["f64_finite"]) is float # int passes through unchanged assert out["i64"] == np.int64(7) +def test_scrub_non_finite_coerces_numpy_float64_to_native_float() -> None: + """numpy.float64 needs an explicit cast; every other numpy scalar gets one free. + + numpy.float64 is the only numpy scalar type that subclasses Python + ``float``, so it matches the ``isinstance(obj, float)`` branch and would + otherwise be returned as-is instead of falling through to the duck-typed + ``float()`` coercion the other numpy types take. + """ + out = scrub_non_finite({"v": np.float64(3.14)}) + assert type(out["v"]) is float + + +def test_scrub_non_finite_output_is_orjson_serializable_for_numpy_float64() -> None: + """scrub_non_finite is the pre-orjson guard, so its output must always dump. + + orjson rejects numpy.float64 with "Type is not JSON serializable", which + aborts the exporter that called it (nvbugs 6683575). + """ + import orjson + + out = scrub_non_finite( + {"scalar": np.float64(3.14), "nested": [{"deep": np.float64(1.0)}]} + ) + parsed = orjson.loads(orjson.dumps(out)) + + assert parsed["scalar"] == pytest.approx(3.14) + assert parsed["nested"][0]["deep"] == pytest.approx(1.0) + + def test_scrub_non_finite_does_not_recurse_into_str_bytes() -> None: """A literal 'nan' string is not a numeric NaN and must survive untouched.""" payload = { diff --git a/tests/unit/exporters/test_search_history_numpy_guard.py b/tests/unit/exporters/test_search_history_numpy_guard.py new file mode 100644 index 0000000000..32c2c9c3d7 --- /dev/null +++ b/tests/unit/exporters/test_search_history_numpy_guard.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""search_history.json must survive numpy scalars reaching the exporter. + +``SearchIteration`` is a plain dataclass with ``variation_values: +dict[str, Any]``, so nothing between a planner and the exporter coerces +scalar types. Planners are expected to hand back native Python numbers, +but the export boundary must not be the thing that fails when one +doesn't -- ``scrub_non_finite`` is the guard that makes that true. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +from aiperf.common.enums import OptimizationDirection +from aiperf.config.sweep import ( + AdaptiveSearchSweep, + Objective, + SearchSpaceDimension, +) +from aiperf.exporters.search_history import write_search_history +from aiperf.orchestrator.search_planner.base import SearchIteration + + +def _real_dim_cfg() -> AdaptiveSearchSweep: + return AdaptiveSearchSweep( + search_space=[ + SearchSpaceDimension(path="endpoint.timeout", lo=1.0, hi=100.0, kind="real") + ], + objectives=[ + Objective( + metric="output_token_throughput", + direction=OptimizationDirection.MAXIMIZE, + ) + ], + max_iterations=10, + ) + + +def test_write_search_history_serializes_numpy_variation_values(tmp_path: Path) -> None: + """A numpy.float64 in variation_values must not abort the export. + + Regression for nvbugs 6683575: orjson rejects numpy.float64 outright + ("Type is not JSON serializable: numpy.float64"), which failed the whole + run rather than just the export. + """ + history = [ + SearchIteration( + iteration_idx=0, + variation_values={"endpoint.timeout": np.float64(12.5)}, + objective_value=np.float64(10.0), + objective_values=[np.float64(10.0)], + feasible=True, + ) + ] + + write_search_history(tmp_path, history, _real_dim_cfg()) + + payload = json.loads((tmp_path / "search_history.json").read_text()) + assert payload["iterations"][0]["variation_values"][ + "endpoint.timeout" + ] == pytest.approx(12.5) + + +def test_write_search_history_maps_numpy_nan_objective_to_null(tmp_path: Path) -> None: + """A numpy NaN must land as JSON null, not as a coerced-but-unscrubbed value. + + The guard has to preserve the module's NaN discipline: converting numpy + scalars without re-checking finiteness would let orjson turn a NaN into + ``null`` by its own silent coercion, which is right by accident here but + wrong for the "not scored" contract the exporter relies on. + """ + history = [ + SearchIteration( + iteration_idx=0, + variation_values={"endpoint.timeout": np.float64(12.5)}, + objective_value=np.float64("nan"), + objective_values=[np.float64("nan")], + feasible=False, + ) + ] + + write_search_history(tmp_path, history, _real_dim_cfg()) + + payload = json.loads((tmp_path / "search_history.json").read_text()) + assert payload["iterations"][0]["objective_values"] == [None] diff --git a/tests/unit/orchestrator/search_planner/test_optuna_planner.py b/tests/unit/orchestrator/search_planner/test_optuna_planner.py index 377692542d..6eccdf1c55 100644 --- a/tests/unit/orchestrator/search_planner/test_optuna_planner.py +++ b/tests/unit/orchestrator/search_planner/test_optuna_planner.py @@ -8,6 +8,8 @@ from __future__ import annotations +import numpy as np +import orjson import pytest from pytest import param @@ -126,6 +128,41 @@ def _make_result( ) +class _NumpyScalarSampler(optuna.samplers.RandomSampler): + """Sampler that returns ``numpy.float64`` from float suggestions. + + Stands in for BoTorchSampler without pulling in the torch/botorch stack. + Optuna's ``IntDistribution.to_external_repr`` coerces to native ``int``, + but ``FloatDistribution`` has no such coercion, so whatever scalar type + the sampler produces reaches the caller of ``suggest_float`` unchanged. + """ + + def sample_independent(self, study, trial, param_name, param_distribution): + value = super().sample_independent(study, trial, param_name, param_distribution) + if isinstance(param_distribution, optuna.distributions.FloatDistribution): + return np.float64(value) + return value + + +@pytest.fixture +def numpy_scalar_sampler(monkeypatch): + """Force the planner to build a sampler that emits numpy scalars.""" + monkeypatch.setattr( + "aiperf.orchestrator.search_planner.optuna_planner.build_sampler", + lambda cfg: _NumpyScalarSampler(seed=42), + ) + + +def _real_dim_cfg() -> AdaptiveSearchSweep: + # endpoint.timeout is the float-typed path used by the other real-kind + # tests; phases.profiling.concurrency is int-typed in BenchmarkConfig. + return _cfg( + search_space=[ + SearchSpaceDimension(path="endpoint.timeout", lo=1.0, hi=100.0, kind="real") + ] + ) + + # ---------------------------------------------------------------------------- # 1. Construction. # ---------------------------------------------------------------------------- @@ -212,6 +249,36 @@ def test_ask_float_dim_returns_float(): assert 1.0 <= proposed <= 100.0 +def test_ask_real_dim_returns_native_float_when_sampler_emits_numpy( + numpy_scalar_sampler, +) -> None: + """A numpy-emitting sampler must not leak numpy.float64 into variation values. + + ``isinstance(np.float64(1.0), float)`` is True, so only an exact type check + catches this; orjson rejects the subclass even though isinstance passes. + """ + planner = OptunaSearchPlanner(_base_config(), _real_dim_cfg()) + proposal = planner.ask() + assert proposal is not None + _, variation = proposal + assert type(variation.values["endpoint.timeout"]) is float + + +def test_history_variation_values_are_orjson_serializable_with_numpy_sampler( + numpy_scalar_sampler, +) -> None: + """History must serialize with orjson so write_search_history() can export it.""" + planner = OptunaSearchPlanner(_base_config(), _real_dim_cfg()) + _, variation = planner.ask() + planner.tell(variation, [_make_result(variation, throughput=10.0, ttft_p95=50.0)]) + + encoded = orjson.dumps([h.variation_values for h in planner.history()]) + + assert orjson.loads(encoded)[0]["endpoint.timeout"] == pytest.approx( + variation.values["endpoint.timeout"] + ) + + # ---------------------------------------------------------------------------- # 3. tell() consumes the pending trial. # ---------------------------------------------------------------------------- From e4e1f83be684663fd6988f29665f2a4a7bc05095 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:52:05 -0700 Subject: [PATCH 042/119] fix(finite): keep numpy int and bool identities through the NaN/inf guard (#1345) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 0d85c8c79cd54db2a9ec3f789ca3fd1f8c9e6950) --- docs/dev/patterns.md | 18 +++++- src/aiperf/common/finite.py | 95 ++++++++++++++++++++++---------- tests/unit/common/test_finite.py | 60 +++++++++++++++++++- 3 files changed, 138 insertions(+), 35 deletions(-) diff --git a/docs/dev/patterns.md b/docs/dev/patterns.md index cac76579b5..957f4843cc 100644 --- a/docs/dev/patterns.md +++ b/docs/dev/patterns.md @@ -359,16 +359,23 @@ def export_records_json(records: list[Record], out_path: Path) -> None: `scrub_non_finite` recursively walks `dict`/`list`/`tuple` containers and rewrites non-finite numeric values to `None`. It leaves `str`/`bytes`/`bool` -alone and coerces numpy scalar types to native Python numbers -(`numpy.float32`, `numpy.float64`, `numpy.int64`, ...). +alone and normalizes numpy scalars to the native Python type they actually +mean — `numpy.int64(7)` becomes `7`, not `7.0`, and `numpy.bool_(True)` +becomes `True`, not `1.0`. -That coercion is load-bearing, not incidental: `orjson.dumps` **raises** +That normalization is load-bearing, not incidental: `orjson.dumps` **raises** `TypeError: Type is not JSON serializable: numpy.float64` rather than degrading, so a numpy scalar anywhere in a payload aborts the export and takes the run down with it. Any code that hands values to an exporter (planners, scorers, analysis helpers) should still return native floats — `scrub_non_finite` is the backstop, not the excuse. +Most exporters are shielded by accident: they call +`scrub_non_finite(model.model_dump(mode="json"))`, and Pydantic's JSON-mode +dump already coerces numpy. Payloads assembled as plain dicts from +dataclasses (`search_history.json` is the live example) have no such step, +so `scrub_non_finite` is their only guard. + ### `is_finite_value` for the canonical finiteness check ```python @@ -385,6 +392,11 @@ Use `is_finite_value` instead of `math.isfinite` or `not math.isnan`: `isinstance(x, float)` misses numpy scalar types on some numpy versions, and `math.isfinite` raises on non-numeric inputs. +It rejects booleans — Python's and numpy's — because a bool is not a metric +value. That matters for `nan_safe_mean`/`nan_safe_std`, which filter through +it: admitting `numpy.bool_(True)` as `1.0` would make the same data average +differently depending only on which bool type produced it. + ### `nan_safe_mean` / `nan_safe_std` for aggregation ```python diff --git a/src/aiperf/common/finite.py b/src/aiperf/common/finite.py index c9730a5c74..989fa85511 100644 --- a/src/aiperf/common/finite.py +++ b/src/aiperf/common/finite.py @@ -52,6 +52,25 @@ ] +def _native_scalar(x: Any) -> Any: + """Return the native Python counterpart of a numpy scalar, else ``x``. + + Numpy's ``.item()`` maps each scalar to the Python type it actually + means -- ``numpy.bool_`` to ``bool``, ``numpy.int64`` to ``int``, + ``numpy.float32`` to ``float`` -- which ``float()`` alone would flatten. + Used instead of ``import numpy`` so this module stays import-light; it + is pulled in on per-record paths. Python scalars have no ``.item()`` and + cost one failed attribute lookup. + """ + item = getattr(x, "item", None) + if not callable(item): + return x + try: + return item() + except (TypeError, ValueError): + return x + + def is_finite_value(x: Any) -> bool: """Return True if ``x`` is a finite real number. @@ -61,11 +80,17 @@ def is_finite_value(x: Any) -> bool: ``int``/``float`` and numpy scalar types (``float32``, ``float64``, ``int64``, ...) because ``float(np.float64(...))`` round-trips. + ``numpy.bool_`` is rejected alongside Python ``bool``. It does not + subclass ``bool``, so without an explicit check it would be admitted as + the value ``1.0`` and skew :func:`nan_safe_mean` / :func:`nan_safe_std` + -- the same input would average differently depending only on which + bool type produced it. + Strings, bytes, lists, dicts and other non-numeric types return False (the ``float()`` coercion either raises ``ValueError`` or ``TypeError``, both of which are caught). """ - if x is None or isinstance(x, bool): + if x is None or isinstance(_native_scalar(x), bool): return False try: return math.isfinite(float(x)) @@ -102,16 +127,47 @@ class MetricSummary(AIPerfBaseModel): """ +def _scrub_scalar(obj: Any) -> Any: + """Normalize one non-container value for :func:`scrub_non_finite`. + + Finite numerics come back as their native Python type, non-finite ones + as ``None``, and anything non-numeric passes through untouched. + """ + if isinstance(obj, float): + # ``numpy.float64`` is the one numpy scalar type that subclasses + # ``float``, so it matches here rather than taking the ``.item()`` + # path below. Returning it unconverted leaks it into orjson.dumps(), + # which rejects it outright ("Type is not JSON serializable"). + # float() is a no-op on an exact float -- CPython returns the same + # object, and the call is cheaper than the isfinite() beside it. + return float(obj) if math.isfinite(obj) else None + if not hasattr(obj, "__float__") or isinstance(obj, int): + return obj + # Numpy scalar or other duck-typed number. ``numpy.int64(7)`` must stay + # ``7`` rather than widen to ``7.0``, and ``numpy.bool_(True)`` must stay + # ``True`` rather than collapse to ``1.0``. + native = _native_scalar(obj) + # bool is caught here too, since bool subclasses int; returning + # ``native`` keeps True a bool instead of flattening it to 1. + if isinstance(native, int): + return native + try: + f = float(native) + except (TypeError, ValueError): + return obj + return f if math.isfinite(f) else None + + def scrub_non_finite(obj: Any) -> Any: """Recursively replace non-finite numeric values with ``None``. Walks ``dict``, ``list``, and ``tuple`` containers; leaves ``str``, ``bytes``, and ``bytearray`` alone (a string literal ``"nan"`` is not a - numeric NaN and must not be rewritten). Numpy scalars are handled two - ways: ``numpy.float64`` subclasses ``float`` and is cast explicitly, - while every other numpy type (``numpy.float32``, ``numpy.int64``, ...) - reaches the duck-typed ``__float__``/``float()`` branch that mirrors - :func:`is_finite_value`'s strategy. + numeric NaN and must not be rewritten). Numpy scalars are normalized to + their native Python counterpart rather than uniformly to ``float``, so + ``numpy.int64(7)`` yields ``7`` (not ``7.0``) and ``numpy.bool_(True)`` + yields ``True`` (not ``1.0``). ``numpy.float64`` needs an explicit cast + because it subclasses ``float``; the rest go through ``.item()``. Use before ``orjson.dumps`` on any payload that may contain metric values. This is the guard for two distinct orjson behaviors: it @@ -119,10 +175,9 @@ def scrub_non_finite(obj: Any) -> Any: explicit-None semantics downstream), and it raises outright on numpy scalars ("Type is not JSON serializable: numpy.float64"). - The returned structure preserves the input container types (dict - stays dict, tuple stays tuple); numpy scalars are coerced to Python - ``float``. Booleans are passed through unchanged because they are not - metric values. + The returned structure preserves the input container types (dict stays + dict, tuple stays tuple). Booleans are passed through unchanged because + they are not metric values -- a non-finite check does not apply to them. """ if isinstance(obj, (str, bytes, bytearray)): return obj @@ -134,25 +189,7 @@ def scrub_non_finite(obj: Any) -> Any: return [scrub_non_finite(v) for v in obj] if isinstance(obj, tuple): return tuple(scrub_non_finite(v) for v in obj) - if isinstance(obj, float): - if not math.isfinite(obj): - return None - # ``numpy.float64`` is the one numpy scalar type that subclasses - # ``float``, so it matches here instead of falling through to the - # duck-typed branch below that coerces every other numpy type. - # Returning it unconverted leaks it into orjson.dumps(), which - # rejects it outright ("Type is not JSON serializable"). float() is - # a no-op on an exact float -- CPython hands back the same object. - return float(obj) - # Numpy scalar / other numeric: duck-typed ``float()`` coercion (same - # strategy as is_finite_value). Non-numeric objects fall through unchanged. - if hasattr(obj, "__float__") and not isinstance(obj, int): - try: - f = float(obj) - except (TypeError, ValueError): - return obj - return f if math.isfinite(f) else None - return obj + return _scrub_scalar(obj) def nan_safe_mean(values: Any) -> float | None: diff --git a/tests/unit/common/test_finite.py b/tests/unit/common/test_finite.py index 461e72adb1..d35673635d 100644 --- a/tests/unit/common/test_finite.py +++ b/tests/unit/common/test_finite.py @@ -44,6 +44,10 @@ param(np.float64(1.5), True, id="numpy_float64_finite"), param(np.int64(0), True, id="numpy_int64_zero"), param(np.int64(-100), True, id="numpy_int64_negative"), + # numpy.bool_ does not subclass Python bool, so it misses the + # isinstance(x, bool) rejection and would count as the value 1.0. + param(np.bool_(True), False, id="numpy_bool_true_rejected"), + param(np.bool_(False), False, id="numpy_bool_false_rejected"), ], ) def test_is_finite_value(value: object, expected: bool) -> None: @@ -148,15 +152,54 @@ def test_scrub_non_finite_numpy_scalars() -> None: assert type(out["f64_finite"]) is float # int passes through unchanged assert out["i64"] == np.int64(7) + assert type(out["i64"]) is int + + +def test_scrub_non_finite_preserves_numpy_integer_as_int() -> None: + """numpy integers must stay integers, not widen to float. + + ``numpy.int64`` neither subclasses ``int`` nor matches the ``float`` + branch, so a bare ``float()`` coercion would write ``7.0`` to disk for a + value that is conceptually ``7``. + """ + out = scrub_non_finite({"i64": np.int64(7), "i32": np.int32(-3), "u8": np.uint8(3)}) + assert type(out["i64"]) is int + assert out["i64"] == 7 + assert type(out["i32"]) is int + assert out["i32"] == -3 + assert type(out["u8"]) is int + assert out["u8"] == 3 + + +def test_scrub_non_finite_preserves_numpy_bool_as_bool() -> None: + """numpy booleans must stay booleans. + + The function documents "Booleans are passed through unchanged", but + ``numpy.bool_`` does not subclass Python ``bool``, so it missed that + branch and was coerced to ``1.0``. + """ + out = scrub_non_finite({"t": np.bool_(True), "f": np.bool_(False)}) + assert out["t"] is True + assert out["f"] is False + + +def test_scrub_non_finite_numpy_int_and_bool_json_shape() -> None: + """On disk a numpy int must render as 7 and a numpy bool as true.""" + import orjson + + raw = orjson.dumps( + scrub_non_finite({"count": np.int64(7), "feasible": np.bool_(True)}) + ).decode() + + assert raw == '{"count":7,"feasible":true}' def test_scrub_non_finite_coerces_numpy_float64_to_native_float() -> None: - """numpy.float64 needs an explicit cast; every other numpy scalar gets one free. + """numpy.float64 needs its own cast; every other numpy scalar takes .item(). numpy.float64 is the only numpy scalar type that subclasses Python ``float``, so it matches the ``isinstance(obj, float)`` branch and would - otherwise be returned as-is instead of falling through to the duck-typed - ``float()`` coercion the other numpy types take. + otherwise be returned as-is rather than normalized like the rest. """ out = scrub_non_finite({"v": np.float64(3.14)}) assert type(out["v"]) is float @@ -233,6 +276,17 @@ def test_nan_safe_mean_handles_numpy_inputs() -> None: assert nan_safe_mean(values) == pytest.approx(2.0) +def test_nan_safe_mean_excludes_numpy_bool_like_python_bool() -> None: + """A numpy bool must not be averaged in as 1.0. + + Python ``True`` is already excluded as "not a metric value"; numpy's + bool has to behave identically or the same data yields a different + mean depending only on which bool type produced it. + """ + assert nan_safe_mean([np.bool_(True), 3.0]) == nan_safe_mean([True, 3.0]) + assert nan_safe_mean([np.bool_(True), 3.0]) == pytest.approx(3.0) + + def test_nan_safe_std_empty_returns_none() -> None: """Empty input -> None.""" assert nan_safe_std([]) is None From 227032c4b7c3ae82c94093fb14d624f9a726fe68 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:08:32 -0700 Subject: [PATCH 043/119] fix: close RecordProcessor dataset-configuration race for scaled replicas (#1348) Signed-off-by: Elias Bermudez (cherry picked from commit dfef45f51695eb0b91520f4084928019c1338ded) --- docs/environment-variables.md | 1 + src/aiperf/common/enums/enums.py | 2 + src/aiperf/common/environment.py | 10 + src/aiperf/common/messages/__init__.py | 4 + .../common/messages/dataset_messages.py | 31 +++ .../multiprocess_service_manager.py | 46 ++-- src/aiperf/dataset/dataset_manager.py | 53 +++- src/aiperf/records/dataset_gate.py | 88 +++++- .../records/record_processor_service.py | 16 +- src/aiperf/records/records_manager.py | 16 +- .../test_multiprocess_service_manager.py | 34 +++ tests/unit/dataset/test_dataset_manager.py | 73 +++++ .../property/_numeric_bounds_baseline.txt | 2 + tests/unit/records/test_dataset_gate.py | 252 ++++++++++++++++++ 14 files changed, 593 insertions(+), 35 deletions(-) create mode 100644 tests/unit/records/test_dataset_gate.py diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 5afae4d928..9ae5c55407 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -96,6 +96,7 @@ Dataset loading and configuration. Controls timeouts and behavior for dataset lo | Environment Variable | Default | Constraints | Description | |----------------------|---------|-------------|-------------| | `AIPERF_DATASET_CONFIGURATION_TIMEOUT` | `300.0` | ≥ 1.0, ≤ 100000.0 | Timeout in seconds for dataset configuration operations | +| `AIPERF_DATASET_CATCH_UP_REQUEST_TIMEOUT` | `10.0` | ≥ 0.1, ≤ 100000.0 | Timeout in seconds for the one-shot DatasetConfigStatusRequest a RecordProcessor/RecordsManager sends to DatasetManager when it hasn't yet received DatasetConfiguredNotification. Recovers a subscriber that finished subscribing after DatasetManager already published the one-shot notification, instead of blocking for the full CONFIGURATION_TIMEOUT. | | `AIPERF_DATASET_BASETEN_SESSION_COLUMN` | `'provided_session_id'` | one of: 'provided_session_id' / 'poor_man_session_id' | Session column used by the Baseten trace loader when both supported columns exist. Set to poor_man_session_id for legacy traces. If the selected column is absent, the loader uses the available column. | | `AIPERF_DATASET_MMAP_BASE_PATH` | `None` | — | Base path for memory-mapped dataset files. If None, uses system temp directory. Set to a shared filesystem path for Kubernetes mounted volumes. Example: AIPERF_DATASET_MMAP_BASE_PATH=/mnt/shared-pvc creates files at /mnt/shared-pvc/aiperf_mmap_{benchmark_id}/ | | `AIPERF_DATASET_MMAP_CACHE_ENABLED` | `True` | — | If True, AIPerf reuses memory-mapped dataset files across runs whose input bytes, tokenizer identity, and prompt/input settings are byte-identical. Set to False to force every run to re-tokenize and re-write its mmap files. Cache misses still produce byte-identical mmap files to a non-cached run. | diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index 52f2238589..790b588df4 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -449,6 +449,8 @@ class MessageType(CaseInsensitiveStrEnum): CREDIT_PHASE_START = "credit_phase_start" CREDIT_PHASES_CONFIGURED = "credit_phases_configured" CREDITS_COMPLETE = "credits_complete" + DATASET_CONFIG_STATUS_REQUEST = "dataset_config_status_request" + DATASET_CONFIG_STATUS_RESPONSE = "dataset_config_status_response" DATASET_CONFIGURED_NOTIFICATION = "dataset_configured_notification" DATASET_CONFIGURATION_FAILED = "dataset_configuration_failed" ERROR = "error" diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index cce590d5de..08844b2fca 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -306,6 +306,16 @@ class _DatasetSettings(BaseSettings): default=300.0, description="Timeout in seconds for dataset configuration operations", ) + CATCH_UP_REQUEST_TIMEOUT: float = Field( + ge=0.1, + le=100000.0, + default=10.0, + description="Timeout in seconds for the one-shot DatasetConfigStatusRequest " + "a RecordProcessor/RecordsManager sends to DatasetManager when it hasn't yet " + "received DatasetConfiguredNotification. Recovers a subscriber that finished " + "subscribing after DatasetManager already published the one-shot notification, " + "instead of blocking for the full CONFIGURATION_TIMEOUT.", + ) BASETEN_SESSION_COLUMN: Literal["provided_session_id", "poor_man_session_id"] = ( Field( default="provided_session_id", diff --git a/src/aiperf/common/messages/__init__.py b/src/aiperf/common/messages/__init__.py index c34f5cc8df..5098bb33c5 100644 --- a/src/aiperf/common/messages/__init__.py +++ b/src/aiperf/common/messages/__init__.py @@ -37,6 +37,8 @@ ConversationResponseMessage, ConversationTurnRequestMessage, ConversationTurnResponseMessage, + DatasetConfigStatusRequest, + DatasetConfigStatusResponse, DatasetConfigurationFailedNotification, DatasetConfiguredNotification, ) @@ -96,6 +98,8 @@ "ConversationResponseMessage", "ConversationTurnRequestMessage", "ConversationTurnResponseMessage", + "DatasetConfigStatusRequest", + "DatasetConfigStatusResponse", "DatasetConfigurationFailedNotification", "DatasetConfiguredNotification", "ErrorMessage", diff --git a/src/aiperf/common/messages/dataset_messages.py b/src/aiperf/common/messages/dataset_messages.py index 2628ca1ffa..cedab3bbbf 100644 --- a/src/aiperf/common/messages/dataset_messages.py +++ b/src/aiperf/common/messages/dataset_messages.py @@ -91,6 +91,37 @@ def route_client_metadata(cls, v: Any) -> DatasetClientMetadata: return v +class DatasetConfigStatusRequest(BaseServiceMessage): + """Late-join catch-up request: ask DatasetManager whether dataset + configuration has already completed. + + DatasetConfiguredNotification is a one-shot PUB/SUB broadcast with no + replay. A RecordProcessor or RecordsManager that hasn't received it yet + sends this request to recover deterministically instead of blocking for + the full CONFIGURATION_TIMEOUT -- covers the race where a subscriber + finishes subscribing after the notification was already published. + """ + + message_type: MessageTypeT = MessageType.DATASET_CONFIG_STATUS_REQUEST + + +class DatasetConfigStatusResponse(BaseServiceMessage): + """Response to DatasetConfigStatusRequest. + + ``notification`` carries the already-published DatasetConfiguredNotification + if configuration has completed, else None -- in which case the requester + falls back to waiting on the normal PUB/SUB path. + """ + + message_type: MessageTypeT = MessageType.DATASET_CONFIG_STATUS_RESPONSE + + notification: DatasetConfiguredNotification | None = Field( + default=None, + description="The already-published DatasetConfiguredNotification, if " + "dataset configuration has completed. None if not yet configured.", + ) + + class DatasetConfigurationFailedNotification(BaseServiceMessage): """Notification published by DatasetManager when its PROFILE_CONFIGURE handler raises. diff --git a/src/aiperf/controller/multiprocess_service_manager.py b/src/aiperf/controller/multiprocess_service_manager.py index ce197175b6..4ebcededac 100644 --- a/src/aiperf/controller/multiprocess_service_manager.py +++ b/src/aiperf/controller/multiprocess_service_manager.py @@ -4,6 +4,7 @@ import multiprocessing import os import uuid +from collections import Counter from multiprocessing import Process from multiprocessing.context import SpawnProcess @@ -177,27 +178,35 @@ async def wait_for_all_services_registration( # ProfileConfigureCommand would broadcast before the optionals had # subscribed, leaving them un-configured and their data missing from # the final export. - required_types = set( + # + # Count replicas per type, not per-type presence: a replicated + # service (e.g. RECORD_PROCESSOR, scaled with worker count) has + # multiple multi_process_info entries sharing one service_type. A + # replica that registers after the first can otherwise miss + # DatasetManager's one-shot DatasetConfiguredNotification and hang + # until DATASET.CONFIGURATION_TIMEOUT. + required_counts: Counter[ServiceTypeT] = Counter( info.service_type for info in self.multi_process_info - ) or set(self.required_services.keys()) + ) or Counter(self.required_services) # TODO: Can this be done better by using asyncio.Event()? async def _wait_for_registration(): while not stop_event.is_set(): - # Get all registered service types from the id map - registered_types = { + registered_counts = Counter( service_info.service_type for service_info in self.service_id_map.values() if service_info.registration_status == ServiceRegistrationStatus.REGISTERED - } + ) - # Check if all required types are registered - if required_types.issubset(registered_types): + if all( + registered_counts[service_type] >= count + for service_type, count in required_counts.items() + ): return - self._reap_dead_processes_during_registration(required_types) + self._reap_dead_processes_during_registration(required_counts) # Wait a bit before checking again await asyncio.sleep(0.5) @@ -205,24 +214,25 @@ async def _wait_for_registration(): try: await asyncio.wait_for(_wait_for_registration(), timeout=timeout_seconds) except TimeoutError as e: - # Log which services didn't register in time - registered_types_set = set( + registered_counts = Counter( service_info.service_type for service_info in self.service_id_map.values() if service_info.registration_status == ServiceRegistrationStatus.REGISTERED ) - for service_type in required_types: - if service_type not in registered_types_set: + for service_type, count in required_counts.items(): + registered = registered_counts[service_type] + if registered < count: self.error( - f"Service {service_type} failed to register within timeout" + f"Service {service_type} failed to fully register within " + f"timeout ({registered}/{count} replicas registered)" ) raise AIPerfError("Some services failed to register within timeout") from e def _reap_dead_processes_during_registration( - self, required_types: set[ServiceTypeT] + self, required_counts: "Counter[ServiceTypeT]" ) -> None: """Reap dead processes mid-registration: required dying is fatal, optional dying gets a warning and is dropped from the wait set. @@ -232,9 +242,9 @@ def _reap_dead_processes_during_registration( entire benchmark. ``process is None`` is treated as dead — a None process means the spawn call failed before producing a handle. - Mutates ``required_types`` (discards optional dead types) and - ``self.multi_process_info`` (removes optional dead entries) so the - caller's wait loop can converge. + Mutates ``required_counts`` (decrements the expected replica count + for optional dead types) and ``self.multi_process_info`` (removes + optional dead entries) so the caller's wait loop can converge. Raises: AIPerfError: if any required service has died. @@ -254,7 +264,7 @@ def _reap_dead_processes_during_registration( f"registering (exit code {exit_code}); continuing " f"benchmark without it." ) - required_types.discard(info.service_type) + required_counts[info.service_type] -= 1 self.multi_process_info.remove(info) async def _wait_for_process(self, info: MultiProcessRunInfo) -> None: diff --git a/src/aiperf/dataset/dataset_manager.py b/src/aiperf/dataset/dataset_manager.py index bfc095ae34..e059ed7d00 100644 --- a/src/aiperf/dataset/dataset_manager.py +++ b/src/aiperf/dataset/dataset_manager.py @@ -35,6 +35,8 @@ ConversationResponseMessage, ConversationTurnRequestMessage, ConversationTurnResponseMessage, + DatasetConfigStatusRequest, + DatasetConfigStatusResponse, DatasetConfigurationFailedNotification, DatasetConfiguredNotification, ProfileConfigureCommand, @@ -133,6 +135,16 @@ def __init__( # writes under the same key the lookup would have used. self._cache_key_for_run: str | None = None self._cache_hit_used: bool = False + # Last DatasetConfiguredNotification published this run, if any. + # DatasetConfiguredNotification is a one-shot PUB/SUB broadcast with + # no replay; a subscriber that finishes subscribing after it was + # published never receives it. Cached here so DATASET_CONFIG_STATUS_REQUEST + # (sent by a RecordProcessor/RecordsManager that hasn't seen the + # notification yet) can hand back the already-published config + # instead of the requester blocking until CONFIGURATION_TIMEOUT. + self._last_dataset_configured_notification: ( + DatasetConfiguredNotification | None + ) = None def _is_kubernetes_run(self) -> bool: """Return whether the optional KUBERNETES service-run plugin is active.""" @@ -856,13 +868,13 @@ async def _configure_from_cache_hit(self, hit: mmap_cache.CacheHit) -> None: f"unique conversations: {len(self.dataset_metadata.conversations)}, " f"unique turn count: {self.dataset_metadata.total_turn_count}" ) - await self.publish( - DatasetConfiguredNotification( - service_id=self.service_id, - metadata=self.dataset_metadata, - client_metadata=client_metadata, - ) + notification = DatasetConfiguredNotification( + service_id=self.service_id, + metadata=self.dataset_metadata, + client_metadata=client_metadata, ) + self._last_dataset_configured_notification = notification + await self.publish(notification) def _populate_cache_after_run(self) -> None: """Write the just-finalized run's mmap files into the cache.""" @@ -998,12 +1010,29 @@ async def _configure_dataset(self) -> None: # Note: dataset_configured event is set in _configure_dataset_client_and_free_memory() # after the dataset client is initialized, to avoid a race condition where fallback # requests arrive before the client is ready. - await self.publish( - DatasetConfiguredNotification( - service_id=self.service_id, - metadata=self.dataset_metadata, - client_metadata=client_metadata, - ) + notification = DatasetConfiguredNotification( + service_id=self.service_id, + metadata=self.dataset_metadata, + client_metadata=client_metadata, + ) + self._last_dataset_configured_notification = notification + await self.publish(notification) + + @on_request(MessageType.DATASET_CONFIG_STATUS_REQUEST) + async def _handle_dataset_config_status_request( + self, message: DatasetConfigStatusRequest + ) -> DatasetConfigStatusResponse: + """Late-join catch-up: hand back the already-published + DatasetConfiguredNotification, if any, to a requester that hasn't + received the one-shot PUB/SUB broadcast. + """ + self.debug( + lambda: f"Handling dataset config status request from {message.service_id}" + ) + return DatasetConfigStatusResponse( + service_id=self.service_id, + request_id=message.request_id, + notification=self._last_dataset_configured_notification, ) @on_request(MessageType.CONVERSATION_REQUEST) diff --git a/src/aiperf/records/dataset_gate.py b/src/aiperf/records/dataset_gate.py index fcf3bf1b71..38663393e1 100644 --- a/src/aiperf/records/dataset_gate.py +++ b/src/aiperf/records/dataset_gate.py @@ -11,21 +11,103 @@ from __future__ import annotations import asyncio +from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING from aiperf.common.environment import Environment from aiperf.common.messages import BaseServiceErrorMessage +from aiperf.common.messages.dataset_messages import ( + DatasetConfigStatusRequest, + DatasetConfigStatusResponse, + DatasetConfiguredNotification, +) from aiperf.common.models.error_models import ErrorDetails if TYPE_CHECKING: from aiperf.common.base_component_service import BaseComponentService + from aiperf.common.protocols import RequestClientProtocol + + +class DatasetConfigCatchUp: + """One-shot late-join catch-up for ``DatasetConfiguredNotification``. + + ``DatasetConfiguredNotification`` is a single one-shot PUB/SUB broadcast: + DatasetManager sends it once and never replays it. A subscriber (e.g. a + RecordProcessor replica) that finishes subscribing *after* DatasetManager + already published it never receives it, and without this class blocks + until ``Environment.DATASET.CONFIGURATION_TIMEOUT``. + + This issues one bounded request/response query to DatasetManager asking + whether configuration has already completed, and if so applies it + immediately via ``on_configured`` instead of waiting out the full + timeout. It is a self-heal for a missed broadcast, not the primary path + -- the normal PUB/SUB notification is still expected to be the common + case, and a failed or negative catch-up simply falls back to waiting on + it. Concurrent callers (multiple records arriving before the gate opens) + share a single in-flight attempt via the lock; only the first pays the + request/response round trip. + """ + + def __init__( + self, + request_client: RequestClientProtocol, + on_configured: Callable[[DatasetConfiguredNotification], Awaitable[None]], + service_id: str, + ) -> None: + self._request_client = request_client + self._on_configured = on_configured + self._service_id = service_id + self._lock = asyncio.Lock() + self._attempted = False + + async def try_once(self, event: asyncio.Event) -> None: + """Attempt the catch-up query exactly once for this instance's lifetime. + + No-ops immediately (without taking the lock) once ``event`` is set or + a prior attempt has already run, so the hot path (event already set) + never pays the lock acquisition after the first call. + """ + if event.is_set() or self._attempted: + return + async with self._lock: + if event.is_set() or self._attempted: + return + self._attempted = True + try: + response = await self._request_client.request( + DatasetConfigStatusRequest(service_id=self._service_id), + timeout=Environment.DATASET.CATCH_UP_REQUEST_TIMEOUT, + ) + except Exception: + # Not fatal: DatasetManager may not be reachable yet, or the + # request may simply time out. The caller falls back to + # waiting on the normal PUB/SUB notification. + return + # The normal PUB/SUB notification may have arrived (and applied + # configuration) while this request was in flight -- recheck + # before applying the cached copy, or a concurrent delivery would + # apply configuration to producers/observers twice. + if ( + not event.is_set() + and isinstance(response, DatasetConfigStatusResponse) + and response.notification is not None + ): + await self._on_configured(response.notification) async def await_dataset_configured( - service: BaseComponentService, event: asyncio.Event + service: BaseComponentService, + event: asyncio.Event, + catch_up: DatasetConfigCatchUp | None = None, ) -> bool: """Block until the dataset-configured ``event`` is set. + If ``catch_up`` is provided and the event isn't already set, makes one + bounded attempt to actively pull the configuration from DatasetManager + before falling back to the timed wait -- recovers a subscriber that + missed the one-shot ``DatasetConfiguredNotification`` broadcast instead + of always blocking for the full ``CONFIGURATION_TIMEOUT``. + Returns True once configured. On timeout, treats a missing dataset configuration as a fatal misconfiguration: reports it via a BaseServiceErrorMessage (so the run exits non-zero) and kills the service so @@ -37,6 +119,10 @@ async def await_dataset_configured( # wait_for timer allocation on the hot path. if event.is_set(): return True + if catch_up is not None: + await catch_up.try_once(event) + if event.is_set(): + return True try: await asyncio.wait_for( event.wait(), timeout=Environment.DATASET.CONFIGURATION_TIMEOUT diff --git a/src/aiperf/records/record_processor_service.py b/src/aiperf/records/record_processor_service.py index 2218f84c26..3c46fd8e6b 100644 --- a/src/aiperf/records/record_processor_service.py +++ b/src/aiperf/records/record_processor_service.py @@ -42,7 +42,7 @@ RecordProcessorProtocol, ) from aiperf.post_processors.record_observer_context import RecordObserverContext -from aiperf.records.dataset_gate import await_dataset_configured +from aiperf.records.dataset_gate import DatasetConfigCatchUp, await_dataset_configured from aiperf.records.inference_result_parser import InferenceResultParser if TYPE_CHECKING: @@ -102,6 +102,14 @@ def __init__( # this event so processors are configured (e.g. accuracy ground truths) before # any record is graded. self._dataset_configured_event: asyncio.Event = asyncio.Event() + self._dataset_config_catch_up = DatasetConfigCatchUp( + request_client=self.comms.create_request_client( + address=CommAddress.DATASET_MANAGER_PROXY_FRONTEND, + bind=False, + ), + on_configured=self._on_dataset_configured, + service_id=self.service_id, + ) # Stage 1 - PRODUCERS: parse a record and emit one typed result on the # record_type channel declared in plugins.yaml metadata. Grouped by that @@ -286,7 +294,11 @@ async def _on_inference_results(self, message: InferenceResultsMessage) -> None: the one exception: its False path has already killed the service and aborted the run, so no barrier is left waiting. """ - if not await await_dataset_configured(self, self._dataset_configured_event): + if not await await_dataset_configured( + self, + self._dataset_configured_event, + getattr(self, "_dataset_config_catch_up", None), + ): return record = message.record diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index b8c2a7c386..bad2a3acb8 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -91,7 +91,7 @@ UIType, ) from aiperf.records import records_manager_processing -from aiperf.records.dataset_gate import await_dataset_configured +from aiperf.records.dataset_gate import DatasetConfigCatchUp, await_dataset_configured from aiperf.records.error_tracker import ErrorTracker from aiperf.records.records_manager_processing import ( LoadedAnalyzer, @@ -499,6 +499,14 @@ def __init__( # this event so results processors are configured (e.g. accuracy task names) # before any record is accumulated. self._dataset_configured_event: asyncio.Event = asyncio.Event() + self._dataset_config_catch_up = DatasetConfigCatchUp( + request_client=self.comms.create_request_client( + address=CommAddress.DATASET_MANAGER_PROXY_FRONTEND, + bind=False, + ), + on_configured=self._on_dataset_configured, + service_id=self.service_id, + ) self._previous_realtime_records: int | None = None # Server-metric snapshot from the prior realtime tick. The realtime block @@ -777,7 +785,11 @@ async def _on_records(self, message: RecordsMessage) -> None: envelope (``message.metadata`` / ``message.error``), never off sniffing a record type. """ - if not await await_dataset_configured(self, self._dataset_configured_event): + if not await await_dataset_configured( + self, + self._dataset_configured_event, + getattr(self, "_dataset_config_catch_up", None), + ): return if self.is_trace_enabled: self.trace(f"Received records: {message}") diff --git a/tests/unit/controller/test_multiprocess_service_manager.py b/tests/unit/controller/test_multiprocess_service_manager.py index c51baba5f1..33b0e6847c 100644 --- a/tests/unit/controller/test_multiprocess_service_manager.py +++ b/tests/unit/controller/test_multiprocess_service_manager.py @@ -298,6 +298,40 @@ async def test_wait_blocks_until_optional_services_register( stop_event=asyncio.Event(), timeout_seconds=1.0 ) + @pytest.mark.asyncio + async def test_wait_blocks_until_all_replicas_of_a_type_register( + self, service_manager: MultiProcessServiceManager, mock_alive_process: MagicMock + ): + """A scaled service type (e.g. RECORD_PROCESSOR, replicated based on + worker count) must have every replica registered before the wait + returns -- not just the first one.""" + from aiperf.common.enums import ServiceRegistrationStatus + from aiperf.common.models.service_models import ServiceRunInfo + + replica_1 = MultiProcessRunInfo.model_construct( + process=mock_alive_process, + service_type=ServiceType.RECORD_PROCESSOR, + service_id="record_processor_1", + ) + replica_2 = MultiProcessRunInfo.model_construct( + process=mock_alive_process, + service_type=ServiceType.RECORD_PROCESSOR, + service_id="record_processor_2", + ) + service_manager.multi_process_info = [replica_1, replica_2] + service_manager.service_id_map = { + "record_processor_1": ServiceRunInfo( + service_type=ServiceType.RECORD_PROCESSOR, + registration_status=ServiceRegistrationStatus.REGISTERED, + service_id="record_processor_1", + ), + } + + with pytest.raises(AIPerfError, match="failed to register within timeout"): + await service_manager.wait_for_all_services_registration( + stop_event=asyncio.Event(), timeout_seconds=1.0 + ) + @pytest.mark.asyncio async def test_stop_event_cancels_registration_wait( self, service_manager: MultiProcessServiceManager, mock_alive_process: MagicMock diff --git a/tests/unit/dataset/test_dataset_manager.py b/tests/unit/dataset/test_dataset_manager.py index 087127448e..2c2618f81b 100644 --- a/tests/unit/dataset/test_dataset_manager.py +++ b/tests/unit/dataset/test_dataset_manager.py @@ -12,6 +12,7 @@ from aiperf.common.messages import ( ConversationRequestMessage, ConversationTurnRequestMessage, + DatasetConfigStatusRequest, DatasetConfiguredNotification, ) from aiperf.common.messages.command_messages import ProfileConfigureCommand @@ -1045,3 +1046,75 @@ async def test_cache_hit_allows_payload_bytes_when_clean( assert dm.run.cfg.get_cache_bust_target() == CacheBustTarget.NONE dm._reject_body_mutators_for_payload_bytes(MemoryMapFormat.PAYLOAD_BYTES) + + +class TestDatasetConfigStatusRequest: + """Late-join catch-up for DatasetConfiguredNotification. + + DatasetConfiguredNotification is a one-shot PUB/SUB broadcast with no + replay. A RecordProcessor/RecordsManager that finishes subscribing after + DatasetManager already published it must be able to recover the config + via DatasetConfigStatusRequest instead of blocking for the full + CONFIGURATION_TIMEOUT. + """ + + @pytest.mark.asyncio + async def test_before_configuration_returns_no_notification( + self, initialized_dataset_manager + ) -> None: + """Before PROFILE_CONFIGURE has run, no notification has been + published yet -- the response must say so (None), so the requester + falls back to waiting on the normal PUB/SUB path.""" + dm = initialized_dataset_manager + + response = await dm._handle_dataset_config_status_request( + DatasetConfigStatusRequest(service_id="late_record_processor") + ) + + assert response.notification is None + + @pytest.mark.asyncio + async def test_echoes_request_id_so_the_dealer_client_can_match_it( + self, initialized_dataset_manager + ) -> None: + """Regression: ZMQDealerRequestClient matches an in-flight request to + its response by ``response_message.request_id`` (see + ``aiperf.zmq.dealer_request_client``). A response that doesn't echo + the request's ``request_id`` is silently unmatched -- the caller's + request never resolves and every catch-up attempt times out instead + of self-healing.""" + dm = initialized_dataset_manager + + response = await dm._handle_dataset_config_status_request( + DatasetConfigStatusRequest( + service_id="late_record_processor", request_id="req-123" + ) + ) + + assert response.request_id == "req-123" + + @pytest.mark.asyncio + async def test_after_configuration_replays_last_notification( + self, configured_dataset_manager + ) -> None: + """After PROFILE_CONFIGURE has run and DatasetConfiguredNotification + was published, a late-joining requester must get back the exact + notification that was broadcast -- this is what lets a RecordProcessor + that missed the broadcast self-heal instead of hanging until the + 300s dataset-configuration timeout.""" + dm = configured_dataset_manager + + published = [ + call.args[0] + for call in dm.publish.call_args_list + if isinstance(call.args[0], DatasetConfiguredNotification) + ] + assert len(published) == 1 + + response = await dm._handle_dataset_config_status_request( + DatasetConfigStatusRequest(service_id="late_record_processor") + ) + + assert response.notification is not None + assert response.notification.metadata == published[0].metadata + assert response.notification.client_metadata == published[0].client_metadata diff --git a/tests/unit/property/_numeric_bounds_baseline.txt b/tests/unit/property/_numeric_bounds_baseline.txt index fb2ae4bd26..fdfc96ec03 100644 --- a/tests/unit/property/_numeric_bounds_baseline.txt +++ b/tests/unit/property/_numeric_bounds_baseline.txt @@ -123,6 +123,8 @@ CreditPhaseSendingCompleteMessage.request_ns: numeric field has no ge/gt/le/lt b CreditPhaseStartMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. CreditsCompleteMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. DagSpawn.join_at: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. +DatasetConfigStatusRequest.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. +DatasetConfigStatusResponse.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. DatasetConfigurationFailedNotification.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. DatasetConfiguredNotification.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. DatasetManifest.seed: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. diff --git a/tests/unit/records/test_dataset_gate.py b/tests/unit/records/test_dataset_gate.py new file mode 100644 index 0000000000..5daefd1476 --- /dev/null +++ b/tests/unit/records/test_dataset_gate.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for the dataset-configuration gate. + +DatasetConfiguredNotification is a one-shot PUB/SUB broadcast with no +replay. A RecordProcessor/RecordsManager that finishes subscribing after +DatasetManager already published it never receives it and, without +DatasetConfigCatchUp, blocks until CONFIGURATION_TIMEOUT (300s default). +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from aiperf.common.messages.dataset_messages import ( + DatasetConfigStatusRequest, + DatasetConfigStatusResponse, + DatasetConfiguredNotification, +) +from aiperf.records.dataset_gate import DatasetConfigCatchUp, await_dataset_configured + + +def _make_notification() -> DatasetConfiguredNotification: + return DatasetConfiguredNotification.model_construct( + service_id="dataset_manager", + metadata=MagicMock(), + client_metadata=MagicMock(), + ) + + +class TestDatasetConfigCatchUp: + @pytest.mark.asyncio + async def test_try_once_noop_if_event_already_set(self): + """No request should be sent once the normal PUB/SUB path already + configured the event -- the catch-up is purely a fallback.""" + request_client = AsyncMock() + on_configured = AsyncMock() + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + event = asyncio.Event() + event.set() + + await catch_up.try_once(event) + + request_client.request.assert_not_called() + on_configured.assert_not_called() + + @pytest.mark.asyncio + async def test_try_once_applies_notification_when_already_configured(self): + """The core self-heal: DatasetManager reports it already published + the notification, so the late joiner applies it immediately instead + of waiting out the full timeout.""" + notification = _make_notification() + request_client = AsyncMock() + request_client.request.return_value = DatasetConfigStatusResponse( + service_id="dataset_manager", notification=notification + ) + on_configured = AsyncMock() + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + event = asyncio.Event() + + await catch_up.try_once(event) + + request_client.request.assert_awaited_once() + sent_message = request_client.request.await_args.args[0] + assert isinstance(sent_message, DatasetConfigStatusRequest) + assert sent_message.service_id == "rp-1" + on_configured.assert_awaited_once_with(notification) + + @pytest.mark.asyncio + async def test_try_once_falls_back_when_not_yet_configured(self): + """DatasetManager hasn't configured yet: no-op, caller falls back to + waiting on the normal PUB/SUB notification.""" + request_client = AsyncMock() + request_client.request.return_value = DatasetConfigStatusResponse( + service_id="dataset_manager", notification=None + ) + on_configured = AsyncMock() + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + event = asyncio.Event() + + await catch_up.try_once(event) + + on_configured.assert_not_called() + assert not event.is_set() + + @pytest.mark.asyncio + async def test_try_once_swallows_request_failure(self): + """DatasetManager unreachable/timeout: not fatal, falls back to the + normal PUB/SUB wait instead of propagating.""" + request_client = AsyncMock() + request_client.request.side_effect = TimeoutError + on_configured = AsyncMock() + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + event = asyncio.Event() + + await catch_up.try_once(event) + + on_configured.assert_not_called() + + @pytest.mark.asyncio + async def test_try_once_only_queries_once_across_concurrent_callers(self): + """Multiple records arriving concurrently before the gate opens must + share a single in-flight request, not one each. + + Uses an explicit release gate (rather than a bare ``asyncio.sleep``, + which this suite's autouse fixture makes resolve instantly) so the + request is provably still in flight while the other four callers run + ``try_once`` -- deterministic proof of single-flight, not an + artifact of task-scheduling order. + """ + release = asyncio.Event() + request_client = AsyncMock() + + async def slow_request(*args, **kwargs): + await release.wait() + return DatasetConfigStatusResponse(service_id="dataset_manager") + + request_client.request.side_effect = slow_request + on_configured = AsyncMock() + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + event = asyncio.Event() + + tasks = [asyncio.create_task(catch_up.try_once(event)) for _ in range(5)] + await asyncio.sleep(0) # let every task reach the in-flight request + + request_client.request.assert_awaited_once() + + release.set() + await asyncio.gather(*tasks) + + request_client.request.assert_awaited_once() + + @pytest.mark.asyncio + async def test_try_once_does_not_requery_after_a_negative_attempt(self): + """A negative catch-up result (not yet configured) must not trigger a + second request on a later call -- it's a one-shot attempt per + instance; the normal PUB/SUB path is the source of truth after that.""" + request_client = AsyncMock() + request_client.request.return_value = DatasetConfigStatusResponse( + service_id="dataset_manager", notification=None + ) + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=AsyncMock(), + service_id="rp-1", + ) + event = asyncio.Event() + + await catch_up.try_once(event) + await catch_up.try_once(event) + + request_client.request.assert_awaited_once() + + @pytest.mark.asyncio + async def test_try_once_does_not_reapply_if_notification_wins_the_race(self): + """Regression: if the normal PUB/SUB notification is delivered (and + applied, setting the event) while the catch-up request is still in + flight, the cached response that arrives afterward must NOT be + re-applied -- otherwise producers/observers get configured twice.""" + entered = asyncio.Event() + release = asyncio.Event() + + async def slow_request(*args, **kwargs): + entered.set() + await release.wait() + return DatasetConfigStatusResponse( + service_id="dataset_manager", notification=_make_notification() + ) + + request_client = AsyncMock() + request_client.request.side_effect = slow_request + on_configured = AsyncMock() + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + event = asyncio.Event() + + task = asyncio.create_task(catch_up.try_once(event)) + await entered.wait() + + # The normal PUB/SUB path wins the race: it applies configuration + # and sets the event while the catch-up request is still pending. + event.set() + release.set() + await task + + on_configured.assert_not_called() + + +class TestAwaitDatasetConfiguredWithCatchUp: + @pytest.mark.asyncio + async def test_catch_up_unblocks_immediately_without_waiting_out_timeout( + self, monkeypatch + ): + """End-to-end: a late joiner passed a catch_up that successfully + recovers the config must return True immediately, never touching the + CONFIGURATION_TIMEOUT wait path.""" + notification = _make_notification() + request_client = AsyncMock() + request_client.request.return_value = DatasetConfigStatusResponse( + service_id="dataset_manager", notification=notification + ) + + service = MagicMock() + event = asyncio.Event() + + applied: list[DatasetConfiguredNotification] = [] + + async def on_configured(msg: DatasetConfiguredNotification) -> None: + applied.append(msg) + event.set() + + catch_up = DatasetConfigCatchUp( + request_client=request_client, + on_configured=on_configured, + service_id="rp-1", + ) + + async def fail_if_called(*args, **kwargs): + raise AssertionError( + "should not fall through to the timed wait after a successful catch-up" + ) + + monkeypatch.setattr(asyncio, "wait_for", fail_if_called) + + result = await await_dataset_configured(service, event, catch_up) + + assert result is True + assert applied == [notification] From d109379193c17a3cbc242fa80901043a77617af5 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:22:42 -0700 Subject: [PATCH 044/119] fix(config): stop silently dropping CLI flags passed with --config (#1280) Signed-off-by: Elias Bermudez (cherry picked from commit b61435e76882e503a0325225a6f607a947e8a4e6) --- .cursor/rules/python.mdc | 4 + .github/copilot-instructions.md | 4 + AGENTS.md | 4 + CLAUDE.md | 4 + docs/cli-options.md | 4 +- docs/dev/global-invariants.md | 125 ++++ .../config/flags/_config_flag_routing.py | 491 +++++++++++++ src/aiperf/config/flags/_converter_dataset.py | 439 +++++++++-- .../config/flags/_converter_profiling.py | 39 +- src/aiperf/config/flags/_converter_runtime.py | 9 +- .../config/flags/_converter_telemetry.py | 33 +- src/aiperf/config/flags/_converter_warmup.py | 46 +- src/aiperf/config/flags/_section_fields.py | 2 + src/aiperf/config/flags/cli_config.py | 7 +- src/aiperf/config/flags/resolver.py | 636 +++++++++++----- src/aiperf/config/loader/parsing.py | 24 +- ...st_random_pool_batch_size_yaml_override.py | 114 ++- .../test_random_seed_envelope_override.py | 75 ++ .../common/config/test_config_validators.py | 29 + tests/unit/config/test_config_flag_routing.py | 317 ++++++++ .../config/test_config_input_overrides.py | 694 ++++++++++++++++++ .../config/test_config_override_invariants.py | 554 ++++++++++++++ .../config/test_config_phase_overrides.py | 354 +++++++++ .../config/test_config_telemetry_overrides.py | 251 +++++++ tests/unit/config/test_trace_flag_routing.py | 36 +- 25 files changed, 3969 insertions(+), 326 deletions(-) create mode 100644 src/aiperf/config/flags/_config_flag_routing.py create mode 100644 tests/unit/cli_runner/test_random_seed_envelope_override.py create mode 100644 tests/unit/config/test_config_flag_routing.py create mode 100644 tests/unit/config/test_config_input_overrides.py create mode 100644 tests/unit/config/test_config_override_invariants.py create mode 100644 tests/unit/config/test_config_phase_overrides.py create mode 100644 tests/unit/config/test_config_telemetry_overrides.py diff --git a/.cursor/rules/python.mdc b/.cursor/rules/python.mdc index bdf5cce17f..d532653a40 100644 --- a/.cursor/rules/python.mdc +++ b/.cursor/rules/python.mdc @@ -51,6 +51,10 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo Numeric metric values crossing a serialization boundary or feeding a numerical algorithm must be finite or explicitly `None`. Use `aiperf.common.finite` (`FiniteFloat`, `scrub_non_finite`, `nan_safe_mean`/`std`, `is_finite_value`). Mechanical CI invariants in `tests/unit/property/test_finite_invariants.py` reject new violations and ratchet existing debt to zero via baseline files. See [`docs/dev/patterns.md`](docs/dev/patterns.md) § "NaN/Inf Discipline Pattern" and [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) for the full contract. +## CLI Flag Routing Under `--config` + +A CLI flag passed alongside `--config` MUST either change the resolved config or raise an error naming the flag — never be silently ignored, which for a benchmarking tool means publishing numbers that do not match what was asked for. `CLIConfig` is the source of truth, structurally: `resolve_config` receives a `CLIConfig` and a path and nothing else, so a flag reaches `AIPerfConfig` only by being a field on it. Every field must be classified in `src/aiperf/config/flags/_config_flag_routing.py` as `ROUTED_UNDER_CONFIG`, `UNROUTED_UNDER_CONFIG`, `EXEMPT_FROM_CONFIG_ROUTING`, or one of the conditional sets (`COMPANION_ROUTED`, `MAGIC_LIST_ONLY_UNDER_CONFIG`). Adding a flag without classifying it fails `test_every_cli_config_field_is_classified`; claiming a flag is routed when nothing routes it fails `test_routed_field_never_silently_no_ops`. Prefer routing the flag (check whether a builder already exists) over listing it as unrouted. See [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) § "CLI flag routing under `--config`". + ## Build and Test Commands ```bash diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 35afb07d47..a14e7de782 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -46,6 +46,10 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo Numeric metric values crossing a serialization boundary or feeding a numerical algorithm must be finite or explicitly `None`. Use `aiperf.common.finite` (`FiniteFloat`, `scrub_non_finite`, `nan_safe_mean`/`std`, `is_finite_value`). Mechanical CI invariants in `tests/unit/property/test_finite_invariants.py` reject new violations and ratchet existing debt to zero via baseline files. See [`docs/dev/patterns.md`](docs/dev/patterns.md) § "NaN/Inf Discipline Pattern" and [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) for the full contract. +## CLI Flag Routing Under `--config` + +A CLI flag passed alongside `--config` MUST either change the resolved config or raise an error naming the flag — never be silently ignored, which for a benchmarking tool means publishing numbers that do not match what was asked for. `CLIConfig` is the source of truth, structurally: `resolve_config` receives a `CLIConfig` and a path and nothing else, so a flag reaches `AIPerfConfig` only by being a field on it. Every field must be classified in `src/aiperf/config/flags/_config_flag_routing.py` as `ROUTED_UNDER_CONFIG`, `UNROUTED_UNDER_CONFIG`, `EXEMPT_FROM_CONFIG_ROUTING`, or one of the conditional sets (`COMPANION_ROUTED`, `MAGIC_LIST_ONLY_UNDER_CONFIG`). Adding a flag without classifying it fails `test_every_cli_config_field_is_classified`; claiming a flag is routed when nothing routes it fails `test_routed_field_never_silently_no_ops`. Prefer routing the flag (check whether a builder already exists) over listing it as unrouted. See [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) § "CLI flag routing under `--config`". + ## Build and Test Commands ```bash diff --git a/AGENTS.md b/AGENTS.md index 59643c536e..ed011257f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,10 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo Numeric metric values crossing a serialization boundary or feeding a numerical algorithm must be finite or explicitly `None`. Use `aiperf.common.finite` (`FiniteFloat`, `scrub_non_finite`, `nan_safe_mean`/`std`, `is_finite_value`). Mechanical CI invariants in `tests/unit/property/test_finite_invariants.py` reject new violations and ratchet existing debt to zero via baseline files. See [`docs/dev/patterns.md`](docs/dev/patterns.md) § "NaN/Inf Discipline Pattern" and [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) for the full contract. +## CLI Flag Routing Under `--config` + +A CLI flag passed alongside `--config` MUST either change the resolved config or raise an error naming the flag — never be silently ignored, which for a benchmarking tool means publishing numbers that do not match what was asked for. `CLIConfig` is the source of truth, structurally: `resolve_config` receives a `CLIConfig` and a path and nothing else, so a flag reaches `AIPerfConfig` only by being a field on it. Every field must be classified in `src/aiperf/config/flags/_config_flag_routing.py` as `ROUTED_UNDER_CONFIG`, `UNROUTED_UNDER_CONFIG`, `EXEMPT_FROM_CONFIG_ROUTING`, or one of the conditional sets (`COMPANION_ROUTED`, `MAGIC_LIST_ONLY_UNDER_CONFIG`). Adding a flag without classifying it fails `test_every_cli_config_field_is_classified`; claiming a flag is routed when nothing routes it fails `test_routed_field_never_silently_no_ops`. Prefer routing the flag (check whether a builder already exists) over listing it as unrouted. See [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) § "CLI flag routing under `--config`". + ## Build and Test Commands ```bash diff --git a/CLAUDE.md b/CLAUDE.md index d8f9360b7b..923a42389b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,10 @@ Python 3.11+ async AI benchmarking tool for measuring LLM inference server perfo Numeric metric values crossing a serialization boundary or feeding a numerical algorithm must be finite or explicitly `None`. Use `aiperf.common.finite` (`FiniteFloat`, `scrub_non_finite`, `nan_safe_mean`/`std`, `is_finite_value`). Mechanical CI invariants in `tests/unit/property/test_finite_invariants.py` reject new violations and ratchet existing debt to zero via baseline files. See [`docs/dev/patterns.md`](docs/dev/patterns.md) § "NaN/Inf Discipline Pattern" and [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) for the full contract. +## CLI Flag Routing Under `--config` + +A CLI flag passed alongside `--config` MUST either change the resolved config or raise an error naming the flag — never be silently ignored, which for a benchmarking tool means publishing numbers that do not match what was asked for. `CLIConfig` is the source of truth, structurally: `resolve_config` receives a `CLIConfig` and a path and nothing else, so a flag reaches `AIPerfConfig` only by being a field on it. Every field must be classified in `src/aiperf/config/flags/_config_flag_routing.py` as `ROUTED_UNDER_CONFIG`, `UNROUTED_UNDER_CONFIG`, `EXEMPT_FROM_CONFIG_ROUTING`, or one of the conditional sets (`COMPANION_ROUTED`, `MAGIC_LIST_ONLY_UNDER_CONFIG`). Adding a flag without classifying it fails `test_every_cli_config_field_is_classified`; claiming a flag is routed when nothing routes it fails `test_routed_field_never_silently_no_ops`. Prefer routing the flag (check whether a builder already exists) over listing it as unrouted. See [`docs/dev/global-invariants.md`](docs/dev/global-invariants.md) § "CLI flag routing under `--config`". + ## Build and Test Commands ```bash diff --git a/docs/cli-options.md b/docs/cli-options.md index 9da10cea6b..6d20aa8dae 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -545,7 +545,7 @@ Fraction of trace sessions to keep for replay, sampled whole-session to preserve #### `-f`, `--config` `` -Path to a YAML configuration file. CLI flags override values from the config file. +Path to a YAML configuration file. CLI flags override values from the config file. A flag that cannot be applied on top of a config file is rejected by name rather than ignored, so a run never silently differs from what was asked for. ### Fixed Schedule @@ -2109,7 +2109,7 @@ Fraction of trace sessions to keep for replay, sampled whole-session to preserve #### `-f`, `--config` `` -Path to a YAML configuration file. CLI flags override values from the config file. +Path to a YAML configuration file. CLI flags override values from the config file. A flag that cannot be applied on top of a config file is rejected by name rather than ignored, so a run never silently differs from what was asked for. ### Fixed Schedule diff --git a/docs/dev/global-invariants.md b/docs/dev/global-invariants.md index 0b33af7f49..005653fe3a 100644 --- a/docs/dev/global-invariants.md +++ b/docs/dev/global-invariants.md @@ -152,6 +152,131 @@ When you fix a field, just delete its line from the baseline; the test starts enforcing the constraint on that field on the next CI run. +## CLI flag routing under `--config` + +`resolve_config` merges a YAML config file with explicitly-set CLI +flags. Historically it only knew how to route a subset of `CLIConfig` +into the merged dict, and **every other flag was discarded without a +word** — `aiperf profile -f base.yaml --random-seed 42` ran with a +different seed than the user asked for and said nothing. For a +benchmarking tool that silently corrupts published numbers. + +The invariant now in force: + +> A CLI flag passed alongside `--config` either changes the resolved +> config, or raises an error naming the flag. It is never silently +> ignored. + +### `CLIConfig` is the source of truth + +The guarantee is scoped to what `resolve_config` can see, and its +signature is the boundary: + +```python +def resolve_config(cli_config: CLIConfig, config_file: Path | None = None) -> AIPerfConfig +``` + +Both entry points — `aiperf profile` and `aiperf service` — call it with +nothing but their `CLIConfig` and a path. So a CLI option can influence +the resolved `AIPerfConfig` **only** by being a field on `CLIConfig`, +and that is a structural property of the call, not a convention someone +has to remember. It is what lets the classification test enumerate +`CLIConfig.model_fields` and claim completeness: there is no second +source of flags for it to miss. + +Two consequences worth stating plainly: + +- **Command-level parameters are out of scope, by construction.** + `aiperf service` declares `--service-id`, `--health-host`, + `--health-port` and its `service_type` argument directly on the + command function rather than on `CLIConfig`. They are service-instance + plumbing, never passed to `resolve_config`, and so cannot be dropped + by this mechanism — they never enter it. Anything that *is* benchmark + configuration belongs on `CLIConfig`, precisely so this guarantee + covers it. +- **The assumption is falsifiable, and this is how it would break.** + Giving `resolve_config` another parameter that carries user intent, or + having it read flags from module state or the environment, puts values + into the resolved config that no test enumerates. If you need to do + that, extend the classification to cover the new source in the same + breath — otherwise the suite will keep reporting a completeness it no + longer has. + +Every field on `CLIConfig` must be classified in +[`_config_flag_routing.py`](https://github.com/ai-dynamo/aiperf/tree/main/src/aiperf/config/flags/_config_flag_routing.py): + +| Set | Meaning | +| --- | --- | +| `ROUTED_UNDER_CONFIG` | Reaches `AIPerfConfig`. Derived from the resolver's own routing tables where possible, so it cannot drift from them. | +| `UNROUTED_UNDER_CONFIG` | Known not to route. Raises `ConfigurationError` naming the flag. | +| `EXEMPT_FROM_CONFIG_ROUTING` | Not benchmark config at all (`--config` itself). Each entry needs a stated reason. | +| `COMPANION_ROUTED` | Routes only alongside another flag (`--model-selection-strategy` needs `--model-names`). Rejected when the companion is absent. | +| `MAGIC_LIST_ONLY_UNDER_CONFIG` | Routes in list form only (`--isl 128 256` becomes a sweep parameter; scalar `--isl 128` goes to the dataset). Decided per value. | + +### Why the guarantee holds + +Four layers, each covering the previous one's blind spot: + +1. **The universe is derived, not maintained.** Every set is + subtracted from `frozenset(CLIConfig.model_fields)`. You cannot add + a CLI option without adding a field, so nothing is checked against + a list someone has to remember to update. +2. **Runtime is default-deny.** `reject_unrouted_cli_flags` computes + `model_fields_set - ROUTED - EXEMPT` and raises on the remainder — + "is this known-good?", not "is this known-bad?". It gates on + `model_fields_set`, never truthiness, so `--prompt-batch-size 0` + counts as set. +3. **Classification is mandatory.** + `test_every_cli_config_field_is_classified` fails when a field + belongs to none of the sets, so a new flag fails CI at authoring + time. +4. **The classification is verified, not trusted.** Layer 3 only + proves someone applied a label. + `test_routed_field_never_silently_no_ops` drives every routed field + against both a synthetic and a file dataset and asserts it changes + the config or raises. This is what catches whole-section + classifications (`OUTPUT`/`TOKENIZER`/`ACCURACY`/`SWEEPING` are + marked routed wholesale, so a new member is auto-classified and + layer 3 passes vacuously). + +Two further invariants guard the merge itself: +`test_override_emits_no_key_the_user_did_not_set` runs `build_dataset` +twice with different values for one field — any emitted key identical +across both runs is a materialized default that would overwrite a YAML +value the user never mentioned. `test_routed_dataset_field_actually_changes_the_resolved_config` +covers the dataset block specifically. + +### Adding a new CLI flag + +1. Add the field to `CLIConfig` as usual. CI now fails with + `CLI fields are unclassified for the --config path: [...]`. +2. Decide what the flag should do under `--config`: + - **Route it** (preferred). Dataset-shaping fields flow through + `build_dataset` automatically. Otherwise wire it into + `build_cli_overrides` — check first whether a builder already + exists, as `build_mlflow`/`build_otel`/`build_network_latency` + did while going uncalled. + - **Or list it in `UNROUTED_UNDER_CONFIG`**, so users get an error + naming the flag rather than a wrong benchmark. +3. If the invariant suite cannot generate a value from the + annotation (`typing.Any`, free-form strings), add one to + `FIELD_PROBE_VALUES` or list the field in `UNDRIVABLE_FIELDS` with + a reason. Do not let it report as skipped — a skip is invisible in + CI, which is exactly how a coverage hole hides. + +### Known gaps + +`--sweep-type` and `--disable-auto-fixed-schedule` are unrouted and +loud: the first needs a sweep block to attach to, the second is +consumed during phase construction, which this path does not rebuild. + +The flags in `SWEEP_FIELDS_NOT_ROUTED` (`--concurrency-min/max/steps`, +`--isl-*`/`--osl-*`, the `*-sla-ms` filters, `--parameter-sweep-*`) +resolve cleanly and change nothing, so they are rejected. Verified +individually — notably `--ttft-sla-ms` does **not** take effect even +alongside `--search-recipe` and `--streaming`. Routing them is +follow-up work; until then the failure is loud. + ## Extending the suite ### Adding a new mechanical invariant diff --git a/src/aiperf/config/flags/_config_flag_routing.py b/src/aiperf/config/flags/_config_flag_routing.py new file mode 100644 index 0000000000..f650a82f96 --- /dev/null +++ b/src/aiperf/config/flags/_config_flag_routing.py @@ -0,0 +1,491 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Which CLI flags actually reach ``AIPerfConfig`` when ``--config`` is used. + +``resolve_config`` merges a YAML base with explicitly-set CLI flags, but it +only knows how to route a subset of ``CLIConfig`` into the merged dict. Any +flag outside that subset used to be discarded without a word: a user running +``aiperf profile -f base.yaml --random-seed 42`` got a different seed than +they asked for and no diagnostic. For a benchmarking tool that silently +corrupts published numbers. + +``CLIConfig`` is the source of truth, and that is structural rather than a +convention: ``resolve_config`` takes a ``CLIConfig`` and a path, and both +``aiperf profile`` and ``aiperf service`` call it with nothing else. A flag +can therefore reach ``AIPerfConfig`` only by being a field on ``CLIConfig``, +which is what lets the classification test enumerate ``model_fields`` and +claim completeness. Give ``resolve_config`` another source of user intent +and that claim quietly stops holding -- extend the classification in the +same change. (``aiperf service``'s own ``--service-id`` / ``--health-*`` +parameters live on the command, never reach this path, and are out of scope +by construction.) + +This module makes the boundary explicit and enforceable: + +``ROUTED_UNDER_CONFIG`` + Fields the resolver genuinely routes. Derived from the same maps the + resolver and converters use, so it cannot drift from them. + +``UNROUTED_UNDER_CONFIG`` + Fields known to be dropped. Written out by hand on purpose -- a computed + complement would make ``test_every_section_field_is_classified`` vacuous. + A newly-added CLI flag lands in neither set and fails that test at + authoring time, which is the whole point. + +:func:`reject_unrouted_cli_flags` turns the drop into a ``ConfigurationError`` +naming the offending flags. Repairing the routing (so entries move from +``UNROUTED`` to ``ROUTED``) is separate work; until then users get a loud +error instead of a wrong benchmark. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from aiperf.config.flags._section_fields import ( + ACCURACY_FIELDS, + ENDPOINT_FIELDS, + INPUT_FIELDS, + LOADGEN_FIELDS, + OUTPUT_FIELDS, + SWEEPING_FIELDS, + TOKENIZER_FIELDS, +) + +if TYPE_CHECKING: + from aiperf.config.flags import CLIConfig + +ALL_SECTION_FIELDS: frozenset[str] = ( + ACCURACY_FIELDS + | ENDPOINT_FIELDS + | INPUT_FIELDS + | LOADGEN_FIELDS + | OUTPUT_FIELDS + | SWEEPING_FIELDS + | TOKENIZER_FIELDS +) + + +# Fields that are not benchmark configuration at all, so "routed under +# --config" does not apply to them. Each needs a reason: an unexplained +# exemption is a silent drop with extra steps. +EXEMPT_FROM_CONFIG_ROUTING: frozenset[str] = frozenset( + { + # The --config flag itself; it selects the file being merged. + "config_file", + # Alias input consumed during CLIConfig validation rather than routed + # onto AIPerfConfig; it populates its canonical sibling, which is + # itself classified. + "stream", + } +) + +# Fields outside every section frozenset that nonetheless reach AIPerfConfig, +# verified by resolving with the flag set and diffing against the no-flag +# baseline. The section sets are opt-in and incomplete; this records what is +# genuinely routed rather than letting absence imply anything. +_ROUTED_OUTSIDE_SECTIONS: frozenset[str] = frozenset( + { + # service runtime / logging, via build_logging_runtime + "api_host", + "api_port", + "extra_verbose", + "log_level", + "record_processor_service_count", + "stats_interval", + "ui_type", + "verbose", + "workers_max", + "zmq_dual_bind", + "zmq_ipc_path", + "zmq_tcp_host", + # telemetry collectors already wired into build_cli_overrides + "gpu_telemetry", + "no_gpu_telemetry", + "server_metrics", + "no_server_metrics", + "server_metrics_formats", + # wandb, via build_wandb (the non-project flags are guarded loudly + # rather than dropped when --wandb-project is absent) + "wandb_project", + "wandb_entity", + "wandb_run_name", + "wandb_tags", + # agentic phase fields, via _apply_agentic_replay_fields + "agentic_cache_warmup_duration", + "agentic_warmup_grace_period", + # publishing targets, via build_mlflow / build_otel / + # build_network_latency -- the builders the CLI-only converter has + # always used, now called by build_cli_overrides too + "mlflow_artifact_globs", + "mlflow_experiment", + "mlflow_parent_run_id", + "mlflow_run_name", + "mlflow_tags", + "mlflow_tracking_uri", + "otel_resource_attributes", + "otel_url", + "gen_ai_provider", + "network_latency_automatic", + "network_latency_mean", + "network_latency_ping_interval", + # scenario lock, via _apply_scenario_overrides + "scenario", + "unsafe_override", + # inter-turn delay cap, carried onto trace datasets by build_dataset + "inter_turn_delay_cap_seconds", + # verbatim system prompt, carried onto the dataset by + # _VERBATIM_DATASET_FIELDS (added on main by #1268) + "system_prompt", + "system_prompt_file", + # baseten_trace dataset knobs carried by _VERBATIM_DATASET_FIELDS; + # against another format they raise rather than drop + "force_min_tokens", + "max_idle_gap_cap_seconds", + "omit_kv_hints", + "open_loop_replay", + "open_loop_strict", + "replay_speedup", + "trace_session_sample_ratio", + } +) + + +# SWEEPING members that resolve cleanly under --config and change nothing, +# each verified by resolving with the flag and diffing the result. Notably +# --ttft-sla-ms does NOT take effect even alongside --search-recipe plus +# --streaming, contrary to the example in resolver's module docstring. +SWEEP_FIELDS_NOT_ROUTED: frozenset[str] = frozenset( + { + "concurrency_max", + "concurrency_min", + "concurrency_steps", + "convergence_mode", + "convergence_stat", + "degradation_metric_tag", + "degradation_stat", + "e2e_sla_ms", + "isl_max", + "isl_min", + "isl_steps", + "itl_sla_ms", + "osl_max", + "osl_min", + "osl_steps", + "parameter_sweep_cooldown_seconds", + "parameter_sweep_mode", + "parameter_sweep_same_seed", + "search_style", + "sweep_variants", + "tpot_sla_ms", + "ttft_sla_ms", + } +) + + +# Flags that only route when a companion flag is also present, verified by +# resolving with and without the companion. Without it they resolve cleanly +# and change nothing, so they must be rejected rather than quietly ignored. +COMPANION_ROUTED: dict[str, frozenset[str]] = { + # _apply_endpoint_overrides writes `models.strategy` only inside the + # `model_names` branch. + "model_selection_strategy": frozenset({"model_names"}), + # _apply_phase_loadgen_overrides consults arrival_pattern only when + # rewriting a phase for --request-rate-series. + "arrival_pattern": frozenset({"request_rate_series"}), + # EndpointConfig._validate_per_chunk_usage requires server token counts, + # streaming, and a chat endpoint; on its own the flag can only be set to + # its default. + "per_chunk_usage": frozenset({"use_server_token_count", "streaming"}), +} + + +# Flags that would replace the dataset the config file declared rather than +# shape it: its source file, its public-dataset identity, its format. The YAML +# owns those, and build_dataset drops the corresponding keys in override mode, +# so the flags must keep erroring instead of appearing to work. +DATASET_SOURCE_FIELDS: frozenset[str] = frozenset( + { + # Switch the dataset TYPE, not just a value inside it. The variants + # barely overlap -- FileDataset carries a dozen fields PublicDataset + # rejects -- so accepting these would silently invalidate whole groups + # of keys the user wrote in the config file. + "public_dataset", + "hf_weka_dataset", + } +) + +# Source flags that change a value WITHIN the declared type: the trace file +# and the loader that reads it. Overriding these is the "same benchmark, +# different trace" loop, and is no more surprising than --model overriding +# models.items. build_dataset applies them only when the config file declares +# type: file; against any other type they raise, because that would be a type +# switch wearing a different hat. +DATASET_SOURCE_IN_TYPE_FIELDS: frozenset[str] = frozenset( + {"input_file", "custom_dataset_type"} +) + +# INPUT_FIELDS members that build_dataset does not carry: they land on the +# slos block or the profiling phase instead. They ARE routed (see +# _apply_phase_loadgen_overrides and the goodput handling in +# build_cli_overrides); this set exists to keep them out of the dataset +# reconciliation, which would otherwise report them as inert because +# build_dataset legitimately emits nothing for them. +_INPUT_NOT_ON_DATASET: frozenset[str] = frozenset( + { + # --goodput lands on the benchmark `slos` block. + "goodput", + # The fixed-schedule flags select and shape the profiling phase, so + # they are routed by _apply_phase_loadgen_overrides rather than + # carried on the dataset -- and must stay out of the dataset + # reconciliation, which would otherwise report them as inert. + "fixed_schedule", + "fixed_schedule_auto_offset", + "fixed_schedule_end_offset", + "fixed_schedule_start_offset", + } +) + + +# Dataset-carried fields that are not INPUT_FIELDS members. build_dataset +# reads them (via _VERBATIM_DATASET_FIELDS and the trace helpers), so the +# per-flag reconciliation in _apply_dataset_overrides must consider them too; +# keying that check on INPUT_FIELDS alone let them no-op silently. +DATASET_FIELDS_OUTSIDE_INPUT: frozenset[str] = frozenset( + { + "force_min_tokens", + "inter_turn_delay_cap_seconds", + "max_idle_gap_cap_seconds", + "omit_kv_hints", + "open_loop_replay", + "open_loop_strict", + "replay_speedup", + "system_prompt", + "system_prompt_file", + "trace_idle_gap_cap_seconds", + "trace_session_sample_ratio", + } +) + +# INPUT fields that _apply_dataset_overrides carries onto the dataset block. +# Used to decide whether a config file lacking a dataset is an error for this +# invocation or simply nothing to do. +DATASET_OVERRIDE_FIELDS: frozenset[str] = ( + INPUT_FIELDS - DATASET_SOURCE_FIELDS - _INPUT_NOT_ON_DATASET +) - {"headers", "extra_inputs"} + + +def _build_routed_under_config() -> frozenset[str]: + """Derive the routed set from the resolver's own routing tables. + + Imports are local: ``resolver`` imports this module's reject helper + lazily, and reaching back into it at module scope would close the cycle. + Deriving rather than restating means adding an entry to + ``_ENDPOINT_FIELD_MAP`` or ``_LOADGEN_PHASE_FIELD_MAP`` automatically + counts as routed here. + """ + from aiperf.config.flags._converter_endpoint import _ENDPOINT_FIELD_MAP + from aiperf.config.flags._converter_profiling import ( + _AGENTIC_REPLAY_ROUTES, + _GAMMA_ONLY_ROUTES, + _RAMP_FIELDS, + ) + from aiperf.config.flags.resolver import _LOADGEN_PHASE_FIELD_MAP + + # _apply_endpoint_overrides: the field map, plus --url and the + # --model-names/--model-selection-strategy pair that lands on `models`. + endpoint = ( + set(_ENDPOINT_FIELD_MAP) + | { + "urls", + "model_names", + "model_selection_strategy", + } + | { + # probe sub-blocks, via _maybe_build_reset_kv_cache / + # _maybe_build_server_profiler + field + for field in ENDPOINT_FIELDS + if field.startswith(("reset_kv_cache", "server_profiler")) + } + ) + + # _apply_input_overrides routes headers/extra onto the endpoint block; + # every other dataset-shaping field goes through _apply_dataset_overrides, + # which delegates to build_dataset in override mode. + # + # The exceptions are the fields that would replace the dataset the config + # file declared rather than shape it -- source, type, and format. The YAML + # owns those, so the flags stay rejected (see DATASET_SOURCE_FIELDS). + inputs = (INPUT_FIELDS - DATASET_SOURCE_FIELDS) - _INPUT_NOT_ON_DATASET + inputs |= DATASET_SOURCE_IN_TYPE_FIELDS + + # _apply_phase_loadgen_overrides: the phase field map, the rate-series + # pair, and the AGENTIC_REPLAY fields shared with BasePhaseConfig. + loadgen = ( + {attr for attr, _ in _LOADGEN_PHASE_FIELD_MAP} + | {"request_rate_series", "arrival_pattern"} + | (set(_AGENTIC_REPLAY_ROUTES) & LOADGEN_FIELDS) + # Phase shaping, applied by _apply_phase_shaping_overrides through the + # converter's own helpers. Derived from the same tables those helpers + # read, so a new ramp or gamma-only route counts as routed here too. + | {attr for attr, _ in _RAMP_FIELDS} + | {attr for _, attr in _GAMMA_ONLY_ROUTES} + | {"request_cancellation_rate", "request_cancellation_delay"} + # Warmup phase shaping, applied by _apply_warmup_overrides. + | {field for field in LOADGEN_FIELDS if field.startswith("warmup_")} + # Fixed-schedule phase selection and its offsets, and --goodput, which + # lands on the benchmark slos block. + | {field for field in INPUT_FIELDS if field.startswith("fixed_schedule")} + | {"goodput"} + ) + + # Whole-section builders consumed by build_cli_overrides: build_artifacts + # + resolve_auto_plot, build_tokenizer, build_accuracy. + whole_sections = OUTPUT_FIELDS | TOKENIZER_FIELDS | ACCURACY_FIELDS + + # SWEEPING minus the members verified to resolve cleanly while changing + # nothing (see SWEEP_FIELDS_NOT_ROUTED). The rest -- --search-recipe, + # --search-space and friends -- do take effect. build_sweep / expand_search_recipe + # consume the section, but only conditionally -- e.g. --concurrency-min / + # --concurrency-max / --concurrency-steps under --config verifiably + # produce no sweep at all, while --ttft-sla-ms does take effect alongside + # --search-recipe. Separating genuine drops from "needs a companion flag" + # requires a combination audit that is out of scope here, and erroring on + # the whole section would break valid recipe invocations. Tracked as + # follow-up work; see AIP-1133. + sweeping = set(SWEEPING_FIELDS) - SWEEP_FIELDS_NOT_ROUTED + + return frozenset( + endpoint + | inputs + | loadgen + | whole_sections + | sweeping + | _ROUTED_OUTSIDE_SECTIONS + | DATASET_FIELDS_OUTSIDE_INPUT + ) + + +ROUTED_UNDER_CONFIG: frozenset[str] = _build_routed_under_config() + + +def _build_magic_list_only_fields() -> frozenset[str]: + """Dataset fields whose list form routes as a sweep parameter. + + ``promote_benchmark_magic_lists`` hoists a list-shaped ``--isl 128 256`` + into a ``sweep`` parameter; the scalar form routes onto the dataset block + through ``_apply_dataset_overrides``. Both forms take effect, so these + fields are routed either way. The set records which fields have the two + distinct destinations, so tests (and ``MAGIC_LIST_ONLY_UNDER_CONFIG`` + consumers generally) can assert against the right one. + """ + from aiperf.config.flags.converter import _CLI_DATASET_MAGIC_LIST_PATHS + + return frozenset(attr for attr, _ in _CLI_DATASET_MAGIC_LIST_PATHS) + + +MAGIC_LIST_ONLY_UNDER_CONFIG: frozenset[str] = _build_magic_list_only_fields() + +# Fields silently discarded when --config is supplied. Listed explicitly so +# that a new CLI flag belongs to neither set and trips the classification +# test. Entries move to ROUTED_UNDER_CONFIG as routing is repaired. +UNROUTED_UNDER_CONFIG: frozenset[str] = frozenset( + { + # ----- endpoint ----- + # ----- input: dataset identity, owned by the config file ----- + # Every other INPUT field now routes through _apply_dataset_overrides; + # these would swap the dataset the YAML declared rather than shape it. + *DATASET_SOURCE_FIELDS, + # ----- sweep flags that resolve cleanly and do nothing ----- + *SWEEP_FIELDS_NOT_ROUTED, + # ----- outside every section frozenset ----- + # Still dropped: --sweep-type needs a sweep block to attach to, and + # --disable-auto-fixed-schedule is consumed by phase construction + # which this path does not rebuild. Both need their own routing. + "sweep_type", + "disable_auto_fixed_schedule", + # ----- loadgen: ramps, pacing, cancellation ----- + } +) + + +def flag_names_for(field: str) -> tuple[str, ...]: + """Return the CLI flag spellings for a ``CLIConfig`` field. + + Reads the ``CLIParameter`` annotation metadata rather than kebab-casing + the attribute, because several fields do not follow from their name -- + ``conversation_num`` is ``--num-conversations``, ``warmup_request_count`` + is also ``--num-warmup-requests``. Error messages have to name flags the + user can actually type. + + Returns an empty tuple when the field carries no ``CLIParameter``. + """ + from aiperf.config.flags import CLIConfig + + info = CLIConfig.model_fields.get(field) + if info is None: + return () + # Fields declare their flags via CLIParameter or via cyclopts' Parameter + # (--parameter-sweep-same-seed uses the latter). Match structurally on a + # `name` tuple of dash-prefixed spellings rather than on either class, so + # a third declaration style still yields real flag names. + for meta in info.metadata: + names = getattr(meta, "name", None) + if isinstance(names, (tuple, list)) and any( + isinstance(n, str) and n.startswith("--") for n in names + ): + return tuple(n for n in names if isinstance(n, str)) + return () + + +def _describe(field: str) -> str: + """Render a field as every spelling it accepts, falling back to the name. + + Naming only the first declared spelling meant a user who typed + ``--num-warmup-requests`` was told about ``--warmup-request-count``, and + ``--num-conversations`` was reported as ``--conversation-num`` -- flags + absent from their shell history. This layer cannot see which spelling was + typed (the CLIConfig it receives has already been parsed), so it names + them all rather than guessing. + """ + names = flag_names_for(field) + return "/".join(names) if names else field + + +def reject_unrouted_cli_flags(cli: CLIConfig) -> None: + """Raise when explicitly-set CLI flags cannot be routed under ``--config``. + + Gating is on ``cli.model_fields_set`` -- a flag the user did not pass can + never trigger this, and a flag passed with a falsy value (``--concurrency 0``) + still counts as set. + + Args: + cli: the parsed ``CLIConfig`` for this invocation. + + Raises: + ConfigurationError: naming every offending flag at once, so a user + fixing a long command line learns about all of them in one run. + """ + from aiperf.config.loader.errors import ConfigurationError + + # Keyed on every CLIConfig field rather than on the section frozensets: + # those are opt-in, and a field nobody sectioned was invisible here. + unrouted = cli.model_fields_set - ROUTED_UNDER_CONFIG - EXEMPT_FROM_CONFIG_ROUTING + # Companion-routed flags are inert on their own, so treat a missing + # companion the same as no routing at all. + unrouted |= { + field + for field, companions in COMPANION_ROUTED.items() + if field in cli.model_fields_set and not (companions & cli.model_fields_set) + } + if not unrouted: + return + + flags = ", ".join(sorted(_describe(field) for field in unrouted)) + raise ConfigurationError( + f"These CLI flags cannot be combined with --config and would " + f"otherwise be ignored: {flags}. Set the equivalent values in the " + f"YAML config file, or drop --config and pass the run entirely on " + f"the command line." + ) diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index 2ad16b3476..6fa3b75c74 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -359,6 +359,19 @@ def _resolve_entries(cli: CLIConfig) -> int | None: return None +# Per-modality subtables only SyntheticDataset carries. File and public +# datasets take their values from the trace, and leaking one of these onto +# them trips extra_forbidden at validation. +_SYNTHETIC_ONLY_SUBTABLES: tuple[str, ...] = ( + "prompts", + "prefix_prompts", + "rankings", + "audio", + "images", + "video", +) + + def _apply_dataset_type(d: dict[str, Any], cli: CLIConfig, needs_text: bool) -> None: from aiperf.common.enums import DatasetType @@ -376,14 +389,7 @@ def _apply_dataset_type(d: dict[str, Any], cli: CLIConfig, needs_text: bool) -> d["entries"] = entries d["_entries_explicit"] = entries_explicit # PublicDataset doesn't carry per-modality subtables. - for key in ( - "prompts", - "prefix_prompts", - "rankings", - "audio", - "images", - "video", - ): + for key in _SYNTHETIC_ONLY_SUBTABLES: d.pop(key, None) return if cli.input_file: @@ -393,14 +399,7 @@ def _apply_dataset_type(d: dict[str, Any], cli: CLIConfig, needs_text: bool) -> # FileDataset only carries synthesis + osl as auxiliary fields. The # synthetic-only subtables are dropped here; --osl is handled by # _apply_file_osl. - for key in ( - "prompts", - "prefix_prompts", - "rankings", - "audio", - "images", - "video", - ): + for key in _SYNTHETIC_ONLY_SUBTABLES: d.pop(key, None) return d["type"] = DatasetType.SYNTHETIC @@ -466,7 +465,15 @@ def _apply_random_corpus_style_and_range_ratio( d.setdefault("prompts", {})["random_range_ratio"] = cli.prompt_random_range_ratio -def _apply_turns(d: dict[str, Any], cli: CLIConfig) -> None: +def _apply_turns(d: dict[str, Any], cli: CLIConfig, *, only_set: bool = False) -> None: + """Route conversation turn shaping onto the dataset dict. + + ``only_set`` suppresses the companion value each pair normally carries. + Setting just ``--conversation-turn-mean`` emits ``stddev`` too, which is + the right default when the CLI builds a whole dataset from nothing, but + would overwrite a YAML ``turns.stddev`` the user never mentioned when the + result is merged as an override. + """ fields_set = cli.model_fields_set if ( "conversation_turn_mean" in fields_set @@ -476,18 +483,22 @@ def _apply_turns(d: dict[str, Any], cli: CLIConfig) -> None: # placeholder; the sweep block carries the full list. v = cli.conversation_turn_mean turn_mean = v[0] if isinstance(v, list) and v else v - d["turns"] = { - "mean": turn_mean, - "stddev": cli.conversation_turn_stddev, - } + turns: dict[str, Any] = {} + if not only_set or "conversation_turn_mean" in fields_set: + turns["mean"] = turn_mean + if not only_set or "conversation_turn_stddev" in fields_set: + turns["stddev"] = cli.conversation_turn_stddev + d["turns"] = turns if ( "conversation_turn_delay_mean" in fields_set or "conversation_turn_delay_stddev" in fields_set ): - d["turn_delay"] = { - "mean": cli.conversation_turn_delay_mean, - "stddev": cli.conversation_turn_delay_stddev, - } + turn_delay: dict[str, Any] = {} + if not only_set or "conversation_turn_delay_mean" in fields_set: + turn_delay["mean"] = cli.conversation_turn_delay_mean + if not only_set or "conversation_turn_delay_stddev" in fields_set: + turn_delay["stddev"] = cli.conversation_turn_delay_stddev + d["turn_delay"] = turn_delay if "conversation_turn_delay_ratio" in fields_set: d["turn_delay_ratio"] = cli.conversation_turn_delay_ratio @@ -554,6 +565,34 @@ def _apply_implicit_media_batch(d: dict[str, Any], cli: CLIConfig) -> None: media["batch_size"] = 1 +def apply_implicit_media_batch_override( + override: dict[str, Any], existing_dataset: dict[str, Any] +) -> None: + """Materialize batch_size=1 for a media block the override newly shapes, + when neither the override nor the existing YAML dataset already carries + a batch_size for that modality. + + ``_apply_implicit_media_batch`` is CLI-only-path behavior: applied + blindly as an override it would reset a YAML batch size the user never + mentioned. But when the YAML has no block for that modality at all, + there is nothing to protect, and the model default ``batch_size=0`` + means "disabled" -- a media-shape flag (e.g. ``--image-width-mean``) + with no batch_size anywhere then resolves to an *included but empty* + modality instead of the batch size of 1 the CLI-only path applies for + the same flags, and the request silently degrades to text-only. + """ + for media_key in ("images", "audio", "video"): + media = override.get(media_key) + if not media or "batch_size" in media: + continue + existing = existing_dataset.get(media_key) + if isinstance(existing, dict) and ( + "batch_size" in existing or "batchSize" in existing + ): + continue + media["batch_size"] = 1 + + # --- file-dataset incompatibility validation ----------------------------- @@ -615,7 +654,12 @@ def _apply_implicit_media_batch(d: dict[str, Any], cli: CLIConfig) -> None: ) -def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: +def _reject_file_dataset_incompatible( + cli: CLIConfig, + *, + declared_type: Any = None, + declared_format: Any = None, +) -> None: """Reject synthetic-only flags on FILE or PUBLIC (trace) datasets. Flags rejected: prefix prompts, ISL shaping (--isl/--isl-stddev/ @@ -636,15 +680,29 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: ``FileDataset.osl`` / ``PublicDataset.osl`` by ``_apply_file_osl`` as a per-record fallback. """ - if not cli.input_file and not _implies_public_dataset(cli): + from aiperf.common.enums import DatasetType + from aiperf.plugin.enums import CustomDatasetType + + # Under --config the dataset type comes from the YAML, so --input-file is + # absent even when the target IS a file dataset. Trust the declared value + # when the caller supplies one; fall back to flag inference otherwise. + if declared_type is not None: + if declared_type not in (DatasetType.FILE, DatasetType.PUBLIC): + return + elif not cli.input_file and not _implies_public_dataset(cli): return s = cli.model_fields_set - from aiperf.plugin.enums import CustomDatasetType - is_random_pool = ( - cli.input_file is not None - and cli.custom_dataset_type == CustomDatasetType.RANDOM_POOL - ) + if declared_type is not None: + is_random_pool = ( + declared_type == DatasetType.FILE + and declared_format == CustomDatasetType.RANDOM_POOL + ) + else: + is_random_pool = ( + cli.input_file is not None + and cli.custom_dataset_type == CustomDatasetType.RANDOM_POOL + ) batch_size_attrs = _RANDOM_POOL_BATCH_SIZE_FLAGS non_batch_violations = [ flag @@ -656,9 +714,47 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: for attr, flag in _FILE_DATASET_INCOMPATIBLE_TRIGGERS if attr in s and attr in batch_size_attrs and not is_random_pool ] + _raise_file_dataset_violation( + cli, + non_batch_violations, + batch_violations, + declared_type=declared_type, + declared_format=declared_format, + ) + + +def _raise_file_dataset_violation( + cli: CLIConfig, + non_batch_violations: list[str], + batch_violations: list[str], + *, + declared_type: Any, + declared_format: Any, +) -> None: + """Raise the message that fits how this dataset was chosen. + + Split out of _reject_file_dataset_incompatible to keep it under the + complexity gate; the branching is inherent to the three ways a dataset + can be selected (config file, --input-file, --public-dataset). + + Under --config the user passed neither --input-file nor --public-dataset + -- the config file owns the dataset, and both flags are rejected on that + path -- so the CLI-only advice cannot be followed. + """ + declared = declared_type is not None if non_batch_violations: + flags = ", ".join(non_batch_violations) + if declared: + raise ValueError( + f"{flags} is only supported with synthetic datasets, but the " + f"config file declares type: {declared_type}" + f"{f' (format: {declared_format})' if declared_format else ''}. " + "Set a synthetic dataset in the config file to apply " + "synthetic-only prompt shaping (ISL, prefix prompts, multimodal " + "generation, multi-turn conversation, etc)." + ) raise ValueError( - f"{', '.join(non_batch_violations)} is only supported with synthetic datasets; " + f"{flags} is only supported with synthetic datasets; " "remove --input-file / --public-dataset (use a synthetic dataset) to " "apply synthetic-only prompt shaping (ISL, prefix prompts, multimodal " "generation, multi-turn conversation, etc)." @@ -669,10 +765,13 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: # not first told to add --custom-dataset-type random_pool and only then rejected # at load time. The loader repeats an equivalent check for the pool shapes that # are only visible once the files are parsed (inline records, named entries). + s = cli.model_fields_set set_batch_flags = [ flag for attr, flag in _FILE_DATASET_INCOMPATIBLE_TRIGGERS - if attr in s and attr in batch_size_attrs and getattr(cli, attr) != 1 + if attr in s + and attr in _RANDOM_POOL_BATCH_SIZE_FLAGS + and getattr(cli, attr) != 1 ] if set_batch_flags and cli.input_file is not None and Path(cli.input_file).is_dir(): raise ValueError( @@ -681,6 +780,22 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: "flattens them into one anonymous pool per modality. Point --input-file " "at a single random_pool file, or drop the batch-size flags." ) + if batch_violations: + flags = ", ".join(batch_violations) + if declared: + # The config file owns the format; --custom-dataset-type and + # --input-file are the CLI-only remedies suggested below. + # An omitted `format:` reads back as None from the raw YAML dict + # even though FileDataset.format defaults to single_turn; report + # the effective format rather than a misleading "None". + from aiperf.plugin.enums import DatasetFormat + + effective = declared_format or DatasetFormat.SINGLE_TURN + raise ValueError( + f"{flags} requires a random_pool dataset, but the config file " + f"declares format: {effective}. Set format: random_pool " + "in the config file, or drop these flags." + ) if batch_violations and cli.input_file is not None: remedy = ( "Either add --custom-dataset-type random_pool to keep --input-file" @@ -689,7 +804,7 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: "to random_pool to keep --input-file" ) raise ValueError( - f"{', '.join(batch_violations)} requires --custom-dataset-type random_pool " + f"{flags} requires --custom-dataset-type random_pool " f"when used with --input-file. {remedy}, or remove --input-file to use a " "synthetic dataset." ) @@ -701,15 +816,27 @@ def _reject_file_dataset_incompatible(cli: CLIConfig) -> None: ) -def _apply_random_pool_batch_sizes(d: dict[str, Any], cli: CLIConfig) -> None: +def _apply_random_pool_batch_sizes( + d: dict[str, Any], + cli: CLIConfig, + *, + declared_format: Any = None, +) -> None: """Route batch-size CLI flags onto FileDataset fields when format is random_pool. FileDataset has no prompts/images/audio/video sub-configs, so batch sizes live as flat fields and are threaded directly to RandomPoolDatasetLoader. + + ``declared_format`` carries the format when a YAML config declared it, in + which case ``--input-file`` and ``--custom-dataset-type`` are absent from + the CLI and cannot be used to recognize a random_pool dataset. """ from aiperf.plugin.enums import CustomDatasetType - if not cli.input_file or cli.custom_dataset_type != CustomDatasetType.RANDOM_POOL: + if declared_format is not None: + if declared_format != CustomDatasetType.RANDOM_POOL: + return + elif not cli.input_file or cli.custom_dataset_type != CustomDatasetType.RANDOM_POOL: return s = cli.model_fields_set if "prompt_batch_size" in s: @@ -738,7 +865,11 @@ def _apply_random_pool_batch_sizes(d: dict[str, Any], cli: CLIConfig) -> None: ) -def _reject_baseten_only_trace_flags(cli: CLIConfig) -> None: +def _reject_baseten_only_trace_flags( + cli: CLIConfig, + *, + declared_format: Any = None, +) -> None: """Reject baseten_trace-only replay knobs on incompatible datasets. These knobs are only consumed by the baseten_trace loader; on any other @@ -763,6 +894,15 @@ def _reject_baseten_only_trace_flags(cli: CLIConfig) -> None: if not set_flags: return msg = f"{', '.join(set_flags)} is only supported by the baseten_trace loader" + if declared_format is not None: + # A config file declared the loader, and --input-file / + # --custom-dataset-type are rejected on that path, so inferring from + # them would reject a combination the YAML explicitly supports. + if declared_format == CustomDatasetType.BASETEN_TRACE: + return + raise ValueError( + f"{msg}, but the config file declares format: {declared_format}." + ) if _implies_public_dataset(cli) or not cli.input_file: raise ValueError( f"{msg}; provide --input-file and --custom-dataset-type baseten_trace." @@ -839,7 +979,11 @@ def _reject_baseten_trace_unsupported_synthesis( ) -def _reject_baseten_trace_extra_input_collisions(cli: CLIConfig) -> None: +def _reject_baseten_trace_extra_input_collisions( + cli: CLIConfig, + *, + declared_format: Any = None, +) -> None: """Reject --extra-inputs keys the baseten_trace loader injects per-turn. Loader-injected per-turn values (``min_tokens`` from the recorded output @@ -851,7 +995,13 @@ def _reject_baseten_trace_extra_input_collisions(cli: CLIConfig) -> None: """ from aiperf.plugin.enums import CustomDatasetType - if cli.custom_dataset_type != CustomDatasetType.BASETEN_TRACE: + # The format can come from the CLI or from a config file; the loader + # injects the same per-turn values either way, so the collision is the + # same collision. + resolved_format = ( + declared_format if declared_format is not None else cli.custom_dataset_type + ) + if resolved_format != CustomDatasetType.BASETEN_TRACE: return extra = dict(cli.extra_inputs or ()) collisions: list[tuple[str, str]] = [] @@ -1174,7 +1324,142 @@ def _determine_needs_text(cli: CLIConfig) -> bool: # --- public entrypoint ---------------------------------------------------- -def build_dataset(cli: CLIConfig) -> dict[str, Any]: +def _reject_source_override_type_switch(cli: CLIConfig, declared_type: Any) -> None: + """Refuse --input-file / --custom-dataset-type unless the YAML says file. + + Both change a value inside a file dataset. Against a synthetic or public + config they would switch the dataset type, invalidating whole groups of + keys the config file declares -- so the user gets a message naming the + declared type instead of a validation error listing fields they never + touched. + """ + from aiperf.common.enums import DatasetType + + if declared_type is None or declared_type == DatasetType.FILE: + return + offenders = [ + flag + for attr, flag in ( + ("input_file", "--input-file"), + ("custom_dataset_type", "--custom-dataset-type"), + ) + if attr in cli.model_fields_set and getattr(cli, attr) is not None + ] + if not offenders: + return + raise ValueError( + f"{', '.join(offenders)} applies to a file dataset, but the config " + f"file declares type: {declared_type}. Change the type in the config " + f"file, or drop these flags -- switching the dataset type from the " + f"command line would invalidate the rest of the dataset block." + ) + + +def _run_dataset_rejections( + cli: CLIConfig, + *, + declared_type: Any = None, + declared_format: Any = None, +) -> None: + """Run the flag-compatibility guards for the resolved dataset identity. + + ``declared_type``/``declared_format`` are set when a YAML config file + declared them, in which case the guards evaluate against those rather + than against flags like ``--input-file`` that the ``--config`` path + rejects and therefore never sees. + """ + from aiperf.common.enums import DatasetType + + override_mode = declared_type is not None + + _reject_file_dataset_incompatible( + cli, declared_type=declared_type, declared_format=declared_format + ) + _reject_source_override_type_switch(cli, declared_type) + _reject_baseten_only_trace_flags( + cli, declared_format=declared_format if override_mode else None + ) + if override_mode: + source = ( + f"--custom-dataset-type {declared_format}" + if "custom_dataset_type" in cli.model_fields_set + else f"YAML format: {declared_format}" + ) + _reject_baseten_trace_unsupported_synthesis( + cli, + declared_format, + dataset_format_source=source, + ) + else: + _reject_baseten_trace_unsupported_synthesis(cli, cli.custom_dataset_type) + _reject_baseten_trace_extra_input_collisions( + cli, declared_format=declared_format if override_mode else None + ) + + if cli.dataset_filters and not ( + _implies_public_dataset(cli) or declared_type == DatasetType.PUBLIC + ): + if override_mode: + # Telling a --config user to pass --public-dataset is bad advice: + # the config file owns the type, and that flag is rejected here. + raise ValueError( + f"--dataset-filter requires a public dataset, but the config " + f"file declares type: {declared_type}." + ) + raise ValueError("--dataset-filter requires --public-dataset") + + +def _seed_declared_identity( + d: dict[str, Any], + cli: CLIConfig, + declared_type: Any, + declared_format: Any, +) -> None: + """Stand in for ``_apply_dataset_type`` when the YAML declared the type. + + Seeds type/format so the type-aware helpers downstream route correctly + (they are stripped again before the override is returned), carries the + ``entries`` count that ``_apply_dataset_type`` would otherwise have + emitted, and drops the synthetic-only subtables for file/public datasets. + """ + from aiperf.common.enums import DatasetType + + d["type"] = declared_type + if declared_format is not None: + d.setdefault("format", declared_format) + + # --num-conversations feeds `entries`. Take it only from the flags that + # name a conversation count: _resolve_entries also falls back to + # --request-count, which makes sense when building a dataset from nothing + # but as an override would reset a YAML entry count from an unrelated + # loadgen flag. + fields_set = cli.model_fields_set + if ( + "conversation_num" in fields_set + and "conversation_num_dataset_entries" not in fields_set + ): + conversations = cli.conversation_num + if isinstance(conversations, list): + conversations = max(conversations) if conversations else None + if conversations is not None: + d["entries"] = conversations + + if declared_type in (DatasetType.FILE, DatasetType.PUBLIC): + # Same strip _apply_dataset_type performs for these types on the + # CLI-only path: neither carries the synthetic per-modality subtables, + # and leaking one trips extra_forbidden. Helpers below + # (--prompt-corpus, random_pool batch sizes) re-attach what these + # types legitimately support. + for key in _SYNTHETIC_ONLY_SUBTABLES: + d.pop(key, None) + + +def build_dataset( + cli: CLIConfig, + *, + declared_type: Any = None, + declared_format: Any = None, +) -> dict[str, Any]: """Build a single dataset entry (without the wrapping ``name`` field). Discriminates among synthetic / file / public based on the populated @@ -1184,27 +1469,61 @@ def build_dataset(cli: CLIConfig) -> dict[str, Any]: when --input-file is set, except batch-size flags when ``--custom-dataset-type random_pool`` is the custom dataset type. + Args: + cli: the parsed CLIConfig for this invocation. + declared_type: dataset ``type`` when the caller already knows it -- + i.e. a YAML config file declared it. Supplying it switches this + function into *override mode*: the type is read rather than + inferred from which flags happen to be populated, defaults are + not materialized for values the user did not set, and the keys + the config file owns (``type``/``format``/``path``/``dataset``) + are stripped from the result. That makes the output safe to + deep-merge onto a YAML dataset block. + declared_format: dataset ``format`` alongside ``declared_type``, so + format-sensitive routing (random_pool batch sizes, baseten trace + guards) evaluates against the declared value. + Returns: - A dict suitable for ``DatasetConfig.model_validate({"name": "main", **out})``. + A dict suitable for ``DatasetConfig.model_validate({"name": "main", **out})``, + or -- in override mode -- for deep-merging onto an existing dataset dict. """ + + override_mode = declared_type is not None + # --custom-dataset-type legitimately overrides the YAML-declared format + # (see _seed_declared_identity / _reject_source_override_type_switch), so + # the format-sensitive guards below must judge the overridden value, not + # the stale pre-override declared_format -- otherwise a guard can reject + # the exact override it is checking for. + effective_format = ( + cli.custom_dataset_type + if "custom_dataset_type" in cli.model_fields_set + else declared_format + ) + needs_text = _determine_needs_text(cli) - _reject_file_dataset_incompatible(cli) - _reject_baseten_only_trace_flags(cli) - _reject_baseten_trace_unsupported_synthesis(cli, cli.custom_dataset_type) - _reject_baseten_trace_extra_input_collisions(cli) - if cli.dataset_filters and not _implies_public_dataset(cli): - raise ValueError("--dataset-filter requires --public-dataset") + _run_dataset_rejections( + cli, declared_type=declared_type, declared_format=effective_format + ) d = _flat_dataset_fields(cli) _attach_subtables(d, cli) - _apply_dataset_type(d, cli, needs_text) + if override_mode: + _seed_declared_identity(d, cli, declared_type, declared_format) + else: + _apply_dataset_type(d, cli, needs_text) _apply_sequence_distribution(d, cli) _apply_random_corpus_style_and_range_ratio(d, cli) - _apply_turns(d, cli) + _apply_turns(d, cli, only_set=override_mode) _apply_synthesis(d, cli) - _apply_implicit_media_batch(d, cli) + if not override_mode: + # Materializes batch_size=1 whenever a media-shape flag is set. As an + # override that would silently reset a YAML batch size the user never + # mentioned, so it is CLI-only-path behavior. + _apply_implicit_media_batch(d, cli) _apply_file_osl(d, cli) - _apply_random_pool_batch_sizes(d, cli) + _apply_random_pool_batch_sizes( + d, cli, declared_format=effective_format if override_mode else None + ) # block_size for FILE hash-id traces is owned by _apply_block_size (which # also rejects weka / non-hash-id formats). Do not also call # _apply_file_block_size — that helper is broader and redundant here. @@ -1215,6 +1534,22 @@ def build_dataset(cli: CLIConfig) -> dict[str, Any]: _apply_corpus_and_cache_bust(d, cli) if "random_seed" in cli.model_fields_set: d["random_seed"] = cli.random_seed + if override_mode: + # The config file owns the dataset TYPE and its public-dataset + # identity; --public-dataset / --hf-weka-dataset are rejected up + # front, so dropping these keys loses nothing. + for owned in ("type", "dataset"): + d.pop(owned, None) + # `path` and `format` are different: --input-file and + # --custom-dataset-type legitimately override them within a file + # dataset (see _reject_source_override_type_switch). Keep them only + # when the user actually asked, so an unset flag cannot clobber the + # config file's value with the declared one seeded earlier. + fields_set = cli.model_fields_set + if "input_file" not in fields_set: + d.pop("path", None) + if "custom_dataset_type" not in fields_set: + d.pop("format", None) return d diff --git a/src/aiperf/config/flags/_converter_profiling.py b/src/aiperf/config/flags/_converter_profiling.py index fef1c2e81c..4100616441 100644 --- a/src/aiperf/config/flags/_converter_profiling.py +++ b/src/aiperf/config/flags/_converter_profiling.py @@ -392,6 +392,29 @@ def _seed_rate_dimension( prof["rate"] = rate_lo +def apply_cancellation(prof: dict[str, Any], cli: CLIConfig) -> None: + """Route --request-cancellation-rate/-delay onto a phase dict. + + Shared with the YAML+CLI resolver so both paths carry the same shape and + the same dependency guard. + """ + delay_set = "request_cancellation_delay" in cli.model_fields_set + if cli.request_cancellation_rate: + cancel: dict[str, Any] = {"rate": cli.request_cancellation_rate} + if delay_set: + cancel["delay"] = cli.request_cancellation_delay + prof["cancellation"] = cancel + elif delay_set: + # Mirror --arrival-smoothness gating: refuse to silently drop a + # user-supplied flag whose dependency wasn't met. + raise ValueError( + "--request-cancellation-delay requires --request-cancellation-rate " + "to be set (cancellation is disabled when rate is unset). " + "Pass --request-cancellation-rate > 0 to enable cancellation, or " + "drop --request-cancellation-delay." + ) + + def _apply_profiling_ramps(prof: dict[str, Any], cli: CLIConfig) -> None: fields_set = cli.model_fields_set for field, key in _RAMP_FIELDS: @@ -559,21 +582,7 @@ def _validate_profiling(prof: dict[str, Any], cli: CLIConfig) -> None: # Deliberate override of the PhaseConfig default (which would # leave it unbounded). prof.setdefault("requests", 10) - delay_set = "request_cancellation_delay" in cli.model_fields_set - if cli.request_cancellation_rate: - cancel: dict[str, Any] = {"rate": cli.request_cancellation_rate} - if delay_set: - cancel["delay"] = cli.request_cancellation_delay - prof["cancellation"] = cancel - elif delay_set: - # Mirror --arrival-smoothness gating: refuse to silently drop a - # user-supplied flag whose dependency wasn't met. - raise ValueError( - "--request-cancellation-delay requires --request-cancellation-rate " - "to be set (cancellation is disabled when rate is unset). " - "Pass --request-cancellation-rate > 0 to enable cancellation, or " - "drop --request-cancellation-delay." - ) + apply_cancellation(prof, cli) def _maybe_auto_promote_trace( diff --git a/src/aiperf/config/flags/_converter_runtime.py b/src/aiperf/config/flags/_converter_runtime.py index 2dbca45609..aa70cb3f2f 100644 --- a/src/aiperf/config/flags/_converter_runtime.py +++ b/src/aiperf/config/flags/_converter_runtime.py @@ -150,6 +150,8 @@ def _build_communication(cli: CLIConfig) -> dict[str, Any] | None: def build_logging_runtime( cli: CLIConfig, + *, + base_api_port: bool = False, ) -> tuple[dict[str, Any], dict[str, Any]]: """Build (logging, runtime) dicts for AIPerfConfig from a CLIConfig. @@ -162,8 +164,11 @@ def build_logging_runtime( Pydantic defaults on ``RuntimeConfig`` / ``LoggingConfig``. Verbose-driven log-level/UI promotion still writes (it's a derived effect, not a default). """ - # api_host requires api_port to be set explicitly (or via env). - if cli.api_host is not None and cli.api_port is None: + # api_host requires api_port to be set explicitly (or via env, or -- when + # a config file is in play -- by the file itself: base_api_port says the + # YAML already supplies runtime.api_port, so demanding the flag would + # reject a supported combination). + if cli.api_host is not None and cli.api_port is None and not base_api_port: raise ValueError( "api_host requires api_port (or AIPERF_API_SERVER_PORT) to be set" ) diff --git a/src/aiperf/config/flags/_converter_telemetry.py b/src/aiperf/config/flags/_converter_telemetry.py index 3a785325a4..7a4626f568 100644 --- a/src/aiperf/config/flags/_converter_telemetry.py +++ b/src/aiperf/config/flags/_converter_telemetry.py @@ -333,13 +333,23 @@ def _resolve_stream_domains(value: Any) -> tuple[bool, bool]: ) -def build_otel(cli: CLIConfig) -> dict[str, Any]: - """Translate OTel CLI flags into the first-class OTel config dict.""" +def build_otel(cli: CLIConfig, *, base_metrics_url: bool = False) -> dict[str, Any]: + """Translate OTel CLI flags into the first-class OTel config dict. + + ``base_metrics_url`` says a YAML config file already supplies + ``otel.metrics_url``. Without it, ``-f base.yaml --gen-ai-provider X`` + is rejected for missing a flag whose value the config file provides -- + the same shape as ``build_mlflow``'s ``base_tracking_uri``. + """ otel: dict[str, Any] = {} cli_set = cli.model_fields_set if "otel_url" in cli_set and cli.otel_url is not None: otel["metrics_url"] = _normalize_otel_metrics_url(cli.otel_url) + elif base_metrics_url: + # The config file owns metrics_url; emit only the overrides so the + # deep-merge leaves it in place. + pass else: # ``--stream`` and ``--gen-ai-provider`` are OTel-only secondary # flags: they only take effect when ``--otel-url`` is set. Refuse @@ -425,13 +435,18 @@ def _apply_mlflow_secondary_fields(out: dict[str, Any], cli: CLIConfig) -> None: out["parent_run_id"] = cli.mlflow_parent_run_id -def build_mlflow(cli: CLIConfig) -> dict[str, Any]: +def build_mlflow(cli: CLIConfig, *, base_tracking_uri: bool = False) -> dict[str, Any]: """Translate MLflow CLI flags into the first-class MLflow config dict. Ports v1 ``_validate_mlflow_config``: refuses secondary MLflow flags without ``--mlflow-tracking-uri``, rejects empty strings on tracking_uri/experiment/artifact_glob entries, and normalizes whitespace on tracking_uri/experiment/run_name/artifact_globs. + + ``base_tracking_uri`` says a YAML config file already supplies + ``mlflow.tracking_uri``. Without it, ``-f base.yaml --mlflow-experiment X`` + would be rejected for missing a flag whose value the config file already + provides -- the same shape as ``build_wandb``'s ``base_enabled``. """ # Normalize artifact-glob entries first so an "empty glob" error # surfaces before the missing-tracking-uri error. @@ -442,13 +457,21 @@ def build_mlflow(cli: CLIConfig) -> dict[str, Any]: secondary_present = any( key in cli.model_fields_set for key in _MLFLOW_SECONDARY_FIELDS ) - if secondary_present: + if not secondary_present: + return {} + if not base_tracking_uri: raise ValueError( "--mlflow-experiment, --mlflow-run-name, --mlflow-tag, " "--mlflow-artifact-glob, and --mlflow-parent-run-id require " "--mlflow-tracking-uri to be set." ) - return {} + # The config file owns tracking_uri; emit only the overrides so the + # deep-merge leaves it in place. + from_yaml_base: dict[str, Any] = {} + _apply_mlflow_secondary_fields(from_yaml_base, cli) + if artifact_globs is not None: + from_yaml_base["artifact_globs"] = artifact_globs + return from_yaml_base out: dict[str, Any] = {"tracking_uri": tracking_uri} _apply_mlflow_secondary_fields(out, cli) diff --git a/src/aiperf/config/flags/_converter_warmup.py b/src/aiperf/config/flags/_converter_warmup.py index ab582971b8..51798a1600 100644 --- a/src/aiperf/config/flags/_converter_warmup.py +++ b/src/aiperf/config/flags/_converter_warmup.py @@ -76,6 +76,30 @@ def _warmup_pattern_type(w: dict[str, Any], cli: CLIConfig, s: set[str]) -> None w["concurrency"] = warmup_concurrency if warmup_concurrency is not None else 1 +def _warmup_override_pattern(w: dict[str, Any], cli: CLIConfig, s: set[str]) -> None: + """Emit only the pattern fields the user set, for an existing warmup phase. + + ``_warmup_pattern_type`` derives ``type``/``rate``/``concurrency`` together, + falling back to the non-warmup ``--concurrency`` / ``--request-rate`` / + ``--arrival-pattern`` when the warmup-specific flag is absent. That is right + when building a phase from nothing and wrong as an override: it would + recompute the phase type the config file declared from flags aimed at the + profiling phase. + """ + if "warmup_concurrency" in s and not isinstance(cli.warmup_concurrency, list): + w["concurrency"] = cli.warmup_concurrency + if "warmup_request_rate" in s and cli.warmup_request_rate is not None: + w["rate"] = cli.warmup_request_rate + if "warmup_arrival_pattern" in s: + match cli.warmup_arrival_pattern: + case ArrivalPattern.GAMMA: + w["type"] = PhaseType.GAMMA + case ArrivalPattern.CONSTANT: + w["type"] = PhaseType.CONSTANT + case _: + w["type"] = PhaseType.POISSON + + def _warmup_ramps(w: dict[str, Any], cli: CLIConfig, s: set[str]) -> None: def _pick(warmup_field: str, fallback_field: str) -> Any: if warmup_field in s: @@ -106,7 +130,7 @@ def _pick(warmup_field: str, fallback_field: str) -> Any: w["rate_ramp"] = {"duration": rr} -def build_warmup(cli: CLIConfig) -> dict[str, Any] | None: +def build_warmup(cli: CLIConfig, *, base_warmup: bool = False) -> dict[str, Any] | None: """Build a warmup phase dict from CLIConfig, or return None. The warmup phase is only emitted when the caller explicitly set one of the @@ -124,7 +148,13 @@ def build_warmup(cli: CLIConfig) -> dict[str, Any] | None: # "concurrency": 10, "requests": 50} """ s = cli.model_fields_set - if not ({"warmup_request_count", "warmup_num_sessions", "warmup_duration"} & s): + # base_warmup: a YAML config file already declares a warmup phase, so the + # trigger the CLI-only path requires is already satisfied -- the flags + # have somewhere to land, and dropping them would be the silent-ignore + # behavior this exists to prevent. + if not base_warmup and not ( + {"warmup_request_count", "warmup_num_sessions", "warmup_duration"} & s + ): # No warmup trigger -> no warmup phase. Refuse to silently drop # secondary warmup-only flags the user supplied — except under a # --scenario, where the auto-synthesized agentic warmup consumes @@ -139,9 +169,15 @@ def build_warmup(cli: CLIConfig) -> dict[str, Any] | None: "the grace period, or drop --warmup-grace-period." ) return None - w: dict[str, Any] = {"exclude_from_results": True} + # Overriding an existing phase emits only what the user set: the config + # file owns exclude_from_results, and the phase type it declares must not + # be recomputed from flags the user did not pass. + w: dict[str, Any] = {} if base_warmup else {"exclude_from_results": True} _warmup_count_field(w, cli) - _warmup_pattern_type(w, cli, s) + if base_warmup: + _warmup_override_pattern(w, cli, s) + else: + _warmup_pattern_type(w, cli, s) _warmup_ramps(w, cli, s) _apply_agentic_replay_fields(w, cli) if "warmup_prefill_concurrency" in s: @@ -156,7 +192,7 @@ def build_warmup(cli: CLIConfig) -> dict[str, Any] | None: # has a non-None default and so cannot be reliably distinguished from # explicit user input downstream), warmup_grace_period defaults to None, # so a non-None value means the user explicitly asked for it. - if "duration" not in w: + if "duration" not in w and not base_warmup: if cli.scenario is not None: # Under a scenario the agentic warmup barrier consumes the # grace via _apply_agentic_replay_fields; the user-declared diff --git a/src/aiperf/config/flags/_section_fields.py b/src/aiperf/config/flags/_section_fields.py index adc1a144ce..d6ae303811 100644 --- a/src/aiperf/config/flags/_section_fields.py +++ b/src/aiperf/config/flags/_section_fields.py @@ -83,6 +83,8 @@ "prompt_prefix_length", "prompt_prefix_shared_system_length", "prompt_prefix_user_context_length", + "prompt_random_corpus_style", + "prompt_random_range_ratio", "prompt_sequence_distribution", # ----- image modality ----- "image_width_mean", diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 01e7d10d68..3d2eb8d601 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -738,8 +738,11 @@ def url(self) -> str: Field( default=None, description=( - "Path to a YAML configuration file. " - "CLI flags override values from the config file." + "Path to a YAML configuration file. CLI flags override " + "values from the config file. A flag that cannot be applied " + "on top of a config file is rejected by name rather than " + "ignored, so a run never silently differs from what was " + "asked for." ), ), CLIParameter( diff --git a/src/aiperf/config/flags/resolver.py b/src/aiperf/config/flags/resolver.py index decb5c3546..5d5eadc576 100644 --- a/src/aiperf/config/flags/resolver.py +++ b/src/aiperf/config/flags/resolver.py @@ -7,9 +7,16 @@ (``aiperf profile`` and ``aiperf service``). When both are supplied, the YAML supplies the base configuration and any explicitly-set CLI flags on ``cli_config`` are deep-merged on top before AIPerfConfig validation -- so -``aiperf profile --config foo.yaml --search-recipe X --ttft-sla-ms 200`` -works the way users intuit instead of throwing +``aiperf profile --config foo.yaml --streaming --search-recipe X`` works the +way users intuit instead of throwing ``CLIConfig.endpoint.modelNames: Field required``. + +Not every flag can be applied this way. Anything this path cannot route is +rejected up front by ``reject_unrouted_cli_flags`` with an error naming the +flag, rather than being silently discarded -- see ``_config_flag_routing`` +and ``docs/dev/global-invariants.md`` for the classification and the tests +that keep it honest. ``--ttft-sla-ms`` is one such flag today: it does not +take effect under ``--config`` even alongside a recipe, so it errors. """ from __future__ import annotations @@ -18,8 +25,6 @@ import logging from typing import TYPE_CHECKING, Any -from pydantic.alias_generators import to_camel - from aiperf.common.enums import DatasetType from aiperf.common.phase import infer_legacy_phase_kind from aiperf.config.flags._resolver_gpu_telemetry import ( @@ -38,7 +43,7 @@ OUTPUT_FIELDS, SWEEPING_FIELDS, ) -from aiperf.plugin.enums import ArrivalPattern, DatasetFormat, PhaseType +from aiperf.plugin.enums import ArrivalPattern, PhaseType if TYPE_CHECKING: from pathlib import Path @@ -83,8 +88,14 @@ def resolve_config( return convert_cli_to_aiperf(cli_config) from aiperf.config import AIPerfConfig + from aiperf.config.flags._config_flag_routing import reject_unrouted_cli_flags from aiperf.config.loader import load_config_dict + # Fail before any merging: a flag this path cannot route would otherwise + # be dropped without a word, handing the user a benchmark that silently + # ignored what they asked for. + reject_unrouted_cli_flags(cli_config) + yaml_dict = load_config_dict(config_file) _normalize_loaded_benchmark_shorthands(yaml_dict) # Build the recipe's view of BenchmarkConfig from YAML + the @@ -111,10 +122,9 @@ def resolve_config( yaml_dict = normalize_gpu_telemetry_base_for_override(yaml_dict, overrides) yaml_dict = normalize_server_metrics_base_for_override(yaml_dict, overrides) merged = deep_merge(yaml_dict, overrides) if overrides else yaml_dict - _apply_dataset_synthesis_overrides(merged, cli_config) - _apply_dataset_filter_overrides(merged, cli_config) - _apply_random_pool_batch_size_overrides(merged, cli_config) + _apply_dataset_overrides(merged, cli_config) _apply_phase_loadgen_overrides(merged, cli_config) + _apply_warmup_overrides(merged, cli_config) promote_benchmark_magic_lists( merged, cli_config, @@ -178,7 +188,12 @@ def build_cli_overrides( build_tokenizer, ) from aiperf.config.flags._converter_runtime import build_logging_runtime - from aiperf.config.flags._converter_telemetry import build_wandb + from aiperf.config.flags._converter_telemetry import ( + build_mlflow, + build_network_latency, + build_otel, + build_wandb, + ) out: dict[str, Any] = {} _apply_endpoint_overrides(out, cli) @@ -193,21 +208,78 @@ def build_cli_overrides( _apply_optional_section( out, "wandb", build_wandb(cli, base_enabled=wandb_base_enabled) ) + # These three builders already existed and are used by the CLI-only + # converter; this path simply never called them, so --mlflow-*, + # --otel-url, and the network-latency flags were dropped whenever a + # config file was supplied. They gate on model_fields_set, so an unset + # flag leaves the YAML block alone. + _apply_optional_section(out, "network_latency", build_network_latency(cli)) + otel_base_url = benchmark_config is not None and bool( + benchmark_config.otel.metrics_url + ) + _apply_optional_section( + out, "otel", build_otel(cli, base_metrics_url=otel_base_url) + ) + mlflow_base_uri = benchmark_config is not None and bool( + benchmark_config.mlflow.tracking_uri + ) + _apply_optional_section( + out, "mlflow", build_mlflow(cli, base_tracking_uri=mlflow_base_uri) + ) + _apply_scenario_overrides(out, cli) + + if cli.goodput: + # Recipe-emitted SLOs win on key collision, matching + # convert_cli_to_aiperf: a goodput-style recipe owns the SLO contract + # for its run, so a stray --goodput must not override its thresholds. + slos = dict(cli.goodput) + slos.update(out.get("slos") or {}) + out["slos"] = slos if "no_sweep_table" in cli.model_fields_set: out["no_sweep_table"] = cli.no_sweep_table + # random_seed is an envelope key distinct from the dataset's own seed + # field (written by build_dataset onto the dataset block, which only + # feeds SessionIDGenerator). AIPerfConfig.random_seed is what + # resolve_run_seed threads into rng.init(...) for every child service + # process, so every other rng.derive(...) consumer -- synthetic prompt + # content, media generation, per-conversation turn shaping -- needs it + # too, matching _assemble_optional on the CLI-only path. + if "random_seed" in cli.model_fields_set: + out["random_seed"] = cli.random_seed + # Service-runtime CLI flags (--ui, --log-level, --verbose, ZMQ knobs) # land on RuntimeConfig / LoggingConfig in AIPerfConfig. build_logging_runtime # already gates on cli.model_fields_set, so YAML defaults stay # intact when the user didn't pass these flags. - logging_dict, runtime_dict = build_logging_runtime(cli) + runtime_base_port = benchmark_config is not None and ( + benchmark_config.runtime.api_port is not None + ) + logging_dict, runtime_dict = build_logging_runtime( + cli, base_api_port=runtime_base_port + ) _apply_optional_section(out, "logging", logging_dict) _apply_optional_section(out, "runtime", runtime_dict) return out +def _apply_scenario_overrides(out: dict[str, Any], cli: CLIConfig) -> None: + """Mirror the converter's scenario-lock fields onto the override dict. + + ``scenario`` and ``unsafe_override`` are plain data on ``BenchmarkConfig`` + rather than envelope keys, so ``_wrap_under_envelope`` moves them under + ``benchmark:`` exactly as ``_apply_scenario_fields`` relies on for the + CLI-only path. + """ + set_fields = cli.model_fields_set + if "scenario" in set_fields: + out["scenario"] = cli.scenario + if "unsafe_override" in set_fields: + out["unsafe_override"] = cli.unsafe_override + + def _apply_optional_section( out: dict[str, Any], key: str, value: dict[str, Any] | None ) -> None: @@ -339,7 +411,11 @@ def _apply_endpoint_overrides(out: dict[str, Any], cli: CLIConfig) -> None: ``--model-names`` lives on the CLIConfig endpoint section but maps to the ``models.items`` block on AIPerfConfig; everything else stays on ``endpoint``. """ - from aiperf.config.flags._converter_endpoint import _ENDPOINT_FIELD_MAP + from aiperf.config.flags._converter_endpoint import ( + _ENDPOINT_FIELD_MAP, + _maybe_build_reset_kv_cache, + _maybe_build_server_profiler, + ) ep_set = cli.model_fields_set & ENDPOINT_FIELDS if not ep_set: @@ -350,6 +426,15 @@ def _apply_endpoint_overrides(out: dict[str, Any], cli: CLIConfig) -> None: for cli_field, aiperf_key in _ENDPOINT_FIELD_MAP.items(): if cli_field in ep_set: endpoint[aiperf_key] = getattr(cli, cli_field) + # The probe sub-blocks have builders the CLI-only path already uses; this + # path simply never called them. They emit only fields the user set, so a + # YAML block keeps whatever the flags do not mention. + for key, built in ( + ("reset_kv_cache", _maybe_build_reset_kv_cache(cli)), + ("server_profiler", _maybe_build_server_profiler(cli)), + ): + if built is not None: + endpoint[key] = built if endpoint: out["endpoint"] = endpoint if "model_names" in ep_set and cli.model_names: @@ -376,219 +461,213 @@ def _apply_input_overrides(out: dict[str, Any], cli: CLIConfig) -> None: out.pop("endpoint", None) -def _apply_dataset_filter_overrides(merged: dict[str, Any], cli: CLIConfig) -> None: - if "dataset_filters" not in cli.model_fields_set: - return - - from aiperf.config.flags._converter_dataset import _parse_dataset_filters +def _locate_yaml_dataset(merged: dict[str, Any]) -> dict[str, Any] | None: + """Return the YAML dataset dict that CLI overrides apply to. + Accepts both the ``benchmark.dataset`` shorthand and the canonical + ``benchmark.datasets`` list. The multi-dataset branch below anticipates a + cap that has not been lifted yet, not current behavior: ``AIPerfConfig + .datasets`` is ``max_length=1``, and ``resolve_config`` validates before + overrides run, so a YAML with more than one dataset entry fails + validation before this function is ever reached. Kept as future-proofing + -- matching the long-standing convention of the dataset-filter and + synthesis overlays this function replaces, from when the cap did not + exist -- for whenever the limit lifts. + """ benchmark = merged.get("benchmark") if not isinstance(benchmark, dict): - raise ValueError("--dataset-filter requires a public dataset") - dataset = benchmark.get("dataset") - if not isinstance(dataset, dict): - datasets = benchmark.get("datasets") - if not isinstance(datasets, list) or not datasets: - raise ValueError("--dataset-filter requires a public dataset") - if len(datasets) > 1: - logger.warning( - "--dataset-filter with multiple YAML datasets applies only to " - "the first dataset" - ) - dataset = datasets[0] - if not isinstance(dataset, dict): - raise ValueError("--dataset-filter requires a public dataset") - if dataset.get("type") != DatasetType.PUBLIC: - raise ValueError("--dataset-filter requires a public dataset") - filters = dataset.setdefault("filters", {}) - filters.update(_parse_dataset_filters(cli.dataset_filters)) - - -def _first_yaml_dataset( - benchmark: dict[str, Any], *, warn_context: str -) -> dict[str, Any] | None: - """Resolve the singular ``dataset`` or first entry of ``datasets`` from a - merged YAML ``benchmark`` mapping. Returns ``None`` if neither is present. - - ``warn_context`` names the flag/feature in the "multiple datasets" warning - (e.g. ``"Batch-size flags"``), consistent with the convention shared by - ``_apply_dataset_filter_overrides`` and ``_apply_dataset_synthesis_overrides``. - """ + return None dataset = benchmark.get("dataset") if isinstance(dataset, dict): return dataset - datasets = benchmark.get("datasets") if not isinstance(datasets, list) or not datasets: return None if len(datasets) > 1: logger.warning( - "%s with multiple YAML datasets apply only to the first dataset", - warn_context, + "Dataset CLI flags with multiple YAML datasets apply only to the " + "first dataset" ) - dataset = datasets[0] - return dataset if isinstance(dataset, dict) else None + first = datasets[0] + return first if isinstance(first, dict) else None -# Maps CLIConfig attribute name -> (FileDataset field name, CLI flag display name). -# Used by _apply_random_pool_batch_size_overrides for gating and error messages. -_RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP: tuple[tuple[str, str, str], ...] = ( - ("prompt_batch_size", "prompt_batch_size", "--prompt-batch-size"), - ("image_batch_size", "image_batch_size", "--image-batch-size"), - ("audio_batch_size", "audio_batch_size", "--audio-batch-size"), - ("video_batch_size", "video_batch_size", "--video-batch-size"), -) +def _snake_to_camel(name: str) -> str: + head, *rest = name.split("_") + return head + "".join(word[:1].upper() + word[1:] for word in rest) + + +def _drop_alias_spellings(base: dict[str, Any], override: dict[str, Any]) -> None: + """Remove the camelCase spelling of every key the override sets. + + YAML configs address fields by their camelCase alias (``maxOsl``) while + ``build_dataset`` emits the snake_case field name (``max_osl``). Merging + the two as-is would leave both spellings of the same field in the dict and + let validation pick a winner -- which is exactly the kind of quiet + ambiguity this work exists to remove. Deleting the alias first makes the + CLI value win outright, whichever spelling the config file used. + """ + for key, value in override.items(): + alias = _snake_to_camel(key) + if alias != key and alias in base: + if isinstance(value, dict) and isinstance(base[alias], dict): + # Same field, different spelling: keep the YAML's siblings by + # folding its sub-dict onto the canonical key before merging. + base[key] = deep_merge(base.pop(alias), base.get(key) or {}) + else: + base.pop(alias, None) + if isinstance(value, dict) and isinstance(base.get(key), dict): + _drop_alias_spellings(base[key], value) + + +def _inert_dataset_flags( + cli: CLIConfig, + declared_type: Any, + declared_format: Any, + candidates: set[str], +) -> list[str]: + """Return the flags in ``candidates`` that contribute nothing on their own. + + Attribution is by re-running the real builder with exactly one flag set, + rather than by inspecting the combined result: a flag that emits nothing + alone emits nothing in company either, and asking ``build_dataset`` + directly means the answer cannot drift from what it actually does. + + A flag whose solo build raises is treated as contributing -- it is loud, + which is the property that matters, and the combined build already + succeeded. + """ + # CLIConfig is a TYPE_CHECKING-only import at module scope; it has to be + # imported for real here, and the narrow `except` below is what would + # have surfaced that rather than silently treating every flag as routed. + from aiperf.config.flags import CLIConfig as _CLIConfig + from aiperf.config.flags._converter_dataset import build_dataset + from aiperf.config.loader.errors import ConfigurationError + + inert: list[str] = [] + for field in candidates: + # Solo construction is deliberately outside the try: a CLIConfig + # cross-field validator failing on this single field alone would be + # a real bug in this function's premise (the combined CLIConfig + # already validated), and must surface rather than be swallowed by + # the except below, which exists for build_dataset only. + solo = _CLIConfig(**{field: getattr(cli, field)}) + try: + emitted = build_dataset( + solo, declared_type=declared_type, declared_format=declared_format + ) + except (ValueError, ConfigurationError): + # Loud on its own, which is the property that matters; the + # combined build already succeeded. Deliberately narrow: a broad + # `except` here would turn a bug in this function into a silently + # disabled guard. + continue + if not emitted: + inert.append(field) + return inert -def _apply_random_pool_batch_size_overrides( - merged: dict[str, Any], cli: CLIConfig +def _reject_inert_dataset_flags( + cli: CLIConfig, dataset: dict[str, Any], declared_type: Any ) -> None: - """Overlay explicit batch-size CLI flags onto a YAML-supplied random_pool dataset. - - In the YAML+CLI path ``_apply_input_overrides`` only routes ``headers`` and - ``extra_inputs``; every other ``INPUT_FIELDS`` member (including the four - batch-size fields added by this PR) was silently discarded. This function - closes that gap for the four fields that ``RandomPoolDatasetLoader`` consumes. - - Gating is on ``cli.model_fields_set`` — not truthiness, not ``is not None`` - against the field value — so an unset flag never clobbers a YAML-supplied value. - Zero is a valid value (``image/audio/video_batch_size=0`` disables that modality). - - Only applies to ``type: file`` datasets. Synthetic and public datasets have no - ``format`` field at all, so this function must not touch or reject them here. - Note this does NOT mean the flags take effect there: nothing in the YAML+CLI - path currently routes batch-size flags onto ``SyntheticDataset.prompts/ - images/audio/video.batch_size`` (``_apply_input_overrides`` only handles - ``headers``/``extra_inputs``), so a batch-size flag against a synthetic YAML - dataset has no effect -- a pre-existing gap this function does not close and - is out of scope to fix here. It logs a warning rather than dropping the flag - silently, since the neighbouring wrong-format case raises loudly. The CLI-only - path (no ``--config``) applies these flags correctly; only the YAML+CLI overlay - drops them. - - For a ``type: file`` dataset that isn't ``format: random_pool``, a ``ValueError`` - is raised with a message that names the flag and the format, matching the - friendly error the CLI-only path produces instead of letting the ``FileDataset`` - model validator fire a raw Pydantic trace. - - With multiple YAML datasets the override applies to the first dataset only, - consistent with the convention in ``_apply_dataset_synthesis_overrides`` and - ``_apply_dataset_filter_overrides``. + """Raise for dataset flags that resolve cleanly while doing nothing. + + Checking only whether the whole override came back empty let any inert + flag ride along with a routable one: the result was non-empty, the guard + never fired, and the inert flag was dropped exactly as before this work. + Every multi-flag command line touching the dataset escaped the guarantee, + so the reconciliation is per flag. + + ``declared_type`` is the caller's already-defaulted value (``dataset.get( + "type") or DatasetType.SYNTHETIC``), passed in rather than re-derived so + the error message reports the same type the check actually ran against + -- re-deriving it here previously meant the message rendered the raw, + possibly-``None`` YAML value instead. """ - set_fields = cli.model_fields_set & { - cli_attr for cli_attr, _, _ in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP - } - if not set_fields: - return + from aiperf.config.flags._config_flag_routing import ( + DATASET_FIELDS_OUTSIDE_INPUT, + DATASET_OVERRIDE_FIELDS, + flag_names_for, + ) + from aiperf.config.loader.errors import ConfigurationError - benchmark = merged.get("benchmark") - if not isinstance(benchmark, dict): + candidates = cli.model_fields_set & ( + DATASET_OVERRIDE_FIELDS | DATASET_FIELDS_OUTSIDE_INPUT + ) + if not candidates: return - dataset = _first_yaml_dataset(benchmark, warn_context="Batch-size flags") - if dataset is None: - return - if dataset.get("type") != DatasetType.FILE: - # Adjacent to a loud ValueError for a file dataset of the wrong format, so - # do not drop this one in silence: the flag genuinely has no effect here. - logger.warning( - "%s ignored: batch-size flags are only applied to a YAML dataset with " - "type: file and format: random_pool (got type: %s). The CLI-only path " - "(no --config) applies them normally.", - ", ".join( - flag - for cli_attr, _, flag in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP - if cli_attr in set_fields - ), - dataset.get("type"), - ) + inert = _inert_dataset_flags( + cli, + declared_type, + dataset.get("format"), + candidates, + ) + if not inert: return - # dataset.get("format") reads the raw pre-validation YAML dict, so an omitted - # `format:` key reads back as None here even though FileDataset.format defaults - # to DatasetFormat.SINGLE_TURN -- fall back to that default so the error message - # below reports the actual effective format instead of a misleading "None". - fmt = dataset.get("format") or DatasetFormat.SINGLE_TURN - if fmt != DatasetFormat.RANDOM_POOL: - flag_names = ", ".join( - flag - for cli_attr, _, flag in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP - if cli_attr in set_fields - ) - raise ValueError( - f"{flag_names} requires format: random_pool on the YAML dataset " - f"(got format: {fmt}). Either set format: random_pool in the dataset " - "config, or remove these flags." - ) + names = sorted("/".join(flag_names_for(f) or (f,)) for f in inert) + raise ConfigurationError( + f"These CLI flags have no effect on a dataset of type " + f"{str(declared_type)!r}: {', '.join(names)}. Remove them, or use a " + f"dataset type that supports them." + ) - for cli_attr, dataset_field, _ in _RANDOM_POOL_BATCH_SIZE_OVERRIDE_MAP: - if cli_attr in set_fields: - # FileDataset uses alias_generator=to_camel with extra="forbid": if the - # YAML already supplied this field under its camelCase alias (e.g. - # promptBatchSize, the shipped template idiom), writing the snake_case - # key here leaves both present and Pydantic rejects the snake_case one - # as extra. Drop whichever spelling is already there before writing. - dataset.pop(to_camel(dataset_field), None) - dataset.pop(dataset_field, None) - dataset[dataset_field] = getattr(cli, cli_attr) - - -def _apply_dataset_synthesis_overrides(merged: dict[str, Any], cli: CLIConfig) -> None: - """Overlay explicit synthesis flags onto a YAML-supplied dataset.""" - if not any( - field == "allow_dataset_wrap" or field.startswith("synthesis_") - for field in cli.model_fields_set - ): - return - from aiperf.config.dataset.trace import SynthesisConfig +def _apply_dataset_overrides(merged: dict[str, Any], cli: CLIConfig) -> None: + """Overlay explicitly-set dataset flags onto the YAML-supplied dataset. + + Delegates to ``build_dataset`` -- the same builder the CLI-only path uses + -- in override mode, so the two paths share one implementation of how a + flag maps onto the dataset shape. Previously this file carried a + hand-written routing function per field class (synthesis, filters, + random_pool batch sizes), each of which had to be kept in step with the + converter by hand; anything nobody wrote a function for was silently + dropped. + + ``build_dataset`` emits only keys backed by ``cli.model_fields_set``, so + an unset flag cannot clobber a YAML value, and the config file keeps + ownership of dataset type, format, and source. + """ + from aiperf.config.flags._config_flag_routing import DATASET_OVERRIDE_FIELDS from aiperf.config.flags._converter_dataset import ( - _apply_synthesis, - _reject_baseten_trace_unsupported_synthesis, + apply_implicit_media_batch_override, + build_dataset, ) - benchmark = merged.get("benchmark") - datasets = benchmark.get("datasets") if isinstance(benchmark, dict) else None - if not isinstance(datasets, list) or not datasets: - raise ValueError("synthesis flags require a file or public dataset") - if len(datasets) > 1: - logger.warning( - "Synthesis flags with multiple YAML datasets apply only to the first dataset" - ) - dataset = datasets[0] - if not isinstance(dataset, dict): - raise ValueError("synthesis flags require a file or public dataset") - if dataset.get("type") not in (DatasetType.FILE, DatasetType.PUBLIC): - logger.warning( - "Synthesis flags require a file or public dataset; ignoring them " - "for dataset type %r", - dataset.get("type"), - ) + dataset = _locate_yaml_dataset(merged) + if dataset is None: + if cli.model_fields_set & DATASET_OVERRIDE_FIELDS: + from aiperf.config.loader.errors import ConfigurationError + + raise ConfigurationError( + "Dataset CLI flags require a dataset in the config file, but " + "none was found under benchmark.dataset / benchmark.datasets." + ) return - if dataset.get("type") == DatasetType.FILE: - _reject_baseten_trace_unsupported_synthesis( - cli, - dataset.get("format"), - dataset_format_source="YAML format: baseten_trace", - ) + # A YAML dataset may omit `type`; validation resolves that to synthetic. + # Passing None here would instead switch build_dataset back to inferring + # the type from flags -- the CLI-only path, which materializes defaults + # meant for building a dataset from nothing. Default it so both spellings + # of "synthetic" take the same route. + declared_type = dataset.get("type") or DatasetType.SYNTHETIC + override = build_dataset( + cli, + declared_type=declared_type, + declared_format=dataset.get("format"), + ) + _reject_inert_dataset_flags(cli, dataset, declared_type) + if not override: + return - override = {"type": dataset.get("type")} - _apply_synthesis(override, cli) - if synthesis := override.get("synthesis"): - base = SynthesisConfig.model_validate( - dataset.get("synthesis") or {} - ).model_dump(by_alias=True, exclude_unset=True) - update = SynthesisConfig.model_validate(synthesis).model_dump( - by_alias=True, exclude_unset=True - ) - dataset["synthesis"] = deep_merge(base, update) + apply_implicit_media_batch_override(override, dataset) + _drop_alias_spellings(dataset, override) + merged_dataset = deep_merge(dataset, override) + dataset.clear() + dataset.update(merged_dataset) -# CLI loadgen flag -> phase field. Each entry is (loadgen_attr, phase_key). -# The CLI help promises "CLI flags override values from the config file"; -# this table makes that real for YAML-supplied phase shapes by overlaying -# the explicit CLI value onto the resolved profiling phase. _LOADGEN_PHASE_FIELD_MAP: tuple[tuple[str, str], ...] = ( ("request_count", "requests"), ("benchmark_duration", "duration"), @@ -598,6 +677,22 @@ def _apply_dataset_synthesis_overrides(merged: dict[str, Any], cli: CLIConfig) - ("request_rate", "rate"), ("user_centric_rate", "rate"), ("num_users", "users"), + # --num-conversations sets the profiling phase's session count as well as + # the dataset entry count, matching convert_cli_to_aiperf. It lives in + # INPUT_FIELDS rather than LOADGEN_FIELDS, hence the explicit mention in + # the gate below. + ("conversation_num", "sessions"), +) + +# Fields routed onto the profiling phase that are not LOADGEN_FIELDS members. +_NON_LOADGEN_PHASE_FIELDS: frozenset[str] = frozenset( + { + "conversation_num", + "fixed_schedule", + "fixed_schedule_auto_offset", + "fixed_schedule_start_offset", + "fixed_schedule_end_offset", + } ) @@ -626,7 +721,7 @@ def _apply_phase_loadgen_overrides(merged: dict[str, Any], cli: CLIConfig) -> No _apply_agentic_replay_fields, ) - loadgen_set = cli.model_fields_set & LOADGEN_FIELDS + loadgen_set = cli.model_fields_set & (LOADGEN_FIELDS | _NON_LOADGEN_PHASE_FIELDS) agentic_set = cli.model_fields_set.intersection(_AGENTIC_REPLAY_ROUTES) if not loadgen_set and not agentic_set: return @@ -673,7 +768,164 @@ def _apply_phase_loadgen_overrides(merged: dict[str, Any], cli: CLIConfig) -> No ): target["grace_period"] = cli.benchmark_grace_period + if cli.fixed_schedule: + # _profiling_phase_type does this when building a phase from flags. + # Set it before _apply_phase_shaping_overrides so the fixed-schedule + # offset routes, which validate against the phase type, can apply. + target["type"] = PhaseType.FIXED_SCHEDULE + _apply_agentic_replay_fields(target, cli) + _apply_phase_shaping_overrides(target, cli) + + +def _apply_phase_shaping_overrides(target: dict[str, Any], cli: CLIConfig) -> None: + """Route ramps, cancellation and arrival smoothness onto the phase. + + These were built only by the CLI-only converter, so under ``--config`` + they were dropped and later rejected. Reuse the converter's own helpers + rather than restating the field maps here: ``_RAMP_FIELDS`` and the + gamma-only routes stay in one place, and a new entry in either is picked + up by both paths. + + ``_apply_phase_specific_routes`` validates against ``prof["type"]``, which + on this path is whatever the YAML declared -- so ``--arrival-smoothness`` + against a non-gamma YAML phase is rejected with the converter's message + instead of being silently dropped. + """ + from aiperf.config.flags._converter_profiling import ( + _apply_phase_specific_routes, + _apply_profiling_ramps, + apply_cancellation, + ) + + _apply_profiling_ramps(target, cli) + apply_cancellation(target, cli) + if "type" in target: + _apply_phase_specific_routes(target, cli) + # An explicit start offset contradicts auto_offset, whose default is + # True; build_profiling clears it the same way at + # _converter_profiling.py:531. Without this the user gets a raw + # "auto_offset cannot be True when start_offset is set" from + # validation for a combination the CLI-only path accepts. + if target["type"] == PhaseType.FIXED_SCHEDULE and "start_offset" in target: + target.setdefault("auto_offset", False) + + +# CLI flags that shape the warmup phase. Derived from the section frozenset +# rather than restated so a new warmup_* flag is covered automatically. +_WARMUP_FIELDS: frozenset[str] = frozenset( + field for field in LOADGEN_FIELDS if field.startswith("warmup_") +) + + +def _find_warmup_phase(phases: list[Any]) -> dict[str, Any] | None: + """Return the YAML-declared warmup phase, if there is one.""" + for entry in phases: + if not isinstance(entry, dict): + continue + kind = infer_legacy_phase_kind(entry.get("name"), entry.get("kind")) + if kind == "warmup": + return entry + return None + + +def _apply_warmup_overrides(merged: dict[str, Any], cli: CLIConfig) -> None: + """Overlay explicitly-set ``--warmup-*`` flags onto the warmup phase. + + ``_apply_phase_loadgen_overrides`` deliberately targets only the profiling + phase, so that ``--request-count`` cannot clobber a warmup ramp. That left + the warmup flags with nowhere to go: they were dropped before the + classification gate and rejected after it. + + Three cases, matching what the CLI-only path does where it can: + + - the config file declares a warmup phase -> merge the set flags onto it, + leaving everything the user did not mention alone; + - it does not, but a trigger flag (--warmup-request-count / + --warmup-num-sessions / --warmup-duration) is set -> build the phase, as + ``convert_cli_to_aiperf`` does; + - neither -> raise, because a secondary flag such as + ``--warmup-concurrency`` has nothing to attach to and would otherwise be + silently ignored. + """ + from aiperf.config.flags._converter_warmup import build_warmup + from aiperf.config.loader.errors import ConfigurationError + + warmup_set = cli.model_fields_set & _WARMUP_FIELDS + if not warmup_set: + return + + benchmark = merged.get("benchmark") + if not isinstance(benchmark, dict): + return + phases = benchmark.get("phases") + if not isinstance(phases, list) or not phases: + return + + existing = _find_warmup_phase(phases) + built = build_warmup(cli, base_warmup=existing is not None) + if built is None: + from aiperf.config.flags._config_flag_routing import flag_names_for + + names = sorted("/".join(flag_names_for(f) or (f,)) for f in warmup_set) + raise ConfigurationError( + f"{', '.join(names)} needs a warmup phase to apply to. Declare one " + f"in the config file, or pass a warmup trigger " + f"(--warmup-request-count / --warmup-num-sessions / " + f"--warmup-duration)." + ) + + if existing is None: + phases.insert(0, {"name": "warmup", "kind": "warmup", **built}) + return + + _drop_alias_spellings(existing, built) + merged_warmup = deep_merge(existing, built) + _reject_incompatible_warmup_transition(merged_warmup, warmup_set) + existing.clear() + existing.update(merged_warmup) + + +def _reject_incompatible_warmup_transition( + merged_warmup: dict[str, Any], warmup_set: frozenset[str] | set[str] +) -> None: + """Raise before writing back a warmup override that would merge into an + invalid phase. + + ``_warmup_override_pattern`` emits ``rate``/``concurrency``/``type`` + independently, so a flag that only touches one side of a type transition + can produce a structurally invalid phase: ``--warmup-request-rate`` onto + an existing concurrency phase adds ``rate``, which ``ConcurrencyPhase`` + forbids as an extra field; ``--warmup-arrival-pattern`` onto one switches + ``type`` to a rate phase without a ``rate``, which that phase requires. + Passing both together is a complete, valid transition and is not + rejected here. + """ + from aiperf.config.loader.errors import ConfigurationError + + final_type = merged_warmup.get("type") + if ( + "warmup_request_rate" in warmup_set + and final_type == PhaseType.CONCURRENCY + and merged_warmup.get("rate") is not None + ): + raise ConfigurationError( + "--warmup-request-rate has no effect: the warmup phase is " + "type: concurrency, which has no rate field. Pass " + "--warmup-arrival-pattern too to switch it to a rate-controlled " + "phase, or drop --warmup-request-rate." + ) + if ( + "warmup_arrival_pattern" in warmup_set + and final_type != PhaseType.CONCURRENCY + and merged_warmup.get("rate") is None + and merged_warmup.get("rate_series") is None + ): + raise ConfigurationError( + f"--warmup-arrival-pattern switches the warmup phase to " + f"type: {final_type}, which requires a rate. Pass " + "--warmup-request-rate too, or drop --warmup-arrival-pattern." + ) def _reject_loadgen_target_collisions(fields_set: set[str]) -> None: diff --git a/src/aiperf/config/loader/parsing.py b/src/aiperf/config/loader/parsing.py index 3f154fc70f..20a851c12d 100644 --- a/src/aiperf/config/loader/parsing.py +++ b/src/aiperf/config/loader/parsing.py @@ -292,32 +292,38 @@ def parse_str_or_list_of_positive_values(input: Any) -> list[Any]: return output -def parse_file(value: str | None) -> Path | None: +def parse_file(value: str | Path | None) -> Path | None: """Parse an existing file/directory path from a CLI value. Args: - value: Path string from CLI/config input. ``None`` or an empty string - disables the path and returns ``None``. + value: Path string from CLI/config input, or a ``Path`` this + function already returned -- accepted as a pass-through so the + validator is idempotent when a caller round-trips an + already-parsed field value back through it (e.g. reconstructing + a model from another instance's field values). ``None`` or an + empty string disables the path and returns ``None``. Returns: A ``Path`` pointing to an existing file or directory, or ``None`` when no value was provided. Raises: - ValueError: If ``value`` is not a string, or if the path does not exist as - a file or directory. + ValueError: If ``value`` is not a string or Path, or if the path does not + exist as a file or directory. """ if not value: return None + elif isinstance(value, Path): + path = value elif not isinstance(value, str): raise ValueError(f"Expected a string, but got {type(value).__name__}") else: path = Path(value) - if path.is_file() or path.is_dir(): - return path - else: - raise ValueError(f"'{value}' is not a valid file or directory") + if path.is_file() or path.is_dir(): + return path + else: + raise ValueError(f"'{value}' is not a valid file or directory") def validate_sequence_distribution(v: str | None) -> str | None: diff --git a/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py b/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py index 108bf77693..f3a3a0efcc 100644 --- a/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py +++ b/tests/unit/cli_runner/test_random_pool_batch_size_yaml_override.py @@ -238,7 +238,7 @@ def test_batch_size_on_yaml_with_omitted_format_reports_effective_default( yaml_path = tmp_path / "no_format.yaml" yaml_path.write_text(yaml_content) cli = _cli(prompt_batch_size=4) - with pytest.raises(ValueError, match=re.escape("got format: single_turn")): + with pytest.raises(ValueError, match=re.escape("declares format: single_turn")): resolve_config(cli, yaml_path) @@ -298,39 +298,36 @@ def test_batch_size_flag_on_synthetic_yaml_does_not_raise( assert cfg.benchmark.datasets[0].type == "synthetic" -def test_prompt_batch_size_flag_on_synthetic_yaml_is_dropped_with_warning( - tmp_path: Path, caplog: pytest.LogCaptureFixture +def test_prompt_batch_size_flag_overrides_synthetic_yaml( + tmp_path: Path, ) -> None: - """Pins the known (pre-existing, out-of-scope-to-fix-here) gap: nothing in - the YAML+CLI path routes --prompt-batch-size onto a synthetic dataset's - prompts.batch_size, unlike the CLI-only path (no --config) where the same - flag applies correctly. The YAML value survives untouched and the explicit - CLI flag has no effect, inverting normal CLI-overrides-YAML precedence -- - so it must at least warn, since the adjacent wrong-format file-dataset case - raises loudly for the same flag. If the value assertion starts failing, the - gap has been closed -- update this test to assert the override took effect. + """The gap this used to pin is closed. + + Nothing in the YAML+CLI path routed --prompt-batch-size onto a synthetic + dataset's prompts.batch_size, so the flag was silently ignored and the + YAML value survived -- inverting normal CLI-overrides-YAML precedence. + build_dataset now serves both paths, so the flag applies here exactly as + it does without --config. """ yaml_path = _write_synthetic_yaml(tmp_path, batch_size=1) cli = _cli(prompt_batch_size=4) - with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): - cfg = resolve_config(cli, yaml_path) - assert cfg.benchmark.datasets[0].prompts.batch_size == 1 - assert "--prompt-batch-size ignored" in caplog.text + cfg = resolve_config(cli, yaml_path) + assert cfg.benchmark.datasets[0].prompts.batch_size == 4 -def test_image_batch_size_flag_on_synthetic_yaml_is_dropped_with_warning( - tmp_path: Path, caplog: pytest.LogCaptureFixture +def test_image_batch_size_flag_creates_the_images_block( + tmp_path: Path, ) -> None: - """Companion to the text case: --image-batch-size against a synthetic YAML - dataset with no `images:` block doesn't even create one -- the flag has - zero effect end to end, not just a value mismatch, and warns accordingly. + """Companion to the text case. + + --image-batch-size against a synthetic YAML with no `images:` block used + not to even create one: zero effect end to end, not just a value + mismatch. It now applies. """ yaml_path = _write_synthetic_yaml(tmp_path, batch_size=1) cli = _cli(image_batch_size=2) - with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): - cfg = resolve_config(cli, yaml_path) - assert cfg.benchmark.datasets[0].images is None - assert "--image-batch-size ignored" in caplog.text + cfg = resolve_config(cli, yaml_path) + assert cfg.benchmark.datasets[0].images.batch_size == 2 def test_batch_size_flag_applies_without_cli_custom_dataset_type( @@ -351,3 +348,72 @@ def test_batch_size_flag_applies_without_cli_custom_dataset_type( assert "custom_dataset_type" not in cli.model_fields_set cfg = resolve_config(cli, yaml_path) assert _dataset(cfg).prompt_batch_size == 4 + + +def _write_single_turn_yaml(tmp_path: Path, pool_path: Path) -> Path: + """Write a minimal YAML config with a non-random_pool file dataset.""" + yaml_content = f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: single_turn + path: {pool_path} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + cfg_path = tmp_path / "single_turn.yaml" + cfg_path.write_text(yaml_content) + return cfg_path + + +def test_batch_size_flag_applies_when_custom_dataset_type_overrides_yaml_format( + tmp_path: Path, +) -> None: + """--custom-dataset-type random_pool overrides a non-random_pool YAML + format; the batch-size guard must judge the overridden format, not the + stale pre-override YAML value. + + Regression: declared_format was read straight off the raw YAML dict + before build_dataset applies the --custom-dataset-type override, so this + raised "requires a random_pool dataset... declares format: single_turn" + even though --custom-dataset-type random_pool is exactly the format the + guard is checking for -- the guard and the resolver disagreed about what + the format was. + """ + pool = tmp_path / "pool.jsonl" + pool.touch() + yaml_path = _write_single_turn_yaml(tmp_path, pool) + cli = _cli(custom_dataset_type="random_pool", prompt_batch_size=4) + cfg = resolve_config(cli, yaml_path) + dataset = _dataset(cfg) + assert dataset.format == "random_pool" + assert dataset.prompt_batch_size == 4 + + +def test_image_shape_flag_on_synthetic_yaml_with_no_images_block_still_generates_images( + tmp_path: Path, +) -> None: + """--image-width-mean against a synthetic YAML with no `images:` block must + include images, not silently resolve to batch_size=0 ("disabled"). + + Regression: _apply_implicit_media_batch (which materializes batch_size=1 + whenever a media-shape flag is set without an explicit batch size) was + suppressed wholesale in override mode to avoid stomping a YAML batch size + the user never mentioned. But when the YAML has no images: block at all, + there is nothing to stomp, and the model default batch_size=0 means + "disabled" -- a multimodal benchmark request silently degrades to + text-only with no error, since endpoints/base_endpoint.py skips falsy + content strings. + """ + yaml_path = _write_synthetic_yaml(tmp_path) + cli = _cli(image_width_mean=64) + cfg = resolve_config(cli, yaml_path) + images = cfg.benchmark.datasets[0].images + assert images is not None + assert images.batch_size == 1 diff --git a/tests/unit/cli_runner/test_random_seed_envelope_override.py b/tests/unit/cli_runner/test_random_seed_envelope_override.py new file mode 100644 index 0000000000..b4cb47bf6c --- /dev/null +++ b/tests/unit/cli_runner/test_random_seed_envelope_override.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Regression: ``--random-seed`` under ``--config`` must reach the envelope seed. + +``build_dataset`` writes ``cli.random_seed`` onto the dataset block, but +``AIPerfConfig.random_seed`` -- the envelope-level seed that +``resolve_run_seed`` feeds into ``rng.init(...)`` for every child service +process -- is a distinct field that ``build_cli_overrides`` never wrote in +the YAML+CLI path. The dataset seed only drives ``SessionIDGenerator``; +every other ``rng.derive(...)`` consumer (synthetic prompt content, +image/audio/video generation, per-conversation turn count and delay) reads +off the envelope seed instead, so it stayed non-reproducible despite +``--random-seed`` appearing to take effect. +""" + +from __future__ import annotations + +from pathlib import Path + +from aiperf.config.flags import CLIConfig +from aiperf.config.flags.resolver import resolve_config + + +def _cli(**kwargs: object) -> CLIConfig: + """Build a CLIConfig with only the supplied fields in model_fields_set.""" + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +def _write_minimal_yaml(tmp_path: Path) -> Path: + yaml_content = """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + cfg_path = tmp_path / "base.yaml" + cfg_path.write_text(yaml_content) + return cfg_path + + +def test_random_seed_flag_sets_the_envelope_seed(tmp_path: Path) -> None: + """--random-seed under --config must reach AIPerfConfig.random_seed, + not just the dataset's own seed field.""" + yaml_path = _write_minimal_yaml(tmp_path) + cli = _cli(random_seed=42) + cfg = resolve_config(cli, yaml_path) + assert cfg.random_seed == 42 + + +def test_random_seed_flag_still_reaches_the_dataset_seed(tmp_path: Path) -> None: + """The existing dataset-seed routing must not regress while fixing the + envelope gap -- SessionIDGenerator takes the dataset seed with + precedence, so both fields must carry the flag's value.""" + yaml_path = _write_minimal_yaml(tmp_path) + cli = _cli(random_seed=42) + cfg = resolve_config(cli, yaml_path) + assert cfg.benchmark.datasets[0].random_seed == 42 + + +def test_random_seed_flag_absent_leaves_envelope_seed_unset(tmp_path: Path) -> None: + """An unset --random-seed must not clobber the YAML with a materialized + default; the envelope seed stays whatever the YAML (or its own default) + supplies.""" + yaml_path = _write_minimal_yaml(tmp_path) + cli = _cli() + assert "random_seed" not in cli.model_fields_set + cfg = resolve_config(cli, yaml_path) + assert cfg.random_seed is None diff --git a/tests/unit/common/config/test_config_validators.py b/tests/unit/common/config/test_config_validators.py index 9e03a09897..2370ac7286 100644 --- a/tests/unit/common/config/test_config_validators.py +++ b/tests/unit/common/config/test_config_validators.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from pathlib import Path from typing import Any import pytest @@ -9,12 +10,40 @@ coerce_value, normalize_http_url, normalize_http_urls, + parse_file, parse_str_as_numeric_dict, parse_str_or_dict_as_tuple_list, parse_str_or_list_of_positive_values, ) +class TestParseFile: + """Test suite for the parse_file function.""" + + def test_parse_file_path_output_returns_same_path(self, tmp_path: Path) -> None: + """Must be idempotent: re-applying to its own output must not raise. + + Regression: CLIConfig.input_file's BeforeValidator only accepted + ``str`` input, so re-constructing a CLIConfig from an + already-parsed ``cli.input_file`` (a Path, per the return type) + raised "Expected a string, but got PosixPath" -- surfaced by + _inert_dataset_flags's solo-reconstruction pattern + (`_CLIConfig(**{field: getattr(cli, field)})`), which round-trips + an already-validated field's value back through its own validator. + """ + target = tmp_path / "pool.jsonl" + target.touch() + parsed = parse_file(str(target)) + assert parse_file(parsed) == parsed + + def test_parse_file_none_input_returns_none(self) -> None: + assert parse_file(None) is None + + def test_parse_file_missing_path_raises_error(self) -> None: + with pytest.raises(ValueError, match="not a valid file or directory"): + parse_file("/no/such/path/here") + + class TestCoerceValue: """Test suite for the coerce_value function.""" diff --git a/tests/unit/config/test_config_flag_routing.py b/tests/unit/config/test_config_flag_routing.py new file mode 100644 index 0000000000..eb9068c05c --- /dev/null +++ b/tests/unit/config/test_config_flag_routing.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CLI flags combined with ``--config`` must never be silently ignored. + +``resolve_config`` historically routed only a small subset of ``CLIConfig`` +fields into the YAML-supplied base config; every other explicitly-set flag was +silently discarded. A user passing ``-f base.yaml --random-seed 42`` got a run +with a different seed than they asked for, with no diagnostic. + +These tests pin the guarantee that replaces that behavior: a flag that cannot +be routed under ``--config`` raises a ``ConfigurationError`` naming the flag, +rather than being dropped. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aiperf.config.flags import CLIConfig +from aiperf.config.flags._section_fields import ( + ACCURACY_FIELDS, + ENDPOINT_FIELDS, + INPUT_FIELDS, + LOADGEN_FIELDS, + OUTPUT_FIELDS, + SWEEPING_FIELDS, + TOKENIZER_FIELDS, +) +from aiperf.config.flags.resolver import resolve_config +from aiperf.config.loader.errors import ConfigurationError + +ALL_SECTIONS = ( + ACCURACY_FIELDS + | ENDPOINT_FIELDS + | INPUT_FIELDS + | LOADGEN_FIELDS + | OUTPUT_FIELDS + | SWEEPING_FIELDS + | TOKENIZER_FIELDS +) + + +@pytest.fixture +def base_yaml(tmp_path: Path) -> Path: + """A minimal, valid YAML config with a synthetic dataset.""" + cfg = tmp_path / "base.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +def cli(**kwargs: object) -> CLIConfig: + """Build a CLIConfig whose model_fields_set is exactly ``kwargs``.""" + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Unrouted flags must be loud +# --------------------------------------------------------------------------- + + +def test_unrouted_input_flag_raises_naming_the_flag(base_yaml: Path) -> None: + """--public-dataset would switch the dataset TYPE, so it errors. + + (--input-file, which this case used to cover, now overrides the path + within a file dataset -- switching the type is the part that stays + rejected.) + """ + with pytest.raises(ConfigurationError, match=r"--public-dataset"): + resolve_config(cli(public_dataset="sharegpt"), base_yaml) + + +def test_unrouted_sweeping_flag_raises(base_yaml: Path) -> None: + """Sweep bounds resolve cleanly and change nothing, so they must error. + + (The warmup flags this used to check are routed now.) + """ + with pytest.raises(ConfigurationError, match=r"--concurrency-min"): + resolve_config(cli(concurrency_min=2), base_yaml) + + +def test_dataset_type_switch_flag_raises(base_yaml: Path) -> None: + """--hf-weka-dataset implies a public dataset, switching the type.""" + with pytest.raises(ConfigurationError, match=r"--hf-weka-dataset"): + resolve_config(cli(hf_weka_dataset="some/dataset"), base_yaml) + + +def test_error_names_every_offending_flag(base_yaml: Path) -> None: + """All unrouted flags are reported at once, not one per run.""" + with pytest.raises(ConfigurationError) as excinfo: + resolve_config(cli(concurrency_min=2, public_dataset="sharegpt"), base_yaml) + message = str(excinfo.value) + assert "--concurrency-min" in message + assert "--public-dataset" in message + + +def test_error_mentions_config_flag_as_the_cause(base_yaml: Path) -> None: + """The message must tell users WHY the flag was rejected.""" + with pytest.raises(ConfigurationError, match=r"--config"): + resolve_config(cli(concurrency_min=2), base_yaml) + + +# --------------------------------------------------------------------------- +# Routed flags keep working +# --------------------------------------------------------------------------- + + +def test_routed_endpoint_flag_still_applies(base_yaml: Path) -> None: + """--streaming is routed; it must override YAML without raising.""" + resolved = resolve_config(cli(streaming=True), base_yaml) + assert resolved.benchmark.endpoint.streaming is True + + +def test_routed_output_flag_still_applies(base_yaml: Path, tmp_path: Path) -> None: + """--artifact-dir is routed through build_artifacts; must not raise.""" + target = tmp_path / "artifacts" + resolved = resolve_config(cli(artifact_directory=target), base_yaml) + assert Path(resolved.benchmark.artifacts.dir) == target + + +def test_no_flags_with_config_is_fine(base_yaml: Path) -> None: + """A bare --config invocation must not trip the guard.""" + resolved = resolve_config(cli(), base_yaml) + assert resolved.benchmark.models.items[0].name == "test-model" + + +# --------------------------------------------------------------------------- +# Magic lists: routed as sweep parameters, but only when list-shaped +# --------------------------------------------------------------------------- + + +def test_magic_list_isl_is_routed_as_sweep_parameter(base_yaml: Path) -> None: + """A list-shaped --isl becomes a sweep parameter, so it must not raise.""" + resolved = resolve_config(cli(prompt_input_tokens_mean=[128, 256]), base_yaml) + assert resolved.sweep is not None + assert resolved.sweep.parameters["datasets.default.prompts.isl.mean"] == [128, 256] + + +def test_scalar_isl_routes_onto_the_dataset(base_yaml: Path) -> None: + """The scalar form reaches the dataset block rather than a sweep. + + The magic-list form becomes a sweep parameter; the scalar form is routed + by _apply_dataset_overrides. Both must take effect -- neither may be + silently dropped. + """ + resolved = resolve_config(cli(prompt_input_tokens_mean=128), base_yaml) + assert resolved.benchmark.datasets[0].prompts.isl.mean == 128 + + +# --------------------------------------------------------------------------- +# The CLI-only path is untouched +# --------------------------------------------------------------------------- + + +def test_unrouted_flag_without_config_is_allowed() -> None: + """Without --config the converter handles every flag; no guard applies.""" + resolved = resolve_config( + cli( + random_seed=1234, + model_names=["m"], + urls=["http://localhost:8000"], + ), + None, + ) + assert resolved.benchmark.datasets[0].random_seed == 1234 + + +# --------------------------------------------------------------------------- +# Totality: no field may be silently unclassified +# --------------------------------------------------------------------------- + + +def test_every_cli_config_field_is_classified() -> None: + """EVERY CLIConfig field must be classified, not just sectioned ones. + + The section frozensets are opt-in: adding a field to ``CLIConfig`` + requires nobody to touch ``_section_fields.py``, and + ``test_section_fields_partition_cli_config`` only checks that section + entries exist on CLIConfig -- never the reverse. So an unsectioned field + was invisible to the guard and silently dropped under ``--config``, which + is how 17 fields (all mlflow_*, --otel-url, network_latency_*, --scenario) + came to be dropped without anyone noticing. + + Keying this on CLIConfig.model_fields instead makes classification + mandatory: a newly-added flag belongs to no set and fails here, at + authoring time. + """ + from aiperf.config.flags._config_flag_routing import ( + EXEMPT_FROM_CONFIG_ROUTING, + ROUTED_UNDER_CONFIG, + UNROUTED_UNDER_CONFIG, + ) + + unclassified = ( + frozenset(CLIConfig.model_fields) + - ROUTED_UNDER_CONFIG + - UNROUTED_UNDER_CONFIG + - EXEMPT_FROM_CONFIG_ROUTING + ) + assert not unclassified, ( + f"CLI fields are unclassified for the --config path: " + f"{sorted(unclassified)}. Every flag must be routed, listed in " + f"UNROUTED_UNDER_CONFIG so users get a loud error, or exempted in " + f"EXEMPT_FROM_CONFIG_ROUTING with a reason." + ) + + +def test_exempt_fields_are_not_also_classified() -> None: + """An exemption must not overlap the routed or unrouted sets.""" + from aiperf.config.flags._config_flag_routing import ( + EXEMPT_FROM_CONFIG_ROUTING, + ROUTED_UNDER_CONFIG, + UNROUTED_UNDER_CONFIG, + ) + + overlap = EXEMPT_FROM_CONFIG_ROUTING & (ROUTED_UNDER_CONFIG | UNROUTED_UNDER_CONFIG) + assert not overlap, f"exempt fields also classified: {sorted(overlap)}" + + +def test_every_section_field_is_classified() -> None: + """Each section field is either routed under --config or known-unrouted. + + This is the regression gate. A newly-added CLI flag that nobody wired into + the resolver lands in neither set and fails here, at authoring time, + instead of being silently dropped for users. + """ + from aiperf.config.flags._config_flag_routing import ( + ROUTED_UNDER_CONFIG, + UNROUTED_UNDER_CONFIG, + ) + + unclassified = ALL_SECTIONS - ROUTED_UNDER_CONFIG - UNROUTED_UNDER_CONFIG + assert not unclassified, ( + f"CLI fields are neither routed nor listed as unrouted under --config: " + f"{sorted(unclassified)}. Route them in resolver.py, or add them to " + f"UNROUTED_UNDER_CONFIG so users get a loud error instead of silence." + ) + + +def test_routed_and_unrouted_are_disjoint() -> None: + """A field cannot be both routed and rejected.""" + from aiperf.config.flags._config_flag_routing import ( + ROUTED_UNDER_CONFIG, + UNROUTED_UNDER_CONFIG, + ) + + assert not (ROUTED_UNDER_CONFIG & UNROUTED_UNDER_CONFIG) + + +def test_every_unrouted_field_has_a_real_flag_name() -> None: + """Error messages must name actual CLI flags, not guessed kebab-case.""" + from aiperf.config.flags._config_flag_routing import ( + UNROUTED_UNDER_CONFIG, + flag_names_for, + ) + + missing = sorted(f for f in UNROUTED_UNDER_CONFIG if not flag_names_for(f)) + assert not missing, f"No CLIParameter flag name found for: {missing}" + + +# --------------------------------------------------------------------------- +# Error messages must name a flag the user can find in their shell history +# --------------------------------------------------------------------------- + + +def test_dataset_source_in_type_fields_names_input_file_and_custom_dataset_type() -> ( + None +): + """``DATASET_SOURCE_IN_TYPE_FIELDS`` must actually name the two flags its + own docstring describes ("the trace file and the loader that reads it"), + not sit empty. + + Regression: `14e94535f` moved ``input_file``/``custom_dataset_type`` out + of ``DATASET_SOURCE_FIELDS`` and introduced this set as their documented + new home, but never populated it -- routing was (and is) still correct, + since both land in ``ROUTED_UNDER_CONFIG`` via the ``INPUT_FIELDS - + DATASET_SOURCE_FIELDS`` subtraction regardless of this set's contents, + which made the emptiness easy to miss. An empty set under a docstring + describing exactly this behavior misleads anyone auditing routing for + these flags into concluding the opposite of what actually happens. + """ + from aiperf.config.flags._config_flag_routing import ( + DATASET_SOURCE_IN_TYPE_FIELDS, + ROUTED_UNDER_CONFIG, + ) + + assert {"input_file", "custom_dataset_type"} == DATASET_SOURCE_IN_TYPE_FIELDS + assert DATASET_SOURCE_IN_TYPE_FIELDS <= ROUTED_UNDER_CONFIG + + +def test_error_lists_every_spelling_of_a_multi_alias_flag(base_yaml: Path) -> None: + """Naming only the first declared spelling sends users looking for a flag + they never typed. + + ``--sweep-variant`` was reported as ``--variant``. Since the resolver + cannot see which spelling was typed, it names them all. + """ + with pytest.raises(ConfigurationError) as excinfo: + resolve_config(cli(sweep_variants=["concurrency=2"]), base_yaml) + message = str(excinfo.value) + assert "--variant" in message + assert "--sweep-variant" in message diff --git a/tests/unit/config/test_config_input_overrides.py b/tests/unit/config/test_config_input_overrides.py new file mode 100644 index 0000000000..431e0d8ca7 --- /dev/null +++ b/tests/unit/config/test_config_input_overrides.py @@ -0,0 +1,694 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Dataset-shaping CLI flags must override a YAML-supplied dataset. + +Rather than hand-writing a routing function per field class, the YAML+CLI +path reuses ``build_dataset`` -- the same builder the CLI-only path uses -- +with the dataset type supplied by the YAML instead of inferred from flags. + +Two properties matter and are pinned here: + +1. an explicitly-set flag reaches the resolved dataset, and +2. a flag the user did NOT set never overwrites the YAML value, including + the defaults ``build_dataset`` materializes for the CLI-only path + (``images.batch_size``, ``turns.stddev``). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aiperf.config import AIPerfConfig +from aiperf.config.flags import CLIConfig +from aiperf.config.flags.resolver import resolve_config +from aiperf.config.loader.errors import ConfigurationError + + +def cli(**kwargs: object) -> CLIConfig: + """Build a CLIConfig whose model_fields_set is exactly ``kwargs``.""" + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +def dataset(cfg: AIPerfConfig): + return cfg.benchmark.datasets[0] + + +@pytest.fixture +def synthetic_yaml(tmp_path: Path) -> Path: + """Synthetic dataset with values a CLI flag could clobber.""" + cfg = tmp_path / "synthetic.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + random_seed: 11 + entries: 16 + prompts: + isl: + mean: 32 + stddev: 4 + osl: + mean: 8 + stddev: 1 + turns: + mean: 3 + stddev: 2 + images: + width: + mean: 64 + batch_size: 4 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +@pytest.fixture +def random_pool_yaml(tmp_path: Path) -> Path: + pool = tmp_path / "pool.jsonl" + pool.write_text('{"text": "hi"}\n') + cfg = tmp_path / "random_pool.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: random_pool + path: {pool} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +# --------------------------------------------------------------------------- +# Flags reach the resolved dataset +# --------------------------------------------------------------------------- + + +def test_random_seed_overrides_yaml(synthetic_yaml: Path) -> None: + assert ( + dataset(resolve_config(cli(random_seed=99), synthetic_yaml)).random_seed == 99 + ) + + +def test_image_width_mean_overrides_yaml(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(image_width_mean=128), synthetic_yaml)) + assert resolved.images.width.mean == 128 + + +def test_scalar_isl_overrides_yaml(synthetic_yaml: Path) -> None: + """The scalar form must now route, not just the magic-list form.""" + resolved = dataset( + resolve_config(cli(prompt_input_tokens_mean=256), synthetic_yaml) + ) + assert resolved.prompts.isl.mean == 256 + + +def test_prefix_pool_size_overrides_yaml(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(prompt_prefix_pool_size=7), synthetic_yaml)) + assert resolved.prefix_prompts.pool_size == 7 + + +def test_conversation_turn_mean_overrides_yaml(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(conversation_turn_mean=5), synthetic_yaml)) + assert resolved.turns.mean == 5 + + +def test_audio_length_mean_overrides_yaml(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(audio_length_mean=2.5), synthetic_yaml)) + assert resolved.audio.length.mean == 2.5 + + +def test_video_duration_overrides_yaml(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(video_duration=9), synthetic_yaml)) + assert resolved.video.duration == 9 + + +def test_rankings_passages_mean_overrides_yaml(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(rankings_passages_mean=6), synthetic_yaml)) + assert resolved.rankings.passages.mean == 6 + + +# --------------------------------------------------------------------------- +# Unset flags must never clobber YAML +# --------------------------------------------------------------------------- + + +def test_unset_fields_leave_yaml_untouched(synthetic_yaml: Path) -> None: + resolved = dataset(resolve_config(cli(), synthetic_yaml)) + assert resolved.random_seed == 11 + assert resolved.prompts.isl.mean == 32 + assert resolved.turns.stddev == 2 + assert resolved.images.batch_size == 4 + + +def test_image_width_flag_does_not_reset_yaml_batch_size(synthetic_yaml: Path) -> None: + """build_dataset materializes images.batch_size=1 for the CLI-only path. + + Leaking that default into the override would silently reset a YAML batch + size the user never mentioned -- the exact class of bug this work exists + to remove. + """ + resolved = dataset(resolve_config(cli(image_width_mean=128), synthetic_yaml)) + assert resolved.images.width.mean == 128 + assert resolved.images.batch_size == 4 + + +def test_turn_mean_flag_does_not_reset_yaml_turn_stddev(synthetic_yaml: Path) -> None: + """Likewise for the turns.stddev=0 default.""" + resolved = dataset(resolve_config(cli(conversation_turn_mean=5), synthetic_yaml)) + assert resolved.turns.mean == 5 + assert resolved.turns.stddev == 2 + + +def test_isl_flag_does_not_reset_yaml_osl(synthetic_yaml: Path) -> None: + resolved = dataset( + resolve_config(cli(prompt_input_tokens_mean=256), synthetic_yaml) + ) + assert resolved.prompts.osl.mean == 8 + + +def test_isl_mean_flag_does_not_reset_yaml_isl_stddev(synthetic_yaml: Path) -> None: + """Likewise for prompts.isl.stddev -- --isl only sets prompts.isl.mean.""" + resolved = dataset( + resolve_config(cli(prompt_input_tokens_mean=256), synthetic_yaml) + ) + assert resolved.prompts.isl.mean == 256 + assert resolved.prompts.isl.stddev == 4 + + +def test_osl_mean_flag_does_not_reset_yaml_osl_stddev(synthetic_yaml: Path) -> None: + """Likewise for prompts.osl.stddev -- --osl only sets prompts.osl.mean.""" + resolved = dataset( + resolve_config(cli(prompt_output_tokens_mean=64), synthetic_yaml) + ) + assert resolved.prompts.osl.mean == 64 + assert resolved.prompts.osl.stddev == 1 + + +# --------------------------------------------------------------------------- +# The YAML owns the dataset type and source +# --------------------------------------------------------------------------- + + +def test_synthetic_only_flag_on_file_dataset_is_rejected( + random_pool_yaml: Path, +) -> None: + """A synthetic-shaping flag against a YAML file dataset must say so. + + The declared type comes from the YAML, so the converter's file-dataset + guard has to fire on it rather than on --input-file being absent. + """ + with pytest.raises((ValueError, ConfigurationError)) as excinfo: + resolve_config(cli(prompt_prefix_pool_size=4), random_pool_yaml) + assert "prefix" in str(excinfo.value).lower() + + +def test_random_pool_batch_size_still_routes(random_pool_yaml: Path) -> None: + """Regression guard for PR #1274 behavior after the refactor.""" + resolved = dataset(resolve_config(cli(prompt_batch_size=7), random_pool_yaml)) + assert resolved.prompt_batch_size == 7 + + +def test_zero_batch_size_overrides_yaml(tmp_path: Path) -> None: + """``--prompt-batch-size 0`` must win over a non-zero YAML value. + + Zero disables the text modality on random_pool (FileDataset allows ge=0 + as of the base branch). It is also falsy, so an override path gating on + truthiness rather than ``model_fields_set`` would drop it and leave the + YAML batch size in place -- text still enabled, contrary to the request. + """ + pool = tmp_path / "pool.jsonl" + pool.write_text('{"text": "hi"}\n') + cfg = tmp_path / "rp.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: random_pool + path: {pool} + prompt_batch_size: 4 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + assert dataset(resolve_config(cli(prompt_batch_size=0), cfg)).prompt_batch_size == 0 + # ...and an unset flag still leaves the YAML value alone. + assert dataset(resolve_config(cli(), cfg)).prompt_batch_size == 4 + + +def test_public_dataset_flag_remains_rejected(synthetic_yaml: Path) -> None: + """--public-dataset would swap the dataset type the YAML declared.""" + with pytest.raises(ConfigurationError, match=r"--public-dataset"): + resolve_config(cli(public_dataset="sharegpt"), synthetic_yaml) + + +# --------------------------------------------------------------------------- +# --num-conversations: must match CLI-only semantics, not error +# --------------------------------------------------------------------------- + + +def test_conversation_num_sets_entries_and_sessions(synthetic_yaml: Path) -> None: + """``--num-conversations`` must do under --config what it does without it. + + On the CLI-only path the flag sets both ``dataset.entries`` and the + profiling phase's ``sessions``. Under --config it produced neither, and + the resolver rejected it with "no effect on a dataset of type + 'synthetic'" -- factually wrong, on one of the most-used flags in the + tool. + """ + resolved = resolve_config(cli(conversation_num=13), synthetic_yaml) + assert dataset(resolved).entries == 13 + profiling = [p for p in resolved.benchmark.phases if p.name == "profiling"] + assert profiling and profiling[0].sessions == 13 + + +def test_conversation_num_dataset_entries_sets_entries(synthetic_yaml: Path) -> None: + """The explicit entry-count flag wins for ``entries``.""" + resolved = resolve_config(cli(conversation_num_dataset_entries=21), synthetic_yaml) + assert dataset(resolved).entries == 21 + + +def test_request_count_does_not_become_dataset_entries(synthetic_yaml: Path) -> None: + """--request-count must not silently reset a YAML ``entries``. + + _resolve_entries falls back to request_count on the CLI-only path, where + it builds a dataset from nothing. As an override that fallback would + overwrite the config file's entry count from an unrelated loadgen flag. + """ + resolved = resolve_config(cli(request_count=50), synthetic_yaml) + assert dataset(resolved).entries == 16 # the YAML value + + +# --------------------------------------------------------------------------- +# Per-flag reconciliation: a flag is not excused by its neighbours +# --------------------------------------------------------------------------- + + +def test_inert_flag_paired_with_routable_one_still_errors( + synthetic_yaml: Path, +) -> None: + """An unroutable dataset flag must be caught even in company. + + The guard used to fire only when build_dataset produced nothing at all, + so pairing an inert flag with a working one left the result non-empty + and the inert flag was discarded exactly as before this work. Every + multi-flag command line touching the dataset escaped the guarantee. + """ + with pytest.raises(ConfigurationError, match=r"--synthesis-max-isl"): + resolve_config( + cli(prompt_input_tokens_mean=128, synthesis_max_isl=512), synthetic_yaml + ) + + +def test_inert_media_flag_paired_on_file_dataset_errors( + random_pool_yaml: Path, +) -> None: + """Same, for the media flags against a file dataset.""" + with pytest.raises(ConfigurationError, match=r"--image-width-mean"): + resolve_config(cli(prompt_batch_size=4, image_width_mean=128), random_pool_yaml) + + +def test_two_routable_flags_together_are_fine(synthetic_yaml: Path) -> None: + """The reconciliation must not reject flags that do take effect.""" + resolved = dataset( + resolve_config( + cli(prompt_input_tokens_mean=128, image_width_mean=64), synthetic_yaml + ) + ) + assert resolved.prompts.isl.mean == 128 + assert resolved.images.width.mean == 64 + + +def test_dataset_flag_outside_input_fields_is_reconciled( + synthetic_yaml: Path, +) -> None: + """--inter-turn-delay-cap-seconds is dataset-carried but not INPUT_FIELDS. + + Fields outside INPUT_FIELDS were never considered by the guard at all, + so this one no-oped silently on a synthetic dataset. + """ + with pytest.raises(ConfigurationError, match=r"--inter-turn-delay-cap-seconds"): + resolve_config(cli(inter_turn_delay_cap_seconds=3.0), synthetic_yaml) + + +# --------------------------------------------------------------------------- +# YAML-declared baseten_trace: the guards must read the declared identity +# --------------------------------------------------------------------------- + + +@pytest.fixture +def baseten_yaml(tmp_path: Path) -> Path: + """A file dataset that declares format: baseten_trace in the YAML.""" + trace = tmp_path / "baseten.jsonl" + trace.write_text('{"text": "hi"}\n') + cfg = tmp_path / "baseten.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: baseten_trace + path: {trace} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("replay_speedup", 2.0), + ("max_idle_gap_cap_seconds", 3.0), + ("trace_session_sample_ratio", 0.5), + ("open_loop_replay", True), + ("omit_kv_hints", True), + ], +) +def test_baseten_only_flags_accepted_on_yaml_declared_baseten_trace( + baseten_yaml: Path, field: str, value: object +) -> None: + """These are baseten_trace-only, and the YAML declares baseten_trace. + + The guard inferred the loader from --input-file / --custom-dataset-type, + which the --config path rejects outright because the config file owns + the dataset source and format. So a supported combination failed before + the run started. + """ + resolved = dataset(resolve_config(cli(**{field: value}), baseten_yaml)) + assert getattr(resolved, field) == value + + +def test_baseten_only_flag_still_rejected_on_other_trace_format( + random_pool_yaml: Path, +) -> None: + """The guard must still fire when the YAML declares a different loader.""" + with pytest.raises(ValueError, match="baseten_trace"): + resolve_config(cli(replay_speedup=2.0), random_pool_yaml) + + +def test_baseten_extra_input_collision_rejected_from_yaml_format( + baseten_yaml: Path, +) -> None: + """Loader-injected extras collide whether the format came from YAML or CLI. + + The collision guard keyed off cli.custom_dataset_type, so under --config + it never ran and the user's value was clobbered on the wire instead. + """ + with pytest.raises(ValueError, match="min_tokens"): + resolve_config(cli(extra_inputs=["min_tokens:5"]), baseten_yaml) + + +def test_synthetic_only_flag_message_is_actionable_under_config( + random_pool_yaml: Path, +) -> None: + """The advice must be something a --config user can act on. + + The CLI-only wording says to remove --input-file / --public-dataset. On + this path the user passed neither, and both are rejected outright, so + that advice is unfollowable. Point at the config file instead -- the same + correction already made for --dataset-filter one function over. + """ + with pytest.raises(ValueError) as excinfo: + resolve_config(cli(prompt_prefix_pool_size=4), random_pool_yaml) + message = str(excinfo.value) + assert "--input-file" not in message + assert "--public-dataset" not in message + assert "config file" in message + + +def test_batch_size_message_is_actionable_under_config(tmp_path: Path) -> None: + """Same for the batch-size variant of the guard.""" + trace = tmp_path / "t.jsonl" + trace.write_text('{"text": "hi"}\n') + cfg = tmp_path / "mooncake.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: mooncake_trace + path: {trace} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + with pytest.raises(ValueError) as excinfo: + resolve_config(cli(image_batch_size=2), cfg) + message = str(excinfo.value) + assert "--input-file" not in message + assert "config file" in message + + +def test_dataset_without_explicit_type_uses_override_mode(tmp_path: Path) -> None: + """A YAML dataset may omit ``type``; validation resolves it to synthetic. + + Passing that absent type through as None switched build_dataset back to + inferring from flags -- the CLI-only path, which materializes defaults + built for constructing a dataset from nothing. Both spellings of + "synthetic" must take the same route. + """ + cfg = tmp_path / "no_type.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + entries: 16 + turns: + mean: 3 + stddev: 2 + images: + width: + mean: 64 + batch_size: 4 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + resolved = dataset(resolve_config(cli(image_width_mean=128), cfg)) + assert resolved.type == "synthetic" + assert resolved.images.width.mean == 128 + # Nothing the user did not set may be disturbed. + assert resolved.images.batch_size == 4 + assert resolved.turns.stddev == 2 + assert resolved.entries == 16 + + +def test_trace_idle_gap_cap_routes_on_a_trace_dataset(tmp_path: Path) -> None: + """--trace-idle-gap-cap-seconds is carried by build_dataset for traces. + + It was classified unrouted from a probe run only against a synthetic + dataset, where it correctly emits nothing -- so this PR hard-rejected a + flag that works. + """ + trace = tmp_path / "t.jsonl" + trace.write_text('{"text": "hi"}\n') + cfg = tmp_path / "trace.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: mooncake_trace + path: {trace} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + resolved = dataset(resolve_config(cli(trace_idle_gap_cap_seconds=3.0), cfg)) + assert resolved.trace_idle_gap_cap_seconds == 3.0 + + +def test_trace_idle_gap_cap_still_errors_on_synthetic(synthetic_yaml: Path) -> None: + """On a synthetic dataset it genuinely carries nothing, so it must be loud.""" + with pytest.raises(ConfigurationError, match=r"--trace-idle-gap-cap-seconds"): + resolve_config(cli(trace_idle_gap_cap_seconds=3.0), synthetic_yaml) + + +# --------------------------------------------------------------------------- +# Dataset source: override within the declared type, never switch it +# --------------------------------------------------------------------------- + + +def test_input_file_swaps_the_path_on_a_file_dataset( + random_pool_yaml: Path, tmp_path: Path +) -> None: + """Running the same benchmark against a different trace is a normal loop. + + --model already overrides models.items and --url overrides + endpoint.urls, so the config file does not own identity in general; + there is no reason the dataset path should be the exception. + """ + other = tmp_path / "other.jsonl" + other.write_text('{"text": "other"}\n') + resolved = dataset( + resolve_config(CLIConfig(input_file=str(other)), random_pool_yaml) + ) + assert Path(resolved.path) == other + + +def test_input_file_leaves_the_rest_of_the_dataset_alone( + tmp_path: Path, +) -> None: + """Swapping the path must not disturb the rest of the block.""" + pool = tmp_path / "pool.jsonl" + pool.write_text('{"text": "hi"}\n') + other = tmp_path / "other.jsonl" + other.write_text('{"text": "other"}\n') + cfg = tmp_path / "rp.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: random_pool + path: {pool} + prompt_batch_size: 4 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + resolved = dataset(resolve_config(CLIConfig(input_file=str(other)), cfg)) + assert Path(resolved.path) == other + assert resolved.format == "random_pool" + assert resolved.prompt_batch_size == 4 + + +def test_custom_dataset_type_swaps_the_loader(tmp_path: Path) -> None: + """--custom-dataset-type changes the loader within type: file.""" + trace = tmp_path / "t.jsonl" + trace.write_text('{"text": "hi"}\n') + cfg = tmp_path / "mooncake.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: mooncake_trace + path: {trace} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + resolved = dataset(resolve_config(cli(custom_dataset_type="random_pool"), cfg)) + assert resolved.format == "random_pool" + + +def test_input_file_against_a_synthetic_config_errors( + synthetic_yaml: Path, tmp_path: Path +) -> None: + """Switching the dataset TYPE is a different thing, and stays rejected. + + The variants barely overlap -- FileDataset carries a dozen fields + PublicDataset rejects -- so a type switch silently invalidates whole + groups of keys the user wrote in the config file. + """ + other = tmp_path / "other.jsonl" + other.write_text('{"text": "other"}\n') + with pytest.raises(ValueError, match=r"synthetic"): + resolve_config(CLIConfig(input_file=str(other)), synthetic_yaml) + + +def test_public_dataset_still_switches_type_and_errors( + random_pool_yaml: Path, +) -> None: + with pytest.raises(ConfigurationError, match=r"--public-dataset"): + resolve_config(cli(public_dataset="sharegpt"), random_pool_yaml) + + +def test_inert_flag_message_reports_synthetic_for_an_omitted_yaml_type( + tmp_path: Path, +) -> None: + """An inert flag against a YAML dataset with no `type:` key must name the + type it actually resolves to (synthetic), not the raw None. + + Regression: the inert-flag check correctly defaults `dataset.get("type") + or DatasetType.SYNTHETIC` for the check itself, but the error message + interpolated the raw (unset) value, so an omitted `type:` -- legal, and + validated as synthetic -- printed "dataset of type None". Needs a + non-dataset flag (--streaming) alongside the inert one: without it, + ``build_cli_overrides`` returns nothing, ``pre_merged`` and ``merged`` + alias the same dict, and the preliminary ``AIPerfConfig.model_validate`` + call's own type-defaulting mutates the shared dict in place before this + check runs, masking the bug. + """ + cfg_path = tmp_path / "no_type.yaml" + cfg_path.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: {} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + with pytest.raises(ConfigurationError, match=r"dataset of type 'synthetic'"): + resolve_config(cli(synthesis_max_isl=100, streaming=True), cfg_path) diff --git a/tests/unit/config/test_config_override_invariants.py b/tests/unit/config/test_config_override_invariants.py new file mode 100644 index 0000000000..f0a9e24195 --- /dev/null +++ b/tests/unit/config/test_config_override_invariants.py @@ -0,0 +1,554 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Mechanical invariants over every dataset flag, not a hand-picked sample. + +Two properties are checked for all of ``INPUT_FIELDS`` at once, so a field +added later is covered without anyone remembering to write a test for it: + +**No spontaneous keys.** In override mode ``build_dataset`` must emit only +values the user actually supplied. The detector runs the builder twice with +two *different* values for the same field: any emitted key whose value is +identical across both runs cannot be derived from the input, so it is a +materialized default. Merged onto a YAML dataset, such a key silently +overwrites a value the user never mentioned -- how ``images.batch_size=1`` +and ``turns.stddev=0`` would have leaked before this was guarded. + +**No dead routing.** Every field the registry claims is routed under +``--config`` must actually change the resolved config. ``ROUTED_UNDER_CONFIG`` +computes the input section as a complement, so a newly-added field is +classified as routed automatically; this test is what stops that convenience +from quietly re-opening the bug for a field ``build_dataset`` does not carry. +""" + +from __future__ import annotations + +import enum +import types +import typing +from pathlib import Path +from typing import Any + +import pytest + +from aiperf.common.enums import DatasetType +from aiperf.config.flags import CLIConfig +from aiperf.config.flags._config_flag_routing import ( + DATASET_OVERRIDE_FIELDS, + MAGIC_LIST_ONLY_UNDER_CONFIG, + ROUTED_UNDER_CONFIG, +) +from aiperf.config.flags._converter_dataset import build_dataset +from aiperf.config.flags.resolver import resolve_config +from aiperf.config.loader.errors import ConfigurationError + +# Every field the registry claims to route, which is what the guarantee is +# about -- not just the dataset ones. +_ROUTED_FIELDS_UNDER_TEST = ROUTED_UNDER_CONFIG + + +def cli(**kwargs: object) -> CLIConfig: + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +def _with_companion(field: str, value: Any) -> dict[str, Any]: + """Flag kwargs for ``field``, plus the companion its schema requires. + + ``images.width`` and friends validate as a normal distribution, so a + stddev with no mean is rejected outright. Driving the pair keeps these + fields covered instead of erroring into a skip. + """ + kwargs: dict[str, Any] = {field: value} + if field.endswith("_stddev"): + mean_field = field[: -len("_stddev")] + "_mean" + if mean_field in CLIConfig.model_fields: + kwargs[mean_field] = 8 + return kwargs + + +def _unwrap(annotation: Any) -> Any: + while True: + origin = typing.get_origin(annotation) + if origin is typing.Annotated: + annotation = typing.get_args(annotation)[0] + continue + if origin in (typing.Union, types.UnionType): + args = [a for a in typing.get_args(annotation) if a is not type(None)] + if not args: + return None + # Magic-list flags are declared `list[int] | int | None`. Prefer + # the scalar member: it is the form this path routes, and taking + # args[0] would report the field as undrivable. + scalars = [a for a in args if typing.get_origin(a) is not list] + annotation = scalars[0] if scalars else args[0] + continue + return annotation + + +# Values for fields whose annotation cannot produce them: ``typing.Any`` +# (the magic-list flags), or types whose contents must parse. Without these +# the fields would report as undrivable and go untested. +FIELD_PROBE_VALUES: dict[str, list[Any]] = { + "request_count": [2, 3], + "concurrency": [2, 3], + "num_users": [2, 3], + "prefill_concurrency": [2, 3], + "request_rate": [2.0, 3.0], + "benchmark_duration": [2.0, 3.0], + "headers": [["x-probe-a:1"], ["x-probe-b:2"]], + "extra_inputs": [["probe_a:1"], ["probe_b:2"]], + "mlflow_tags": [["probe_a:1"], ["probe_b:2"]], + "server_metrics": [ + ["http://localhost:9090/metrics"], + ["http://localhost:9091/metrics"], + ], + "gpu_telemetry": [ + ["http://localhost:9400/metrics"], + ["http://localhost:9401/metrics"], + ], + "isl_osl_pairs": [["128,16"], ["256,32"]], + # --goodput takes space-separated "metric:value" pairs, not a list. + "goodput": ["ttft:200", "ttft:300"], + "server_metrics_formats": [["json"], ["csv"]], + "sweep_variants": [["concurrency=2"], ["concurrency=4"]], + # --search-recipe names a registered plugin, not an arbitrary string. + "search_recipe": ["prefill-ttft-curve", "concurrency-ramp"], +} + + +def _candidate_values(field: str) -> list[Any]: + """Valid values to drive ``field`` with, or [] if none can be generated. + + Returning [] (rather than guessing) keeps the test honest: a field it + cannot drive is reported as skipped instead of silently passing. + """ + if field in FIELD_PROBE_VALUES: + return FIELD_PROBE_VALUES[field] + info = CLIConfig.model_fields.get(field) + if info is None: + return [] + annotation = _unwrap(info.annotation) + if typing.get_origin(annotation) is typing.Literal: + return list(typing.get_args(annotation)) + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + return list(annotation) + if annotation is bool: + return [True, False] + if annotation is int: + return [2, 3] + if annotation is float: + return [2.0, 3.0] + if annotation is str: + return ["aiperf-probe-a", "aiperf-probe-b"] + if annotation is Path: + return [Path("/tmp/aiperf-probe-a"), Path("/tmp/aiperf-probe-b")] + # list[str] and friends: a one-element list is enough to vary the value. + # Fields whose contents must parse (--header k:v, --search-sla ...) reject + # these and raise, which the invariant accepts as loud. + if typing.get_origin(annotation) is list: + args = typing.get_args(annotation) + if args and args[0] is str: + return [["aiperf-probe-a"], ["aiperf-probe-b"]] + return [] + + +def _value_pair(field: str) -> tuple[Any, Any] | None: + """Two distinct valid values for ``field``, or None if unavailable.""" + values = _candidate_values(field) + return (values[0], values[1]) if len(values) >= 2 else None + + +def _leaves(value: Any, prefix: str = "") -> dict[str, Any]: + """Flatten a nested dict into ``{dotted.path: leaf}``.""" + if not isinstance(value, dict): + return {prefix: value} + out: dict[str, Any] = {} + for key, sub in value.items(): + out.update(_leaves(sub, f"{prefix}.{key}" if prefix else str(key))) + return out + + +# Fields the value generator cannot drive from their annotation (lists, +# free-form strings, parsed DSL strings). Listing them explicitly rather than +# skipping on the fly means a newly-added field of an unsupported type fails +# this suite instead of quietly reporting "skipped" -- a skip is invisible in +# CI, which is how a coverage hole hides. +UNDRIVABLE_FIELDS: frozenset[str] = frozenset( + { + # list-shaped + "audio_depths", + "audio_sample_rates", + "conversation_num", + "conversation_turn_mean", + "dataset_filters", + "prompt_input_tokens_mean", + "prompt_input_tokens_stddev", + "prompt_output_tokens_mean", + "prompt_output_tokens_stddev", + "video_audio_depth", + # needs a path that exists on disk, which a module-level constant + # cannot provide; covered directly by the dataset-source tests in + # test_config_input_overrides.py + "input_file", + # free-form / parsed strings with no safe auto-generated value + "hf_dataset_subset", + "prompt_sequence_distribution", + "video_codec", + } +) + + +def _require_drivable(field: str) -> None: + """Fail rather than skip when a field cannot be driven and is not listed.""" + assert field in UNDRIVABLE_FIELDS, ( + f"{field} cannot be driven from its annotation, so it is untested. " + f"Extend _candidate_values to cover its type, or add it to " + f"UNDRIVABLE_FIELDS with a reason." + ) + + +# Fields whose emitted shape legitimately carries a constant companion key. +# Each entry needs a reason; an unexplained entry is a bug being suppressed. +# Empty today -- no field currently needs an exemption; add one here (with a +# reason) if a future field's constant-leaf check is a legitimate structural +# wrapper key rather than a materialized default. +_CONSTANT_KEY_ALLOWLIST: dict[str, set[str]] = {} + + +@pytest.mark.parametrize("field", sorted(DATASET_OVERRIDE_FIELDS)) +def test_override_emits_no_key_the_user_did_not_set(field: str) -> None: + """An emitted key that ignores the input value is a materialized default.""" + pair = _value_pair(field) + if pair is None: + _require_drivable(field) + pytest.skip(f"{field} is listed in UNDRIVABLE_FIELDS") + first, second = pair + + try: + low = build_dataset(cli(**{field: first}), declared_type=DatasetType.SYNTHETIC) + high = build_dataset( + cli(**{field: second}), declared_type=DatasetType.SYNTHETIC + ) + except (ValueError, ConfigurationError): + pytest.skip(f"{field} is not valid against a synthetic dataset alone") + + low_leaves, high_leaves = _leaves(low), _leaves(high) + constant = { + path + for path in low_leaves.keys() & high_leaves.keys() + if low_leaves[path] == high_leaves[path] + } + constant -= _CONSTANT_KEY_ALLOWLIST.get(field, set()) + + assert not constant, ( + f"build_dataset emitted {sorted(constant)} for --{field.replace('_', '-')} " + f"with a value independent of the flag, so it is a default rather than " + f"something the user asked for. Merged onto a YAML dataset it would " + f"silently overwrite the config file's value. Suppress it in override " + f"mode, as _apply_implicit_media_batch is." + ) + + +@pytest.fixture +def trace_yaml(tmp_path: Path) -> Path: + """A file/trace dataset, for the flags that only apply to one.""" + pool = tmp_path / "trace.jsonl" + pool.write_text('{"text": "hi", "hash_ids": [1]}\n') + cfg = tmp_path / "trace.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: mooncake_trace + path: {pool} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +@pytest.fixture +def rich_yaml(tmp_path: Path) -> Path: + """A synthetic dataset carrying values an override could clobber.""" + cfg = tmp_path / "rich.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + random_seed: 11 + entries: 16 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +@pytest.mark.parametrize("field", sorted(DATASET_OVERRIDE_FIELDS)) +def test_routed_dataset_field_actually_changes_the_resolved_config( + field: str, rich_yaml: Path, trace_yaml: Path +) -> None: + """A field claimed as routed must have an observable effect. + + Checked against both a synthetic and a file dataset: many flags apply to + only one (synthesis_* is file/public-only, prefix prompts are + synthetic-only), so requiring an effect on a single type would flag + correct routing as broken. + """ + if field in MAGIC_LIST_ONLY_UNDER_CONFIG: + pytest.skip(f"{field} routes as a sweep parameter in its list form") + candidates = _candidate_values(field) + if not candidates: + _require_drivable(field) + pytest.skip(f"{field} is listed in UNDRIVABLE_FIELDS") + + # Try every candidate: one may coincide with the model's own default + # (DatasetSamplingStrategy.SEQUENTIAL does), which would look like "no + # effect" without meaning the flag is dropped. + resolutions = [] + errors: list[str] = [] + changed = None + for config_yaml in (rich_yaml, trace_yaml): + baseline = resolve_config(cli(), config_yaml).model_dump(mode="json") + for value in candidates: + try: + resolved = resolve_config( + cli(**_with_companion(field, value)), config_yaml + ).model_dump(mode="json") + except (ValueError, ConfigurationError) as exc: + errors.append(str(exc)) + continue + resolutions.append(resolved) + if resolved != baseline: + changed = resolved + if not resolutions: + # "has no effect on a dataset of type X" is OUR error for a field + # nothing routes -- the bug, not a domain guard. Anything else (weka + # -only, baseten-only, needs a companion flag) is a real constraint + # and the flag is loud rather than dropped, so skipping is right. + assert not all("have no effect" in e for e in errors), ( + f"--{field.replace('_', '-')} is classified as routed but nothing " + f"routes it: resolving raises {errors[0]!r}. Route it, or list it " + f"in UNROUTED_UNDER_CONFIG so the error names the flag properly." + ) + pytest.skip(f"{field} is not valid against a synthetic dataset alone") + assert changed is not None, ( + f"--{field.replace('_', '-')} is classified as routed under --config but " + f"setting it changed nothing in the resolved config -- it is being " + f"silently dropped. Route it, or list it in UNROUTED_UNDER_CONFIG so " + f"users get an error." + ) + + +# --------------------------------------------------------------------------- +# The whole-CLIConfig guarantee +# --------------------------------------------------------------------------- + +# Enums with a single member: no second value exists to prove an effect with. +SINGLE_VALUED_FIELDS: frozenset[str] = frozenset( + { + # URLSelectionStrategy has exactly one member. + "url_selection_strategy", + # Every mode but the default fails validation on its own, and setting + # a field to the value it already has cannot change anything -- so no + # observation here can distinguish routed from dropped. + "wait_for_model_mode", + } +) + + +def _companions_for(field: str) -> dict[str, Any]: + """Flags that must accompany ``field`` for it to mean anything.""" + from aiperf.config.flags._config_flag_routing import COMPANION_ROUTED + + companions: dict[str, Any] = {} + for companion in COMPANION_ROUTED.get(field, ()): # type: ignore[call-overload] + if companion == "model_names": + companions["model_names"] = ["companion-model"] + elif companion in ("use_server_token_count", "streaming"): + companions[companion] = True + if field.endswith("_stddev"): + mean_field = field[: -len("_stddev")] + "_mean" + if mean_field in CLIConfig.model_fields: + companions[mean_field] = 8 + return companions + + +@pytest.mark.parametrize("field", sorted(_ROUTED_FIELDS_UNDER_TEST)) +def test_routed_field_never_silently_no_ops( + field: str, rich_yaml: Path, trace_yaml: Path +) -> None: + """Every routed flag must change the config or raise -- never both quiet. + + This is the guarantee in one assertion, and it covers all of + ROUTED_UNDER_CONFIG rather than the dataset subset. Sections whose + builders consume the whole section (OUTPUT/TOKENIZER/ACCURACY/SWEEPING) + are marked routed wholesale, so a newly-added member of one of them is + classified as routed automatically; without this test nothing would ever + check whether that claim is true, and the flag would be silently dropped + exactly as before. + + Raising is acceptable: a flag rejected for needing a companion, a + different dataset type, or a streaming endpoint is loud, and loud is the + property we care about. The forbidden outcome is resolving successfully + while ignoring what the user asked for. + """ + if field in MAGIC_LIST_ONLY_UNDER_CONFIG: + pytest.skip(f"{field} routes as a sweep parameter in its list form") + if field in SINGLE_VALUED_FIELDS: + pytest.skip(f"{field} has a single valid value; no effect to observe") + candidates = _candidate_values(field) + if not candidates: + _require_drivable(field) + pytest.skip(f"{field} is listed in UNDRIVABLE_FIELDS") + + companions = _companions_for(field) + # Assert per YAML, not across the union: a field that works on a trace + # dataset and silently no-ops on a synthetic one is a silent drop for + # every user of a synthetic config, and accumulating `changed` across + # both fixtures hid exactly that. + observed_baseline = False + for config_yaml in (rich_yaml, trace_yaml): + try: + baseline = resolve_config(cli(**companions), config_yaml).model_dump( + mode="json" + ) + except (ValueError, TypeError, ConfigurationError): + # TypeError: _build_adaptive_search's "--search-* flags require + # --search-space" -- a real, intentional loud signal for the + # ROUTED_UNDER_CONFIG-wide field set this test covers, not just + # the dataset-scoped ValueError/ConfigurationError pair other + # loops in this file use. + continue + observed_baseline = True + changed = silent_noop = False + for value in candidates: + try: + resolved = resolve_config( + cli(**{field: value}, **companions), config_yaml + ).model_dump(mode="json") + except (ValueError, TypeError, ConfigurationError): + continue # loud: acceptable + if resolved != baseline: + changed = True + else: + silent_noop = True + + assert changed or not silent_noop, ( + f"--{field.replace('_', '-')} is classified as routed under " + f"--config, but with {config_yaml.name} it resolves successfully " + f"and changes nothing -- it is silently ignored. Route it, or move " + f"it to UNROUTED_UNDER_CONFIG so the user gets an error naming it." + ) + assert observed_baseline, ( + f"neither rich_yaml nor trace_yaml produced a baseline for " + f"--{field.replace('_', '-')} (both raised on companions alone), so " + f"this case checked nothing. Fix the companion/fixture combination, " + f"or move the field to UNDRIVABLE_FIELDS with a reason." + ) + + +# One routable flag per dataset type, used as an anchor so the flag under +# test is never the only thing driving the override. +_ANCHOR_FLAGS: dict[str, dict[str, Any]] = { + "rich": {"random_seed": 4242}, + "trace": {"random_seed": 4242}, +} + + +@pytest.mark.parametrize("field", sorted(DATASET_OVERRIDE_FIELDS)) +def test_dataset_flag_is_not_excused_by_a_neighbouring_flag( + field: str, rich_yaml: Path, trace_yaml: Path +) -> None: + """A flag must not become silently droppable by being passed with others. + + The single-flag parametrization above cannot see this: the guard used to + fire only when the whole override came back empty, so any inert flag + riding along with a routable one was discarded with no diagnostic. Every + multi-flag command line touching the dataset escaped the guarantee. + """ + if field in MAGIC_LIST_ONLY_UNDER_CONFIG or field in SINGLE_VALUED_FIELDS: + pytest.skip(f"{field} is covered by the single-flag invariant only") + candidates = _candidate_values(field) + if not candidates: + _require_drivable(field) + pytest.skip(f"{field} is listed in UNDRIVABLE_FIELDS") + + companions = _companions_for(field) + for label, config_yaml in (("rich", rich_yaml), ("trace", trace_yaml)): + anchor = _ANCHOR_FLAGS[label] + if field in anchor: + continue + try: + baseline = resolve_config( + cli(**anchor, **companions), config_yaml + ).model_dump(mode="json") + except Exception: + continue + changed = silent_noop = False + # Every candidate, not just one: a bool's False and an enum's first + # member frequently ARE the resolved default, and setting a field to + # the value it already holds cannot change anything. + for value in candidates: + try: + resolved = resolve_config( + cli(**{field: value}, **anchor, **companions), config_yaml + ).model_dump(mode="json") + except Exception: + continue # loud: acceptable + if resolved != baseline: + changed = True + else: + silent_noop = True + assert changed or not silent_noop, ( + f"--{field.replace('_', '-')} resolves cleanly and changes nothing " + f"when passed alongside {sorted(anchor)} against {config_yaml.name}. " + f"Paired with another flag it is silently dropped, even though it " + f"may error correctly on its own." + ) + + +def test_inert_probe_lets_solo_construction_errors_propagate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A field whose SOLO CLIConfig construction fails must propagate as a + real error, not be silently treated as "loud" and skipped by the + inert-flag probe. + + ``_inert_dataset_flags`` used to wrap both the solo ``CLIConfig(...)`` + construction and the ``build_dataset(...)`` call in one ``try``, so a + construction failure -- a ``ValueError``, e.g. from a cross-field + validator on CLIConfig itself; theoretical today, no current field hits + it -- was caught by the same ``except (ValueError, ConfigurationError)`` + meant only for ``build_dataset``, and the field silently vanished from + the inert list instead of the failure surfacing. Simulated via + monkeypatch since no real field currently reaches this path. + """ + import aiperf.config.flags as flags_module + from aiperf.config.flags.resolver import _inert_dataset_flags + + class _RaisingCLIConfig: + def __init__(self, **kwargs: object) -> None: + raise ValueError("boom: solo construction failed") + + monkeypatch.setattr(flags_module, "CLIConfig", _RaisingCLIConfig) + + real_cli = cli(synthesis_max_isl=100) + with pytest.raises(ValueError, match="boom"): + _inert_dataset_flags( + real_cli, DatasetType.SYNTHETIC, None, {"synthesis_max_isl"} + ) diff --git a/tests/unit/config/test_config_phase_overrides.py b/tests/unit/config/test_config_phase_overrides.py new file mode 100644 index 0000000000..f0007f6c3f --- /dev/null +++ b/tests/unit/config/test_config_phase_overrides.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Phase-shaping CLI flags must survive ``--config``. + +The resolver overlaid only the flat loadgen fields onto the profiling phase. +The ramp durations, request cancellation, and arrival smoothness were built +solely by the CLI-only converter, so under ``--config`` they were dropped -- +and then, once the classification gate landed, rejected outright. + +They are routed here by reusing the converter's own helpers rather than +restating the mapping, so the two paths cannot drift. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aiperf.config import AIPerfConfig +from aiperf.config.flags import CLIConfig +from aiperf.config.flags.resolver import resolve_config +from aiperf.config.loader.errors import ConfigurationError +from aiperf.config.phases import PhaseConfig + + +def cli(**kwargs: object) -> CLIConfig: + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +def profiling(cfg: AIPerfConfig) -> PhaseConfig: + return next(p for p in cfg.benchmark.phases if p.name == "profiling") + + +@pytest.fixture +def concurrency_yaml(tmp_path: Path) -> Path: + cfg = tmp_path / "conc.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + phases: + type: concurrency + concurrency: 4 + requests: 5 +""" + ) + return cfg + + +@pytest.fixture +def gamma_yaml(tmp_path: Path) -> Path: + cfg = tmp_path / "gamma.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + phases: + type: gamma + rate: 10 + requests: 5 +""" + ) + return cfg + + +# --------------------------------------------------------------------------- +# Ramps +# --------------------------------------------------------------------------- + + +def test_concurrency_ramp_duration_routes(concurrency_yaml: Path) -> None: + resolved = profiling( + resolve_config(cli(concurrency_ramp_duration=30), concurrency_yaml) + ) + assert resolved.concurrency_ramp is not None + assert resolved.concurrency_ramp.duration == 30 + + +def test_prefill_concurrency_ramp_duration_routes(concurrency_yaml: Path) -> None: + resolved = profiling( + resolve_config(cli(prefill_concurrency_ramp_duration=15), concurrency_yaml) + ) + assert resolved.prefill_ramp is not None + assert resolved.prefill_ramp.duration == 15 + + +def test_ramp_does_not_disturb_other_phase_values(concurrency_yaml: Path) -> None: + resolved = profiling( + resolve_config(cli(concurrency_ramp_duration=30), concurrency_yaml) + ) + assert resolved.concurrency == 4 + assert resolved.requests == 5 + + +# --------------------------------------------------------------------------- +# Cancellation +# --------------------------------------------------------------------------- + + +def test_request_cancellation_rate_routes(concurrency_yaml: Path) -> None: + resolved = profiling( + resolve_config(cli(request_cancellation_rate=25.0), concurrency_yaml) + ) + assert resolved.cancellation is not None + assert resolved.cancellation.rate == 25.0 + + +def test_request_cancellation_delay_routes_with_rate(concurrency_yaml: Path) -> None: + resolved = profiling( + resolve_config( + cli(request_cancellation_rate=25.0, request_cancellation_delay=2.0), + concurrency_yaml, + ) + ) + assert resolved.cancellation.delay == 2.0 + + +def test_cancellation_delay_without_rate_still_errors(concurrency_yaml: Path) -> None: + """The converter's dependency guard must hold on this path too.""" + with pytest.raises(ValueError, match="requires --request-cancellation-rate"): + resolve_config(cli(request_cancellation_delay=2.0), concurrency_yaml) + + +# --------------------------------------------------------------------------- +# Arrival smoothness +# --------------------------------------------------------------------------- + + +def test_arrival_smoothness_routes_on_gamma_phase(gamma_yaml: Path) -> None: + resolved = profiling(resolve_config(cli(arrival_smoothness=0.5), gamma_yaml)) + assert resolved.smoothness == 0.5 + + +def test_arrival_smoothness_rejected_on_non_gamma_phase( + concurrency_yaml: Path, +) -> None: + with pytest.raises(ValueError, match="arrival-smoothness"): + resolve_config(cli(arrival_smoothness=0.5), concurrency_yaml) + + +# --------------------------------------------------------------------------- +# Warmup +# --------------------------------------------------------------------------- + + +def warmup(cfg: AIPerfConfig) -> PhaseConfig | None: + return next((p for p in cfg.benchmark.phases if p.name == "warmup"), None) + + +@pytest.fixture +def warmup_yaml(tmp_path: Path) -> Path: + """A config that already declares a warmup phase.""" + cfg = tmp_path / "warmup.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + phases: + - name: warmup + kind: warmup + type: concurrency + concurrency: 2 + requests: 3 + - name: profiling + kind: profiling + type: concurrency + concurrency: 4 + requests: 5 +""" + ) + return cfg + + +def test_warmup_flag_overrides_existing_warmup_phase(warmup_yaml: Path) -> None: + resolved = resolve_config(cli(warmup_request_count=9), warmup_yaml) + assert warmup(resolved).requests == 9 + + +def test_warmup_flag_leaves_untouched_warmup_values_alone(warmup_yaml: Path) -> None: + """Only what the user set may change on the YAML's warmup phase.""" + resolved = resolve_config(cli(warmup_request_count=9), warmup_yaml) + assert warmup(resolved).concurrency == 2 + + +def test_warmup_flag_does_not_disturb_the_profiling_phase(warmup_yaml: Path) -> None: + resolved = resolve_config(cli(warmup_request_count=9), warmup_yaml) + assert profiling(resolved).requests == 5 + assert profiling(resolved).concurrency == 4 + + +def test_secondary_warmup_flag_applies_to_an_existing_phase( + warmup_yaml: Path, +) -> None: + """--warmup-concurrency has no trigger of its own. + + On the CLI-only path that means "no warmup phase", so the flag is + ignored. Here the config file already declares the phase, so the flag + has somewhere to land and must be applied rather than dropped. + """ + resolved = resolve_config(cli(warmup_concurrency=7), warmup_yaml) + assert warmup(resolved).concurrency == 7 + + +def test_warmup_trigger_creates_a_phase_when_yaml_has_none( + concurrency_yaml: Path, +) -> None: + """A trigger flag builds the warmup phase, as it does CLI-only.""" + resolved = resolve_config(cli(warmup_request_count=4), concurrency_yaml) + created = warmup(resolved) + assert created is not None + assert created.requests == 4 + assert created.exclude_from_results is True + + +def test_secondary_warmup_flag_without_a_phase_or_trigger_errors( + concurrency_yaml: Path, +) -> None: + """Nowhere to land and nothing to create it: must be loud, not dropped.""" + with pytest.raises(ConfigurationError, match=r"warmup"): + resolve_config(cli(warmup_concurrency=7), concurrency_yaml) + + +def test_warmup_request_rate_onto_concurrency_yaml_errors(warmup_yaml: Path) -> None: + """--warmup-request-rate alone onto a YAML concurrency warmup phase must + raise a clear, named error rather than merge into an invalid phase. + + _warmup_override_pattern emits `rate` independently of `type`, so this + used to produce {type: concurrency, rate: 5.0, ...} and fail deep inside + AIPerfConfig.model_validate with a raw "Extra inputs are not permitted" + pydantic error instead of naming the flag. + """ + with pytest.raises(ConfigurationError, match=r"--warmup-request-rate"): + resolve_config(cli(warmup_request_rate=5.0), warmup_yaml) + + +def test_warmup_arrival_pattern_onto_concurrency_yaml_errors(warmup_yaml: Path) -> None: + """--warmup-arrival-pattern alone onto a YAML concurrency warmup phase + must raise a clear, named error rather than merge into an invalid phase. + + _warmup_override_pattern switches `type` to the rate phase without + setting `rate`, so this used to fail deep inside + AIPerfConfig.model_validate with "rate-controlled phases require rate or + rate_series" instead of naming the flag. + """ + with pytest.raises(ConfigurationError, match=r"--warmup-arrival-pattern"): + resolve_config(cli(warmup_arrival_pattern="constant"), warmup_yaml) + + +def test_warmup_request_rate_and_arrival_pattern_together_transitions_cleanly( + warmup_yaml: Path, +) -> None: + """Setting both flags together is a complete, valid transition and must + not be rejected.""" + resolved = resolve_config( + cli(warmup_request_rate=5.0, warmup_arrival_pattern="constant"), + warmup_yaml, + ) + w = warmup(resolved) + assert w.type == "constant" + assert w.rate == 5.0 + + +# --------------------------------------------------------------------------- +# Goodput -> the slos block +# --------------------------------------------------------------------------- + + +def test_goodput_routes_to_slos(concurrency_yaml: Path) -> None: + resolved = resolve_config(cli(goodput="ttft:200"), concurrency_yaml) + assert resolved.benchmark.slos == {"ttft": 200.0} + + +def test_goodput_merges_with_yaml_slos(tmp_path: Path) -> None: + """A YAML slos block keeps the keys the flag does not mention.""" + cfg = tmp_path / "slos.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + slos: + tpot: 50 + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + resolved = resolve_config(cli(goodput="ttft:200"), cfg) + assert resolved.benchmark.slos == {"tpot": 50.0, "ttft": 200.0} + + +# --------------------------------------------------------------------------- +# Fixed schedule +# --------------------------------------------------------------------------- + + +@pytest.fixture +def trace_phase_yaml(tmp_path: Path) -> Path: + trace = tmp_path / "t.jsonl" + trace.write_text('{"timestamp": 0, "text": "hi"}\n') + cfg = tmp_path / "trace_phase.yaml" + cfg.write_text( + f"""\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: file + format: mooncake_trace + path: {trace} + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +def test_fixed_schedule_switches_the_phase_type(trace_phase_yaml: Path) -> None: + resolved = resolve_config(cli(fixed_schedule=True), trace_phase_yaml) + assert profiling(resolved).type == "fixed_schedule" + + +def test_fixed_schedule_offsets_route(trace_phase_yaml: Path) -> None: + resolved = profiling( + resolve_config( + cli(fixed_schedule=True, fixed_schedule_start_offset=5), trace_phase_yaml + ) + ) + assert resolved.start_offset == 5 diff --git a/tests/unit/config/test_config_telemetry_overrides.py b/tests/unit/config/test_config_telemetry_overrides.py new file mode 100644 index 0000000000..438d09c815 --- /dev/null +++ b/tests/unit/config/test_config_telemetry_overrides.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Telemetry and scenario flags must survive ``--config``. + +``build_mlflow``, ``build_otel``, and ``build_network_latency`` have always +existed and are called by the CLI-only converter, but ``build_cli_overrides`` +never called them -- so every ``--mlflow-*`` flag, ``--otel-url``, and the +network-latency flags were silently discarded whenever a config file was +supplied. Same story for ``--scenario`` / ``--unsafe-override``, which the +converter writes onto the benchmark body via ``_apply_scenario_fields``. + +These flags configure where results are *published*. Dropping them silently +means a run reports to the wrong MLflow experiment, or to none at all, with +no diagnostic. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aiperf.config.flags import CLIConfig +from aiperf.config.flags.resolver import resolve_config + + +def cli(**kwargs: object) -> CLIConfig: + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) # type: ignore[arg-type] + + +@pytest.fixture +def base_yaml(tmp_path: Path) -> Path: + cfg = tmp_path / "base.yaml" + cfg.write_text( + """\ +schemaVersion: "2.0" +benchmark: + model: test-model + endpoint: + url: http://localhost:8000 + dataset: + type: synthetic + phases: + type: concurrency + concurrency: 1 + requests: 5 +""" + ) + return cfg + + +# --------------------------------------------------------------------------- +# MLflow +# --------------------------------------------------------------------------- + + +def test_mlflow_tracking_uri_routes(base_yaml: Path) -> None: + resolved = resolve_config(cli(mlflow_tracking_uri="http://mlflow:5000"), base_yaml) + assert resolved.benchmark.mlflow.tracking_uri == "http://mlflow:5000" + + +def test_mlflow_experiment_routes(base_yaml: Path) -> None: + resolved = resolve_config( + cli( + mlflow_tracking_uri="http://mlflow:5000", mlflow_experiment="my-experiment" + ), + base_yaml, + ) + assert resolved.benchmark.mlflow.experiment == "my-experiment" + + +def test_mlflow_run_name_routes(base_yaml: Path) -> None: + resolved = resolve_config( + cli(mlflow_tracking_uri="http://mlflow:5000", mlflow_run_name="run-7"), + base_yaml, + ) + assert resolved.benchmark.mlflow.run_name == "run-7" + + +def test_mlflow_secondary_flag_without_tracking_uri_still_errors( + base_yaml: Path, +) -> None: + """The guard stands when neither the CLI nor the YAML supplies a URI.""" + with pytest.raises(ValueError, match="require --mlflow-tracking-uri"): + resolve_config(cli(mlflow_experiment="orphan"), base_yaml) + + +def test_mlflow_flag_overrides_yaml_value(base_yaml: Path, tmp_path: Path) -> None: + """A YAML-supplied mlflow block must lose to the explicit flag.""" + cfg = tmp_path / "with_mlflow.yaml" + cfg.write_text( + base_yaml.read_text().replace( + " phases:", + " mlflow:\n experiment: from-yaml\n tracking_uri: http://yaml:5000\n phases:", + ) + ) + resolved = resolve_config(cli(mlflow_experiment="from-cli"), cfg) + assert resolved.benchmark.mlflow.experiment == "from-cli" + # The flag the user did not pass must survive untouched. + assert resolved.benchmark.mlflow.tracking_uri == "http://yaml:5000" + + +# --------------------------------------------------------------------------- +# OpenTelemetry +# --------------------------------------------------------------------------- + + +def test_otel_url_routes(base_yaml: Path) -> None: + resolved = resolve_config(cli(otel_url="http://otel:4318"), base_yaml) + assert resolved.benchmark.otel.metrics_url is not None + + +def test_gen_ai_provider_routes(base_yaml: Path) -> None: + resolved = resolve_config( + cli(otel_url="http://otel:4318", gen_ai_provider="nvidia"), base_yaml + ) + assert resolved.benchmark.otel.gen_ai_provider == "nvidia" + + +# --------------------------------------------------------------------------- +# Network latency +# --------------------------------------------------------------------------- + + +def test_network_latency_mean_routes(base_yaml: Path) -> None: + resolved = resolve_config(cli(network_latency_mean=12.5), base_yaml) + assert resolved.benchmark.network_latency.mean_ms == 12.5 + + +def test_network_latency_automatic_routes(base_yaml: Path) -> None: + resolved = resolve_config(cli(network_latency_automatic=True), base_yaml) + assert resolved.benchmark.network_latency.enabled is True + + +# --------------------------------------------------------------------------- +# Scenario lock +# --------------------------------------------------------------------------- + + +def test_unsafe_override_routes(base_yaml: Path) -> None: + resolved = resolve_config(cli(unsafe_override=True), base_yaml) + assert resolved.benchmark.unsafe_override is True + + +# --------------------------------------------------------------------------- +# Runtime: --api-host with a YAML-supplied api_port +# --------------------------------------------------------------------------- + + +@pytest.fixture +def api_port_yaml(base_yaml: Path, tmp_path: Path) -> Path: + cfg = tmp_path / "with_port.yaml" + cfg.write_text( + base_yaml.read_text().replace( + " phases:", " runtime:\n api_port: 19090\n phases:" + ) + ) + return cfg + + +def test_api_host_accepts_port_from_yaml(api_port_yaml: Path) -> None: + """--api-host must not demand a flag the config file already supplies. + + build_logging_runtime raises when api_host is set and api_port is None, + which under --config is exactly the supported case: the port came from + the file. + """ + resolved = resolve_config(cli(api_host="0.0.0.0"), api_port_yaml) + assert resolved.benchmark.runtime.api_host == "0.0.0.0" + assert resolved.benchmark.runtime.api_port == 19090 + + +def test_api_host_without_any_port_still_errors(base_yaml: Path) -> None: + """The guard stands when neither the CLI nor the YAML supplies a port.""" + with pytest.raises(ValueError, match="api_host requires api_port"): + resolve_config(cli(api_host="0.0.0.0"), base_yaml) + + +@pytest.fixture +def otel_yaml(base_yaml: Path, tmp_path: Path) -> Path: + cfg = tmp_path / "with_otel.yaml" + cfg.write_text( + base_yaml.read_text().replace( + " phases:", + " otel:\n metrics_url: http://yaml-otel:4318/v1/metrics\n phases:", + ) + ) + return cfg + + +def test_gen_ai_provider_accepts_otel_url_from_yaml(otel_yaml: Path) -> None: + """Secondary OTel flags must not demand a URL the config file supplies.""" + resolved = resolve_config(cli(gen_ai_provider="nvidia"), otel_yaml) + assert resolved.benchmark.otel.gen_ai_provider == "nvidia" + assert str(resolved.benchmark.otel.metrics_url).startswith("http://yaml-otel") + + +def test_otel_secondary_flag_without_any_url_still_errors(base_yaml: Path) -> None: + """The guard stands when neither source supplies a metrics URL.""" + with pytest.raises(ValueError, match="requires --otel-url"): + resolve_config(cli(gen_ai_provider="nvidia"), base_yaml) + + +# --------------------------------------------------------------------------- +# Endpoint probes: reset-kv-cache and server-profiler +# --------------------------------------------------------------------------- + + +def test_reset_kv_cache_routes(base_yaml: Path) -> None: + resolved = resolve_config(cli(reset_kv_cache=True), base_yaml) + assert resolved.benchmark.endpoint.reset_kv_cache is not None + + +def test_reset_kv_cache_path_routes(base_yaml: Path) -> None: + resolved = resolve_config( + cli(reset_kv_cache=True, reset_kv_cache_path="/reset"), base_yaml + ) + assert resolved.benchmark.endpoint.reset_kv_cache.path == "/reset" + + +def test_server_profiler_routes(base_yaml: Path) -> None: + resolved = resolve_config( + cli(server_profiler=True, server_profiler_start_path="/start"), base_yaml + ) + assert resolved.benchmark.endpoint.server_profiler.start_path == "/start" + + +def test_probe_flag_overrides_yaml_block(base_yaml: Path, tmp_path: Path) -> None: + """A YAML-supplied probe block must lose to the explicit flag, field-wise.""" + cfg = tmp_path / "with_probe.yaml" + cfg.write_text( + base_yaml.read_text() + .replace( + " dataset:", + " endpoint2_placeholder: 0\n dataset:", + ) + .replace(" endpoint2_placeholder: 0\n", "") + ) + cfg.write_text( + base_yaml.read_text().replace( + " url: http://localhost:8000", + " url: http://localhost:8000\n" + " reset_kv_cache:\n" + " path: /from-yaml\n" + " timeout_seconds: 30", + ) + ) + resolved = resolve_config(cli(reset_kv_cache_path="/from-cli"), cfg) + assert resolved.benchmark.endpoint.reset_kv_cache.path == "/from-cli" + # The field the user did not pass survives. + assert resolved.benchmark.endpoint.reset_kv_cache.timeout_seconds == 30 diff --git a/tests/unit/config/test_trace_flag_routing.py b/tests/unit/config/test_trace_flag_routing.py index 0b3ae2633e..c108177b6e 100644 --- a/tests/unit/config/test_trace_flag_routing.py +++ b/tests/unit/config/test_trace_flag_routing.py @@ -15,9 +15,10 @@ from aiperf.config.flags.cli_config import CLIConfig from aiperf.config.flags.converter import convert_cli_to_aiperf from aiperf.config.flags.resolver import ( - _apply_dataset_synthesis_overrides, + _apply_dataset_overrides, resolve_config, ) +from aiperf.config.loader.errors import ConfigurationError from aiperf.plugin.enums import CustomDatasetType, PublicDatasetType _WEKA_HF = PublicDatasetType.SEMIANALYSIS_CC_TRACES_WEKA_WITH_SUBAGENTS @@ -253,9 +254,9 @@ def test_invalid_yaml_datasets_raise(self, datasets: list[object]) -> None: merged = {"benchmark": {"datasets": datasets}} with pytest.raises( - ValueError, match="synthesis flags require a file or public dataset" + ConfigurationError, match="require a dataset in the config file" ): - _apply_dataset_synthesis_overrides(merged, CLIConfig(synthesis_max_osl=512)) + _apply_dataset_overrides(merged, CLIConfig(synthesis_max_osl=512)) def test_multiple_yaml_datasets_warns_and_updates_only_first( self, caplog: pytest.LogCaptureFixture @@ -268,24 +269,25 @@ def test_multiple_yaml_datasets_warns_and_updates_only_first( merged = {"benchmark": {"datasets": [first, second]}} with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): - _apply_dataset_synthesis_overrides( - merged, CLIConfig(synthesis_max_osl=12000) - ) + _apply_dataset_overrides(merged, CLIConfig(synthesis_max_osl=12000)) assert "apply only to the first dataset" in caplog.text - assert first["synthesis"] == {"speedupRatio": 2.0, "maxOsl": 12000} + assert first["synthesis"] == {"speedupRatio": 2.0, "max_osl": 12000} assert second["synthesis"] == {"maxOsl": 8000} - def test_non_trace_yaml_dataset_warns_and_is_unchanged( - self, caplog: pytest.LogCaptureFixture - ) -> None: + def test_non_trace_yaml_dataset_is_rejected(self) -> None: + """Synthesis has no meaning on a synthetic dataset, so say so. + + This previously warned and dropped the flag; a warning on stderr is + not enough when the result is a benchmark that ignored what the user + asked for. + """ dataset = {"type": "synthetic"} merged = {"benchmark": {"datasets": [dataset]}} - with caplog.at_level("WARNING", logger="aiperf.config.flags.resolver"): - _apply_dataset_synthesis_overrides(merged, CLIConfig(synthesis_max_osl=512)) + with pytest.raises(ConfigurationError, match="have no effect"): + _apply_dataset_overrides(merged, CLIConfig(synthesis_max_osl=512)) - assert "require a file or public dataset" in caplog.text assert dataset == {"type": "synthetic"} @pytest.mark.parametrize( @@ -311,7 +313,7 @@ def test_baseten_yaml_rejects_unsupported_synthesis( } with pytest.raises(ValueError, match=match) as exc_info: - _apply_dataset_synthesis_overrides(merged, CLIConfig(**cli_kwargs)) + _apply_dataset_overrides(merged, CLIConfig(**cli_kwargs)) assert "YAML format: baseten_trace" in str(exc_info.value) assert "--custom-dataset-type" not in str(exc_info.value) @@ -320,11 +322,9 @@ def test_baseten_yaml_accepts_output_len_synthesis(self) -> None: dataset = {"type": "file", "format": "baseten_trace"} merged = {"benchmark": {"datasets": [dataset]}} - _apply_dataset_synthesis_overrides( - merged, CLIConfig(synthesis_output_len_multiplier=2.0) - ) + _apply_dataset_overrides(merged, CLIConfig(synthesis_output_len_multiplier=2.0)) - assert dataset["synthesis"] == {"outputLenMultiplier": 2.0} + assert dataset["synthesis"] == {"output_len_multiplier": 2.0} class TestOslFallbackRouting: From 25d4fb30d409f5f88dfea22c9e60038b26ea8bc3 Mon Sep 17 00:00:00 2001 From: Ilana Nguyen Date: Mon, 31 Aug 2026 14:57:53 -0700 Subject: [PATCH 045/119] fix(plot): clarify pareto frontier connectors (#1344) Signed-off-by: inguyen (cherry picked from commit 04802735fecc2acea650e38cd7da29e1295ecc8b) --- src/aiperf/plot/core/plot_generator.py | 18 +++ tests/unit/plot/test_plot_generator.py | 173 +++++++++++++++++++++++++ tests/unit/plot/test_png_exporter.py | 66 ++++++++++ 3 files changed, 257 insertions(+) diff --git a/src/aiperf/plot/core/plot_generator.py b/src/aiperf/plot/core/plot_generator.py index 630cee3d64..93d8cd1bc9 100644 --- a/src/aiperf/plot/core/plot_generator.py +++ b/src/aiperf/plot/core/plot_generator.py @@ -712,6 +712,24 @@ def create_pareto_plot( # Apply NVIDIA branding layout layout = self._get_base_layout(title, x_label, y_label) + layout["margin"]["b"] = max(layout["margin"].get("b", 0), 120) + pareto_note = { + "x": 0.5, + "y": 0, + "xref": "paper", + "yref": "paper", + "yshift": -65, + "text": "Lines connect Pareto frontier points only; other points remain markers.", + "showarrow": False, + "font": { + "size": 11, + "family": PLOT_FONT_FAMILY, + "color": self.colors["secondary"], + }, + "xanchor": "center", + "yanchor": "top", + } + layout["annotations"] = list(layout.get("annotations", [])) + [pareto_note] fig.update_layout(layout) return fig diff --git a/tests/unit/plot/test_plot_generator.py b/tests/unit/plot/test_plot_generator.py index e3ba167c66..67e1a47240 100644 --- a/tests/unit/plot/test_plot_generator.py +++ b/tests/unit/plot/test_plot_generator.py @@ -179,6 +179,179 @@ def test_create_scatter_line_plot_auto_labels(self, plot_generator, multi_run_df assert "Time to First Token" in fig.layout.xaxis.title.text assert "Inter Token Latency" in fig.layout.yaxis.title.text + def test_scatter_line_connects_all_points_in_concurrency_groups( + self, plot_generator: PlotGenerator + ) -> None: + """Test non-Pareto scatter_line plots connect every point in a group.""" + df = pd.DataFrame( + { + "model": ["model-a", "model-b", "model-c", "model-d", "model-e"], + "concurrency": [50, 50, 50, 50, 50], + "time_to_first_token": [52000, 6000, 18000, 34000, 44000], + "request_throughput": [25, 123, 88, 61, 42], + } + ) + + fig = plot_generator.create_scatter_line_plot( + df=df, + x_metric="time_to_first_token", + y_metric="request_throughput", + label_by="model", + group_by="concurrency", + ) + + main_trace = fig.data[1] + assert main_trace.mode == "lines+markers+text" + assert main_trace.name == "50" + assert list(main_trace.x) == [6000, 18000, 34000, 44000, 52000] + assert list(main_trace.y) == [123, 88, 61, 42, 25] + + def test_configured_concurrency_groups_use_clean_legend_names( + self, plot_generator: PlotGenerator + ) -> None: + """Test labels and groups stay distinct for custom multi-run plots.""" + df = pd.DataFrame( + { + "model": ["model-a", "model-b"] * 3, + "concurrency": [1, 1, 10, 10, 50, 50], + "request_latency": [9000, 4000, 16000, 5000, 52000, 6000], + "output_token_throughput_per_gpu": [3, 7, 16, 56, 25, 123], + } + ) + + fig = plot_generator.create_pareto_plot( + df=df, + x_metric="request_latency", + y_metric="output_token_throughput_per_gpu", + label_by="model", + group_by="concurrency", + ) + + legend_traces = [trace for trace in fig.data if trace.showlegend] + assert [trace.name for trace in legend_traces] == ["1", "10", "50"] + assert [list(trace.text) for trace in legend_traces] == [ + ["model-b", "model-a"], + ["model-b", "model-a"], + ["model-b", "model-a"], + ] + + def test_pareto_plot_concurrency_groups_only_connect_frontier( + self, plot_generator: PlotGenerator + ) -> None: + """Test concurrency-grouped Pareto plots still connect only frontier points.""" + df = pd.DataFrame( + { + "model": ["model-a", "model-b", "model-c", "model-d", "model-e"], + "concurrency": [50, 50, 50, 50, 50], + "request_latency": [52000, 6000, 18000, 34000, 44000], + "output_token_throughput_per_gpu": [25, 80, 123, 61, 42], + } + ) + + fig = plot_generator.create_pareto_plot( + df=df, + x_metric="request_latency", + y_metric="output_token_throughput_per_gpu", + label_by="model", + group_by="concurrency", + ) + + main_line_trace = fig.data[1] + assert main_line_trace.mode == "lines" + assert main_line_trace.name is None + assert list(main_line_trace.x) == [6000, 18000] + assert list(main_line_trace.y) == [80, 123] + assert len(fig.layout.annotations) == 1 + pareto_note = fig.layout.annotations[0] + assert ( + pareto_note.text + == "Lines connect Pareto frontier points only; other points remain markers." + ) + assert pareto_note.y == 0 + assert pareto_note.yshift == -65 + assert fig.layout.margin.b >= 120 + + def test_pareto_plot_experiment_group_only_connects_frontier( + self, plot_generator: PlotGenerator + ) -> None: + """Test classification-style experiment groups stay true Pareto plots.""" + df = pd.DataFrame( + { + "model": ["model-a", "model-b", "model-c", "model-d", "model-e"], + "experiment_group": ["concurrency_50"] * 5, + "request_latency": [52000, 6000, 18000, 34000, 44000], + "output_token_throughput_per_gpu": [25, 80, 123, 61, 42], + } + ) + + fig = plot_generator.create_pareto_plot( + df=df, + x_metric="request_latency", + y_metric="output_token_throughput_per_gpu", + label_by="model", + group_by="experiment_group", + ) + + main_line_trace = fig.data[1] + assert main_line_trace.mode == "lines" + assert list(main_line_trace.x) == [6000, 18000] + assert list(main_line_trace.y) == [80, 123] + + def test_pareto_plot_swept_request_rate_only_connects_frontier( + self, plot_generator: PlotGenerator + ) -> None: + """Test other swept groupings also preserve Pareto-frontier connectors.""" + df = pd.DataFrame( + { + "model": ["model-a", "model-b", "model-c", "model-d", "model-e"] * 2, + "request_rate": [5] * 5 + [25] * 5, + "request_latency": [52000, 6000, 18000, 34000, 44000] * 2, + "output_token_throughput_per_gpu": [25, 80, 123, 61, 42] * 2, + } + ) + + fig = plot_generator.create_pareto_plot( + df=df, + x_metric="request_latency", + y_metric="output_token_throughput_per_gpu", + label_by="model", + group_by="request_rate", + ) + + main_lines = [ + trace + for trace in fig.data + if trace.mode == "lines" and not trace.showlegend and trace.line.width == 3 + ] + assert [list(trace.x) for trace in main_lines] == [[6000, 18000], [6000, 18000]] + assert [list(trace.y) for trace in main_lines] == [[80, 123], [80, 123]] + + def test_pareto_plot_default_model_groups_only_connect_frontier( + self, plot_generator: PlotGenerator + ) -> None: + """Test default Pareto grouping preserves frontier-only connector lines.""" + df = pd.DataFrame( + { + "model": ["model-a"] * 5, + "concurrency": [10, 20, 30, 40, 50], + "request_latency": [52000, 6000, 18000, 34000, 44000], + "output_token_throughput_per_gpu": [25, 80, 123, 61, 42], + } + ) + + fig = plot_generator.create_pareto_plot( + df=df, + x_metric="request_latency", + y_metric="output_token_throughput_per_gpu", + label_by="concurrency", + group_by="model", + ) + + main_line_trace = fig.data[1] + assert main_line_trace.mode == "lines" + assert list(main_line_trace.x) == [6000, 18000] + assert list(main_line_trace.y) == [80, 123] + def test_create_time_series_scatter(self, plot_generator, single_run_df): """Test time series scatter plot creation.""" fig = plot_generator.create_time_series_scatter( diff --git a/tests/unit/plot/test_png_exporter.py b/tests/unit/plot/test_png_exporter.py index ce35ef49f4..29668e84d4 100644 --- a/tests/unit/plot/test_png_exporter.py +++ b/tests/unit/plot/test_png_exporter.py @@ -792,6 +792,72 @@ def test_runs_to_dataframe( assert df["concurrency"].tolist() == [1, 4] assert df["request_latency"].tolist() == [100.0, 150.0] # p50 values + def test_concurrency_group_legend_uses_metadata_not_run_name( + self, multi_run_exporter, tmp_path: Path + ) -> None: + """Regression: run-name prefixes must not leak into concurrency legends.""" + models = ["Qwen/Qwen3-32B-FP8", "Llama-3.3-70B-Instruct-FP8-dynamic"] + runs = [] + for concurrency in [1, 10, 50]: + for index, model in enumerate(models): + safe_model = model.replace("/", "_") + runs.append( + RunData( + metadata=RunMetadata( + run_name=f"{safe_model}-openai-chat-concurrency{concurrency}", + run_path=tmp_path + / f"{safe_model}-openai-chat-concurrency{concurrency}", + model=model, + concurrency=concurrency, + ), + requests=None, + aggregated={ + "request_latency": { + "avg": 1000.0 * concurrency + index, + "unit": "ms", + }, + "output_token_throughput_per_gpu": { + "avg": 100.0 / concurrency + index, + "unit": "tokens/s/gpu", + }, + }, + timeslices=None, + slice_duration=None, + ) + ) + + df = multi_run_exporter._runs_to_dataframe( + runs, {"display_names": {}, "units": {}} + ) + spec = PlotSpec( + name="pareto_by_concurrency", + plot_type=PlotType.PARETO, + metrics=[ + MetricSpec( + name="request_latency", source=DataSource.AGGREGATED, axis="x" + ), + MetricSpec( + name="output_token_throughput_per_gpu", + source=DataSource.AGGREGATED, + axis="y", + ), + ], + title="Pareto by Concurrency", + filename="pareto_by_concurrency.png", + label_by="model", + group_by="concurrency", + ) + + fig = multi_run_exporter._create_plot_from_spec( + spec, df, {"display_names": {}, "units": {}} + ) + + legend_traces = [trace for trace in fig.data if trace.showlegend] + assert [trace.name for trace in legend_traces] == ["1", "10", "50"] + assert all( + "openai-chat-concurrency" not in trace.name for trace in legend_traces + ) + class TestSingleRunPNGExporter: """Tests for SingleRunPNGExporter class.""" From a0d1ab8996e932c1ac925e61fc0b1a8533988e09 Mon Sep 17 00:00:00 2001 From: Jacob M Date: Mon, 31 Aug 2026 23:41:04 -0400 Subject: [PATCH 046/119] feat: Support stateful responses API with previous_response_id chaining (#1347) Signed-off-by: Jacob Murry (cherry picked from commit ea33cc5a57902d656e2b146d2688fcf437534989) --- docs/tutorials/openai-responses.md | 62 ++++- src/aiperf/common/models/record_models.py | 5 + src/aiperf/common/utils.py | 9 + src/aiperf/config/endpoint.py | 27 ++ .../endpoints/_openai_responses_replay.py | 16 ++ src/aiperf/endpoints/base_endpoint.py | 8 + src/aiperf/endpoints/openai_responses.py | 105 +++++++- src/aiperf/workers/session_manager.py | 22 ++ src/aiperf/workers/worker.py | 18 ++ tests/unit/config/test_endpoint.py | 47 ++++ tests/unit/endpoints/conftest.py | 2 + .../unit/endpoints/test_responses_endpoint.py | 245 +++++++++++++++++- tests/unit/workers/test_session_manager.py | 88 ++++++- tests/unit/workers/test_worker.py | 67 ++++- 14 files changed, 707 insertions(+), 14 deletions(-) diff --git a/docs/tutorials/openai-responses.md b/docs/tutorials/openai-responses.md index 383dc67e07..7a791ce912 100644 --- a/docs/tutorials/openai-responses.md +++ b/docs/tutorials/openai-responses.md @@ -234,7 +234,7 @@ aiperf profile \ --- -## Multi-Turn Conversations +## Multi-Turn Conversations & Stateful Chaining Benchmark multi-turn conversations using the Responses API: @@ -251,6 +251,61 @@ aiperf profile \ --url localhost:8000 ``` +### Stateful Chaining with `previous_response_id` + +Stateful chaining is **opt-in and driven by requesting storage**. Enable it with: + +```bash +--extra-inputs '{"store": true}' +``` + +`store` is a *request* parameter, not a standard field of the Responses object, +so AIPerf keys chaining off the storage you requested rather than off an echoed +response field — the OpenAI spec does not include `store` on the response, and +servers such as vLLM's agentic-api accept it on the request but never serialize +it back. (If a server *does* echo `store: true` on the response object, that is +honored too.) + +Some backends also require a server-side flag (e.g. vLLM with +`VLLM_ENABLE_RESPONSES_API_STORE=1`) to actually persist responses. If the +server does not persist a requested response, the next chained request fails +against the missing `previous_response_id`; use a storing backend when enabling +this feature. + +> **Startup requirement:** requesting `store: true` on `--endpoint-type responses` +> is rejected at startup unless [`--use-server-token-count`](#server-token-counts) +> is also set. Chaining sends only the newest turn on the wire, so client-side ISL +> would undercount the server-side prompt; server-reported token counts are the +> only accurate source. Add `--use-server-token-count`, or drop `store: true` to +> keep sending the full history client-side. + +When chaining is active with `--endpoint-type responses`: +- On **Turn 0**, AIPerf sends the initial prompt and captures the server-generated `response.id` (e.g. `resp_`) from the response object — because `store: true` was requested for the run. +- On **Turn 1+**, AIPerf sets `previous_response_id: ` and sends only the single newest turn in the `input` array rather than re-sending the entire accumulated conversation history. + +**Scope:** chaining is applied only in the default delta context mode +(`deltas_without_responses`), where the newest turn is a genuine delta. The +`*_with_responses` context modes carry full per-turn history and are left +unchained to avoid sending the conversation twice. + +Chaining is also limited to the session-driven request path. Pre-encoded +datasets (`--input-file` payloads sent verbatim) bypass session tracking, so +their requests are sent exactly as authored and are never chained — author +`previous_response_id` into those payloads yourself if you need it. + +The startup requirement above inspects the endpoint-level `--extra-inputs`. If +`store: true` is instead supplied per turn (via a dataset row's `extra`), +it cannot be seen at startup, but chaining still triggers the one-time runtime +warning that client-side ISL undercounts the server-side prompt — enable +`--use-server-token-count` in that case too. + +> **Input Sequence Length note:** because a chained turn only puts the newest turn +> on the wire (the prior history lives server-side), the default client-side ISL +> reflects just that turn and undercounts the prompt the server actually prefills. +> Use [`--use-server-token-count`](#server-token-counts) for accurate multi-turn +> ISL when chaining is enabled. AIPerf emits a one-time warning if chaining runs +> without it. + See the [Multi-Turn Conversations](multi-turn.md) tutorial for details on conversation control parameters. --- @@ -348,10 +403,13 @@ For reference, AIPerf processes these Responses API streaming events: | Event Type | Data Extracted | |---|---| +| `response.created` | Server-generated response ID (`resp_`) for stateful session chaining | | `response.output_text.delta` | Text content delta | | `response.reasoning_text.delta` | Reasoning content delta | +| `response.function_call_arguments.delta` | Tool call arguments delta | | `response.output_text.done` | Final text (fallback for providers that emit text only in done events) | -| `response.completed` | Usage statistics | +| `response.completed` | Usage statistics and response ID | | All other events | Skipped | This enables accurate measurement of TTFT, ITL, and token throughput metrics when streaming is enabled. + diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 8970e24727..fa8aae0536 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -1074,6 +1074,11 @@ class RequestInfo(RecordContext): description="Index of the URL to use when multiple --url values are configured. " "None means use the default (first) URL. Used for round-robin load balancing.", ) + previous_response_id: str | None = Field( + default=None, + description="Response ID from the previous turn (e.g. 'resp_') " + "used for stateful chaining in the Responses API.", + ) class RequestRecord(AIPerfBaseModel): diff --git a/src/aiperf/common/utils.py b/src/aiperf/common/utils.py index 0da86e8e35..3d4fd820d3 100644 --- a/src/aiperf/common/utils.py +++ b/src/aiperf/common/utils.py @@ -25,6 +25,15 @@ def is_tty() -> bool: return sys.stdout is not None and getattr(sys.stdout, "isatty", lambda: False)() +def is_truthy_flag(value: Any) -> bool: + """Whether a config/extra value represents a truthy boolean flag. + + ``--extra-inputs`` values may arrive as a real bool or as a string + (e.g. ``store:true``), so accept ``True`` and case-insensitive ``"true"``. + """ + return value is True or (isinstance(value, str) and value.strip().lower() == "true") + + async def call_all_functions_self( self_: object, funcs: list[Callable], *args, **kwargs ) -> None: diff --git a/src/aiperf/config/endpoint.py b/src/aiperf/config/endpoint.py index a39d6079ad..652e6abff9 100644 --- a/src/aiperf/config/endpoint.py +++ b/src/aiperf/config/endpoint.py @@ -26,6 +26,7 @@ ModelSelectionStrategy, RequestContentType, ) +from aiperf.common.utils import is_truthy_flag from aiperf.config.base import BaseConfig from aiperf.config.loader.parsing import normalize_http_urls from aiperf.plugin.enums import ( @@ -547,6 +548,32 @@ def _validate_per_chunk_usage(self) -> Self: ) return self + @model_validator(mode="after") + def _validate_responses_store_requires_server_token_count(self) -> Self: + """Require server token counts when Responses stateful storage is requested. + + Passing ``store: true`` via ``--extra-inputs`` opts the Responses API into + server-side persistence, which enables ``previous_response_id`` chaining: + prior turns live server-side and only the newest turn is sent on the wire. + Client-side ISL (the default) then reflects only that newest turn and + undercounts the combined prompt the model actually prefills. The true count + is only recoverable from the server's ``usage.prompt_tokens``, so require + ``--use-server-token-count``. + """ + if ( + self.type == EndpointType.RESPONSES + and is_truthy_flag(self.extra.get("store")) + and not self.use_server_token_count + ): + raise ValueError( + "Responses endpoint with --extra-inputs store:true enables " + "previous_response_id chaining, which sends only the newest turn " + "on the wire; client-side Input Sequence Length would undercount " + "the server-side prompt. Add --use-server-token-count so ISL comes " + "from the server's usage.prompt_tokens, or remove store:true." + ) + return self + @model_validator(mode="after") def _validate_wait_for_model_coherent(self) -> Self: """Reject configurations where probe sub-options are set to non-default diff --git a/src/aiperf/endpoints/_openai_responses_replay.py b/src/aiperf/endpoints/_openai_responses_replay.py index 1205eb598c..01fef32a2b 100644 --- a/src/aiperf/endpoints/_openai_responses_replay.py +++ b/src/aiperf/endpoints/_openai_responses_replay.py @@ -38,12 +38,28 @@ {"response.failed", "response.incomplete", "response.error", "error"} ) +# Terminal failure states on a Response object itself (non-streaming body, or +# the nested ``response`` of a streaming lifecycle event). Streaming carries +# these as ``type`` events (above); a non-streaming HTTP 200 body carries no +# ``type`` and instead reports the failure via top-level ``status``. +_FAILURE_STATUSES = frozenset({"failed", "incomplete"}) + def is_failure_event(json_obj: JsonObject) -> bool: """True when ``json_obj`` is a Responses-API stream-end failure event.""" return json_obj.get("type") in _FAILURE_EVENT_TYPES +def is_failure_status(response_obj: JsonObject) -> bool: + """True when a Response object reports a terminal failure via ``status``. + + Non-streaming responses return HTTP 200 with a body such as + ``{"status": "failed", ...}`` and no ``type`` field, so ``is_failure_event`` + does not catch them. This inspects the object's own ``status``. + """ + return response_obj.get("status") in _FAILURE_STATUSES + + def collect_response_items( json_obj: JsonObject, items_by_key: dict[str, dict[str, Any]], diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index 2888805e5f..76fe36d105 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -86,6 +86,14 @@ def extract_response_data(self, record: RequestRecord) -> list[ParsedResponse]: record._parsed_responses_cache = parsed_responses return parsed_responses + def extract_response_id(self, record: RequestRecord) -> str | None: + """Extract server-generated response ID for stateful session chaining. + + Default returns None (stateless). Subclasses implementing stateful protocols + (e.g. ResponsesEndpoint) override this. + """ + return None + def process_responses( self, record: RequestRecord, diff --git a/src/aiperf/endpoints/openai_responses.py b/src/aiperf/endpoints/openai_responses.py index 28623bf93e..f5b8960324 100644 --- a/src/aiperf/endpoints/openai_responses.py +++ b/src/aiperf/endpoints/openai_responses.py @@ -18,6 +18,7 @@ Turn, ) from aiperf.common.types import JsonObject +from aiperf.common.utils import is_truthy_flag from aiperf.endpoints import _openai_responses_replay as _replay from aiperf.endpoints.base_endpoint import BaseEndpoint @@ -189,15 +190,20 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: # lives in top-level ``instructions``. The per-conversation # ``user_context_message`` is prepended as a leading user item. input_items: list[dict[str, Any]] = [] - if request_info.user_context_message: - input_items.append( - { - "type": "message", - "role": self.DEFAULT_TURN_ROLE, - "content": request_info.user_context_message, - } - ) - input_items.extend(self.build_messages(turns)) + if request_info.previous_response_id: + # Stateful chaining: previous_response_id points to server-side history. + self._warn_chaining_isl_once() + input_items.extend(self.build_messages([turns[-1]])) + else: + if request_info.user_context_message: + input_items.append( + { + "type": "message", + "role": self.DEFAULT_TURN_ROLE, + "content": request_info.user_context_message, + } + ) + input_items.extend(self.build_messages(turns)) # Conversation-level fields walk turns from the end so FORK-mode # children whose final turn lacks model/tools still inherit the parent's @@ -211,6 +217,9 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: "model": model_name or model_endpoint.primary_model_name, "stream": model_endpoint.endpoint.streaming, } + if request_info.previous_response_id: + payload["previous_response_id"] = request_info.previous_response_id + for key, value in ( ("instructions", request_info.system_message or None), ("max_output_tokens", max_tokens), @@ -229,6 +238,84 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: self.trace(lambda: f"Formatted payload: {payload}") return payload + _warned_chaining_isl: bool = False + + def _warn_chaining_isl_once(self) -> None: + """Warn once that chained turns only send the newest turn on the wire. + + With ``previous_response_id`` the prior history lives server-side, so + client-side Input Sequence Length (the default) reflects only the newest + turn and undercounts the prompt the server actually prefills. Server-side + token counts (``use_server_token_count``) report the true prompt size. + """ + if ( + self._warned_chaining_isl + or self.model_endpoint.endpoint.use_server_token_count + ): + return + self._warned_chaining_isl = True + self.warning( + "Stateful Responses chaining is active (previous_response_id): only " + "the newest turn is sent on the wire, so client-side Input Sequence " + "Length undercounts the server-side prompt for chained turns. Enable " + "--use-server-token-count for accurate multi-turn ISL." + ) + + def extract_response_id(self, record: RequestRecord) -> str | None: + """Extract the server-generated response ID (``resp_``) for stateful + chaining, but only when storage was requested for this run. + + ``store`` is a *request* parameter, not a standard field of the Responses + object: the OpenAI spec does not list it on the response, and real + servers (e.g. vLLM's agentic-api) accept it on the request but never + serialize it back onto ``response.created`` / ``response.completed``. + Gating on the echoed response ``store`` therefore never fires against + those servers and silently disables chaining. We instead chain when + storage was requested via ``--extra-inputs store:true`` (endpoint + ``extra``); a server that *does* echo ``store: true`` is honored too. + When neither is set we return ``None`` so the caller falls back to + sending the full history. + + The response object lives under ``response`` for streaming lifecycle + events (``response.created`` / ``response.completed`` / ...) and at the + top level for a non-streaming response body. + + A request that returns HTTP 200 but fails does not set + ``RequestRecord.error``. Streaming signals this via a + ``response.failed`` / ``response.incomplete`` event; a non-streaming + body signals it via top-level ``status``. In either case the advertised + id belongs to an aborted response, so chaining onto it would corrupt the + next turn and we return ``None`` regardless of the advertised id. + """ + store_requested = self._store_requested() + resp_id: str | None = None + for response in record.responses: + json_obj = response.get_json() + if not isinstance(json_obj, dict): + continue + if _replay.is_failure_event(json_obj): + return None + nested = json_obj.get("response") + source = nested if isinstance(nested, dict) else json_obj + if _replay.is_failure_status(source): + return None + candidate = source.get("id") + if not (isinstance(candidate, str) and candidate): + continue + if store_requested or source.get("store"): + resp_id = candidate + return resp_id + + def _store_requested(self) -> bool: + """Whether the run requested server-side storage via endpoint ``extra``. + + ``EndpointInfo.extra`` is a list of ``(key, value)`` pairs; ``store`` may + arrive as a bool or a string depending on how ``--extra-inputs`` was + supplied. + """ + store = dict(self.model_endpoint.endpoint.extra or []).get("store") + return is_truthy_flag(store) + @staticmethod def _maybe_enable_usage_stream_options( payload: dict[str, Any], model_endpoint: Any diff --git a/src/aiperf/workers/session_manager.py b/src/aiperf/workers/session_manager.py index ca11be3d0e..2f08d83780 100644 --- a/src/aiperf/workers/session_manager.py +++ b/src/aiperf/workers/session_manager.py @@ -95,6 +95,11 @@ class UserSession(AIPerfBaseModel): "find any children to pin yet — orchestrator dispatches them on " "the credit-return path AFTER this terminal eviction runs).", ) + previous_response_id: str | None = Field( + default=None, + description="Response ID from the previous turn (e.g. 'resp_') " + "used for stateful chaining in the Responses API.", + ) def advance_turn(self, turn_index: int) -> Turn: """Append the next turn onto ``turn_list`` and return it. @@ -124,6 +129,12 @@ def advance_turn(self, turn_index: int) -> Turn: ) turn = self.conversation.turns[turn_index] + # A context reset must always break the server-side chain: chaining onto a + # previous_response_id would retain history the reset is meant to discard. + # Clearing is the safe direction (falls back to sending full history). + if turn.reset_context: + self.previous_response_id = None + if self.context_mode == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES: self.turn_list = [turn] else: @@ -162,6 +173,10 @@ def store_response(self, response_turn: Turn) -> None: """ self.turn_list.append(response_turn) + def store_response_id(self, response_id: str | None) -> None: + """Store the response ID from the server for stateful chaining.""" + self.previous_response_id = response_id + class UserSessionManager: """User session manager for multi-turn processing. @@ -326,6 +341,12 @@ def seed_from_parent( assistant responses) into the child so that the request-builder prepends the full parent context before the child's own messages. + The parent's ``previous_response_id`` is copied too so a FORK child + continues the server-side Responses chain instead of replaying + history: replay drops filtered outputs (e.g. reasoning items) that + only survive server-side, so a child that lost the chain would lack + equivalent parent context. + No-op (with a debug-friendly silent return) if either session is already evicted — the FORK-pin refcount usually keeps the parent resident, but late-arriving children may race past eviction. The @@ -337,6 +358,7 @@ def seed_from_parent( if parent is None or child is None: return child.turn_list = list(parent.turn_list) + child.previous_response_id = parent.previous_response_id def release_fork_child(self, x_correlation_id: str) -> None: """Decrement the FORK-pin refcount on the session, floored at 0. diff --git a/src/aiperf/workers/worker.py b/src/aiperf/workers/worker.py index 3d5ddb9bd8..15aecc996d 100644 --- a/src/aiperf/workers/worker.py +++ b/src/aiperf/workers/worker.py @@ -1210,6 +1210,23 @@ def _finalize_session_response( ) if assistant_turn is not None: session.store_response(assistant_turn) + # Response-ID chaining is only coherent for the mode that captures live + # responses and accumulates genuine deltas (DELTAS_WITHOUT_RESPONSES). + # In the *_WITH_RESPONSES modes ``turns[-1]`` already carries the full + # authored history, so chaining would double the context on the wire. + # A failed turn may still emit a ``response.created`` id for a partial or + # aborted response; chaining onto it would corrupt the server context, so + # keep the last good id (from a prior successful turn) instead. + if ( + not record.has_error + and session.should_store_response() + and ( + extract_response_id := getattr( + self.inference_client.endpoint, "extract_response_id", None + ) + ) + ): + session.store_response_id(extract_response_id(record)) self._populate_response_metrics(credit_context, record, parsed_responses) def _populate_response_metrics( @@ -1508,6 +1525,7 @@ def _create_request_info( source_inner_idx=source_turn.source_inner_idx if source_turn else None, source_kind=source_turn.source_kind if source_turn else None, turns=turns, + previous_response_id=session.previous_response_id if session else None, drop_perf_ns=credit_context.drop_perf_ns, credit_issued_ns=credit.issued_at_ns, system_message=system_message, diff --git a/tests/unit/config/test_endpoint.py b/tests/unit/config/test_endpoint.py index 48ae9e1bb8..87cfd982dc 100644 --- a/tests/unit/config/test_endpoint.py +++ b/tests/unit/config/test_endpoint.py @@ -92,6 +92,53 @@ def test_per_chunk_usage_with_server_token_count_is_valid() -> None: assert endpoint.per_chunk_usage is True +@pytest.mark.parametrize( + "store_value", + [ + param(True, id="bool_true"), + param("True", id="str_true_capitalized"), + ], +) # fmt: skip +def test_responses_store_true_requires_server_token_count(store_value) -> None: + with pytest.raises(ValueError, match="Add --use-server-token-count"): + EndpointConfig( + urls=["http://localhost:8000"], + type=EndpointType.RESPONSES, + extra={"store": store_value}, + use_server_token_count=False, + ) + + +def test_responses_store_true_with_server_token_count_is_valid() -> None: + endpoint = EndpointConfig( + urls=["http://localhost:8000"], + type=EndpointType.RESPONSES, + extra={"store": True}, + use_server_token_count=True, + ) + assert endpoint.extra["store"] is True + + +@pytest.mark.parametrize( + "endpoint_type,extra", + [ + param(EndpointType.RESPONSES, {"store": False}, id="responses_store_false"), + param(EndpointType.RESPONSES, {}, id="responses_no_store"), + param(EndpointType.CHAT, {"store": True}, id="chat_store_true"), + ], +) # fmt: skip +def test_store_gate_does_not_reject_when_chaining_inactive( + endpoint_type, extra +) -> None: + endpoint = EndpointConfig( + urls=["http://localhost:8000"], + type=endpoint_type, + extra=extra, + use_server_token_count=False, + ) + assert endpoint.type == endpoint_type + + def _make_model_endpoint( base_url: str, transport: TransportType | None = None ) -> ModelEndpointInfo: diff --git a/tests/unit/endpoints/conftest.py b/tests/unit/endpoints/conftest.py index e2b5401b8c..ecda605d1d 100644 --- a/tests/unit/endpoints/conftest.py +++ b/tests/unit/endpoints/conftest.py @@ -63,6 +63,7 @@ def create_request_info( system_message: str | None = None, user_context_message: str | None = None, cache_bust_target: CacheBustTarget | None = None, + previous_response_id: str | None = None, **turn_kwargs, ) -> RequestInfo: """Helper to create RequestInfo with all required fields. @@ -95,6 +96,7 @@ def create_request_info( system_message=system_message, user_context_message=user_context_message, cache_bust_target=cache_bust_target, + previous_response_id=previous_response_id, ) diff --git a/tests/unit/endpoints/test_responses_endpoint.py b/tests/unit/endpoints/test_responses_endpoint.py index ce0316b97e..16a9b8abcd 100644 --- a/tests/unit/endpoints/test_responses_endpoint.py +++ b/tests/unit/endpoints/test_responses_endpoint.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import orjson import pytest from pytest import param @@ -12,7 +13,12 @@ ModelInfo, ModelListInfo, ) -from aiperf.common.models.record_models import ReasoningResponseData, TextResponseData +from aiperf.common.models.record_models import ( + ReasoningResponseData, + RequestRecord, + TextResponse, + TextResponseData, +) from aiperf.common.models.usage_models import Usage from aiperf.endpoints.openai_responses import ResponsesEndpoint from aiperf.plugin.enums import EndpointType @@ -451,6 +457,7 @@ def test_format_payload_multi_turn(self, endpoint, model_endpoint): assert payload["input"][1]["role"] == "assistant" assert payload["input"][2]["content"] == "Third" assert payload["model"] == "m3" + assert "previous_response_id" not in payload def test_format_payload_filters_empty_multimodal_content( self, endpoint, model_endpoint @@ -960,3 +967,239 @@ def test_non_reset_turns_extend(self, endpoint): ] msgs = endpoint.build_messages(turns) assert [m["content"] for m in msgs] == ["A", "B"] + + +class TestResponsesStatefulChaining: + """ResponsesEndpoint must extract response ID from server responses and thread + it as `previous_response_id` on subsequent turns while sending only the latest turn.""" + + @pytest.fixture + def endpoint(self) -> ResponsesEndpoint: + return create_endpoint_with_mock_transport( + ResponsesEndpoint, create_model_endpoint(EndpointType.RESPONSES) + ) + + @pytest.fixture + def store_endpoint(self) -> ResponsesEndpoint: + return create_endpoint_with_mock_transport( + ResponsesEndpoint, + create_model_endpoint(EndpointType.RESPONSES, extra=[("store", True)]), + ) + + def test_extract_response_id_store_requested_response_omits_store_returns_id( + self, store_endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_no_echo", + "object": "response", + "status": "completed", + }, + } + ).decode(), + ) + ] + ) + assert store_endpoint.extract_response_id(record) == "resp_no_echo" + + def test_extract_response_id_streaming_created_returns_id( + self, endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "type": "response.created", + "response": { + "id": "resp_b6c65395f4fb8c7d", + "object": "response", + "status": "in_progress", + "store": True, + }, + } + ).decode(), + ), + TextResponse( + perf_ns=2, + text=orjson.dumps( + { + "type": "response.output_text.delta", + "delta": "Hello", + } + ).decode(), + ), + ] + ) + assert endpoint.extract_response_id(record) == "resp_b6c65395f4fb8c7d" + + def test_extract_response_id_stream_ends_in_failure_returns_none( + self, endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "type": "response.created", + "response": { + "id": "resp_aborted", + "object": "response", + "status": "in_progress", + "store": True, + }, + } + ).decode(), + ), + TextResponse( + perf_ns=2, + text=orjson.dumps( + { + "type": "response.failed", + "response": { + "id": "resp_aborted", + "object": "response", + "status": "failed", + "store": True, + }, + } + ).decode(), + ), + ] + ) + assert endpoint.extract_response_id(record) is None + + def test_extract_response_id_non_streaming_returns_id( + self, endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "id": "resp_non_stream_456", + "object": "response", + "status": "completed", + "store": True, + } + ).decode(), + ) + ] + ) + assert endpoint.extract_response_id(record) == "resp_non_stream_456" + + @pytest.mark.parametrize( + "status", + [ + param("failed", id="failed"), + param("incomplete", id="incomplete"), + ], + ) # fmt: skip + def test_extract_response_id_non_streaming_failure_status_returns_none( + self, store_endpoint: ResponsesEndpoint, status: str + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "id": "resp_aborted_non_stream", + "object": "response", + "status": status, + } + ).decode(), + ) + ] + ) + assert store_endpoint.extract_response_id(record) is None + + def test_extract_response_id_no_id_returns_none( + self, endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "type": "response.output_text.delta", + "delta": "Hello", + } + ).decode(), + ) + ] + ) + assert endpoint.extract_response_id(record) is None + + def test_extract_response_id_store_false_returns_none( + self, endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_not_stored", + "object": "response", + "status": "completed", + "store": False, + }, + } + ).decode(), + ) + ] + ) + assert endpoint.extract_response_id(record) is None + + def test_extract_response_id_store_absent_returns_none( + self, endpoint: ResponsesEndpoint + ) -> None: + record = RequestRecord( + responses=[ + TextResponse( + perf_ns=1, + text=orjson.dumps( + { + "id": "resp_no_store_field", + "object": "response", + "status": "completed", + } + ).decode(), + ) + ] + ) + assert endpoint.extract_response_id(record) is None + + def test_format_payload_turn1_with_previous_response_id_sends_only_latest( + self, endpoint: ResponsesEndpoint + ) -> None: + turn0 = Turn( + role="user", + texts=[Text(contents=["First message"])], + ) + turn1 = Turn( + role="user", + texts=[Text(contents=["Second message"])], + ) + request_info = create_request_info( + model_endpoint=endpoint.model_endpoint, + turns=[turn0, turn1], + previous_response_id="resp_b6c65395f4fb8c7d", + ) + payload = endpoint.format_payload(request_info) + assert payload["previous_response_id"] == "resp_b6c65395f4fb8c7d" + assert len(payload["input"]) == 1 + assert payload["input"][0]["content"] == "Second message" diff --git a/tests/unit/workers/test_session_manager.py b/tests/unit/workers/test_session_manager.py index f418e1293f..55f3764276 100644 --- a/tests/unit/workers/test_session_manager.py +++ b/tests/unit/workers/test_session_manager.py @@ -10,7 +10,7 @@ from pydantic import ValidationError from pytest import param -from aiperf.common.enums import ConversationContextMode +from aiperf.common.enums import ConversationBranchMode, ConversationContextMode from aiperf.common.models import Conversation, Turn from aiperf.common.models.dataset_models import DatasetMetadata from aiperf.plugin.enums import DatasetSamplingStrategy @@ -421,3 +421,89 @@ def test_dataset_metadata_rejects_unsupported_default_mode(self) -> None: sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL, default_context_mode=ConversationContextMode.MESSAGE_ARRAY_WITHOUT_RESPONSES, ) + + +class TestSessionPreviousResponseId: + """Tests for UserSession previous_response_id storage and reset_context handling.""" + + @pytest.mark.parametrize( + "reset_turn", + [ + param( + Turn( + raw_messages=[{"role": "user", "content": "Q2"}], + reset_context=True, + ), + id="with_raw_messages", + ), + param(Turn(reset_context=True), id="synthetic_no_raw_messages"), + ], + ) # fmt: skip + def test_advance_turn_clears_previous_response_id_on_reset_context( + self, reset_turn: Turn + ) -> None: + conv = Conversation( + conversation_id="test-conv-reset", + turns=[ + Turn(raw_messages=[{"role": "user", "content": "Q1"}]), + reset_turn, + ], + ) + session = UserSession( + x_correlation_id="test-corr", + num_turns=2, + conversation=conv, + context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, + ) + session.advance_turn(0) + session.store_response_id("resp_turn0") + assert session.previous_response_id == "resp_turn0" + + # A context reset always breaks the wire chain, regardless of whether the + # turn carries raw_messages: chaining would retain discarded history. + session.advance_turn(1) + assert session.previous_response_id is None + + def test_advance_turn_preserves_previous_response_id_without_reset_context( + self, + ) -> None: + conv = Conversation( + conversation_id="test-conv-normal", + turns=[ + Turn(messages=[{"role": "user", "content": "Q1"}]), + Turn(messages=[{"role": "user", "content": "Q2"}]), + ], + ) + session = UserSession( + x_correlation_id="test-corr", + num_turns=2, + conversation=conv, + context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, + ) + session.advance_turn(0) + session.store_response_id("resp_turn0") + assert session.previous_response_id == "resp_turn0" + + session.advance_turn(1) + assert session.previous_response_id == "resp_turn0" + + def test_seed_from_parent_copies_previous_response_id(self) -> None: + manager = UserSessionManager() + conv = Conversation( + conversation_id="test-conv-fork", + turns=[Turn(messages=[{"role": "user", "content": "Q1"}])], + ) + parent = manager.create_and_store("parent-corr", conv, num_turns=1) + parent.store_response_id("resp_parent_last") + child = manager.create_and_store( + "child-corr", + conv, + num_turns=1, + parent_correlation_id="parent-corr", + branch_mode=ConversationBranchMode.FORK, + ) + assert child.previous_response_id is None + + manager.seed_from_parent("child-corr", "parent-corr") + + assert child.previous_response_id == "resp_parent_last" diff --git a/tests/unit/workers/test_worker.py b/tests/unit/workers/test_worker.py index b9d6d56ef7..a22cc546f8 100644 --- a/tests/unit/workers/test_worker.py +++ b/tests/unit/workers/test_worker.py @@ -7,7 +7,7 @@ import pytest from pytest import param -from aiperf.common.enums import CreditPhase +from aiperf.common.enums import ConversationContextMode, CreditPhase from aiperf.common.models import ( Conversation, ErrorDetails, @@ -20,6 +20,7 @@ from aiperf.config.phases import ConcurrencyPhase from aiperf.credit.structs import Credit, CreditContext from aiperf.dataset.memory_map_utils import PayloadTurnData +from aiperf.workers.session_manager import UserSession from aiperf.workers.worker import ( Worker, _is_terminal_context_overflow, @@ -989,6 +990,70 @@ async def test_retrieve_conversation_for_session_restores_scalars( } +@pytest.mark.asyncio +class TestFinalizeSessionResponseChaining: + """`_finalize_session_response` must chain only onto successful responses.""" + + def _chaining_session(self) -> UserSession: + conv = Conversation( + conversation_id="test-conv-chain", + turns=[Turn(raw_messages=[{"role": "user", "content": "Q1"}])], + ) + return UserSession( + x_correlation_id="test-corr", + num_turns=1, + conversation=conv, + context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, + ) + + async def test_finalize_session_response_error_keeps_last_good_id( + self, + mock_worker: Worker, + sample_credit_context: CreditContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """A failed turn may emit a partial ``response.created`` id; chaining onto + it would corrupt server context, so the last good id must be preserved.""" + monkeypatch.setattr( + mock_worker, "_process_responses_for_record", Mock(return_value=([], None)) + ) + monkeypatch.setattr(mock_worker, "_populate_response_metrics", Mock()) + extract = Mock(return_value="resp_partial") + monkeypatch.setattr( + mock_worker.inference_client.endpoint, "extract_response_id", extract + ) + session = self._chaining_session() + session.store_response_id("resp_prev_good") + record = RequestRecord(error=ErrorDetails(message="boom", type="TestError")) + + mock_worker._finalize_session_response(session, sample_credit_context, record) + + assert session.previous_response_id == "resp_prev_good" + extract.assert_not_called() + + async def test_finalize_session_response_success_stores_new_id( + self, + mock_worker: Worker, + sample_credit_context: CreditContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr( + mock_worker, "_process_responses_for_record", Mock(return_value=([], None)) + ) + monkeypatch.setattr(mock_worker, "_populate_response_metrics", Mock()) + monkeypatch.setattr( + mock_worker.inference_client.endpoint, + "extract_response_id", + Mock(return_value="resp_new"), + ) + session = self._chaining_session() + record = RequestRecord(timestamp_ns=1, start_perf_ns=1, end_perf_ns=2) + + mock_worker._finalize_session_response(session, sample_credit_context, record) + + assert session.previous_response_id == "resp_new" + + @pytest.mark.asyncio class TestMakeFirstTokenCallback: """Coverage for the REAL Worker._make_first_token_callback factory.""" From 96abbf6cca7de0a424f078b5511cfe8b9e5b74af Mon Sep 17 00:00:00 2001 From: Manu Nicholas Jacob <118115267+manunicholasjacob@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:38:51 -0500 Subject: [PATCH 047/119] test: stop asserting across a Rich line wrap in agentic_code_gen CLI test (#1340) Signed-off-by: Manu Nicholas Jacob (cherry picked from commit 960b9b9f86b832e5f60b7d76feec34bf01d41ef0) --- tests/unit/dataset/agentic_code_gen/test_cli.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/dataset/agentic_code_gen/test_cli.py b/tests/unit/dataset/agentic_code_gen/test_cli.py index 27dd8bbaeb..e95ca192d0 100644 --- a/tests/unit/dataset/agentic_code_gen/test_cli.py +++ b/tests/unit/dataset/agentic_code_gen/test_cli.py @@ -49,4 +49,9 @@ def test_validate_missing_file_exits_cleanly( validate(missing) assert exc.value.code == 1 - assert "is not a file" in capsys.readouterr().out + # The message is printed through Rich, which hard-wraps to the console + # width, and it embeds tmp_path, whose length varies with the username + # and the test name. Either can push the wrap into the middle of the + # phrase being matched, so collapse the wrapping before asserting. + out = " ".join(capsys.readouterr().out.split()) + assert "is not a file" in out From 6751655efa8d1573a1425ccc86e5b3b88e80dd54 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Tue, 1 Sep 2026 20:57:41 -0700 Subject: [PATCH 048/119] ci: increase Windows job timeout to 90 minutes (#1384) Signed-off-by: Anthony Casagrande (cherry picked from commit a84b7b3229eac17a37a7984f367ab2683ffa1a1f) --- .github/workflows/run-unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 561845153e..f90a6e0d43 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -15,7 +15,7 @@ jobs: build: runs-on: ${{ matrix.runner }} # Windows x64 runners can finish the component tier just past 30 minutes. - timeout-minutes: 45 + timeout-minutes: 90 # Force CPython UTF-8 mode so file reads default to UTF-8 instead of the # Windows cp1252 code page (no-op on POSIX). Without it, reading a UTF-8 # file on Windows raises UnicodeDecodeError on bytes like 0x8f. From 9e0036cf814766e05a19c2cdd42df72ad8f37c61 Mon Sep 17 00:00:00 2001 From: Karen Chung Date: Wed, 2 Sep 2026 16:35:20 -0700 Subject: [PATCH 049/119] fix(synthesize): fix agentic-code-gen dataset reports' cumulative ISL counts (#1364) Signed-off-by: Karen Chung Co-authored-by: Harshini Komali <157742537+lkomali@users.noreply.github.com> (cherry picked from commit 911a1aa8f7cb72ec1119503c34f294b6c44e78cd) --- .../agentic_code_gen/reporting/metrics.py | 14 +++++--- .../agentic_code_gen/reporting/plot_report.py | 7 ++-- .../agentic_code_gen/reporting/trace.py | 19 ++++++++--- .../dataset/agentic_code_gen/test_report.py | 32 +++++++++++++++++++ 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/aiperf/dataset/agentic_code_gen/reporting/metrics.py b/src/aiperf/dataset/agentic_code_gen/reporting/metrics.py index e98df74d41..6a73f225d5 100644 --- a/src/aiperf/dataset/agentic_code_gen/reporting/metrics.py +++ b/src/aiperf/dataset/agentic_code_gen/reporting/metrics.py @@ -20,7 +20,10 @@ SynthesizedSession, percentile_stats, ) -from aiperf.dataset.agentic_code_gen.reporting.trace import ParsedTurn +from aiperf.dataset.agentic_code_gen.reporting.trace import ( + ParsedTurn, + reconstruct_cumulative_context, +) def _pct_error(target: float, observed: float) -> float: @@ -103,8 +106,11 @@ def extract_metrics( for turns in sessions.values(): turns_per_session.append(float(len(turns))) session_lat = 0.0 - for i, turn in enumerate(turns): - total_isl.append(float(turn.input_length)) + cumulative_contexts = reconstruct_cumulative_context(turns) + for i, (turn, cumulative_context) in enumerate( + zip(turns, cumulative_contexts, strict=True) + ): + total_isl.append(cumulative_context) total_osl.append(float(turn.output_length)) generation_length.append(float(turn.output_length)) hash_id_block_count.append(float(len(turn.hash_ids))) @@ -116,7 +122,7 @@ def extract_metrics( session_lat += turn.delay_ms + lat if i == 0: - initial_context.append(float(turn.input_length)) + initial_context.append(cumulative_context) else: new_tokens_per_turn.append(float(turn.input_length)) inter_turn_delay.append(turn.delay_ms / 1000.0) diff --git a/src/aiperf/dataset/agentic_code_gen/reporting/plot_report.py b/src/aiperf/dataset/agentic_code_gen/reporting/plot_report.py index a866b94063..9521e6ca11 100644 --- a/src/aiperf/dataset/agentic_code_gen/reporting/plot_report.py +++ b/src/aiperf/dataset/agentic_code_gen/reporting/plot_report.py @@ -12,7 +12,10 @@ import plotly.io as pio from aiperf.dataset.agentic_code_gen.reporting.templates import render_template -from aiperf.dataset.agentic_code_gen.reporting.trace import ParsedTurn +from aiperf.dataset.agentic_code_gen.reporting.trace import ( + ParsedTurn, + reconstruct_cumulative_context, +) from aiperf.plot.constants import NVIDIA_GREEN _HISTOGRAM_PLOTS: list[tuple[str, str]] = [ @@ -121,7 +124,7 @@ def _context_growth_figure( fig.add_trace( go.Scatter( x=list(range(len(turns))), - y=[t.input_length for t in turns], + y=reconstruct_cumulative_context(turns), mode="lines", name=sid[:8], ) diff --git a/src/aiperf/dataset/agentic_code_gen/reporting/trace.py b/src/aiperf/dataset/agentic_code_gen/reporting/trace.py index 306187d0c6..a52b976c21 100644 --- a/src/aiperf/dataset/agentic_code_gen/reporting/trace.py +++ b/src/aiperf/dataset/agentic_code_gen/reporting/trace.py @@ -34,6 +34,17 @@ class ParsedTurn(AIPerfBaseModel): ) +def reconstruct_cumulative_context(turns: list[ParsedTurn]) -> list[int]: + """Reconstruct the cumulative input length observed for each turn.""" + cumulative = 0 + context_lengths: list[int] = [] + for turn in turns: + cumulative += turn.input_length + context_lengths.append(cumulative) + cumulative += turn.output_length + return context_lengths + + def iter_jsonl_rows(path: Path) -> list[dict[str, Any]]: """Read non-empty JSONL rows as dictionaries.""" rows: list[dict[str, Any]] = [] @@ -125,20 +136,18 @@ def load_simulation_sessions(jsonl_path: Path) -> list[dict[str, Any]]: grouped = group_sessions(load_jsonl(jsonl_path)) result: list[dict[str, Any]] = [] for session_id, turns in grouped.items(): - cumulative = 0 sim_turns: list[dict[str, Any]] = [] - for turn in turns: - cumulative += turn.input_length + cumulative_context = reconstruct_cumulative_context(turns) + for turn, input_length in zip(turns, cumulative_context, strict=True): sim_turns.append( { "input_length": turn.input_length, "output_length": turn.output_length, "delay_ms": turn.delay_ms, "hash_ids": turn.hash_ids, - "cumulative_input_length": cumulative, + "cumulative_input_length": input_length, } ) - cumulative += turn.output_length first = turns[0] result.append( diff --git a/tests/unit/dataset/agentic_code_gen/test_report.py b/tests/unit/dataset/agentic_code_gen/test_report.py index 8fc8d84a4c..4f5b597b1b 100644 --- a/tests/unit/dataset/agentic_code_gen/test_report.py +++ b/tests/unit/dataset/agentic_code_gen/test_report.py @@ -178,6 +178,38 @@ def test_new_tokens_use_incremental_jsonl_input_length(self) -> None: assert metrics["initial_context"].tolist() == [1000.0] assert metrics["new_tokens_per_turn"].tolist() == [75.0, 125.0] + def test_total_isl_reconstructs_cumulative_context_across_turns(self) -> None: + sessions = { + "s1": [ + ParsedTurn( + session_id="s1", + input_length=1000, + output_length=20, + hash_ids=[], + delay_ms=0.0, + ), + ParsedTurn( + session_id="s1", + input_length=75, + output_length=10, + hash_ids=[], + delay_ms=1.0, + ), + ParsedTurn( + session_id="s1", + input_length=125, + output_length=10, + hash_ids=[], + delay_ms=1.0, + ), + ] + } + + metrics = extract_metrics(sessions) + + # turn0: 1000. turn1: 1000 + 20 + 75. turn2: 1095 + 10 + 125. + assert metrics["total_isl"].tolist() == [1000.0, 1095.0, 1230.0] + class TestBuildReportData: def test_comparisons_include_target_metrics(self, run_dir: Path) -> None: From 5d96244151a222fe2abe72f44e25fb0e2aed77d2 Mon Sep 17 00:00:00 2001 From: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Date: Thu, 3 Sep 2026 20:44:29 -0700 Subject: [PATCH 050/119] feat(export): imply outputs.json from --export-level raw (#1365) (cherry picked from commit 520397bdf098a9c2e19997274553941b9a85b18a) --- docs/reference/json-export-schema.md | 32 +++ .../tutorials/working-with-profile-exports.md | 1 + src/aiperf/config/artifacts.py | 66 +++--- src/aiperf/config/config.py | 33 ++- src/aiperf/config/flags/cli_config.py | 24 ++- src/aiperf/exporters/outputs_json_exporter.py | 50 ++++- .../outputs_json_record_processor.py | 7 +- .../cli_runner/test_on_complete_callbacks.py | 5 +- tests/unit/config/test_auto_plot_fields.py | 12 +- tests/unit/config/test_outputs_json_prefix.py | 96 +++++++++ .../test_outputs_json_prefix_collision.py | 77 ------- tests/unit/config/test_plot_envelope.py | 30 +++ .../config/test_raw_implies_outputs_json.py | 176 ++++++++++++++++ .../exporters/test_outputs_json_exporter.py | 189 +++++++++++++++++- .../test_outputs_json_record_processor.py | 53 +++-- 15 files changed, 707 insertions(+), 144 deletions(-) create mode 100644 tests/unit/config/test_outputs_json_prefix.py delete mode 100644 tests/unit/config/test_outputs_json_prefix_collision.py create mode 100644 tests/unit/config/test_raw_implies_outputs_json.py diff --git a/docs/reference/json-export-schema.md b/docs/reference/json-export-schema.md index dac629cf1f..8d9c65e619 100644 --- a/docs/reference/json-export-schema.md +++ b/docs/reference/json-export-schema.md @@ -155,6 +155,38 @@ The current schema version is exported as the top-level `schema_version` field o The `schema_version` documented on this page applies only to `profile_export_aiperf.json`. The other files evolve on their own cadence. +### `outputs.json` schema + +`outputs.json` holds the generated response text for each request. It is written when `--export-outputs-json` is set, which `--export-level raw` implies (pass `--no-export-outputs-json` to opt out). With `--profile-export-prefix foo` the file is named `foo_outputs.json`. + +It carries its own `schema_version` (`OutputsJsonExporter.SCHEMA_VERSION`, currently `"1.1"`), independent of the one documented on this page. + +```json +{ + "schema_version": "1.1", + "data": [ + { + "session_num": 0, + "conversation_id": "964a5c5e-9415-4a69-8c81-6ab51671ff39", + "turn_index": 0, + "x_request_id": "ebeeb5d1-2afa-4d58-bacf-2866e69d28e2", + "benchmark_phase": "profiling", + "request_start_ns": 1788226305237111762, + "request_end_ns": 1788226305315824596, + "metrics": {"input_sequence_length": 20, "output_token_count": 10}, + "response_text": "tys field Thy youths proud livery so gazed" + } + ], + "warmup": [] +} +``` + +- **`data` is profiling-only.** Warmup responses live in the separate top-level `warmup` array, so consumers that aggregate over `data` never count warmup traffic. Both arrays are sorted by `(session_num, turn_index)`. +- **`benchmark_phase`** is `profiling` or `warmup` and matches the array the entry sits in. Only a known-warmup phase is routed to `warmup`; an unrecognized phase stays in `data`. +- **`response_text`** is the concatenated generated text. For models that emit a separate reasoning channel, reasoning and answer are concatenated. It is `null` when the response carried no content. +- **`metrics`** holds an allowlisted subset in display units (`input_sequence_length`, `output_token_count`, `output_sequence_length`, `request_latency`, `time_to_first_token`, `inter_token_latency`). Streaming-only metrics are absent from non-streaming records. +- Schema `1.1` added `warmup` and `benchmark_phase`; both are additive over `1.0`. + ### Named phase artifacts Named multi-phase workflows keep the root exports (`profile_export_aiperf.{json,csv}`, `server_metrics_export.{json,csv}`, and related files) as backward-compatible aggregate artifacts. Phase-scoped artifacts are additive and are referenced from `phase_manifest.json`, for example `phases//profile_export_aiperf.json`, `phases//profile_export_aiperf.csv`, and, when server metrics produced data for that phase, `phases//server_metrics.json`. diff --git a/docs/tutorials/working-with-profile-exports.md b/docs/tutorials/working-with-profile-exports.md index ebb73f65c1..4e4beec12e 100644 --- a/docs/tutorials/working-with-profile-exports.md +++ b/docs/tutorials/working-with-profile-exports.md @@ -16,6 +16,7 @@ AIPerf generates multiple output formats after each benchmark run, each optimize - [**`profile_export.jsonl`**](#per-request-records-jsonl) - Per-request metric records in JSON Lines format with one record per line - [**`profile_export_aiperf.json`**](#aggregated-statistics-json) - Aggregated statistics and user configuration as a single JSON object - [**`profile_export_aiperf.csv`**](#aggregated-statistics-csv) - Aggregated statistics in CSV format +- **`outputs.json`** - Generated response text per request, with a small set of per-request metrics. Written when `--export-outputs-json` is set, which `--export-level raw` implies. See [`outputs.json` schema](../reference/json-export-schema.md#outputsjson-schema). ## Data Models diff --git a/src/aiperf/config/artifacts.py b/src/aiperf/config/artifacts.py index cf46ad4c6c..20516bddf9 100644 --- a/src/aiperf/config/artifacts.py +++ b/src/aiperf/config/artifacts.py @@ -96,12 +96,13 @@ class ArtifactsConfig(BaseConfig): description="Base filename override applied to ALL profile and server-metrics " "exports. With prefix='foo' every output becomes `foo.csv`, `foo.json`, " "`foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, " - "`foo_gpu_telemetry.jsonl`, `foo_server_metrics.{jsonl,json,csv,parquet}`. " + "`foo_outputs.json`, `foo_gpu_telemetry.jsonl`, " + "`foo_server_metrics.{jsonl,json,csv,parquet}`. " "When unset (the default), historical per-file names are used: " "`profile_export_aiperf.csv/json`, `profile_export.jsonl`, " - "`profile_export_raw.jsonl`, `gpu_telemetry_export.jsonl`, " + "`profile_export_raw.jsonl`, `outputs.json`, `gpu_telemetry_export.jsonl`, " "`server_metrics_export.{jsonl,json,csv,parquet}`. Known suffixes " - "(`_raw.jsonl`, `_timeslices.{csv,json}`, `_gpu_telemetry.jsonl`, " + "(`_raw.jsonl`, `_outputs.json`, `_timeslices.{csv,json}`, `_gpu_telemetry.jsonl`, " "`_server_metrics.{jsonl,json,csv,parquet}`, `.csv`/`.json`/`.jsonl`/`.parquet`) " "are stripped from the supplied value so `--profile-export-prefix foo_raw.jsonl` " "still yields a clean `foo` base.", @@ -176,14 +177,15 @@ class ArtifactsConfig(BaseConfig): ] auto_plot: Annotated[ - bool, + bool | None, Field( - default=False, + default=None, description=( "Auto-invoke `aiperf plot` against the artifact directory after the " - "benchmark completes. Resolved by the CLI converter from the " - "tri-state CLI flag and the active search recipe's auto_plot_default; " - "by the time it lands here it is a plain bool." + "benchmark completes. Unset (the default) means a `plot:` section " + "implies True; set False to suppress plots even with one, True to " + "force them. `AIPerfConfig` resolves this to a concrete bool, so any " + "config that came through the loader carries a plain bool." ), ), ] @@ -201,10 +203,14 @@ class ArtifactsConfig(BaseConfig): ] export_outputs_json: Annotated[ - bool, + bool | None, Field( - default=False, - description="Export generated response text to outputs.json after the run.", + default=None, + description="Export generated response text after the run, to " + "`outputs.json` or `_outputs.json` when `prefix` is set. " + "Unset (the default) means the `raw` export level implies it; set " + "False to opt out at raw, True to force it at any level. Always a " + "concrete bool once validation has run.", ), ] @@ -221,20 +227,28 @@ def validate_artifacts(self) -> ArtifactsConfig: ) if self.slice_duration is not None and self.slice_duration <= 0: raise ValueError("slice_duration must be > 0") - if self.export_outputs_json and self.prefix is not None: - self._check_outputs_json_collision() + self._apply_raw_implies_outputs_json() return self - def _check_outputs_json_collision(self) -> None: - """Reject prefix values that collide with the outputs.json path.""" - if self.profile_export_json_file.resolve() == self.outputs_json_file.resolve(): - base = self._base() - raise ValueError( - f"--profile-export-prefix resolves to '{base}' which produces " - f"'{base}.json', colliding with --export-outputs-json " - f"(also '{OutputDefaults.OUTPUTS_JSON_FILE.name}'). " - f"Use a different prefix." - ) + def _apply_raw_implies_outputs_json(self) -> None: + """Resolve the tri-state ``export_outputs_json`` to a concrete bool. + + ``None`` means "decide from the export level": raw implies the file, + because if you asked for the full request/response bodies you want the + generated text too. An explicit True or False is honored as-is, so + raw-level debugging can decline the second copy. + + Tri-state rather than ``model_fields_set`` on a ``False`` default: the + Kubernetes apiserver materializes CRD ``default:`` values on write, so a + defaulted-to-False field is indistinguishable from a user-authored one + and would silently defeat the implication for every AIPerfJob. + """ + if self.export_outputs_json is None: + self.export_outputs_json = self.raw + # Resolving is not authoring: keep the field out of + # ``model_fields_set`` so ``exclude_unset`` dumps (the Kubernetes + # ConfigMap payload) do not invent a value the user never wrote. + self.model_fields_set.discard("export_outputs_json") # ========================================================================== # COMPUTED FILE PATH PROPERTIES @@ -254,6 +268,7 @@ def _check_outputs_json_collision(self) -> None: "_timeslices.csv", "_timeslices.json", "_console.txt", + "_outputs.json", "_raw.jsonl", ".parquet", ".csv", @@ -320,7 +335,10 @@ def profile_export_jsonl_file(self) -> Path: @property def outputs_json_file(self) -> Path: """Path for the aggregated generated outputs JSON export file.""" - return self.dir / OutputDefaults.OUTPUTS_JSON_FILE + base = self._base() + default = OutputDefaults.OUTPUTS_JSON_FILE.name + name = f"{base}_outputs.json" if base else default + return self.dir / name @property def profile_export_raw_jsonl_file(self) -> Path: diff --git a/src/aiperf/config/config.py b/src/aiperf/config/config.py index 5a2d55b6e7..bb35fb731b 100644 --- a/src/aiperf/config/config.py +++ b/src/aiperf/config/config.py @@ -1075,20 +1075,33 @@ def _apply_consistent_seed_default(self) -> Self: @model_validator(mode="after") def _plot_implies_auto_plot(self) -> Self: - """When ``plot:`` is set and the user didn't explicitly set - ``artifacts.auto_plot``, flip it to True. Explicit ``auto_plot: false`` - wins; we just log an info-level breadcrumb so the silence doesn't - surprise users. + """Resolve the tri-state ``artifacts.auto_plot`` to a concrete bool. + + ``None`` means the user did not decide, so a ``plot:`` section implies + True. An explicit False wins; we log an info-level breadcrumb so the + silence doesn't surprise users. + + Tri-state rather than ``model_fields_set``: the Kubernetes apiserver + materializes CRD ``default:`` values on write, so a defaulted-to-False + field is indistinguishable from a user-authored one and would silently + suppress plots for every AIPerfJob carrying a ``plot:`` section. """ - if self.plot is None: - return self - if "auto_plot" in self.benchmark.artifacts.model_fields_set: - if self.benchmark.artifacts.auto_plot is False: + artifacts = self.benchmark.artifacts + authored = "auto_plot" in artifacts.model_fields_set + if self.plot is not None: + if artifacts.auto_plot is None: + artifacts.auto_plot = True + elif artifacts.auto_plot is False: _logger.info( "plot section present but artifacts.auto_plot=false " "explicitly; plots will not auto-render. Use ``aiperf plot " "`` after the run to generate them." ) - return self - self.benchmark.artifacts.auto_plot = True + if artifacts.auto_plot is None: + artifacts.auto_plot = False + if not authored: + # Resolving is not authoring: keep the field out of + # ``model_fields_set`` so ``exclude_unset`` dumps (the Kubernetes + # ConfigMap payload) do not invent a value the user never wrote. + artifacts.model_fields_set.discard("auto_plot") return self diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 3d2eb8d601..6051ad1103 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -2657,12 +2657,13 @@ def url(self) -> str: Field( description="Base filename for ALL exported files. With prefix='foo' every " "output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, " - "`foo.jsonl`, `foo_raw.jsonl`, `foo_gpu_telemetry.jsonl`, and " - "`foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), " + "`foo.jsonl`, `foo_raw.jsonl`, `foo_outputs.json`, `foo_gpu_telemetry.jsonl`, " + "and `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), " "historical per-file names are used: `profile_export_aiperf.{csv,json}` " "for the summary, `profile_export.jsonl` and `profile_export_raw.jsonl` " - "for records, `gpu_telemetry_export.jsonl`, and `server_metrics_export.*`. " - "Known suffixes (e.g. `_raw.jsonl`, `_timeslices.csv`, `_server_metrics.parquet`) " + "for records, `outputs.json`, `gpu_telemetry_export.jsonl`, and " + "`server_metrics_export.*`. Known suffixes (e.g. `_raw.jsonl`, " + "`_outputs.json`, `_timeslices.csv`, `_server_metrics.parquet`) " "are stripped from the supplied value.", ), CLIParameter( @@ -2680,7 +2681,9 @@ def url(self) -> str: description="Controls which output files are generated. " "`summary`: Only aggregate metrics files (`.csv`, `.json`). " "`records`: Includes per-request metrics (`.jsonl`). " - "`raw`: Includes raw request/response data (`_raw.jsonl`).", + "`raw`: Includes raw request/response data (`_raw.jsonl`) and, unless " + "`--no-export-outputs-json` is passed, the generated text " + "(`_outputs.json`).", ), CLIParameter( name=("--export-level", "--profile-export-level"), @@ -2739,14 +2742,19 @@ def url(self) -> str: bool, Field( description=( - "Export generated response text to outputs.json after the run. " - "When enabled, the raw generated-text payload for each request is " - "written to an outputs.json file in the artifact directory." + "Export generated response text after the run. When enabled, the " + "raw generated-text payload for each request is written to " + "`outputs.json` in the artifact directory, or to " + "`_outputs.json` when `--profile-export-prefix` is set " + "(prefix `foo` gives `foo_outputs.json`). Implied by " + "`--export-level raw`; pass `--no-export-outputs-json` to opt out " + "of the extra file while keeping raw export." ), ), CLIParameter( name=("--export-outputs-json",), group=Groups.OUTPUT, + negative="--no-export-outputs-json", ), ] = False diff --git a/src/aiperf/exporters/outputs_json_exporter.py b/src/aiperf/exporters/outputs_json_exporter.py index 6ecd875bf0..2b2909c155 100644 --- a/src/aiperf/exporters/outputs_json_exporter.py +++ b/src/aiperf/exporters/outputs_json_exporter.py @@ -8,6 +8,7 @@ import aiofiles import orjson +from aiperf.common.enums import CreditPhase from aiperf.common.exceptions import DataExporterDisabled from aiperf.common.finite import scrub_non_finite from aiperf.common.mixins import AIPerfLoggerMixin @@ -27,6 +28,17 @@ class OutputsJsonExporter(AIPerfLoggerMixin): Self-disables unless --export-outputs-json is set. """ + # 1.1 added the top-level `warmup` array and the per-entry `benchmark_phase` + # field. Both are additive: `data` is still profiling-only. + SCHEMA_VERSION = "1.1" + + # outputs.json is sorted by (session_num, turn_index), so unlike the raw + # record aggregator it cannot stream -- the whole document is materialized + # to sort it, costing roughly 3x the on-disk fragment size at peak. Warn + # past this many records so a large run sees the cost instead of just + # paying it, since --export-level raw now enables this export by default. + LARGE_EXPORT_RECORD_WARNING_THRESHOLD = 50_000 + def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: super().__init__(**kwargs) self._cfg = exporter_config.cfg @@ -61,32 +73,60 @@ async def export(self) -> None: fragments = await self._read_fragments(fragment_files) records: list[JsonObject] = [] + warmup: list[JsonObject] = [] for frag in fragments: + phase = frag.get("benchmark_phase") entry = { "session_num": frag["session_num"], "conversation_id": frag.get("conversation_id"), "turn_index": frag.get("turn_index"), "x_request_id": frag.get("x_request_id"), + "benchmark_phase": phase, "request_start_ns": frag.get("request_start_ns"), "request_end_ns": frag.get("request_end_ns"), "metrics": frag.get("metrics") or {}, "response_text": frag.get("response_text"), } - records.append(entry) - - records.sort(key=lambda r: (r["session_num"], r.get("turn_index") or 0)) + # `data` stays profiling-only so consumers that sum over it do not + # silently start counting warmup responses. Test explicitly for + # WARMUP rather than for PROFILING: only a known-warmup record may + # leave `data`, so a missing or newly-added phase surfaces there + # instead of silently vanishing from everyone's denominators. + if phase == CreditPhase.WARMUP: + warmup.append(entry) + else: + records.append(entry) + + def _order(record: JsonObject) -> tuple[int, int]: + return (record["session_num"], record.get("turn_index") or 0) + + records.sort(key=_order) + warmup.sort(key=_order) output = { - "schema_version": "1.0", + "schema_version": self.SCHEMA_VERSION, "data": records, + "warmup": warmup, } + total = len(records) + len(warmup) + if total >= self.LARGE_EXPORT_RECORD_WARNING_THRESHOLD: + self.warning( + f"Serializing {total} records to {self._file_path.name}; this " + "export is sorted and therefore held in memory in full, needing " + "roughly 3x the fragment size on disk. Pass " + "--no-export-outputs-json to skip it." + ) + self._file_path.parent.mkdir(parents=True, exist_ok=True) content = orjson.dumps(scrub_non_finite(output), option=orjson.OPT_INDENT_2) async with aiofiles.open(self._file_path, "wb") as f: await f.write(content) - self.info(f"Exported {len(records)} records to {self._file_path}") + self.info( + f"Exported {len(records)} records ({len(warmup)} warmup) " + f"to {self._file_path}" + ) self._cleanup_fragments(fragment_files) diff --git a/src/aiperf/post_processors/outputs_json_record_processor.py b/src/aiperf/post_processors/outputs_json_record_processor.py index cc7e974858..e38155d152 100644 --- a/src/aiperf/post_processors/outputs_json_record_processor.py +++ b/src/aiperf/post_processors/outputs_json_record_processor.py @@ -29,6 +29,10 @@ class OutputFragment(AIPerfBaseModel): turn_index: int = Field(ge=0, description="The turn index within the conversation.") conversation_id: str = Field(description="The conversation identifier.") x_request_id: str = Field(description="The unique request identifier.") + benchmark_phase: CreditPhase = Field( + description="The benchmark phase the request ran in. Warmup and profiling " + "responses are both captured; the exporter keeps them in separate arrays.", + ) response_text: str | None = Field( default=None, description="The concatenated generated text from the model response.", @@ -110,8 +114,6 @@ async def observe(self, ctx: RecordObserverContext) -> None: """Extract response text and allowlisted metrics, and write an output fragment.""" record = ctx.record metadata = ctx.metadata - if metadata.benchmark_phase != CreditPhase.PROFILING: - return parts: list[str] = [] for resp in record.content_responses: @@ -141,6 +143,7 @@ async def observe(self, ctx: RecordObserverContext) -> None: turn_index=metadata.turn_index or 0, conversation_id=metadata.conversation_id or "", x_request_id=metadata.x_request_id or "", + benchmark_phase=metadata.benchmark_phase, response_text=response_text, request_start_ns=metadata.request_start_ns or 0, request_end_ns=metadata.request_end_ns or 0, diff --git a/tests/unit/cli_runner/test_on_complete_callbacks.py b/tests/unit/cli_runner/test_on_complete_callbacks.py index 44088fdd35..81df9f9d81 100644 --- a/tests/unit/cli_runner/test_on_complete_callbacks.py +++ b/tests/unit/cli_runner/test_on_complete_callbacks.py @@ -132,8 +132,9 @@ def test_no_auto_plot_passes_empty_callback_list( from aiperf.cli_runner import run_benchmark plan = _make_plan() - # default for ArtifactsConfig.auto_plot is False - assert plan.configs[0].artifacts.auto_plot is False + # auto_plot is tri-state and unset here: this plan is built directly + # rather than through AIPerfConfig, which is what resolves it to a bool. + assert not plan.configs[0].artifacts.auto_plot run_benchmark(plan) diff --git a/tests/unit/config/test_auto_plot_fields.py b/tests/unit/config/test_auto_plot_fields.py index d347a84f00..4c0dbaaef9 100644 --- a/tests/unit/config/test_auto_plot_fields.py +++ b/tests/unit/config/test_auto_plot_fields.py @@ -57,10 +57,16 @@ def test_v1_output_plot_required_default_is_false() -> None: assert cfg.plot_required is False -def test_v2_artifacts_auto_plot_default_is_false() -> None: - """v2 ``ArtifactsConfig.auto_plot`` is a plain bool defaulting to False.""" +def test_v2_artifacts_auto_plot_default_is_unset() -> None: + """v2 ``ArtifactsConfig.auto_plot`` is tri-state, defaulting to unset. + + ``None`` (rather than False) is what lets a ``plot:`` section imply True + without ``model_fields_set``, which the Kubernetes apiserver defeats by + materializing CRD ``default:`` values on write. ``AIPerfConfig`` resolves + it, so anything through the loader carries a plain bool. + """ cfg = ArtifactsConfig() - assert cfg.auto_plot is False + assert cfg.auto_plot is None assert cfg.plot_required is False diff --git a/tests/unit/config/test_outputs_json_prefix.py b/tests/unit/config/test_outputs_json_prefix.py new file mode 100644 index 0000000000..f28d2b8d2e --- /dev/null +++ b/tests/unit/config/test_outputs_json_prefix.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for --profile-export-prefix applying to the outputs.json export.""" + +from __future__ import annotations + +import pytest +from pytest import param + +from aiperf.config.artifacts import ArtifactsConfig +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.flags.converter import convert_cli_to_aiperf + + +class TestOutputsJsonPrefix: + """ArtifactsConfig.outputs_json_file honors --profile-export-prefix.""" + + def test_outputs_json_file_no_prefix_uses_historical_name(self) -> None: + cfg = ArtifactsConfig() + assert cfg.outputs_json_file.name == "outputs.json" + + def test_outputs_json_file_with_prefix_is_prefixed(self) -> None: + cfg = ArtifactsConfig(prefix="foo") + assert cfg.outputs_json_file.name == "foo_outputs.json" + + @pytest.mark.parametrize( + "prefix", + [ + param("foo", id="bare"), + param("foo.json", id="with-json-suffix"), + param("foo.jsonl", id="with-jsonl-suffix"), + param("foo_raw.jsonl", id="with-raw-suffix"), + param("foo_outputs.json", id="with-outputs-suffix"), + param("foo_timeslices.json", id="with-timeslices-suffix"), + ], + ) # fmt: skip + def test_outputs_json_file_strips_known_suffixes(self, prefix: str) -> None: + cfg = ArtifactsConfig(prefix=prefix) + assert cfg.outputs_json_file.name == "foo_outputs.json" + + def test_outputs_suffix_strips_before_json_suffix(self) -> None: + """`_outputs.json` must win over `.json` (longest match first).""" + cfg = ArtifactsConfig(prefix="foo_outputs.json") + assert cfg._base() == "foo" + + @pytest.mark.parametrize( + "prefix", + [ + param("outputs", id="bare"), + param("outputs.json", id="with-json-suffix"), + param("outputs_raw.jsonl", id="with-raw-suffix"), + ], + ) # fmt: skip + def test_formerly_colliding_prefix_now_resolves_distinctly( + self, prefix: str + ) -> None: + """A prefix of 'outputs' used to be a hard error; the paths no longer collide.""" + cfg = ArtifactsConfig(prefix=prefix, export_outputs_json=True) + assert cfg.profile_export_json_file.name == "outputs.json" + assert cfg.outputs_json_file.name == "outputs_outputs.json" + assert cfg.profile_export_json_file != cfg.outputs_json_file + + def test_outputs_json_file_distinct_from_every_other_export_path(self) -> None: + cfg = ArtifactsConfig(prefix="foo", export_outputs_json=True) + others = { + cfg.profile_export_json_file, + cfg.profile_export_csv_file, + cfg.profile_export_jsonl_file, + cfg.profile_export_raw_jsonl_file, + cfg.profile_export_timeslices_json_file, + } + assert cfg.outputs_json_file not in others + + +class TestOutputsJsonPrefixCLI: + """End-to-end: the CLI converter carries the prefix into outputs.json.""" + + def test_convert_cli_to_aiperf_prefixes_outputs_json(self) -> None: + cli = CLIConfig( + model_names=["m"], + profile_export_prefix="foo", + export_outputs_json=True, + ) + cfg = convert_cli_to_aiperf(cli) + assert cfg.benchmark.artifacts.outputs_json_file.name == "foo_outputs.json" + + def test_convert_cli_to_aiperf_formerly_colliding_prefix_allowed(self) -> None: + cli = CLIConfig( + model_names=["m"], + profile_export_prefix="outputs", + export_outputs_json=True, + ) + cfg = convert_cli_to_aiperf(cli) + artifacts = cfg.benchmark.artifacts + assert artifacts.profile_export_json_file.name == "outputs.json" + assert artifacts.outputs_json_file.name == "outputs_outputs.json" diff --git a/tests/unit/config/test_outputs_json_prefix_collision.py b/tests/unit/config/test_outputs_json_prefix_collision.py deleted file mode 100644 index af943e8037..0000000000 --- a/tests/unit/config/test_outputs_json_prefix_collision.py +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for --profile-export-prefix / --export-outputs-json path collision guard. - -When both flags are active and the prefix resolves to 'outputs', the summary -JSON (`outputs.json`) and the generated-output export (`outputs.json`) would -target the same file. The model validator must reject this combination. -""" - -from __future__ import annotations - -import pytest -from pydantic import ValidationError -from pytest import param - -from aiperf.config.artifacts import ArtifactsConfig -from aiperf.config.flags.cli_config import CLIConfig -from aiperf.config.flags.converter import convert_cli_to_aiperf - - -class TestOutputsJsonPrefixCollision: - """ArtifactsConfig.validate_artifacts rejects colliding prefix + export_outputs_json.""" - - @pytest.mark.parametrize( - "prefix", - [ - param("outputs", id="bare"), - param("outputs.json", id="with-json-suffix"), - param("outputs.jsonl", id="with-jsonl-suffix"), - param("outputs.csv", id="with-csv-suffix"), - param("outputs_raw.jsonl", id="with-raw-suffix"), - param("outputs_timeslices.json", id="with-timeslices-suffix"), - ], - ) # fmt: skip - def test_validate_artifacts_colliding_prefix_raises_error( - self, prefix: str - ) -> None: - with pytest.raises( - ValidationError, match="colliding with --export-outputs-json" - ): - ArtifactsConfig(prefix=prefix, export_outputs_json=True) - - def test_validate_artifacts_colliding_prefix_without_export_allows(self) -> None: - cfg = ArtifactsConfig(prefix="outputs", export_outputs_json=False) - assert cfg.profile_export_json_file.name == "outputs.json" - - def test_validate_artifacts_non_colliding_prefix_with_export_allows(self) -> None: - cfg = ArtifactsConfig(prefix="foo", export_outputs_json=True) - assert cfg.profile_export_json_file.name == "foo.json" - assert cfg.outputs_json_file.name == "outputs.json" - - def test_validate_artifacts_no_prefix_with_export_allows(self) -> None: - cfg = ArtifactsConfig(export_outputs_json=True) - assert cfg.profile_export_json_file.name == "profile_export_aiperf.json" - assert cfg.outputs_json_file.name == "outputs.json" - - -class TestOutputsJsonPrefixCollisionCLI: - """End-to-end: CLI converter propagates the collision to ArtifactsConfig validation.""" - - def test_convert_cli_to_aiperf_colliding_prefix_with_export_raises_error( - self, - ) -> None: - cli = CLIConfig( - model_names=["m"], - profile_export_prefix="outputs", - export_outputs_json=True, - ) - with pytest.raises( - ValidationError, match="colliding with --export-outputs-json" - ): - convert_cli_to_aiperf(cli) - - def test_convert_cli_to_aiperf_colliding_prefix_without_export_allows(self) -> None: - cli = CLIConfig(model_names=["m"], profile_export_prefix="outputs") - cfg = convert_cli_to_aiperf(cli) - assert cfg.benchmark.artifacts.profile_export_json_file.name == "outputs.json" diff --git a/tests/unit/config/test_plot_envelope.py b/tests/unit/config/test_plot_envelope.py index a9e3e916cf..58528adc17 100644 --- a/tests/unit/config/test_plot_envelope.py +++ b/tests/unit/config/test_plot_envelope.py @@ -10,6 +10,8 @@ from pytest import param from aiperf.config import AIPerfConfig # noqa: F401 (used by Task 3 tests below) +from aiperf.config.artifacts import ArtifactsConfig +from aiperf.config.loader import load_config_from_mapping from aiperf.config.loader.core import load_config_from_string from aiperf.config.loader.errors import ConfigurationError from aiperf.config.plot import ( @@ -18,6 +20,7 @@ ServerMetricsDownsampling, load_plot_envelope_from_path, ) +from aiperf.kubernetes.spec_converter import build_config_envelope def test_plot_envelope_minimal_inline_form_loads(): @@ -384,3 +387,30 @@ def test_build_benchmark_plan_no_plot_section(): config = load_config_from_string(_BASE_BENCHMARK_YAML) plan = build_benchmark_plan(config) assert plan.plot is None + + +def test_k8s_crd_defaulting_cannot_suppress_implied_auto_plot() -> None: + """A ``plot:`` section must still imply auto_plot under Kubernetes. + + The apiserver materializes CRD ``default:`` values on write, so gating the + implication on ``model_fields_set`` meant every AIPerfJob stored an explicit + ``autoPlot: false`` and silently rendered no plots. ``auto_plot`` is + tri-state so there is no CRD default to materialize. + """ + assert ArtifactsConfig.model_fields["auto_plot"].default is None + + spec = { + "benchmark": { + "model": "test-model", + "endpoint": {"url": "http://localhost:8000"}, + "dataset": {"type": "synthetic"}, + "phases": {"type": "concurrency", "concurrency": 1, "requests": 5}, + }, + "plot": {"visualization": {}}, + } + config = load_config_from_mapping(build_config_envelope(spec)) + assert config.benchmark.artifacts.auto_plot is True + + spec["benchmark"]["artifacts"] = {"autoPlot": False} + config = load_config_from_mapping(build_config_envelope(spec)) + assert config.benchmark.artifacts.auto_plot is False diff --git a/tests/unit/config/test_raw_implies_outputs_json.py b/tests/unit/config/test_raw_implies_outputs_json.py new file mode 100644 index 0000000000..058e0d63de --- /dev/null +++ b/tests/unit/config/test_raw_implies_outputs_json.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for `--export-level raw` implying `--export-outputs-json`. + +Asking for the full request/response bodies should also give you the generated +text. The implication is a default, not a lock: an explicit +`--no-export-outputs-json` (or `exportOutputsJson: false` in YAML) still wins. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from aiperf.common.enums import ExportFormat, ExportLevel +from aiperf.config.artifacts import ArtifactsConfig +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.flags.converter import convert_cli_to_aiperf +from aiperf.config.flags.resolver import resolve_config +from aiperf.config.loader import load_config_from_mapping +from aiperf.kubernetes.spec_converter import build_config_envelope + + +def _artifacts(**kwargs: Any) -> ArtifactsConfig: + return ArtifactsConfig(records=[ExportFormat.JSONL], **kwargs) + + +class TestRawImpliesOutputsJson: + """ArtifactsConfig resolves the implication during validation.""" + + def test_raw_enables_outputs_json(self) -> None: + assert _artifacts(raw=True).export_outputs_json is True + + def test_raw_with_explicit_false_stays_disabled(self) -> None: + cfg = _artifacts(raw=True, export_outputs_json=False) + assert cfg.export_outputs_json is False + + def test_records_level_does_not_enable_outputs_json(self) -> None: + assert _artifacts(raw=False).export_outputs_json is False + + def test_summary_level_does_not_enable_outputs_json(self) -> None: + cfg = ArtifactsConfig(records=False, raw=False) + assert cfg.export_outputs_json is False + + def test_flag_alone_works_below_raw(self) -> None: + """The flag is still independently usable; summary + text is a valid state.""" + cfg = ArtifactsConfig(records=False, raw=False, export_outputs_json=True) + assert cfg.export_level == ExportLevel.SUMMARY + assert cfg.export_outputs_json is True + + +class TestRawImpliesOutputsJsonCLI: + """End-to-end through the CLI converter.""" + + def test_export_level_raw_enables_outputs_json(self) -> None: + cli = CLIConfig(model_names=["m"], export_level=ExportLevel.RAW) + cfg = convert_cli_to_aiperf(cli) + assert cfg.benchmark.artifacts.export_outputs_json is True + + def test_no_export_outputs_json_opts_out_of_the_implication(self) -> None: + cli = CLIConfig( + model_names=["m"], + export_level=ExportLevel.RAW, + export_outputs_json=False, + ) + cfg = convert_cli_to_aiperf(cli) + artifacts = cfg.benchmark.artifacts + assert artifacts.export_level == ExportLevel.RAW + assert artifacts.export_outputs_json is False + + def test_export_level_records_leaves_outputs_json_off(self) -> None: + cli = CLIConfig(model_names=["m"], export_level=ExportLevel.RECORDS) + cfg = convert_cli_to_aiperf(cli) + assert cfg.benchmark.artifacts.export_outputs_json is False + + +class TestRawImpliesOutputsJsonUnderConfigYaml: + """The implication through `resolve_config` with a real `--config` YAML. + + The `--config` layer is the one the repo's flag-routing invariant singles + out, and the implication depends on a contract no other test states: + `build_artifacts` propagates `export_outputs_json` only when the user set + it, so `model_fields_set` stays empty otherwise. A refactor that always + emitted the field would silently kill the implication. + """ + + @staticmethod + def _yaml(tmp_path: Path, artifacts: str) -> Path: + cfg = tmp_path / "base.yaml" + cfg.write_text( + 'schemaVersion: "2.0"\n' + "benchmark:\n" + " model: test-model\n" + " endpoint:\n" + " url: http://localhost:8000\n" + " dataset:\n" + " type: synthetic\n" + " phases:\n" + " type: concurrency\n" + " concurrency: 1\n" + " requests: 5\n" + f" artifacts:\n{artifacts}" + ) + return cfg + + @staticmethod + def _cli(**kwargs: Any) -> CLIConfig: + """A CLIConfig whose model_fields_set is exactly ``kwargs``.""" + return CLIConfig(**CLIConfig(**kwargs).model_dump(exclude_unset=True)) + + def test_yaml_raw_true_implies_outputs_json(self, tmp_path: Path) -> None: + cfg = resolve_config(self._cli(), self._yaml(tmp_path, " raw: true\n")) + assert cfg.benchmark.artifacts.export_outputs_json is True + + def test_yaml_explicit_false_survives_cli_export_level_raw( + self, tmp_path: Path + ) -> None: + cfg = resolve_config( + self._cli(export_level=ExportLevel.RAW), + self._yaml(tmp_path, " exportOutputsJson: false\n"), + ) + artifacts = cfg.benchmark.artifacts + assert artifacts.export_level == ExportLevel.RAW + assert artifacts.export_outputs_json is False + + def test_yaml_raw_true_with_cli_opt_out(self, tmp_path: Path) -> None: + cfg = resolve_config( + self._cli(export_outputs_json=False), + self._yaml(tmp_path, " raw: true\n"), + ) + artifacts = cfg.benchmark.artifacts + assert artifacts.export_level == ExportLevel.RAW + assert artifacts.export_outputs_json is False + + +class TestRawImpliesOutputsJsonUnderKubernetes: + """The implication must survive Kubernetes CRD structural-schema defaulting. + + A CRD ``default:`` is materialized by the apiserver on write, so a bool + field with a non-null default arrives indistinguishable from a user-authored + value. That is why ``export_outputs_json`` is tri-state rather than gated on + ``model_fields_set``: with a ``False`` default the CRD carried + ``default: false``, every AIPerfJob stored an explicit ``false``, and + ``artifacts.raw: true`` silently produced no outputs.json. + """ + + @staticmethod + def _resolve(artifacts: dict[str, Any]) -> ArtifactsConfig: + spec = { + "benchmark": { + "model": "test-model", + "endpoint": {"url": "http://localhost:8000"}, + "dataset": {"type": "synthetic"}, + "phases": {"type": "concurrency", "concurrency": 1, "requests": 5}, + "artifacts": artifacts, + } + } + return load_config_from_mapping(build_config_envelope(spec)).benchmark.artifacts + + def test_crd_omits_default_for_export_outputs_json(self) -> None: + """No CRD default means the apiserver cannot materialize the field.""" + assert ArtifactsConfig.model_fields["export_outputs_json"].default is None + + def test_k8s_raw_true_implies_outputs_json(self) -> None: + assert self._resolve({"raw": True}).export_outputs_json is True + + def test_k8s_explicit_false_still_opts_out(self) -> None: + artifacts = self._resolve({"raw": True, "exportOutputsJson": False}) + assert artifacts.export_level == ExportLevel.RAW + assert artifacts.export_outputs_json is False + + def test_k8s_non_raw_leaves_outputs_json_off(self) -> None: + assert self._resolve({"raw": False}).export_outputs_json is False + + def test_k8s_flag_alone_works_below_raw(self) -> None: + assert self._resolve({"exportOutputsJson": True}).export_outputs_json is True diff --git a/tests/unit/exporters/test_outputs_json_exporter.py b/tests/unit/exporters/test_outputs_json_exporter.py index 184f85c778..bf1e06c174 100644 --- a/tests/unit/exporters/test_outputs_json_exporter.py +++ b/tests/unit/exporters/test_outputs_json_exporter.py @@ -21,6 +21,7 @@ def _make_fragment( request_start_ns: int = 1000000000, request_end_ns: int = 2000000000, metrics: dict | None = None, + benchmark_phase: str = "profiling", ) -> dict: """Build an output fragment dict suitable for JSONL serialization. @@ -32,6 +33,7 @@ def _make_fragment( "turn_index": turn_index, "conversation_id": conversation_id, "x_request_id": x_request_id, + "benchmark_phase": benchmark_phase, "response_text": response_text, "request_start_ns": request_start_ns, "request_end_ns": request_end_ns, @@ -136,7 +138,7 @@ async def test_export_emits_fragment_metrics(self, tmp_path: Path) -> None: assert outputs_file.exists() data = orjson.loads(outputs_file.read_bytes()) - assert data["schema_version"] == "1.0" + assert data["schema_version"] == "1.1" assert len(data["data"]) == 2 entry1 = data["data"][0] @@ -261,3 +263,188 @@ async def test_export_aggregates_multiple_fragment_files( texts = {r["session_num"]: r["response_text"] for r in data["data"]} assert texts[1] == "from proc1" assert texts[2] == "from proc2" + + +class TestOutputsJsonWarmupPartition: + """Warmup responses are exported, but kept out of `data`.""" + + @pytest.mark.asyncio + async def test_warmup_fragments_go_to_warmup_array(self, tmp_path: Path) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True) + + _write_jsonl( + fragments_dir / "output_fragments_proc1.jsonl", + [ + _make_fragment(session_num=1, response_text="profiled"), + _make_fragment( + session_num=0, + response_text="warmed up", + benchmark_phase="warmup", + ), + ], + ) + + exporter = _make_exporter(tmp_path) + await exporter.export() + + data = orjson.loads((tmp_path / "outputs.json").read_bytes()) + assert [r["response_text"] for r in data["data"]] == ["profiled"] + assert [r["response_text"] for r in data["warmup"]] == ["warmed up"] + + @pytest.mark.asyncio + async def test_entries_carry_benchmark_phase(self, tmp_path: Path) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True) + + _write_jsonl( + fragments_dir / "output_fragments_proc1.jsonl", + [ + _make_fragment(session_num=1), + _make_fragment(session_num=0, benchmark_phase="warmup"), + ], + ) + + exporter = _make_exporter(tmp_path) + await exporter.export() + + data = orjson.loads((tmp_path / "outputs.json").read_bytes()) + assert data["data"][0]["benchmark_phase"] == "profiling" + assert data["warmup"][0]["benchmark_phase"] == "warmup" + + @pytest.mark.asyncio + async def test_warmup_array_present_and_empty_when_no_warmup( + self, tmp_path: Path + ) -> None: + """`warmup` is always emitted so consumers can index it unconditionally.""" + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True) + + _write_jsonl( + fragments_dir / "output_fragments_proc1.jsonl", + [_make_fragment(session_num=1)], + ) + + exporter = _make_exporter(tmp_path) + await exporter.export() + + data = orjson.loads((tmp_path / "outputs.json").read_bytes()) + assert data["warmup"] == [] + assert len(data["data"]) == 1 + + @pytest.mark.asyncio + async def test_fragment_without_phase_treated_as_profiling( + self, tmp_path: Path + ) -> None: + """Only a known-warmup phase may leave `data`. + + Pins the direction of the partition branch: inverting it to test for + PROFILING would silently move records with a missing or newly-added + phase into `warmup`, dropping them from consumers' denominators. + """ + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True) + + fragment = _make_fragment(session_num=1, response_text="legacy") + del fragment["benchmark_phase"] + _write_jsonl(fragments_dir / "output_fragments_proc1.jsonl", [fragment]) + + exporter = _make_exporter(tmp_path) + await exporter.export() + + data = orjson.loads((tmp_path / "outputs.json").read_bytes()) + assert [r["response_text"] for r in data["data"]] == ["legacy"] + assert data["warmup"] == [] + + @pytest.mark.asyncio + async def test_warmup_array_sorted_independently(self, tmp_path: Path) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True) + + _write_jsonl( + fragments_dir / "output_fragments_proc1.jsonl", + [ + _make_fragment(session_num=9, benchmark_phase="warmup"), + _make_fragment(session_num=2, benchmark_phase="warmup"), + _make_fragment(session_num=5), + _make_fragment(session_num=1), + ], + ) + + exporter = _make_exporter(tmp_path) + await exporter.export() + + data = orjson.loads((tmp_path / "outputs.json").read_bytes()) + assert [r["session_num"] for r in data["data"]] == [1, 5] + assert [r["session_num"] for r in data["warmup"]] == [2, 9] + + +class TestLargeExportWarning: + """The large-export warning fires on the record count, not the file size. + + outputs.json is sorted, so it cannot stream and holds the whole document in + memory. The warning is the only signal a user gets that a run is paying that + cost, so pin the boundary rather than leaving log-only behavior untested. + """ + + @staticmethod + def _prepare( + tmp_path: Path, record_count: int, threshold: int + ) -> tuple[list[str], OutputsJsonExporter]: + """Build an exporter over ``record_count`` fragments, capturing warnings.""" + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True, exist_ok=True) + _write_jsonl( + fragments_dir / "output_fragments_proc1.jsonl", + [_make_fragment(session_num=i) for i in range(record_count)], + ) + + exporter = _make_exporter(tmp_path) + exporter.LARGE_EXPORT_RECORD_WARNING_THRESHOLD = threshold + + warnings: list[str] = [] + exporter.warning = warnings.append # type: ignore[method-assign] + return warnings, exporter + + @pytest.mark.asyncio + async def test_export_record_count_at_threshold_warns(self, tmp_path: Path) -> None: + warnings, exporter = self._prepare(tmp_path, 3, threshold=2) + await exporter.export() + + assert len(warnings) == 1 + assert "3 records" in warnings[0] + assert "--no-export-outputs-json" in warnings[0] + + @pytest.mark.asyncio + async def test_export_record_count_below_threshold_does_not_warn( + self, tmp_path: Path + ) -> None: + warnings, exporter = self._prepare(tmp_path, 2, threshold=3) + await exporter.export() + + assert warnings == [] + + @pytest.mark.asyncio + async def test_export_warmup_records_count_toward_threshold_warns( + self, tmp_path: Path + ) -> None: + """Warmup records are held in memory too, so they count.""" + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments_dir.mkdir(parents=True) + _write_jsonl( + fragments_dir / "output_fragments_proc1.jsonl", + [ + _make_fragment(session_num=0), + _make_fragment(session_num=1, benchmark_phase="warmup"), + ], + ) + + exporter = _make_exporter(tmp_path) + exporter.LARGE_EXPORT_RECORD_WARNING_THRESHOLD = 2 + warnings: list[str] = [] + exporter.warning = warnings.append # type: ignore[method-assign] + + await exporter.export() + + assert len(warnings) == 1 + assert "2 records" in warnings[0] diff --git a/tests/unit/post_processors/test_outputs_json_record_processor.py b/tests/unit/post_processors/test_outputs_json_record_processor.py index 1479535442..e2d738e022 100644 --- a/tests/unit/post_processors/test_outputs_json_record_processor.py +++ b/tests/unit/post_processors/test_outputs_json_record_processor.py @@ -5,6 +5,7 @@ from pathlib import Path from unittest.mock import MagicMock, PropertyMock +import orjson import pytest from aiperf.common.exceptions import PostProcessorDisabled @@ -103,8 +104,6 @@ async def test_process_record_writes_fragment(self, tmp_path: Path) -> None: @pytest.mark.asyncio async def test_process_record_extracts_response_text(self, tmp_path: Path) -> None: """Verifies response text is concatenated from content_responses.""" - import orjson - config = _make_config(tmp_path, export_outputs_json=True) record = MagicMock(spec=ParsedResponseRecord) @@ -144,8 +143,6 @@ async def test_process_record_captures_metrics_from_producer( ) -> None: """Allowlisted metrics are captured off ctx.metrics into the fragment, so outputs.json does not depend on the records JSONL export (F2).""" - import orjson - from aiperf.common.messages.inference_messages import MetricRecordsData config = _make_config(tmp_path, export_outputs_json=True) @@ -184,11 +181,9 @@ async def test_process_record_captures_metrics_from_producer( assert fragment["metrics"]["output_token_count"] == 42 @pytest.mark.asyncio - async def test_process_record_skips_non_profiling_phase( - self, tmp_path: Path - ) -> None: - """Warmup (non-profiling) records write no fragment; phase filtering lives - in the processor now that the exporter no longer reads profile_export.jsonl.""" + async def test_process_record_captures_warmup_phase(self, tmp_path: Path) -> None: + """Warmup records are captured and tagged, so raw export's outputs.json + covers the same requests as profile_export_raw.jsonl.""" config = _make_config(tmp_path, export_outputs_json=True) record = MagicMock(spec=ParsedResponseRecord) @@ -213,16 +208,50 @@ async def test_process_record_skips_non_profiling_phase( await proc.observe( RecordObserverContext(record=record, metadata=metadata, produced={}) ) + output_file = proc.output_file - assert proc.lines_written == 0 + assert output_file.exists() + fragment = orjson.loads(output_file.read_text().splitlines()[0]) + assert fragment["response_text"] == "warmup text" + assert fragment["benchmark_phase"] == "warmup" + + @pytest.mark.asyncio + async def test_process_record_tags_profiling_phase(self, tmp_path: Path) -> None: + """Profiling records carry the phase too, so the exporter can partition.""" + config = _make_config(tmp_path, export_outputs_json=True) + + record = MagicMock(spec=ParsedResponseRecord) + resp = MagicMock() + resp.data.get_text.return_value = "profiling text" + type(record).content_responses = PropertyMock(return_value=[resp]) + + metadata = MetricRecordMetadata( + session_num=0, + request_start_ns=1000000000, + request_end_ns=2000000000, + worker_id="worker-1", + record_processor_id="proc-1", + benchmark_phase="profiling", + ) + + processor = OutputsJsonRecordProcessor( + service_id="processor-1", + run=MagicMock(cfg=config), + ) + async with aiperf_lifecycle(processor) as proc: + await proc.observe( + RecordObserverContext(record=record, metadata=metadata, produced={}) + ) + output_file = proc.output_file + + fragment = orjson.loads(output_file.read_text().splitlines()[0]) + assert fragment["benchmark_phase"] == "profiling" @pytest.mark.asyncio async def test_process_record_null_response_text_when_no_content( self, tmp_path: Path ) -> None: """When content_responses is empty, response_text is None.""" - import orjson - config = _make_config(tmp_path, export_outputs_json=True) record = MagicMock(spec=ParsedResponseRecord) From 3e9af216ac19d08b9411668298960cd384759883 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:59:43 -0700 Subject: [PATCH 051/119] fix(config): reject synthetic media and rankings flags on file/public datasets (#1359) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit e1f8e16a53fa154e0bafe4834cff3819ce8c5a44) --- docs/tutorials/custom-dataset.md | 13 +- src/aiperf/config/flags/_converter_dataset.py | 35 +++++ .../config/test_config_input_overrides.py | 8 +- .../test_synthetic_only_flag_rejection.py | 131 ++++++++++++++++++ 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tests/unit/config/test_synthetic_only_flag_rejection.py diff --git a/docs/tutorials/custom-dataset.md b/docs/tutorials/custom-dataset.md index 9bd1351b7f..dd07bf0fba 100644 --- a/docs/tutorials/custom-dataset.md +++ b/docs/tutorials/custom-dataset.md @@ -558,7 +558,18 @@ Behavior worth knowing: `--user-context-prompt-length` for a two-tier shared/per-session structure. > **Note:** `--num-prefix-prompts` and `--prefix-prompt-length` apply only to synthetic -> datasets — they are dropped for file and public datasets. `--system-prompt` is not. +> datasets, and AIPerf rejects them at startup when combined with `--input-file` or +> `--public-dataset` rather than ignoring them. `--system-prompt` has no such restriction. +> +> The same rule now covers every synthetic-generation flag: the media *shape* options +> (`--image-width-mean`/`--image-height-mean`/`--image-format`, `--audio-length-mean`, +> `--audio-format`, `--audio-sample-rates`, the `--video-*` family) and the +> `--rankings-*` options. A file or public dataset supplies its own media and passages, +> so these previously did nothing at all; they are now an error. The per-modality +> batch-size flags remain supported with `--custom-dataset-type random_pool`, where they +> control how many items are packed per request -- but only when `--input-file` points +> at a single random-pool file, not a directory: a directory forms a separately named +> pool per file, and batching across pools has no defined meaning. --- diff --git a/src/aiperf/config/flags/_converter_dataset.py b/src/aiperf/config/flags/_converter_dataset.py index 6fa3b75c74..80bfbb12de 100644 --- a/src/aiperf/config/flags/_converter_dataset.py +++ b/src/aiperf/config/flags/_converter_dataset.py @@ -644,6 +644,41 @@ def apply_implicit_media_batch_override( ("conversation_turn_delay_mean", "--conversation-turn-delay-mean"), ("conversation_turn_delay_stddev", "--conversation-turn-delay-stddev"), ("conversation_turn_delay_ratio", "--conversation-turn-delay-ratio"), + # Synthetic media SHAPE and rankings generation, unlike the batch-size + # members above, have no rescue path and no field on FileDataset/ + # PublicDataset to receive them. Kept in sync with the _build_* + # functions by tests/unit/config/test_synthetic_only_flag_rejection.py. + ("rankings_passages_mean", "--rankings-passages-mean"), + ("rankings_passages_stddev", "--rankings-passages-stddev"), + ("rankings_passages_prompt_token_mean", "--rankings-passages-prompt-token-mean"), + ( + "rankings_passages_prompt_token_stddev", + "--rankings-passages-prompt-token-stddev", + ), + ("rankings_query_prompt_token_mean", "--rankings-query-prompt-token-mean"), + ("rankings_query_prompt_token_stddev", "--rankings-query-prompt-token-stddev"), + ("audio_length_mean", "--audio-length-mean"), + ("audio_length_stddev", "--audio-length-stddev"), + ("audio_format", "--audio-format"), + ("audio_depths", "--audio-depths"), + ("audio_sample_rates", "--audio-sample-rates"), + ("audio_num_channels", "--audio-num-channels"), + ("image_height_mean", "--image-height-mean"), + ("image_height_stddev", "--image-height-stddev"), + ("image_width_mean", "--image-width-mean"), + ("image_width_stddev", "--image-width-stddev"), + ("image_format", "--image-format"), + ("video_duration", "--video-duration"), + ("video_fps", "--video-fps"), + ("video_width", "--video-width"), + ("video_height", "--video-height"), + ("video_synth_type", "--video-synth-type"), + ("video_format", "--video-format"), + ("video_codec", "--video-codec"), + ("video_audio_sample_rate", "--video-audio-sample-rate"), + ("video_audio_channels", "--video-audio-num-channels"), + ("video_audio_codec", "--video-audio-codec"), + ("video_audio_depth", "--video-audio-depth"), ) diff --git a/tests/unit/config/test_config_input_overrides.py b/tests/unit/config/test_config_input_overrides.py index 431e0d8ca7..b269a210b0 100644 --- a/tests/unit/config/test_config_input_overrides.py +++ b/tests/unit/config/test_config_input_overrides.py @@ -329,8 +329,12 @@ def test_inert_flag_paired_with_routable_one_still_errors( def test_inert_media_flag_paired_on_file_dataset_errors( random_pool_yaml: Path, ) -> None: - """Same, for the media flags against a file dataset.""" - with pytest.raises(ConfigurationError, match=r"--image-width-mean"): + """Same, for the media flags against a file dataset. + + Either error type satisfies the guarantee: the flag must not be silently + discarded when paired with a routable one. + """ + with pytest.raises((ConfigurationError, ValueError), match=r"--image-width-mean"): resolve_config(cli(prompt_batch_size=4, image_width_mean=128), random_pool_yaml) diff --git a/tests/unit/config/test_synthetic_only_flag_rejection.py b/tests/unit/config/test_synthetic_only_flag_rejection.py new file mode 100644 index 0000000000..bf21579090 --- /dev/null +++ b/tests/unit/config/test_synthetic_only_flag_rejection.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Every synthetic-only flag is rejected on file/public datasets, not dropped. + +The mechanical test below re-derives the field list from the ``_build_*`` +functions, so a newly added synthetic flag fails here rather than quietly +joining the silently-dropped set. +""" + +from __future__ import annotations + +import ast +import inspect +import textwrap +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from pytest import param + +from aiperf.config.flags import _converter_dataset as conv +from aiperf.config.flags.cli_config import CLIConfig + + +def _cli_fields_referenced(func: Callable[[CLIConfig], dict[str, Any]]) -> set[str]: + """Every CLIConfig field name the builder reads, however it reads it. + + Covers all four access shapes the builders use: ``"field" in s``, + ``cli.field``, ``getattr(cli, key)`` over a dict literal, and field names + passed as string arguments to ``_mean_stddev_pair``. + + Only scans ``func``'s own source, not names it delegates to a + module-level constant. All current builders inline their field names, so + this holds today; a builder refactored to pull field names from a shared + constant would need this scanner updated too. + """ + tree = ast.parse(textwrap.dedent(inspect.getsource(func))) + known = set(CLIConfig.model_fields) + found: set[str] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.value in known + ): + found.add(node.value) + elif ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "cli" + and node.attr in known + ): + found.add(node.attr) + return found + + +@pytest.mark.parametrize( + "builder", + [ + param(conv._build_prefix_prompts, id="prefix_prompts"), + param(conv._build_rankings, id="rankings"), + param(conv._build_audio, id="audio"), + param(conv._build_images, id="images"), + param(conv._build_video, id="video"), + ], +) # fmt: skip +def test_every_synthetic_only_flag_is_rejected_or_rescued( + builder: Callable[[CLIConfig], dict[str, Any]], +) -> None: + """No field of a stripped subtable may be silently accepted. + + ``prompts`` is excluded from this sweep on purpose: it is stripped too, but + several members are re-attached for FILE/PUBLIC (``_apply_file_osl``, + ``_apply_corpus_and_cache_bust``, ``_apply_block_size``, + ``_apply_sequence_distribution``), so --osl/--cache-bust/--prompt-corpus + legitimately work there. + """ + rejected = {attr for attr, _ in conv._FILE_DATASET_INCOMPATIBLE_TRIGGERS} + unhandled = _cli_fields_referenced(builder) - rejected + assert not unhandled, ( + f"{sorted(unhandled)} are read by {builder.__name__} but are neither in " + "_FILE_DATASET_INCOMPATIBLE_TRIGGERS nor rescued, so they are silently " + "dropped on file/public datasets." + ) + + +def _trace(tmp_path: Path) -> str: + trace = tmp_path / "t.jsonl" + trace.write_text('{"text": "hi"}\n') + return str(trace) + + +class TestNewlyRejected: + @pytest.mark.parametrize( + "field,value,expected_flag", + [ + param("image_width_mean", 64, "--image-width-mean", id="image_shape"), + param("image_format", "png", "--image-format", id="image_format"), + param("audio_length_mean", 5.0, "--audio-length-mean", id="audio_shape"), + param("video_duration", 3.0, "--video-duration", id="video"), + param( + "rankings_passages_mean", 4, "--rankings-passages-mean", id="rankings" + ), + ], + ) # fmt: skip + def test_rejected_with_input_file( + self, tmp_path: Path, field: str, value: object, expected_flag: str + ) -> None: + cli = CLIConfig( + model_names=["m"], input_file=_trace(tmp_path), **{field: value} + ) + with pytest.raises(ValueError, match=expected_flag): + conv.build_dataset(cli) + + def test_rejected_with_public_dataset(self) -> None: + cli = CLIConfig( + model_names=["m"], public_dataset="sharegpt", image_width_mean=64 + ) + with pytest.raises(ValueError, match="--image-width-mean"): + conv.build_dataset(cli) + + +class TestUnchangedBehavior: + """Guardrails: this must not widen beyond the silently-dropped set.""" + + def test_synthetic_dataset_still_accepts_them(self) -> None: + cli = CLIConfig(model_names=["m"], image_width_mean=64, audio_length_mean=5.0) + out = conv.build_dataset(cli) + assert out["images"]["width"]["mean"] == 64 + assert out["audio"]["length"]["mean"] == 5.0 From 46e3db16778b010e5c265324204680d9e4915799 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:15:11 -0700 Subject: [PATCH 052/119] feat(metrics): add E2E Normalized Interactivity metric (#1270) Signed-off-by: lkomali Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> (cherry picked from commit b61bb73c8c9cbc94b31ecf204cc376d189ef105b) --- docs/metrics-reference.md | 29 ++ src/aiperf/metrics/accumulator.py | 14 + .../e2e_normalized_interactivity_analyzer.py | 175 ++++++++++ .../e2e_normalized_interactivity_metrics.py | 120 +++++++ ...est_e2e_normalized_interactivity_metric.py | 64 ++++ .../test_e2e_normalized_interactivity.py | 308 ++++++++++++++++++ ...test_timeslice_metric_results_processor.py | 45 ++- 7 files changed, 743 insertions(+), 12 deletions(-) create mode 100644 src/aiperf/metrics/e2e_normalized_interactivity_analyzer.py create mode 100644 src/aiperf/metrics/types/e2e_normalized_interactivity_metrics.py create mode 100644 tests/component_integration/test_e2e_normalized_interactivity_metric.py create mode 100644 tests/unit/metrics/test_e2e_normalized_interactivity.py diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index 4cb2341fb4..3699a5f877 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -32,6 +32,7 @@ This document provides a comprehensive reference of all metrics available in AIP - [Total Output Sequence Length](#total-output-sequence-length) - [Total Input Sequence Length](#total-input-sequence-length) - [E2E Output Token Throughput](#e2e-output-token-throughput) + - [E2E Normalized Interactivity](#e2e-normalized-interactivity) - [Output Token Throughput](#output-token-throughput) - [Total Token Throughput](#total-token-throughput) - [Image Metrics](#image-metrics) @@ -584,6 +585,34 @@ e2e_output_token_throughput = output_sequence_length / request_latency_seconds - Available for non-streaming responses (unlike Output Token Throughput Per User which requires streaming). - Flags: `PRODUCES_TOKENS_ONLY | LARGER_IS_BETTER` - Depends on Output Sequence Length and Request Latency metrics. +- Its `p10`/`p25` percentiles are numerically the same order statistic as [E2E Normalized Interactivity](#e2e-normalized-interactivity)'s `p90`/`p75` — see that metric for when to prefer the named InferenceX scalar. + +--- + +### E2E Normalized Interactivity + +**Type:** [Derived Metric](#derived-metrics) (injected post-aggregation) + +> [!NOTE] +> Emitted as two named percentile scalars, `e2e_normalized_interactivity_p90` and `e2e_normalized_interactivity_p75`, in tokens/sec/user. Reproduces the x-axis InferenceX renders for AgentX Pareto curves, so `aiperf profile` yields the point directly with no post-processing. Available for both streaming and non-streaming responses. + +The rate at which a user receives output tokens **including** the prefill wait (TTFT). Unlike [Output Token Throughput Per User](#output-token-throughput-per-user) (`1 / ITL`, decode-only), it charges the whole end-to-end latency, so a large TTFT relative to the tokens produced pulls it down — prefill-delaying cannot inflate it. + +**Formula:** +```python +# per-request ratio, in seconds per output token +ratio = request_latency_seconds / output_sequence_length +# invert AFTER taking the percentile (slow-tail convention) +e2e_normalized_interactivity_p90 = 1.0 / percentile(ratio, 90) +e2e_normalized_interactivity_p75 = 1.0 / percentile(ratio, 75) +``` + +**Notes:** +- The percentile is taken over the seconds-per-token ratio and then inverted (`1 / p(x)`, **not** `p(1 / x)`). A higher percentile names a slower, more pessimistic tail, so `p90` is the effective token rate of the 90th-percentile worst request. This differs from `p90` of the per-request rate, which would report the fast tail. +- **Relationship to [E2E Output Token Throughput](#e2e-output-token-throughput).** Because `x -> 1/x` is monotone, `1 / p90(latency/OSL)` is the same order statistic as `p10(OSL/latency)` *over the same requests*. The two differ only in the space the linear interpolation happens in: on a 300-request run they agree to roughly `1e-9` relative, widening to about `1e-3` for small (n < 100) heavy-tailed samples. Note this metric additionally requires `TTFT > 0` and `ISL > 0`, whereas `E2EOutputTokenThroughputMetric` drops a record only when `request_latency` is zero, so the two populations can diverge on a run with zero-length inputs. They are kept as a distinct named family because they pin InferenceX's exact convention (invert *after* the percentile). Prefer this metric when you need the exact AgentX Pareto x-axis; the throughput percentiles are equivalent otherwise. +- Request-weighted: every request with positive, finite `request_latency`, `output_sequence_length`, and (when present) `time_to_first_token` and `input_sequence_length` contributes one sample, matching InferenceX's filter. The filter can shrink the sample; `count` is not exported for this derived scalar, so a `debug` log reports how many requests were dropped. +- Percentile uses numpy-linear interpolation, matching InferenceX's `quantile`, so values line up exactly for identical inputs. +- Reported once per run (not per `--slice-duration` timeslice), matching InferenceX's run-level definition. --- diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index f3d42e0702..eac103c962 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -42,6 +42,9 @@ inject_derived_latency_metrics, ) from aiperf.metrics.display_units import to_display_unit +from aiperf.metrics.e2e_normalized_interactivity_analyzer import ( + inject_e2e_normalized_interactivity_metrics, +) from aiperf.metrics.metric_dicts import MetricResultsDict, metric_result_from_array from aiperf.metrics.metric_registry import MetricRegistry from aiperf.metrics.network_adjusted_analyzer import ( @@ -753,6 +756,17 @@ def _summarize_for_export_context( mask=mask, warn_degraded=self._warn_replay_degraded, ) + # E2E Normalized Interactivity (AgentX/InferenceX Pareto x-axis): + # 1 / p(request_latency_s / OSL), request-weighted over the masked + # columns. Reported run-scoped (not per timeslice) to match InferenceX, + # which defines this x-axis at run level only -- unlike the send-lag + # family above, the value is well-defined per slice, so per-slice + # support is a possible follow-up, not a correctness constraint. + inject_e2e_normalized_interactivity_metrics( + self._column_store, + overall_results, + mask=mask, + ) overall_results = self._filter_hidden_metrics(overall_results) self.debug(lambda: f"Summarized {len(overall_results)} metric results") diff --git a/src/aiperf/metrics/e2e_normalized_interactivity_analyzer.py b/src/aiperf/metrics/e2e_normalized_interactivity_analyzer.py new file mode 100644 index 0000000000..77858cf785 --- /dev/null +++ b/src/aiperf/metrics/e2e_normalized_interactivity_analyzer.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""E2E Normalized Interactivity analyzer: compute the +``e2e_normalized_interactivity_{p75,p90}`` family from the column store at +summarize-time. + +The metrics themselves are defined in +:mod:`aiperf.metrics.types.e2e_normalized_interactivity_metrics` and defer their +derivation; this module owns the columnar computation and injection, mirroring +:mod:`aiperf.metrics.replay_sched_lag_analyzer` and +:mod:`aiperf.metrics.derived_latency`. + +The value reproduces InferenceX's derivation exactly: the percentile is taken +over the per-request seconds-per-output-token ratio ``request_latency_s / OSL`` +and then inverted, so ``p90`` is the effective token rate of the 90th-percentile +(slowest) request:: + + p90 interactivity = 1 / p90(request_latency_s / OSL) + +Request-weighted: every request with positive, finite ``request_latency``, +``output_sequence_length``, and -- when the columns are present -- +``time_to_first_token`` and ``input_sequence_length`` contributes one sample, +matching InferenceX's filter (all four fields > 0). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +from numpy.typing import NDArray + +from aiperf.common.aiperf_logger import AIPerfLogger +from aiperf.common.constants import NANOS_PER_SECOND +from aiperf.common.finite import is_finite_value +from aiperf.common.models import MetricResult +from aiperf.metrics.types.e2e_normalized_interactivity_metrics import ( + E2ENormalizedInteractivityP75Metric, + E2ENormalizedInteractivityP90Metric, + E2ENormalizedInteractivityPercentileBase, +) +from aiperf.metrics.types.input_sequence_length_metric import InputSequenceLengthMetric +from aiperf.metrics.types.output_sequence_length_metric import ( + OutputSequenceLengthMetric, +) +from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric +from aiperf.metrics.types.ttft_metric import TTFTMetric + +if TYPE_CHECKING: + from aiperf.metrics.column_store import ColumnStore + +_logger = AIPerfLogger(__name__) + + +# The ``percentile`` each class declares is the single source of truth for both +# its identity and the ratio percentile the analyzer inverts. +_PERCENTILE_METRICS: tuple[type[E2ENormalizedInteractivityPercentileBase], ...] = ( + E2ENormalizedInteractivityP75Metric, + E2ENormalizedInteractivityP90Metric, +) + +# Columns that gate a request in/out of the sample, matching InferenceX's +# "all fields > 0" filter. request_latency and OSL are required (no-op without +# them); TTFT and ISL are applied only when the run produced them. +_FILTER_ONLY_TAGS = (TTFTMetric.tag, InputSequenceLengthMetric.tag) + + +def _compute_finite_ratios( + store: ColumnStore, + mask: NDArray[np.bool_] | None, +) -> NDArray[np.float64] | None: + """Return the finite per-request seconds-per-output-token ratios that pass the + positive/finite filter, or ``None`` when nothing qualifies. + + Matches InferenceX's "all fields > 0" filter: request_latency and OSL are + required (no-op without them); TTFT and ISL are applied only when the run + produced them. Because ``count`` is dropped on export for the derived scalar, + this also logs how many requests the filter dropped so the sample size is not + silently invisible. + """ + tags = store.numeric_tags() + if ( + RequestLatencyMetric.tag not in tags + or OutputSequenceLengthMetric.tag not in tags + ): + return None + + def _column(tag: str) -> NDArray[np.float64]: + col = store.numeric(tag) + if mask is None: + return col + # Cap to mask length before boolean indexing: the mask was snapshotted + # before this call and concurrent ingestion (the asyncio.to_thread race) + # may have grown the column past it, which would raise IndexError and + # abort the export. Mirrors MetricsAccumulator.summarize(). + return col[: len(mask)][mask] + + latency_ns = _column(RequestLatencyMetric.tag) + osl = _column(OutputSequenceLengthMetric.tag) + + keep = np.isfinite(latency_ns) & (latency_ns > 0) & np.isfinite(osl) & (osl > 0) + for tag in _FILTER_ONLY_TAGS: + if tag in tags: + col = _column(tag) + keep &= np.isfinite(col) & (col > 0) + + if not keep.any(): + return None + + # Seconds per output token, per surviving request. Drop any non-finite ratio + # (an extreme latency/OSL can overflow the division) so the percentile and its + # reciprocal stay finite -- injected metric values must be finite per the + # NaN/Inf discipline. + ratio = (latency_ns[keep] / NANOS_PER_SECOND) / osl[keep] + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return None + + dropped = int(latency_ns.size - ratio.size) + if dropped > 0: + total, kept = latency_ns.size, ratio.size + # A failed request still occupies a row, with its metric columns absent + # (NaN), so it is excluded by the same finite check as a genuine + # data-quality problem. Report the two separately: this log is the only + # signal for a shrunken sample (``count`` is pinned to 1 and dropped on + # export), and pointing a user at TTFT/ISL when the real cause is a high + # error rate sends them looking for a problem that isn't there. + missing = int((~np.isfinite(latency_ns) | ~np.isfinite(osl)).sum()) + _logger.debug( + lambda: f"E2E normalized interactivity: {dropped} of {total} requests " + f"dropped ({missing} with no recorded latency/OSL -- typically failed " + f"requests; the remainder for a non-positive or non-finite latency, " + f"OSL, TTFT, or ISL, or a non-finite ratio); percentiles computed " + f"over {kept} requests." + ) + return ratio + + +def inject_e2e_normalized_interactivity_metrics( + store: ColumnStore, + results: dict[str, MetricResult], + mask: NDArray[np.bool_] | None = None, +) -> None: + """Inject the E2E Normalized Interactivity percentile family. + + No-op when ``request_latency`` or ``output_sequence_length`` are absent, or + when no request passes the positive-and-finite filter. Pure side effect on + ``results``. + """ + ratio = _compute_finite_ratios(store, mask) + if ratio is None: + return + + # One introselect pass for both percentiles rather than one call per metric -- + # this runs inside MetricsAccumulator.summarize() on every dashboard tick. + ratio_ps = np.percentile(ratio, [cls.percentile for cls in _PERCENTILE_METRICS]) + for cls, ratio_p in zip(_PERCENTILE_METRICS, ratio_ps, strict=True): + ratio_p = float(ratio_p) + if ratio_p <= 0: + continue + value = 1.0 / ratio_p + # Guard the reciprocal too: a subnormal ratio_p can overflow to inf. + if not is_finite_value(value): + continue + results[cls.tag] = MetricResult( + tag=cls.tag, + header=cls.header, + unit=str(cls.unit), + avg=value, + # count is dropped on export for DERIVED scalars; set 1 to match the + # run-scoped sibling convention (replay_sched_lag_analyzer) rather than + # imply a per-request sample size that never ships. + count=1, + console_group=cls.console_group, + ) diff --git a/src/aiperf/metrics/types/e2e_normalized_interactivity_metrics.py b/src/aiperf/metrics/types/e2e_normalized_interactivity_metrics.py new file mode 100644 index 0000000000..1dba975343 --- /dev/null +++ b/src/aiperf/metrics/types/e2e_normalized_interactivity_metrics.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""E2E Normalized Interactivity metrics (AgentX / InferenceX Pareto x-axis). + +E2E Normalized Interactivity is the per-request rate at which a user receives +output tokens INCLUDING the prefill wait: ``output_sequence_length / +request_latency``. Unlike ``output_token_throughput_per_user`` (``1 / ITL``, +decode-only), it charges the whole end-to-end latency, so a large TTFT relative +to the tokens produced pulls it down -- prefill-delaying cannot inflate it. + +To reproduce InferenceX's number exactly, the percentile is taken over the +seconds-per-output-token ratio ``request_latency / OSL`` and then inverted: + + p90 interactivity = 1 / p90(request_latency_s / OSL) + +This is the slow-tail convention InferenceX enforces for interactivity +(``1 / p(x)``, NOT ``p(1 / x)``): a higher percentile names a slower, more +pessimistic tail, so ``p90`` is the effective token rate of the 90th-percentile +worst request. Inverting the ratio's percentile (rather than percentiling the +per-request rate directly) is required because ``1 / p(x) != p(1 / x)`` -- the +two land the interpolation in different spaces. It also means the family cannot +be expressed as a monotonic percentile band (inversion flips the order), so it +is emitted as named scalars, mirroring InferenceX's ``p75``/``p90`` stored +values. + +The values are injected post-aggregation from the ``request_latency`` and +``output_sequence_length`` columns by +:func:`aiperf.metrics.e2e_normalized_interactivity_analyzer.inject_e2e_normalized_interactivity_metrics`, +mirroring the replay-send-lag and derived-latency families. +""" + +from __future__ import annotations + +from typing import ClassVar + +from aiperf.common.enums import ( + MetricConsoleGroup, + MetricFlags, + MetricOverTimeUnit, +) +from aiperf.common.exceptions import NoMetricValue +from aiperf.metrics.base_derived_metric import BaseDerivedMetric +from aiperf.metrics.metric_dicts import MetricResultsDict +from aiperf.metrics.types.output_sequence_length_metric import ( + OutputSequenceLengthMetric, +) +from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric + + +class _E2ENormalizedInteractivityDeferMixin: + """Deferred-derivation behavior for the injected interactivity metrics. + + Not a metric itself (does not subclass BaseMetric) so it is never + registered. The whole family is ``1 / p(request_latency / OSL)`` over the + per-record columns, which the scalar summarize path cannot express, so the + derive defers and + :func:`aiperf.metrics.e2e_normalized_interactivity_analyzer.inject_e2e_normalized_interactivity_metrics` + fills the values post-aggregation from the column store. + """ + + def _derive_value(self, metric_results: MetricResultsDict) -> float: + raise NoMetricValue( + f"{self.tag} is injected post-aggregation from the " # type: ignore[attr-defined] + "request_latency / output_sequence_length columns" + ) + + +class E2ENormalizedInteractivityPercentileBase( + _E2ENormalizedInteractivityDeferMixin, BaseDerivedMetric[float] +): + """Shared metadata for the E2E Normalized Interactivity percentile metrics. + + ``percentile`` is the single source of truth for both the metric's identity + and the ratio percentile the analyzer inverts. + """ + + __is_abstract__ = True + percentile: float + + unit = MetricOverTimeUnit.TOKENS_PER_SECOND_PER_USER + flags = MetricFlags.PRODUCES_TOKENS_ONLY | MetricFlags.LARGER_IS_BETTER + console_group = MetricConsoleGroup.NONE + # Reported run-scoped to match InferenceX, which defines this x-axis at run + # level only. The value is a plain per-record percentile and is well-defined + # per timeslice, so this is a convention choice (not a run-anchoring + # constraint like the send-lag family); per-slice support is a possible + # follow-up. + timeslice_derivable = False + required_metrics: ClassVar[set[str]] = { + RequestLatencyMetric.tag, + OutputSequenceLengthMetric.tag, + } + + +class E2ENormalizedInteractivityP90Metric(E2ENormalizedInteractivityPercentileBase): + """P90 (slow-tail) E2E Normalized Interactivity, in tokens/sec/user. + + Formula: + 1 / p90(request_latency_s / output_sequence_length) + """ + + __is_abstract__ = False + percentile = 90.0 + tag = "e2e_normalized_interactivity_p90" + header = "E2E Normalized Interactivity p90" + short_header = "E2E Norm Intvty p90" + + +class E2ENormalizedInteractivityP75Metric(E2ENormalizedInteractivityPercentileBase): + """P75 (slow-tail) E2E Normalized Interactivity, in tokens/sec/user. + + Formula: + 1 / p75(request_latency_s / output_sequence_length) + """ + + __is_abstract__ = False + percentile = 75.0 + tag = "e2e_normalized_interactivity_p75" + header = "E2E Normalized Interactivity p75" + short_header = "E2E Norm Intvty p75" diff --git a/tests/component_integration/test_e2e_normalized_interactivity_metric.py b/tests/component_integration/test_e2e_normalized_interactivity_metric.py new file mode 100644 index 0000000000..d8bd6c53d0 --- /dev/null +++ b/tests/component_integration/test_e2e_normalized_interactivity_metric.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Component integration test: the E2E Normalized Interactivity metric is emitted +end-to-end from a real ``aiperf profile`` run against the mock server.""" + +import math + +import pytest + +from aiperf.common.models import JsonExportData +from tests.component_integration.conftest import ( + ComponentIntegrationTestDefaults as defaults, +) +from tests.harness.utils import AIPerfCLI + + +def _metric_avg(json_export: JsonExportData, tag: str) -> float: + """Read a metric's ``avg`` from the JSON export. + + ``JsonExportData`` is ``extra="allow"``, so metrics not declared on the model + (like this one) come through as raw dicts rather than typed results. + """ + metric = getattr(json_export, tag, None) + assert metric is not None, f"{tag} missing from the JSON export" + return metric["avg"] if isinstance(metric, dict) else metric.avg + + +@pytest.mark.component_integration +class TestE2ENormalizedInteractivityMetric: + """Smoke test that the p90/p75 interactivity scalars appear and are finite.""" + + def test_metric_emitted_and_finite(self, cli: AIPerfCLI) -> None: + """A streaming chat run (which has TTFT/ISL/OSL/latency) emits both + interactivity percentiles as finite, positive values, and the slow-tail + ordering holds (p90 <= p75). + + Uses the synchronous ``run_sync``: the component_integration ``cli`` + fixture is an in-process runner and ``_single_run`` calls ``asyncio.run`` + internally, so awaiting it inside the pytest event loop would raise + "asyncio.run() cannot be called from a running event loop".""" + result = cli.run_sync( + f""" + aiperf profile \ + --model {defaults.model} --tokenizer {defaults.tokenizer} \ + --endpoint-type chat --streaming \ + --synthetic-input-tokens-mean 200 --output-tokens-mean 100 \ + --request-count {defaults.request_count} \ + --concurrency {defaults.concurrency} \ + --workers-max {defaults.workers_max} \ + --ui {defaults.ui} + """ + ) + assert result.request_count == defaults.request_count + + p90 = _metric_avg(result.json, "e2e_normalized_interactivity_p90") + p75 = _metric_avg(result.json, "e2e_normalized_interactivity_p75") + + # Finite and positive (NaN/Inf discipline; unit is tokens/sec/user). + assert math.isfinite(p90) and p90 > 0, f"p90 not finite/positive: {p90}" + assert math.isfinite(p75) and p75 > 0, f"p75 not finite/positive: {p75}" + + # Slow-tail convention: p90 is the pessimistic tail, so 1/p90(ratio) is + # never larger than 1/p75(ratio). Higher percentile -> lower interactivity. + assert p90 <= p75, f"p90 (slow tail) should be <= p75, got p90={p90}, p75={p75}" diff --git a/tests/unit/metrics/test_e2e_normalized_interactivity.py b/tests/unit/metrics/test_e2e_normalized_interactivity.py new file mode 100644 index 0000000000..e124292dde --- /dev/null +++ b/tests/unit/metrics/test_e2e_normalized_interactivity.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the E2E Normalized Interactivity metrics (AgentX Pareto x-axis). + +The family is injected post-aggregation from the ``request_latency`` and +``output_sequence_length`` columns (see +``inject_e2e_normalized_interactivity_metrics``), so the tests build a +:class:`ColumnStore` and assert on the injected :class:`MetricResult` values. + +A golden-parity test pins the injected value to InferenceX's exact derivation +(``1 / quantile(request_latency_s / OSL)`` with numpy-linear interpolation, from +``derived-agentic-metrics.ts``), so the port is provably equivalent for +identical inputs. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest +from pytest import param + +from aiperf.common.constants import NANOS_PER_SECOND +from aiperf.common.enums import MetricFlags +from aiperf.common.exceptions import NoMetricValue +from aiperf.metrics.column_store import ColumnStore +from aiperf.metrics.e2e_normalized_interactivity_analyzer import ( + inject_e2e_normalized_interactivity_metrics, +) +from aiperf.metrics.metric_dicts import MetricResultsDict +from aiperf.metrics.types.e2e_normalized_interactivity_metrics import ( + E2ENormalizedInteractivityP75Metric, + E2ENormalizedInteractivityP90Metric, +) + +P90 = E2ENormalizedInteractivityP90Metric.tag +P75 = E2ENormalizedInteractivityP75Metric.tag + + +def _infx_quantile(sorted_asc: list[float], q: float) -> float: + """InferenceX's ``quantile`` (agentic-shared.ts): numpy-linear interpolation.""" + if not sorted_asc: + return float("nan") + if len(sorted_asc) == 1: + return sorted_asc[0] + pos = (len(sorted_asc) - 1) * q + lo = int(np.floor(pos)) + hi = int(np.ceil(pos)) + if lo == hi: + return sorted_asc[lo] + return sorted_asc[lo] + (sorted_asc[hi] - sorted_asc[lo]) * (pos - lo) + + +def _store( + *, + latency_s: list[float], + osl: list[float], + ttft_s: list[float] | None = None, + isl: list[float] | None = None, +) -> ColumnStore: + """ColumnStore of per-request columns as the accumulator ingests them. + + ``latency_s`` / ``ttft_s`` are given in seconds for readability and stored as + nanoseconds (the metric's native column unit). + """ + n = len(latency_s) + ttft_s = ttft_s if ttft_s is not None else [0.5] * n + isl = isl if isl is not None else [100.0] * n + store = ColumnStore(initial_capacity=max(n, 1)) + for idx in range(n): + store.ingest( + idx=idx, + record_metrics={ + "request_latency": latency_s[idx] * NANOS_PER_SECOND, + "output_sequence_length": osl[idx], + "time_to_first_token": ttft_s[idx] * NANOS_PER_SECOND, + "input_sequence_length": isl[idx], + }, + start_ns=float(idx), + end_ns=float(idx), + generation_start_ns=None, + ) + return store + + +def _inject(store: ColumnStore, mask: np.ndarray | None = None) -> dict: + results: dict = {} + inject_e2e_normalized_interactivity_metrics(store, results, mask=mask) + return results + + +def test_inject_matches_inverse_of_ratio_percentile() -> None: + """p90/p75 == 1 / percentile(request_latency_s / OSL), the InferenceX way.""" + latency_s = [1.0, 2.0, 3.0, 4.0, 10.0] + osl = [100.0] * 5 + results = _inject(_store(latency_s=latency_s, osl=osl)) + + ratio = np.array(latency_s) / 100.0 + assert results[P90].avg == pytest.approx(1.0 / np.percentile(ratio, 90)) + assert results[P75].avg == pytest.approx(1.0 / np.percentile(ratio, 75)) + assert results[P90].unit == "tokens/sec/user" + + +def test_golden_parity_with_inferencex_quantile() -> None: + """Injected value is byte-for-byte InferenceX's ``1 / quantile(ratio)``.""" + latency_s = [0.7, 1.3, 2.9, 3.1, 5.5, 8.2, 0.4] + osl = [50.0, 120.0, 200.0, 30.0, 512.0, 64.0, 8.0] + results = _inject(_store(latency_s=latency_s, osl=osl)) + + ratios = sorted(latency_s[i] / osl[i] for i in range(len(osl))) + assert results[P90].avg == pytest.approx(1.0 / _infx_quantile(ratios, 0.90)) + assert results[P75].avg == pytest.approx(1.0 / _infx_quantile(ratios, 0.75)) + + +def test_slow_tail_convention_is_not_percentile_of_rate() -> None: + """Locks 1/p(ratio) != p(OSL/latency): the naive per-request-rate percentile + would report the fast tail and must not be what we emit.""" + latency_s = [1.0, 2.0, 3.0, 4.0, 10.0] + osl = [100.0] * 5 + results = _inject(_store(latency_s=latency_s, osl=osl)) + + rate = np.array(osl) / np.array(latency_s) + naive_p90 = float(np.percentile(rate, 90)) # p(1/x): the WRONG convention + # p90 slow-tail interactivity is the pessimistic (smaller) value. + assert results[P90].avg < naive_p90 + + +@pytest.mark.parametrize( + "bad", + [ + param({"latency_s": [1.0, 2.0, 0.0], "osl": [100.0, 100.0, 100.0]}, id="zero_latency"), + param({"latency_s": [1.0, 2.0, 3.0], "osl": [100.0, 100.0, 0.0]}, id="zero_osl"), + ], +) # fmt: skip +def test_filter_drops_nonpositive_core_fields(bad: dict) -> None: + """Requests with non-positive latency or OSL are excluded from the sample.""" + results = _inject(_store(**bad)) + # Third request dropped -> two survive with ratios [0.01, 0.02]. Assert on the + # emitted value (count is not exported for this derived scalar), which also + # proves *which* requests survived, not just how many. + survivors = [0.01, 0.02] + assert results[P90].avg == pytest.approx(1.0 / np.percentile(survivors, 90)) + assert results[P75].avg == pytest.approx(1.0 / np.percentile(survivors, 75)) + + +def test_filter_drops_nonpositive_ttft_and_isl() -> None: + """TTFT<=0 or ISL<=0 excludes the request, matching InferenceX's filter.""" + store = _store( + latency_s=[1.0, 2.0, 3.0], + osl=[100.0, 100.0, 100.0], + ttft_s=[0.5, 0.0, 0.5], + isl=[100.0, 100.0, 0.0], + ) + results = _inject(store) + assert results[P90].avg == pytest.approx(100.0) + + +def test_noop_when_required_columns_absent() -> None: + """No request_latency / OSL columns -> nothing injected.""" + store = ColumnStore(initial_capacity=2) + for idx in range(2): + store.ingest( + idx=idx, + record_metrics={"time_to_first_token": 1.0}, + start_ns=float(idx), + end_ns=float(idx), + generation_start_ns=None, + ) + results = _inject(store) + assert P90 not in results and P75 not in results + + +def test_mask_restricts_sample() -> None: + """Only masked-in requests contribute to the percentile.""" + store = _store(latency_s=[1.0, 2.0, 3.0, 100.0], osl=[100.0] * 4) + mask = np.array([True, True, True, False]) # drop the 100s outlier + results = _inject(store, mask=mask) + # 3 survive with ratios [0.01, 0.02, 0.03] (the 100s outlier masked out). + survivors = [0.01, 0.02, 0.03] + assert results[P90].avg == pytest.approx(1.0 / np.percentile(survivors, 90)) + + +def test_mask_shorter_than_columns_does_not_abort_export() -> None: + """Ingestion can outrun a snapshotted mask; indexing must not raise. + + ``summarize`` runs under ``asyncio.to_thread`` with a mask captured before + the call, so records arriving mid-export grow the columns past it. NumPy + raises IndexError when a boolean mask is shorter than the axis it indexes, + which would abort the whole export -- ``MetricsAccumulator.summarize`` + caps to ``len(mask)`` for this reason, and this injection must match. + """ + store = _store(latency_s=[1.0, 2.0, 3.0, 100.0], osl=[100.0] * 4) + # Mask snapshotted when only three records had been ingested. + stale_mask = np.array([True, True, True]) + + results = _inject(store, mask=stale_mask) + + # The late arrival is excluded, exactly as if it had not been ingested. + survivors = [0.01, 0.02, 0.03] + assert results[P90].avg == pytest.approx(1.0 / np.percentile(survivors, 90)) + + +def test_deferred_derive_raises() -> None: + """The scalar summarize path defers; values come only from injection.""" + metric = E2ENormalizedInteractivityP90Metric() + with pytest.raises(NoMetricValue): + metric._derive_value(MetricResultsDict()) + + +def test_extreme_values_never_inject_nonfinite() -> None: + """Extreme latency/OSL must not inject nan/inf (NaN/Inf discipline): a tiny + latency over a large OSL underflows the ratio so the reciprocal would + overflow to inf; the guard must drop it rather than emit a non-finite value.""" + store = _store(latency_s=[1e-300, 1e-300, 1e-300], osl=[1e9, 1e9, 1e9]) + results = _inject(store) + for tag in (P90, P75): + assert tag not in results or math.isfinite(results[tag].avg) + + +def test_metric_metadata() -> None: + """Percentile identity, unit, and larger-is-better flag are as specified.""" + assert E2ENormalizedInteractivityP90Metric.percentile == 90.0 + assert E2ENormalizedInteractivityP75Metric.percentile == 75.0 + assert E2ENormalizedInteractivityP90Metric.flags & MetricFlags.LARGER_IS_BETTER + + +def test_drop_log_attributes_failed_requests_separately(caplog) -> None: + """A failed request occupies a row with absent (NaN) columns, so it is + excluded by the same finite check as a genuine TTFT/ISL data-quality + problem. The debug log is the only signal for a shrunken sample -- ``count`` + is pinned to 1 and dropped on export -- so it must name the dominant cause + rather than sending the reader after a problem that isn't there.""" + store = ColumnStore(initial_capacity=5) + for idx in range(3): + store.ingest( + idx=idx, + record_metrics={ + "request_latency": 2.0 * NANOS_PER_SECOND, + "output_sequence_length": 100.0, + "time_to_first_token": 0.05 * NANOS_PER_SECOND, + "input_sequence_length": 100.0, + }, + start_ns=float(idx), + end_ns=float(idx), + generation_start_ns=None, + ) + # Error-shaped rows: ingested unconditionally, but carry no metrics. + for idx in (3, 4): + store.ingest( + idx=idx, + record_metrics={}, + start_ns=float(idx), + end_ns=float(idx), + generation_start_ns=None, + ) + + with caplog.at_level("DEBUG"): + _inject(store) + + drop_logs = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()] + assert drop_logs, "expected a debug log reporting the dropped requests" + message = drop_logs[0] + assert "2 of 5 requests dropped" in message + assert "2 with no recorded latency/OSL" in message + assert "computed over 3 requests" in message + + +def test_drop_log_does_not_count_non_positive_values_as_missing(caplog) -> None: + """A finite-but-non-positive latency is a different cause from an absent + one, and the log must not conflate them: ``missing`` counts only rows with + no recorded latency/OSL, so a zero-latency row belongs in the remainder.""" + store = ColumnStore(initial_capacity=5) + for idx in range(3): + store.ingest( + idx=idx, + record_metrics={ + "request_latency": 2.0 * NANOS_PER_SECOND, + "output_sequence_length": 100.0, + }, + start_ns=float(idx), + end_ns=float(idx), + generation_start_ns=None, + ) + store.ingest( + idx=3, + record_metrics={"request_latency": 0.0, "output_sequence_length": 100.0}, + start_ns=3.0, + end_ns=3.0, + generation_start_ns=None, + ) + store.ingest( + idx=4, + record_metrics={}, + start_ns=4.0, + end_ns=4.0, + generation_start_ns=None, + ) + + with caplog.at_level("DEBUG"): + _inject(store) + + message = next( + r.getMessage() for r in caplog.records if "dropped" in r.getMessage() + ) + assert "2 of 5 requests dropped" in message + assert "1 with no recorded latency/OSL" in message + assert "non-positive or non-finite latency" in message diff --git a/tests/unit/post_processors/test_timeslice_metric_results_processor.py b/tests/unit/post_processors/test_timeslice_metric_results_processor.py index 2618cb9a78..fc082d1bf1 100644 --- a/tests/unit/post_processors/test_timeslice_metric_results_processor.py +++ b/tests/unit/post_processors/test_timeslice_metric_results_processor.py @@ -10,6 +10,10 @@ from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.constants import NANOS_PER_MILLIS, NANOS_PER_SECOND from aiperf.metrics.accumulator import MetricsAccumulator +from aiperf.metrics.types.e2e_normalized_interactivity_metrics import ( + E2ENormalizedInteractivityP75Metric, + E2ENormalizedInteractivityP90Metric, +) from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.replay_sched_lag_metrics import ( @@ -273,20 +277,26 @@ class TestMetricsAccumulatorRunScopedDerivedMetrics: accumulator excludes it from per-slice derivation while keeping it in the overall summary.""" + # Every run-scoped derived tag (``timeslice_derivable = False``): derived at + # run level, excluded from per-slice derivation. Shared by both tests so a + # new run-scoped metric is covered by both without updating them separately. + _RUN_SCOPED_TAGS = { + ReplaySchedLagP50Metric.tag, + ReplaySchedLagP90Metric.tag, + ReplaySchedLagP99Metric.tag, + ReplaySchedDegradedMetric.tag, + E2ENormalizedInteractivityP90Metric.tag, + E2ENormalizedInteractivityP75Metric.tag, + } + def test_run_scoped_tags_excluded_from_timeslice_derivation( self, fixed_schedule_slice_run ) -> None: accumulator = MetricsAccumulator(fixed_schedule_slice_run) - run_scoped_tags = { - ReplaySchedLagP50Metric.tag, - ReplaySchedLagP90Metric.tag, - ReplaySchedLagP99Metric.tag, - ReplaySchedDegradedMetric.tag, - } # The run-scoped family derives at run level but is skipped per slice. - assert run_scoped_tags <= set(accumulator._derive_funcs) - assert accumulator._non_timeslice_derived_tags == run_scoped_tags + assert set(accumulator._derive_funcs) >= self._RUN_SCOPED_TAGS + assert accumulator._non_timeslice_derived_tags == self._RUN_SCOPED_TAGS @pytest.mark.asyncio async def test_run_scoped_tags_never_derived_in_timeslice_results( @@ -311,8 +321,19 @@ async def test_run_scoped_tags_never_derived_in_timeslice_results( assert summary.timeslices is not None assert len(summary.timeslices) >= 1 + # Only the send-lag family is asserted here. These records carry no + # request_latency/output_sequence_length, so the interactivity tags + # would be absent whether or not they were excluded -- asserting them + # would look like coverage while proving nothing. Their run-scoped + # contract is enforced at the metadata level by + # test_run_scoped_tags_excluded_from_timeslice_derivation, which does + # fail if the exclusion is removed. + replay_tags = { + ReplaySchedLagP50Metric.tag, + ReplaySchedLagP90Metric.tag, + ReplaySchedLagP99Metric.tag, + ReplaySchedDegradedMetric.tag, + } for ts in summary.timeslices: - assert ReplaySchedLagP50Metric.tag not in ts.metric_results - assert ReplaySchedLagP90Metric.tag not in ts.metric_results - assert ReplaySchedLagP99Metric.tag not in ts.metric_results - assert ReplaySchedDegradedMetric.tag not in ts.metric_results + for tag in replay_tags: + assert tag not in ts.metric_results From 33e647a89fcf621a580174a4403c2589128cd922 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Fri, 4 Sep 2026 17:39:34 -0600 Subject: [PATCH 053/119] fix(timing): surface execution-task exceptions instead of hanging on request-count phases (#1336) Signed-off-by: Amir Fathi (cherry picked from commit 13ae4f6b6b5363007ad52ee2470c3b49c9403b34) --- src/aiperf/timing/phase/runner.py | 43 +++++++- tests/unit/timing/phase/test_runner.py | 143 +++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/src/aiperf/timing/phase/runner.py b/src/aiperf/timing/phase/runner.py index 2cbaadd57f..ca3c077973 100644 --- a/src/aiperf/timing/phase/runner.py +++ b/src/aiperf/timing/phase/runner.py @@ -1022,6 +1022,7 @@ async def _wait_for_sending_complete( self.error( f"Error waiting for phase {self._config.phase} to send all credits: {e!r}" ) + raise finally: preserve_branch_handoff = self._preserve_replay_gate_until_finalize( strategy @@ -1200,7 +1201,10 @@ async def _wait_for_event_with_timeout( timeout: The timeout in seconds. If None, the event will be waited for indefinitely. If timeout is <= 0, returns immediately with timeout. - task_to_cancel: The optional task to cancel when the timeout occurs. + task_to_cancel: The optional task to cancel when the timeout occurs. When + timeout is None, this task is also raced against the event: if it + finishes first with an exception, that exception is raised instead + of waiting forever (see ``_wait_for_event_or_task_exception``). set_event_on_timeout: If True, the event will also be set when the timeout occurs. Returns: @@ -1208,7 +1212,12 @@ async def _wait_for_event_with_timeout( """ if timeout is None: self.debug(lambda: f"Waiting for event '{name}' indefinitely") - await event.wait() + if task_to_cancel is None: + await event.wait() + else: + await self._wait_for_event_or_task_exception( + event, task_to_cancel, name + ) return False def _on_timeout() -> bool: @@ -1236,6 +1245,36 @@ def _on_timeout() -> bool: self.error(f"Error waiting for event '{name}' with timeout: {e!r}") raise + async def _wait_for_event_or_task_exception( + self, event: asyncio.Event, task: asyncio.Task, name: str + ) -> None: + """Wait for ``event``, but fail fast if ``task`` finishes first with an + exception, instead of waiting on it forever. + + An indefinite wait (timeout=None, request-count phases) has no bound to + fall back on, and ``execute_async`` runs ``task`` fire-and-forget: its + done-callback only discards it from the task set, so an exception it + raises is otherwise never retrieved and the wait hangs forever (#1041). + """ + event_task = asyncio.ensure_future(event.wait()) + try: + await asyncio.wait({event_task, task}, return_when=asyncio.FIRST_COMPLETED) + # A failure wins over a same-tick event completion; FIRST_COMPLETED + # only guarantees one of the two is done, so task.done() is checked + # explicitly rather than assumed from event_task's state. + if ( + task.done() + and not task.cancelled() + and (exc := task.exception()) is not None + ): + raise exc + if event_task.done(): + return + await event_task + finally: + if not event_task.done(): + event_task.cancel() + async def _progress_report_loop(self) -> None: """Publish phase progress stats at regular intervals. diff --git a/tests/unit/timing/phase/test_runner.py b/tests/unit/timing/phase/test_runner.py index 3ec7cd22ed..1a28da4a08 100644 --- a/tests/unit/timing/phase/test_runner.py +++ b/tests/unit/timing/phase/test_runner.py @@ -605,6 +605,60 @@ async def test_cancelled_phase_is_not_a_grace_period_timeout( assert result.grace_period_timeout_triggered is False +class TestSendingCompleteExecutionTaskFailure: + async def test_request_count_phase_raises_instead_of_hanging( + self, runner: PhaseRunner + ) -> None: + """A request-count phase has no ``expected_duration_sec``, so the + sending-complete wait has no timeout to fall back on. If the execution + task raises before the last credit is sent, the wait must fail fast and + the failure must propagate out of ``_wait_for_sending_complete`` (not + be logged and swallowed), matching ``run()``'s existing + except -> phase-failure path (#1041).""" + runner._lifecycle.start() + assert runner._lifecycle.time_left_in_seconds() is None + + async def boom() -> None: + raise RuntimeError("send_credit: no workers available for routing") + + runner._execution_task = runner.execute_async(boom()) + + with pytest.raises(RuntimeError, match="no workers available"): + await asyncio.wait_for( + runner._wait_for_sending_complete(MockStrategy()), timeout=5.0 + ) + + # The finally block still ran: sending is marked complete so the rest + # of teardown (returning-wait, stats) does not also hang. + assert runner._lifecycle.is_sending_complete + + async def test_duration_phase_still_times_out_normally( + self, + conv_src: MagicMock, + pub: MagicMock, + router: MagicMock, + conc: MagicMock, + cancel: MagicMock, + cb: MagicMock, + time_traveler: MagicMock, + ) -> None: + """Duration-based phases keep their existing bounded-timeout behavior: + an execution task that never finishes still surfaces as a normal + timeout, not an exception, since #1041 only affects the timeout=None + (request-count) branch.""" + r = make_runner( + cfg(reqs=None, dur=0.001), conv_src, pub, router, conc, cancel, cb + ) + r._lifecycle.start() + never = asyncio.Event() + r._execution_task = r.execute_async(never.wait()) + + await r._wait_for_sending_complete(MockStrategy()) + + assert r._lifecycle.is_sending_complete + assert r._lifecycle.timeout_triggered + + class TestTimeoutHandling: async def test_returns_false_when_event_set( self, runner: PhaseRunner, time_traveler: MagicMock @@ -673,6 +727,95 @@ async def set_later() -> None: ) assert result is False + async def test_indefinite_wait_raises_task_exception_instead_of_hanging( + self, runner: PhaseRunner + ) -> None: + """timeout=None has no bound to fall back on, so a raced task that + raises before the event is set must fail the wait fast (#1041) rather + than hang forever.""" + event = asyncio.Event() + + async def boom() -> None: + raise RuntimeError("send_credit: no workers available for routing") + + task = asyncio.create_task(boom()) + + with pytest.raises(RuntimeError, match="no workers available"): + await asyncio.wait_for( + runner._wait_for_event_with_timeout( + name="t", event=event, timeout=None, task_to_cancel=task + ), + timeout=5.0, + ) + + async def test_indefinite_wait_prioritizes_task_exception_over_simultaneous_event( + self, runner: PhaseRunner + ) -> None: + """If the event is already set and the raced task has already failed + by the time the wait begins, both futures can complete on the same + tick; the failure must still win, or the phase reports success on a + task that raised (CodeRabbit finding on #1041's own fix).""" + event = asyncio.Event() + event.set() + + async def boom() -> None: + raise RuntimeError("send_credit: no workers available for routing") + + task = asyncio.create_task(boom()) + await asyncio.sleep(0) + assert task.done() + + with pytest.raises(RuntimeError, match="no workers available"): + await asyncio.wait_for( + runner._wait_for_event_with_timeout( + name="t", event=event, timeout=None, task_to_cancel=task + ), + timeout=5.0, + ) + + async def test_indefinite_wait_ignores_cancelled_task_and_keeps_waiting( + self, runner: PhaseRunner, time_traveler: MagicMock + ) -> None: + """A cancelled raced task is not a failure: the wait must keep + waiting for the event rather than raising CancelledError.""" + event = asyncio.Event() + never = asyncio.Event() + task = asyncio.create_task(never.wait()) + task.cancel() + + async def set_later() -> None: + await asyncio.sleep(0.01) + event.set() + + asyncio.create_task(set_later()) + result = await runner._wait_for_event_with_timeout( + name="t", event=event, timeout=None, task_to_cancel=task + ) + assert result is False + + async def test_indefinite_wait_falls_through_when_task_finishes_cleanly( + self, runner: PhaseRunner, time_traveler: MagicMock + ) -> None: + """A raced task that finishes with no exception is not a failure + either: the wait must keep waiting for the event rather than + returning early just because the task is done.""" + event = asyncio.Event() + + async def finish_quietly() -> None: + return None + + task = asyncio.create_task(finish_quietly()) + + async def set_later() -> None: + await asyncio.sleep(0.01) + event.set() + + asyncio.create_task(set_later()) + result = await runner._wait_for_event_with_timeout( + name="t", event=event, timeout=None, task_to_cancel=task + ) + assert result is False + class TestStuckSlotsRelease: async def test_release_calls_concurrency_manager( From 9ac5c75e6e4cde25f5103ca139dcde7e4ca6418d Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:34:30 -0700 Subject: [PATCH 054/119] feat: add stale PR/issue auto-close workflow (#1399) Signed-off-by: Elias Bermudez (cherry picked from commit 56945bb52114af818f3e681e21363aa68bc9b381) --- .github/workflows/stale_cleaner.yml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/stale_cleaner.yml diff --git a/.github/workflows/stale_cleaner.yml b/.github/workflows/stale_cleaner.yml new file mode 100644 index 0000000000..93061c2726 --- /dev/null +++ b/.github/workflows/stale_cleaner.yml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# https://github.com/actions/stale + +name: 'Close stale issues and PRs' +on: + schedule: + - cron: '30 9 * * *' # 09:30 UTC daily + +permissions: + contents: write # only for delete-branch option + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 + with: + stale-issue-label: 'stale' + stale-pr-label: 'stale' + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days. As a reminder, please use backlog label to avoid issues being marked as stale.' + close-issue-message: 'This issue has been closed due to inactivity. If you believe this issue is still relevant, please feel free to reopen it with additional context or information.' + stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' + close-pr-message: 'This PR has been closed due to inactivity. If you believe this PR is still relevant, please feel free to reopen it with additional context or information.' + delete-branch: true + days-before-stale: 30 + days-before-close: 5 + any-of-issue-labels: 'bug' + exempt-issue-labels: 'backlog' From bbd3000b39020e98bc9164dabaa44d19dbdc577e Mon Sep 17 00:00:00 2001 From: Pavithra Vijayakrishnan <160681768+pvijayakrish@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:41:03 -0700 Subject: [PATCH 055/119] fix(ci): keep uv.lock consistent with the nightly version patch (#1414) Signed-off-by: pvijayakrish Co-authored-by: Anthony Casagrande (cherry picked from commit ea28b2e81c7367f8403c6f6ebe0837a508795a8d) --- .agents/skills/bump-version/SKILL.md | 48 ++- .../update-pyproject-version/action.yml | 42 +-- .github/workflows/nightly.yml | 9 +- .github/workflows/resolve-check.yml | 3 + tests/unit/tools/test_patch_version.py | 304 ++++++++++++++++++ tools/patch_version.py | 184 +++++++++++ 6 files changed, 541 insertions(+), 49 deletions(-) create mode 100644 tests/unit/tools/test_patch_version.py create mode 100755 tools/patch_version.py diff --git a/.agents/skills/bump-version/SKILL.md b/.agents/skills/bump-version/SKILL.md index f5b52ebebd..a9b8e9f583 100644 --- a/.agents/skills/bump-version/SKILL.md +++ b/.agents/skills/bump-version/SKILL.md @@ -2,7 +2,7 @@ name: bump-version description: Bump the AIPerf package version (e.g. 0.10.0 -> 0.11.0) across pyproject.toml, the mock server, and the version strings shown in docs, then open a PR to main disable-model-invocation: true -allowed-tools: Bash(git fetch *), Bash(git checkout *), Bash(git branch *), Bash(git status *), Bash(git add *), Bash(git commit *), Bash(git push *), Bash(git log *), Bash(grep *), Bash(pre-commit run *), Bash(gh pr create *), Read, Edit, AskUserQuestion +allowed-tools: Bash(git fetch *), Bash(git checkout *), Bash(git branch *), Bash(git status *), Bash(git add *), Bash(git commit *), Bash(git push *), Bash(git log *), Bash(git diff *), Bash(grep *), Bash(uv lock *), Bash(pre-commit run *), Bash(gh pr create *), Read, Edit, AskUserQuestion --- # Bump AIPerf Version @@ -49,7 +49,8 @@ to stay in sync. ``` Review each hit. Update only strings that mean "the AIPerf version". Do NOT touch unrelated version-like strings (dependency pins, `schema_version`, - sample UUIDs, etc.). + sample UUIDs, etc.). `uv.lock` is deliberately outside this grep: other + packages may share the version string, and step 5 regenerates it instead. The canonical set of files (as of 0.10.0 -> 0.11.0) is: - `pyproject.toml` — `version = "..."` under `[project]` @@ -59,14 +60,21 @@ to stay in sync. - `docs/server-metrics/server-metrics-json-schema.md` — every `"aiperf_version": "..."` JSON example and the `(e.g., "...")` description (3 occurrences) + - `uv.lock` — the root `[[package]] name = "aiperf"` entry, regenerated by + `uv lock` in step 5 (never hand-edited) If the grep surfaces a NEW occurrence not in this list, update it too and add it to this skill's canonical list in the same PR. -5. **Do NOT update `uv.lock`:** - `uv.lock` is gitignored and not tracked, so it is not part of the bump. - Tests that need the version import `aiperf.__version__` rather than hard-coding - it; leave those alone. +5. **Regenerate `uv.lock`:** + ```bash + uv lock + ``` + `uv.lock` is tracked and records the root `aiperf` version, so it must move + with `pyproject.toml`. The PR checks `uv sync --locked` in + `run-unit-tests.yml` and `helm-chart.yml` fail on a stale lock, and after + merge the nightly's `uv lock --check` does too. Tests that need the version + import `aiperf.__version__`; leave those alone. 6. **Verify nothing stale remains:** ```bash @@ -74,7 +82,16 @@ to stay in sync. docs/plugins/creating-your-first-plugin.md \ docs/server-metrics/server-metrics-json-schema.md ``` - This must return no matches. Then confirm `NEW_VERSION` is present in each. + This must return no matches. Then confirm `NEW_VERSION` is present in each, + and confirm the lockfile agrees: + ```bash + uv lock --check + git diff uv.lock + ``` + Expect the root `aiperf` `version =` line to change. `uv lock` may also + refresh metadata of already-locked packages (for example add an `sdist =` + line published after the lock was generated); that is fine to commit. + Investigate if any other package's `version =` line changes. 7. **Run pre-commit on the staged files:** ```bash @@ -86,15 +103,15 @@ to stay in sync. Stage only the version files explicitly (never `git add -A` — the working tree may hold unrelated untracked files): ```bash - git add pyproject.toml tests/aiperf_mock_server/pyproject.toml \ + git add pyproject.toml uv.lock tests/aiperf_mock_server/pyproject.toml \ docs/plugins/creating-your-first-plugin.md \ docs/server-metrics/server-metrics-json-schema.md git commit -s -F - <<'EOF' chore: bump aiperf version to - Updates the package and mock-server pyproject.toml versions and refreshes - the version strings shown in the plugin tutorial example and the - server-metrics JSON schema example output. + Updates the package and mock-server pyproject.toml versions, regenerates + uv.lock, and refreshes the version strings shown in the plugin tutorial + example and the server-metrics JSON schema example output. Co-Authored-By: Claude Opus 4.8 EOF @@ -115,14 +132,16 @@ to stay in sync. --body "$(cat <<'EOF' ## Summary - Bump AIPerf package version from `` to ``. - - Updates `pyproject.toml`, the mock-server `pyproject.toml`, and the version - strings in the plugin tutorial and server-metrics JSON schema docs. + - Updates `pyproject.toml`, regenerates `uv.lock`, and updates the mock-server + `pyproject.toml` and the version strings in the plugin tutorial and + server-metrics JSON schema docs. The runtime version is read dynamically from package metadata, so `pyproject.toml` is the source of truth; the other files are doc/example copies. ## Test plan - `grep` confirms no `` strings remain in the bumped files. + - `uv lock --check` passes. - Pre-commit hooks pass. EOF )" @@ -138,5 +157,6 @@ to stay in sync. - NEVER use `--no-verify` or `git commit -n`. - NEVER `git add -A` / `git add .`; stage the version files by explicit path so unrelated untracked files stay out of the commit. -- NEVER touch `uv.lock` (gitignored) or non-AIPerf version-like strings. +- NEVER hand-edit `uv.lock` (regenerate it with `uv lock`) or touch + non-AIPerf version-like strings. - Branch from an up-to-date `main`; one PR = one concern (just the bump). diff --git a/.github/actions/update-pyproject-version/action.yml b/.github/actions/update-pyproject-version/action.yml index da1f53e4e3..73a790c3d9 100644 --- a/.github/actions/update-pyproject-version/action.yml +++ b/.github/actions/update-pyproject-version/action.yml @@ -1,10 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: Update pyproject.toml version +name: Update pyproject.toml and uv.lock version description: > - Validates a PEP 440 version string and patches the version field - in pyproject.toml using Python regex (safer than sed). + Writes a PEP 440 version to pyproject.toml and to the root package entry in + uv.lock via tools/patch_version.py, so `uv sync --locked` stays satisfied. + Requires a checked-out repository and uv on PATH (astral-sh/setup-uv); the + script runs under a uv-provisioned Python 3.13 so its tomllib verification + always runs, on any runner OS. inputs: version: @@ -16,37 +19,8 @@ inputs: runs: using: composite steps: - - name: Patch pyproject.toml version + - name: Patch pyproject.toml and uv.lock version shell: bash env: TARGET_VERSION: ${{ inputs.version }} - run: | - python3 - <<'PYEOF' - import os - import re - - target = os.environ["TARGET_VERSION"] - - # PEP 440: X.Y.Z[(a|b|rc)N][.postN][.devN][+local] - # Local version: [a-zA-Z0-9] segments joined by dots (e.g. "g3e3b3b1"). - if not re.match( - r'^\d+\.\d+\.\d+((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?' - r'(\+[a-zA-Z0-9]+(\.[a-zA-Z0-9]+)*)?$', - target, - ): - raise SystemExit(f"::error::Invalid PEP 440 version '{target}'") - - path = "pyproject.toml" - text = open(path).read() - patched, n = re.subn( - r'^(version\s*=\s*")[^"]*(")', - rf'\g<1>{target}\g<2>', - text, - count=1, - flags=re.MULTILINE, - ) - if n != 1: - raise SystemExit(f"::error::Could not find 'version = \"...\"' in {path}") - open(path, "w").write(patched) - print(f"Patched {path}: version = \"{target}\"") - PYEOF + run: uv run --no-project --python 3.13 python tools/patch_version.py "${TARGET_VERSION}" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 64cecf276a..37f41cffb3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -379,6 +379,9 @@ jobs: - uses: actions/setup-python@v6 with: python-version: '3.13' + # A bare `uv sync` re-locks silently; fail fast on a stale lockfile instead. + - name: Check uv.lock is up to date + run: uv lock --check - name: Validate dependency resolution against required-environments run: uv sync --active --no-install-project @@ -422,11 +425,15 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 - - name: Patch version in pyproject.toml + - name: Patch version in pyproject.toml and uv.lock uses: ./.github/actions/update-pyproject-version with: version: ${{ needs.prepare-nightly.outputs.wheel_version }} + # Catch a bad version patch here, not ten minutes into the Docker build. + - name: Verify uv.lock matches the patched pyproject.toml + run: uv lock --check + # Embed the commit SHA two ways for redundant discoverability: # 1. [project.urls] Repository = in pyproject.toml → # lands in the wheel METADATA as `Project-URL: Repository, ...`. diff --git a/.github/workflows/resolve-check.yml b/.github/workflows/resolve-check.yml index 74863f991e..73be182853 100644 --- a/.github/workflows/resolve-check.yml +++ b/.github/workflows/resolve-check.yml @@ -42,6 +42,9 @@ jobs: - uses: actions/setup-python@v6 with: python-version: '3.13' + # A bare `uv sync` re-locks silently; fail fast on a stale lockfile instead. + - name: Check uv.lock is up to date + run: uv lock --check - name: Validate dependency resolution against required-environments # --no-install-project skips installing aiperf itself; only resolves # and installs the dependency graph, which is all we need to diff --git a/tests/unit/tools/test_patch_version.py b/tests/unit/tools/test_patch_version.py new file mode 100644 index 0000000000..615d5db8f9 --- /dev/null +++ b/tests/unit/tools/test_patch_version.py @@ -0,0 +1,304 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the nightly version patcher used by update-pyproject-version.""" + +import shutil +import tomllib +from collections.abc import Callable +from pathlib import Path + +import pytest +from pytest import param + +import tools.patch_version as patch_version_module +from tools.patch_version import PatchError, main, patch_version + +_REPO_ROOT = Path(__file__).resolve().parents[3] + +# A table with its own `name` precedes [project] so an unscoped regex would +# pick up "pytorch-cpu" instead of the project name. +_PYPROJECT = """\ +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" + +[project] +name = "aiperf" +version = "0.13.0" +dependencies = ["aioitertools"] + +[tool.uv] +required-environments = ["sys_platform == 'linux'"] +""" + +# Mirrors uv's layout: a same-versioned dependency before the root and an inline +# `{ name = "aiperf", ... }` requires-dist reference after it, neither of which +# may be touched. +_LOCK = """\ +version = 1 +revision = 3 + +[[package]] +name = "aioitertools" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "aiperf" +version = "0.13.0" +source = { editable = "." } +dependencies = [ + { name = "aioitertools" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiperf", extras = ["botorch"], marker = "extra == 'optuna'" }, +] +""" + +_NEW = "0.13.0.dev20260910" + + +def _expected_lock(version: str) -> str: + return _LOCK.replace( + 'name = "aiperf"\nversion = "0.13.0"', f'name = "aiperf"\nversion = "{version}"' + ) + + +@pytest.fixture +def tree(tmp_path: Path) -> tuple[Path, Path]: + """A minimal pyproject.toml and uv.lock pair in a temporary directory.""" + pyproject = tmp_path / "pyproject.toml" + lock = tmp_path / "uv.lock" + pyproject.write_text(_PYPROJECT) + lock.write_text(_LOCK) + return pyproject, lock + + +@pytest.mark.parametrize( + "version", + [ + param("0.13.0.dev20260910", id="scheduled-nightly"), + param("0.13.0.dev20260910+manual.34398837728", id="manual-dispatch"), + param("0.14.0rc1", id="release-candidate"), + ], +) # fmt: skip +def test_patch_version_valid_version_updates_only_project_and_root_entries( + tree: tuple[Path, Path], version: str +) -> None: + pyproject, lock = tree + + assert patch_version(version, pyproject, lock) == "aiperf" + + project = tomllib.loads(pyproject.read_text())["project"] + assert project == { + "name": "aiperf", + "version": version, + "dependencies": ["aioitertools"], + } + assert 'name = "pytorch-cpu"' in pyproject.read_text() + assert lock.read_text() == _expected_lock(version) + + +def test_patch_version_preserves_crlf_line_endings(tmp_path: Path) -> None: + pyproject = tmp_path / "pyproject.toml" + lock = tmp_path / "uv.lock" + pyproject.write_bytes(_PYPROJECT.replace("\n", "\r\n").encode()) + lock.write_bytes(_LOCK.replace("\n", "\r\n").encode()) + + patch_version(_NEW, pyproject, lock) + + assert pyproject.read_bytes() == ( + _PYPROJECT.replace('version = "0.13.0"', f'version = "{_NEW}"') + .replace("\n", "\r\n") + .encode() + ) + assert lock.read_bytes() == _expected_lock(_NEW).replace("\n", "\r\n").encode() + + +@pytest.mark.parametrize( + "version", + [ + param("not-a-version", id="garbage"), + param("1.2", id="two-component"), + param("0.13.0+bad_local", id="underscore-in-local"), + param("0.13.0.dev20260910\n", id="trailing-newline"), + ], +) # fmt: skip +def test_patch_version_invalid_version_raises_before_writing( + tree: tuple[Path, Path], version: str +) -> None: + pyproject, lock = tree + + with pytest.raises(PatchError, match="Invalid PEP 440"): + patch_version(version, pyproject, lock) + + assert pyproject.read_text() == _PYPROJECT + assert lock.read_text() == _LOCK + + +def test_patch_version_normalizes_project_name_for_lock_lookup(tmp_path: Path) -> None: + """uv writes the PEP 503 name into uv.lock even when pyproject spells it differently.""" + pyproject = tmp_path / "pyproject.toml" + lock = tmp_path / "uv.lock" + pyproject.write_text(_PYPROJECT.replace('name = "aiperf"', 'name = "AI_Perf.Tool"')) + lock.write_text(_LOCK.replace('name = "aiperf"', 'name = "ai-perf-tool"')) + + assert patch_version(_NEW, pyproject, lock) == "AI_Perf.Tool" + + assert f'name = "ai-perf-tool"\nversion = "{_NEW}"' in lock.read_text() + + +def test_patch_version_missing_lock_raises_with_remedy(tree: tuple[Path, Path]) -> None: + pyproject, lock = tree + lock.unlink() + + with pytest.raises(PatchError, match=r"uv.lock not found; run `uv lock`"): + patch_version(_NEW, pyproject, lock) + + assert pyproject.read_text() == _PYPROJECT + + +@pytest.mark.parametrize( + ("lock_text", "message"), + [ + param( + _LOCK.replace('source = { editable = "." }', 'source = { editable = "src" }'), + "exactly one editable root", + id="no-root-entry", + ), + param( + _LOCK + '\n[[package]]\nname = "aiperf"\nversion = "0.13.0"\nsource = { editable = "." }\n', + "does not carry", + id="two-root-entries", + ), + ], +) # fmt: skip +def test_patch_version_bad_lock_raises_and_writes_nothing( + tree: tuple[Path, Path], lock_text: str, message: str +) -> None: + pyproject, lock = tree + lock.write_text(lock_text) + + with pytest.raises(PatchError, match=message): + patch_version(_NEW, pyproject, lock) + + assert pyproject.read_text() == _PYPROJECT + assert lock.read_text() == lock_text + + +@pytest.mark.parametrize( + ("pyproject_text", "message"), + [ + param('[tool.poetry]\nname = "aiperf"\nversion = "0.13.0"\n', r"no \[project\] table", id="poetry-only"), + param('[project]\nname = "aiperf"\n', r'no version = "..." line', id="no-version"), + param('[project]\nversion = "0.13.0"\n', "no name", id="no-name"), + param('[project\nname = "aiperf"\n', "not valid TOML", id="invalid-toml"), + ], +) # fmt: skip +def test_patch_version_bad_pyproject_raises_and_writes_nothing( + tree: tuple[Path, Path], pyproject_text: str, message: str +) -> None: + pyproject, lock = tree + pyproject.write_text(pyproject_text) + + with pytest.raises(PatchError, match=message): + patch_version(_NEW, pyproject, lock) + + assert pyproject.read_text() == pyproject_text + assert lock.read_text() == _LOCK + + +def test_patch_version_against_repository_files_changes_exactly_two_lines( + tmp_path: Path, +) -> None: + """Bind the textual patterns to the real pyproject.toml and uv.lock layout.""" + pyproject = shutil.copy(_REPO_ROOT / "pyproject.toml", tmp_path / "pyproject.toml") + lock = shutil.copy(_REPO_ROOT / "uv.lock", tmp_path / "uv.lock") + original_pyproject = (_REPO_ROOT / "pyproject.toml").read_text().splitlines() + original_lock = (_REPO_ROOT / "uv.lock").read_text().splitlines() + current = tomllib.loads((_REPO_ROOT / "pyproject.toml").read_text())["project"][ + "version" + ] + # The nightly runs this suite on an already-patched tree, so derive a target + # that can never equal the current version (dev numbers are dates). + target = f"{current.split('.dev')[0].split('+')[0]}.dev99999999" + + assert patch_version(target, Path(pyproject), Path(lock)) == "aiperf" + + changed_pyproject = [ + (a, b) + for a, b in zip( + original_pyproject, Path(pyproject).read_text().splitlines(), strict=True + ) + if a != b + ] + changed_lock = [ + (a, b) + for a, b in zip(original_lock, Path(lock).read_text().splitlines(), strict=True) + if a != b + ] + assert changed_pyproject == [(f'version = "{current}"', f'version = "{target}"')] + assert changed_lock == [(f'version = "{current}"', f'version = "{target}"')] + assert tomllib.loads(Path(pyproject).read_text())["project"]["version"] == target + + +def test_main_success_prints_both_files_and_returns_zero( + tree: tuple[Path, Path], capsys: pytest.CaptureFixture[str] +) -> None: + pyproject, lock = tree + + rc = main([_NEW, "--pyproject", str(pyproject), "--lock", str(lock)]) + + assert rc == 0 + out = capsys.readouterr().out + assert f"Patched {pyproject}" in out and f"Patched {lock}" in out + + +@pytest.mark.parametrize( + ("break_tree", "message"), + [ + param(lambda pyproject, lock: lock.unlink(), "not found", id="missing-lock"), + param(lambda pyproject, lock: (lock.unlink(), lock.mkdir()), "not a regular file", id="lock-is-a-directory"), + ], +) # fmt: skip +def test_main_failure_emits_github_error_annotation( + tree: tuple[Path, Path], + capsys: pytest.CaptureFixture[str], + break_tree: Callable[[Path, Path], object], + message: str, +) -> None: + pyproject, lock = tree + break_tree(pyproject, lock) + + rc = main([_NEW, "--pyproject", str(pyproject), "--lock", str(lock)]) + + assert rc == 1 + assert message in capsys.readouterr().err + assert pyproject.read_text() == _PYPROJECT + + +def test_main_write_failure_reports_error_and_leaves_both_files_untouched( + tree: tuple[Path, Path], + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An OSError while staging the lock must not leave pyproject.toml already patched.""" + pyproject, lock = tree + real_stage = patch_version_module._stage + + def failing_stage(path: Path, text: str) -> Path: + if path == lock: + raise PermissionError(13, "Permission denied", str(path)) + return real_stage(path, text) + + monkeypatch.setattr(patch_version_module, "_stage", failing_stage) + + rc = main([_NEW, "--pyproject", str(pyproject), "--lock", str(lock)]) + + assert rc == 1 + assert capsys.readouterr().err.startswith("::error::") + assert pyproject.read_text() == _PYPROJECT + assert lock.read_text() == _LOCK + assert not (pyproject.parent / "pyproject.toml.tmp").exists() diff --git a/tools/patch_version.py b/tools/patch_version.py new file mode 100755 index 0000000000..4475b5740e --- /dev/null +++ b/tools/patch_version.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Write a PEP 440 version into pyproject.toml and the root entry of uv.lock. + +The nightly pipeline stamps a date-based version onto a checkout before +building. uv.lock records the root project's version too, so patching only +pyproject.toml leaves the lockfile stale: `uv sync --locked` rejects it and a +plain `uv run` rewrites it. Patching both keeps the tree lock-consistent. + +Both rewrites are textual so each file keeps its formatting and line endings +apart from the one version string. The root lock entry is the `[[package]]` +whose name is the PEP 503 form of `[project].name` and whose source is +`{ editable = "." }` (a build-system-backed root). Both patched documents are +re-parsed with tomllib in memory to confirm the intended entries carry the new +version, then staged to temp files and swapped in last, so a parse or +verification failure never touches either file. + +Requires Python 3.11+ (tomllib). + +Usage: + python3 tools/patch_version.py 0.13.0.dev20260910 [--pyproject P] [--lock L] +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import tomllib +from pathlib import Path + +# PEP 440: X.Y.Z[(a|b|rc)N][.postN][.devN][+local]; local segments are [a-zA-Z0-9] joined by dots. +_PEP440 = re.compile( + r"\d+\.\d+\.\d+((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?" + r"(\+[a-zA-Z0-9]+(\.[a-zA-Z0-9]+)*)?" +) +# The [project] table body: from its header to the next table header or EOF. +_PROJECT_TABLE = re.compile( + r"^\[project\][^\r\n]*\r?\n(.*?)(?=^\[|\Z)", re.MULTILINE | re.DOTALL +) +_VERSION_LINE = re.compile(r'^(version\s*=\s*")[^"]*(")', re.MULTILINE) + + +class PatchError(Exception): + """A version patch could not be applied or verified; nothing was written.""" + + +def _canonical(name: str) -> str: + """PEP 503 normalized distribution name, which is what uv writes into uv.lock.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _root_package_pattern(name: str) -> re.Pattern[str]: + """Match the uv.lock `[[package]]` block for the editable root project `name`.""" + return re.compile( + r'(\[\[package\]\]\r?\nname = "' + + re.escape(_canonical(name)) + + r'"\r?\nversion = ")[^"]*("\r?\nsource = \{ editable = "\." \})' + ) + + +def _read(path: Path) -> str: + """Read text verbatim; newline="" keeps CRLF so the rewrite preserves it.""" + with path.open("r", encoding="utf-8", newline="") as handle: + return handle.read() + + +def _stage(path: Path, text: str) -> Path: + """Write text verbatim to a same-directory temp file and return its path.""" + tmp = path.with_name(path.name + ".tmp") + with tmp.open("w", encoding="utf-8", newline="") as handle: + handle.write(text) + return tmp + + +def _patch_pyproject(text: str, version: str) -> tuple[str, str]: + """Return the patched pyproject text and the project name it declares.""" + try: + project = tomllib.loads(text)["project"] + except tomllib.TOMLDecodeError as exc: + raise PatchError(f"pyproject.toml is not valid TOML: {exc}") from exc + except KeyError: + raise PatchError("pyproject.toml has no [project] table") from None + name = project.get("name") + if not isinstance(name, str) or not name: + raise PatchError("[project] has no name") + table = _PROJECT_TABLE.search(text) + if table is None: + raise PatchError("Could not locate the [project] table in pyproject.toml") + body, n = _VERSION_LINE.subn(rf"\g<1>{version}\g<2>", table.group(1), count=1) + if n != 1: + raise PatchError('[project] has no version = "..." line') + return text[: table.start(1)] + body + text[table.end(1) :], name + + +def _patch_lock(text: str, name: str, version: str) -> str: + """Return the lock text with the editable root entry for `name` at `version`.""" + patched, n = _root_package_pattern(name).subn( + rf"\g<1>{version}\g<2>", text, count=1 + ) + if n != 1: + raise PatchError( + f"Could not find exactly one editable root package entry for {name!r} in uv.lock" + ) + return patched + + +def _verify(pyproject_text: str, lock_text: str, name: str, version: str) -> None: + """Re-parse both patched documents and confirm the intended entries carry `version`.""" + try: + project_version = tomllib.loads(pyproject_text)["project"].get("version") + packages = tomllib.loads(lock_text).get("package", []) + except tomllib.TOMLDecodeError as exc: + raise PatchError(f"patched document is not valid TOML: {exc}") from exc + if project_version != version: + raise PatchError(f"[project] version is not {version!r} after patching") + roots = [ + p + for p in packages + if p.get("name") == _canonical(name) + and p.get("source", {}).get("editable") == "." + ] + if len(roots) != 1 or roots[0].get("version") != version: + raise PatchError(f"uv.lock root package {name!r} does not carry {version!r}") + + +def patch_version(version: str, pyproject: Path, lock: Path) -> str: + """Write `version` into `pyproject` and the root package entry of `lock`. + + Both documents are patched and verified in memory, staged to temp files, + and only then swapped into place. Returns the project name. Raises + PatchError on an invalid version, a missing file, or an entry that cannot + be located. + """ + if not _PEP440.fullmatch(version): + raise PatchError(f"Invalid PEP 440 version {version!r}") + for path in (pyproject, lock): + if path.exists() and not path.is_file(): + raise PatchError(f"{path} is not a regular file") + if not pyproject.is_file(): + raise PatchError(f"{pyproject} not found") + if not lock.is_file(): + raise PatchError( + f"{lock} not found; run `uv lock` to create it, and check that the " + "repository is checked out before this step" + ) + + pyproject_new, name = _patch_pyproject(_read(pyproject), version) + lock_new = _patch_lock(_read(lock), name, version) + _verify(pyproject_new, lock_new, name, version) + staged: list[tuple[Path, Path]] = [] + try: + staged.append((_stage(pyproject, pyproject_new), pyproject)) + staged.append((_stage(lock, lock_new), lock)) + except OSError: + for tmp, _ in staged: + tmp.unlink(missing_ok=True) + raise + for tmp, target in staged: + os.replace(tmp, target) + return name + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point; failures print a GitHub Actions `::error::` annotation.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("version", help="PEP 440 version to write") + parser.add_argument("--pyproject", type=Path, default=Path("pyproject.toml")) + parser.add_argument("--lock", type=Path, default=Path("uv.lock")) + args = parser.parse_args(argv) + try: + name = patch_version(args.version, args.pyproject, args.lock) + except (PatchError, OSError) as exc: + print(f"::error::{exc}", file=sys.stderr) + return 1 + print(f'Patched {args.pyproject}: version = "{args.version}"') + print(f'Patched {args.lock}: {name} version = "{args.version}"') + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 971ef4c3249ec56c41dcf01a9cdd05b3f0ed3bf6 Mon Sep 17 00:00:00 2001 From: Dundy Pasupuleti <42792754+dundysm@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:52:18 -0600 Subject: [PATCH 056/119] fix: default GpuSummary.hostname to None for exclude_none telemetry (#1419) (cherry picked from commit de6ca1d6aa109e95ff4c310ce52b2a1ccfa13fca) --- src/aiperf/common/models/export_models.py | 2 +- .../unit/common/models/test_export_models.py | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tests/unit/common/models/test_export_models.py diff --git a/src/aiperf/common/models/export_models.py b/src/aiperf/common/models/export_models.py index 2cbcb792ec..51e93375dd 100644 --- a/src/aiperf/common/models/export_models.py +++ b/src/aiperf/common/models/export_models.py @@ -99,7 +99,7 @@ class GpuSummary(AIPerfBaseModel): default=UNKNOWN_GPU_TELEMETRY_PLATFORM, description="GPU telemetry platform namespace, e.g. 'nvidia', 'amd', or 'unknown'", ) - hostname: str | None + hostname: str | None = None namespace: str | None = None pod_name: str | None = None metrics: dict[str, JsonMetricResult] # metric_key -> {stat_key -> value} diff --git a/tests/unit/common/models/test_export_models.py b/tests/unit/common/models/test_export_models.py new file mode 100644 index 0000000000..67e17128ce --- /dev/null +++ b/tests/unit/common/models/test_export_models.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for export model serialization edge cases.""" + +from datetime import UTC, datetime + +import orjson + +from aiperf.common.models.export_models import ( + EndpointData, + GpuSummary, + TelemetryExportData, + TelemetrySummary, +) + + +class TestGpuSummaryHostnameDefault: + """GpuSummary.hostname must default so exclude_none round-trips survive.""" + + def test_exclude_none_round_trip_with_missing_hostname(self) -> None: + """Reproduce #1417: hostname=None must validate after exclude_none dump. + + dcgm_collector sets hostname from an optional Prometheus label. Message + serialization uses model_dump(exclude_none=True), so a None hostname is + omitted on the wire. Without a default, receive-side model_validate fails + with 'Field required' and profile finalization hangs. + """ + gpus = { + "gpu_0": GpuSummary( + gpu_index=0, + gpu_name="GPU", + gpu_uuid="GPU-0", + hostname=None, + namespace=None, + pod_name=None, + metrics={}, + ) + } + now = datetime.now(UTC) + data = TelemetryExportData( + summary=TelemetrySummary(start_time=now, end_time=now), + endpoints={"localhost:9400": EndpointData(gpus=gpus)}, + ) + + wire = data.model_dump( + exclude_none=True, + mode="json", + context={"include_internal": True}, + ) + gpu0 = wire["endpoints"]["localhost:9400"]["gpus"]["gpu_0"] + assert "hostname" not in gpu0 + + restored = TelemetryExportData.model_validate(orjson.loads(orjson.dumps(wire))) + assert restored.endpoints["localhost:9400"].gpus["gpu_0"].hostname is None From a2e585ce03eae6efdc72442a0ef846630163288f Mon Sep 17 00:00:00 2001 From: Pavithra Vijayakrishnan <160681768+pvijayakrish@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:01:37 -0700 Subject: [PATCH 057/119] ci(nightly): bound the amd64 wheel-test parallelism on the builder (#1418) Signed-off-by: pvijayakrish (cherry picked from commit 7db2ba37a62aa80c882bc90eaf61cc8073e2387b) --- .github/workflows/nightly.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 37f41cffb3..2fd1dd7684 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -600,6 +600,9 @@ jobs: # advisory; this gate sits inside the publish path. - name: Run unit tests against installed nightly wheel (amd64 only) if: matrix.arch == 'amd64' + timeout-minutes: 45 + env: + MALLOC_ARENA_MAX: "2" run: | set -euo pipefail NIGHTLY=$(find dist/ -maxdepth 1 -name "aiperf_nightly-*.whl" -type f | head -1) @@ -627,8 +630,9 @@ jobs: # package is on sys.path via the venv; src/ is NOT on sys.path, # so `import aiperf` resolves to the installed wheel rather than # the source tree. + # -n auto on the many-core builder starved the runner of memory. .venv-wheel-tests/bin/pytest tests/unit \ - -n auto \ + -n 8 --dist=worksteal \ -m 'not performance and not stress' \ --tb=short From e679761f18faf413cc67d7c1d4d01ae66846253e Mon Sep 17 00:00:00 2001 From: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:54:08 -0700 Subject: [PATCH 058/119] feat(spec-decode): build the SPEED-Bench per-category matrix from one run (#1410) (cherry picked from commit 77d3299c23d2cfa0bdc9d1cdda3155aab1695990) --- docs/cli-options.md | 19 +- docs/tutorials/spec-decode-metrics.md | 12 +- docs/tutorials/speed-bench.md | 440 +++++++++---- src/aiperf/analysis/speed_bench_report.py | 404 ++++++++++-- src/aiperf/cli_commands/speed_bench_report.py | 33 +- src/aiperf/common/models/dataset_models.py | 12 +- src/aiperf/common/models/record_models.py | 14 +- src/aiperf/config/templates/README.md | 1 + .../templates/speed_bench_per_request.yaml | 87 +++ .../config/templates/speed_bench_sweep.yaml | 8 + src/aiperf/dataset/loader/speed_bench.py | 27 +- src/aiperf/dataset/memory_map_utils.py | 20 +- src/aiperf/workers/worker.py | 3 +- .../unit/analysis/test_speed_bench_report.py | 620 ++++++++++++++++++ .../cli_commands/test_speed_bench_report.py | 6 +- tests/unit/dataset/loader/test_speed_bench.py | 88 ++- tests/unit/dataset/test_payload_mmap.py | 13 +- tests/unit/workers/test_worker.py | 9 +- 18 files changed, 1603 insertions(+), 213 deletions(-) create mode 100644 src/aiperf/config/templates/speed_bench_per_request.yaml diff --git a/docs/cli-options.md b/docs/cli-options.md index 6d20aa8dae..314767cf91 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -62,7 +62,7 @@ Run an AIPerf service in a single process. ### [`speed-bench-report`](#aiperf-speed-bench-report) -Assemble per-category SPEED-Bench aiperf results into a matrix report. +Assemble SPEED-Bench aiperf results into a matrix report. ### [`synthesize`](#aiperf-synthesize) @@ -3325,13 +3325,16 @@ Select the ZMQ dual-bind communication backend (IPC + TCP). All dual-bind knobs ## `aiperf speed-bench-report` -Assemble per-category SPEED-Bench aiperf results into a matrix report. +Assemble SPEED-Bench aiperf results into a matrix report. -Run ``aiperf profile`` once per SPEED-Bench category, then point this command at the output directories to produce a matrix matching the SPEED-Bench paper format. +Point this command at one or more ``aiperf profile`` output directories. A single run over an aggregate SPEED-Bench split (``speed_bench_qualitative``) already carries every category, and each of its per-request records is stamped with the category it came from, so one run produces the whole matrix. Runs over a single category still work and contribute one column each. **Examples:** ```bash +# One run over all categories: matrix comes from its per-request records +aiperf speed-bench-report ./artifacts/speed_bench_qualitative/ + # Scan a parent directory for per-category run subdirectories aiperf speed-bench-report ./artifacts/ @@ -3341,8 +3344,11 @@ aiperf speed-bench-report ./artifacts/run_coding/ ./artifacts/run_math/ # Acceptance rate matrix (accepted / draft tokens) aiperf speed-bench-report ./artifacts/ --metric accept_rate -# Throughput matrix (output tokens/sec per category) +# Throughput matrix (output tokens/sec per run) aiperf speed-bench-report ./artifacts/ --metric throughput + +# Force the Prometheus scrape, ignoring any per-request records +aiperf speed-bench-report ./artifacts/ --source server ``` #### `--paths`, `--empty-paths` `` _(Required)_ @@ -3364,6 +3370,11 @@ Output format - 'csv', 'table', or 'both'. Defaults to 'both'. Which metric to report - 'accept_length', 'accept_rate', or 'throughput'. Defaults to 'accept_length'.
_Default: `accept_length`_ +### `--source` `` + +Where acceptance numbers come from - 'records' (per-request ``profile_export.jsonl``), 'summary' (the same per-request data reduced to run-level scalars in ``profile_export_aiperf.json``, the only source left at ``--export-level summary``), 'server' (Prometheus scrape in ``server_metrics_export.json``), or 'auto' to try them in that order. Only 'records' can split one run into per-category columns; only 'server' reads server-side data. Defaults to 'auto'. +
_Default: `auto`_ +
## `aiperf synthesize` diff --git a/docs/tutorials/spec-decode-metrics.md b/docs/tutorials/spec-decode-metrics.md index 13e85dd986..1414e8aa68 100644 --- a/docs/tutorials/spec-decode-metrics.md +++ b/docs/tutorials/spec-decode-metrics.md @@ -21,8 +21,12 @@ cleanly when they are not. > reads them inline during a normal `aiperf profile` run. Highest fidelity -- you get > per-request distributions (avg/min/percentiles) and a pooled histogram. Direct-to-vLLM only. > - **Server scrape:** aggregate acceptance is read from the server's Prometheus `/metrics` -> endpoint via `--server-metrics` and assembled with `aiperf speed-bench-report`. See the -> [SPEED-Bench tutorial](speed-bench.md) for that path and for ready-made datasets. +> endpoint via `--server-metrics`. Works on engines with no per-request reporting +> (SGLang, TensorRT-LLM) and behind Dynamo, but the numbers cover the whole server. +> +> For a per-category acceptance matrix over a ready-made speculative-decoding dataset, see the +> [SPEED-Bench tutorial](speed-bench.md), which covers both paths: on the per-request path a +> single run produces the whole matrix, on the scrape path it takes one run per category. For the record shape and the adapter that fills it, see [Per-Request Speculative-Decoding Acceptance](../reference/spec-decode-acceptance.md); for @@ -191,7 +195,7 @@ the expected clean-degradation behavior -- not an error. Common causes: - speculative decoding is off, or the requests had no verify steps; - the server was not started with `--per-request-spec-decode-metrics`; - the server is behind Dynamo, which strips the custom field (use the - [server-scrape path](speed-bench.md) instead); + [server-scrape path](speed-bench.md#portable-path-server-scrape) instead); - the vLLM build predates [PR #48915](https://github.com/vllm-project/vllm/pull/48915). --- @@ -200,5 +204,5 @@ the expected clean-degradation behavior -- not an error. Common causes: - [Speculative Decoding Metrics](../metrics-reference.md#speculative-decoding-metrics) -- metric definitions and formulas. - [Per-Request Speculative-Decoding Acceptance](../reference/spec-decode-acceptance.md) -- the engine-neutral record and adapter architecture. -- [SPEED-Bench tutorial](speed-bench.md) -- the server-scrape acceptance path and ready-made speculative-decoding datasets. +- [SPEED-Bench tutorial](speed-bench.md) -- ready-made speculative-decoding datasets and the per-category acceptance matrix, on either path. - [SpecBench tutorial](spec-bench.md) -- profiling with the SpecBench speculative-decoding dataset. diff --git a/docs/tutorials/speed-bench.md b/docs/tutorials/speed-bench.md index afb3b514ca..02b9594bfc 100644 --- a/docs/tutorials/speed-bench.md +++ b/docs/tutorials/speed-bench.md @@ -8,7 +8,26 @@ sidebar-title: Profile with SPEED-Bench Dataset AIPerf supports benchmarking using [SPEED-Bench](https://huggingface.co/datasets/nvidia/SPEED-Bench) (SPEculative Evaluation Dataset), a benchmark designed for evaluating speculative decoding across diverse semantic domains and input sequence lengths. -This guide covers profiling speculative-decoding-enabled inference servers using SPEED-Bench prompts and collecting server-side acceptance rate metrics per category. +This guide covers profiling speculative-decoding-enabled inference servers with SPEED-Bench prompts and assembling a per-category acceptance matrix with `aiperf speed-bench-report`. + +--- + +## Pick your measurement path first + +Everything downstream depends on how your server reports acceptance, so decide this before you run anything. + +| | Per-request | Server scrape | +|---|---|---| +| **Where the numbers come from** | `metrics.speculative_decoding` on each response body | The server's Prometheus `/metrics` endpoint, sampled during the run | +| **Engines** | vLLM only, direct (not behind Dynamo, which strips the field) | vLLM, SGLang, TensorRT-LLM, NIM-LLM | +| **Runs needed for the 11-category matrix** | **One** | Eleven (one per category) | +| **Scope of the numbers** | Exactly the requests AIPerf issued | The whole server for the sampled window | +| **Extra AIPerf flags** | None | `--server-metrics` (on by default) | +| **Server setup** | `--per-request-spec-decode-metrics summary` | none | + +The per-request path is the better measurement wherever it is available: acceptance is attributed to individual requests rather than inferred from a whole-server counter delta, so nothing another client sends during your run can leak into the result, and you get distributions and a pooled histogram instead of a single ratio. It is also what collapses the eleven-run sweep into one run. + +If you are on SGLang or TensorRT-LLM, or your vLLM sits behind Dynamo, skip to [Portable path: server scrape](#portable-path-server-scrape). Everything else on this page - the datasets, the prepare step, the sampling defaults - applies to both paths. --- @@ -27,9 +46,11 @@ These load all categories combined in a single dataset: | `speed_bench_throughput_16k` | 1,536 | ~16K input tokens, all 3 entropy tiers | | `speed_bench_throughput_32k` | 1,536 | ~32K input tokens, all 3 entropy tiers | +On the per-request path these aggregate splits are the ones you want: every request keeps the category it came from, so one run over `speed_bench_qualitative` still produces a per-category breakdown. + ### Per-Category Qualitative Datasets (80 prompts each) -For per-category acceptance rate measurement, each of the 11 qualitative domains is registered separately: +Each of the 11 qualitative domains is also registered separately, for running one category in isolation: | Dataset Name | Category | |---|---| @@ -75,19 +96,43 @@ This will download all splits into the working directory as JSONL files. Other s * `--config`: select which config to prepare, can be one of the splits in the dataset (e.g., `qualitative`, `throughput_2k`) or `all` to prepare all of the configs. * `--output_dir`: select different output directory to download the dataset to. +> [!IMPORTANT] +> SPEED-Bench stores prompts as references to their upstream source datasets, and the +> prepare script resolves them by downloading each source. **Some of those sources are +> gated on HuggingFace** (for example `cais/hle`), so the script fails partway through +> with `DatasetNotFoundError: Dataset '' is a gated dataset on the Hub` unless you +> have requested access and authenticated: +> +> ```bash +> hf auth login # or: huggingface-cli login (older huggingface_hub) +> # or non-interactively: export HF_TOKEN=hf_... +> ``` +> +> Request access on each gated dataset's Hub page, then re-run the prepare script - it +> reuses the HuggingFace cache, so nothing already downloaded is fetched twice. Expect +> several GB of source data and tens of minutes on a first run. +> +> Do not benchmark a partially prepared file: unresolved rows still hold the literal +> placeholder `FULL BENCHMARK DATA SHOULD BE FETCHED FROM THE SOURCE USING SPECDEC_BENCH`. +> AIPerf rejects those rows at dataset load rather than benchmarking the placeholder text, +> but the run fails at startup rather than producing results. + --- ## Start a Server with Speculative Decoding -Launch an inference server with speculative decoding enabled. For example, with vLLM: +Launch an inference server with speculative decoding enabled. On vLLM, add `--per-request-spec-decode-metrics summary` so each response carries its own acceptance stats: ```bash docker run --gpus all -p 8000:8000 vllm/vllm-openai:latest \ --model meta-llama/Llama-3.1-8B-Instruct \ - --speculative-config '{"model": "meta-llama/Llama-3.2-1B-Instruct", "num_speculative_tokens": 5, "method": "draft_model"}' + --speculative-config '{"model": "meta-llama/Llama-3.2-1B-Instruct", "num_speculative_tokens": 5, "method": "draft_model"}' \ + --per-request-spec-decode-metrics summary ``` -Verify the server is ready: +The flag tracks vLLM [PR #48915](https://github.com/vllm-project/vllm/pull/48915); confirm your build includes it. Use `detailed` instead of `summary` to additionally record the ordered per-step accepted/proposed arrays in the records trace. Omit the flag entirely if you are taking the server-scrape path. + +Verify the server is ready, and that acceptance metrics are attached (look for `metrics.speculative_decoding`): ```bash curl -s localhost:8000/v1/chat/completions \ @@ -95,16 +140,7 @@ curl -s localhost:8000/v1/chat/completions \ -d '{"model":"meta-llama/Llama-3.1-8B-Instruct","messages":[{"role":"user","content":"test"}],"max_tokens":1}' ``` ---- - -## Server Metrics Endpoint - -AIPerf auto-discovers the Prometheus endpoint at `{url}/metrics`. If your server uses a different path, pass it explicitly with `--server-metrics`: - -| Server Type | Metrics Path | Flag Needed | -|---|---|---| -| Standalone vLLM / SGLang | `/metrics` (default) | None (auto-discovered) | -| NIM-LLM containers | `/v1/metrics` | `--server-metrics http://localhost:8000/v1/metrics` | +If that object is absent, the per-request path will silently produce nothing - see [When no acceptance numbers show up](#when-no-acceptance-numbers-show-up). --- @@ -115,103 +151,205 @@ AIPerf auto-discovers the Prometheus endpoint at `{url}/metrics`. If your server For standard (non-reasoning) models, use `temperature=0` and a 4K output length cap: ```bash -aiperf profile \ - --model meta-llama/Llama-3.1-8B-Instruct\ - --endpoint-type chat \ - --streaming \ - --url localhost:8000 \ - --custom-dataset-type speed_bench_coding \ - --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ - --osl 4096 \ - --extra-inputs temperature:0 \ - --concurrency 16 +--osl 4096 --extra-inputs temperature:0 ``` -Do not set `ignore_eos` — let the model stop naturally at its end-of-sequence token. +Do not set `ignore_eos` - let the model stop naturally at its end-of-sequence token. ### Reasoning Models -For reasoning models (e.g., DeepSeek-R1, QwQ), follow the model card's recommended settings for temperature, top_p, and output length. Reasoning models typically require higher output limits and specific sampling parameters. +For reasoning models (e.g. DeepSeek-R1, QwQ), follow the model card's recommended settings for temperature, top_p, and output length. Reasoning models typically require higher output limits and specific sampling parameters. --- -## Per-Category Acceptance Rate Benchmarking +## Per-Category Matrix in One Run -To measure acceptance rates per category (matching the SPEED-Bench paper methodology), run each category separately. Each run collects speculative decoding metrics from the server's Prometheus endpoint. - -### Single Category +Run the aggregate qualitative split once. Every request is tagged with the category of the SPEED-Bench row it came from, so the per-request records carry the breakdown with them: ```bash aiperf profile \ - --model meta/llama-3.1-8b-instruct \ + --model meta-llama/Llama-3.1-8B-Instruct \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ - --custom-dataset-type speed_bench_coding \ + --custom-dataset-type speed_bench_qualitative \ --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ - --server-metrics http://localhost:8000/metrics \ + --request-count $(jq -s '[.[].messages | length] | add' ${SPEED_BENCH_DIR}/qualitative.jsonl) \ --osl 4096 \ --extra-inputs temperature:0 \ --concurrency 16 \ - --output-artifact-dir ./artifacts/speed_bench_coding + --output-artifact-dir ./artifacts/speed_bench_qualitative ``` -### All 11 Categories with Matrix Report +> [!WARNING] +> **Size the run by turns, not by prompts.** `--request-count` counts *requests*, +> and a multi-turn SPEED-Bench row dispatches one request per turn - the qualitative +> split's 880 rows expand to roughly 1,140 requests. Passing `--request-count 880` +> therefore stops partway through the file and **silently drops whole categories**, +> which biases the matrix: the categories that fall off the end are not a random +> sample. The `jq` expression above sizes the run to the split's true turn count. +> Without any count at all, AIPerf defaults to 10 requests. +> +> Verify afterwards that every category is present and none was truncated: +> +> ```bash +> jq -r '.metadata.source_kind' ./artifacts/speed_bench_qualitative/profile_export.jsonl \ +> | sort | uniq -c +> ``` + +No spec-decode flag is needed on the AIPerf side, and `--server-metrics` is unnecessary. The end-of-run console gains a **Spec Decode** section covering the whole run: + +> [!IMPORTANT] +> The output below is from a real run of the exact server command shown above +> (Llama-3.1-8B-Instruct target, Llama-3.2-1B-Instruct drafter, `num_speculative_tokens: 5`, +> greedy decoding) over the full qualitative split at concurrency 16. Treat the numbers as +> one data point, not a specification: acceptance depends on the target/drafter pair, the +> draft budget, sampling temperature, the dataset, and the load. Greedy decoding +> (`temperature: 0`) reduces rejection sampling to plain argmax agreement and is the +> acceptance ceiling - the same run at `temperature: 1` will report lower. + +```text + NVIDIA AIPerf | LLM Metrics: Spec Decode +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━━┓ +┃ Metric ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p50 ┃ std ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━╇━━━━━━━━┩ +│ Acceptance Length (ratio) │ 4.19 │ 1.83 │ 6.00 │ 6.00 │ 5.35 │ 4.13 │ 0.81 │ +│ Token-Weighted Acceptance Length │ 4.63 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │ +│ (ratio) │ │ │ │ │ │ │ │ +│ Draft Acceptance Rate (%) │ 63.83 │ 16.52 │ 100.00 │ 100.00 │ 87.03 │ 62.65 │ 16.23 │ +│ Overall Draft Acceptance Rate (%) │ 72.65 │ N/A │ N/A │ N/A │ N/A │ N/A │ N/A │ +│ Accepted per Verified (ratio) │ 0.70 │ 0.30 │ 1.00 │ 1.00 │ 0.89 │ 0.69 │ 0.14 │ +│ Spec Decode Steps (count) │ 118.53 │ 1.00 │ 821.00 │ 744.78 │ 230.00 │ 71.00 │ 163.53 │ +└───────────────────────────────────┴────────┴───────┴────────┴────────┴────────┴───────┴────────┘ + Accepted drafts per step (% of steps): 0: 13% 1: 9% 2: 7% 3: 6% 4: 4% 5: 61% +``` -Loop through all categories, then assemble results into a per-category matrix: +The accepted-draft histogram is typically **bimodal** rather than smoothly decaying: most verify +steps either accept the whole draft block or almost none of it. That is content-dependent +acceptance - easy stretches where target and drafter agree completely, punctuated by tokens where +they diverge immediately - and it is why the per-request `Acceptance Length` distribution is wide +(`min` 1.83, `max` 6.00 = `num_speculative_tokens + 1`). -```bash -CATEGORIES="coding humanities math multilingual qa rag reasoning roleplay stem summarization writing" -MODEL="meta/llama-3.1-8b-instruct" +Then split that run by category: -for cat in $CATEGORIES; do - echo "=== Running category: $cat ===" - aiperf profile \ - --model "$MODEL" \ - --endpoint-type chat \ - --streaming \ - --url localhost:8000 \ - --custom-dataset-type speed_bench_${cat} \ - --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ - --server-metrics http://localhost:8000/metrics \ - --osl 4096 \ - --extra-inputs temperature:0 \ - --concurrency 16 \ - --output-artifact-dir "./artifacts/speed_bench_${cat}" -done +```bash +aiperf speed-bench-report ./artifacts/speed_bench_qualitative --format both +``` -# Assemble the matrix report -aiperf speed-bench-report ./artifacts/ --format both +```text + Acceptance Length Report +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃ Model ┃ coding ┃ humanities ┃ math ┃ multilingual ┃ qa ┃ rag ┃ reasoning ┃ roleplay ┃ stem ┃ summarization ┃ writing ┃ Overall ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ meta-llama/Llama-3.1-8B-Instruct │ 5.01 │ 4.84 │ 5.24 │ 5.10 │ 4.93 │ 4.51 │ 4.47 │ 3.34 │ 4.78 │ 3.81 │ 4.19 │ 4.56 │ +└──────────────────────────────────┴────────┴────────────┴──────┴──────────────┴──────┴──────┴───────────┴──────────┴──────┴───────────────┴─────────┴─────────┘ +CSV written to speed_bench_report.csv ``` -This produces a CSV (`speed_bench_report.csv`) and console table: +The spread is the point of the matrix: acceptance tracks how predictable each domain's output is. +Structured, low-entropy domains (`math`, `multilingual`, `coding`) sit near the top of the range, +while open-ended generation (`roleplay`, `summarization`) sits well below - a 1.9-point gap here +between `math` and `roleplay`. A matrix that comes back flat across all eleven categories is a +signal to check the drafter configuration, not a result. + +The acceptance-rate matrix works the same way: +```bash +aiperf speed-bench-report ./artifacts/speed_bench_qualitative --metric accept_rate ``` - Acceptance Length Report -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃ Model ┃ coding ┃ humanities ┃ math ┃ writing ┃ Overall ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ meta/llama-3.1-8b-instruct │ 1.80 │ 1.84 │ 1.78 │ 1.76 │ 1.78 │ -└────────────────────────────┴────────┴────────────┴──────┴─────────┴─────────┘ + +The same run is available as a config template if you prefer YAML: + +```bash +aiperf config init --template speed_bench_per_request --output speed_bench.yaml +aiperf profile --config speed_bench.yaml ``` -The report script computes acceptance length from vLLM counter metrics (`accepted_tokens / num_drafts + 1`) and also supports SGLang's direct `spec_accept_length` gauge. +### How the report finds its numbers + +`aiperf speed-bench-report` takes acceptance from the highest-fidelity source each run directory offers, in this order. `--source records|summary|server` pins one explicitly. + +| `--source` | File read | Where the data came from | Columns per run | +|---|---|---|---| +| `records` | `profile_export.jsonl` | per-request `metrics.speculative_decoding` on each **response body** | one **per category** | +| `summary` | `profile_export_aiperf.json` | the **same per-request data**, reduced by AIPerf to run-level scalars | one | +| `server` | `server_metrics_export.json` | the engine's **Prometheus counters**, scraped during the run | one | + +`records` and `summary` share a provenance - both are the per-request stats AIPerf read from response bodies - and differ only in granularity. `server` is the only source that reads server-side data. + +Only `records` can split a single run, because only the per-request trace knows which category each request belonged to. It is written at the default `--export-level records`; a run exported at `--export-level summary` has no trace, so `auto` falls through to `summary` and yields the run-level number as a single column. -Additional report metrics: +`records`, `summary`, and the **counter** branch of `server` (vLLM's `vllm:spec_decode_*` totals) all compute the same two token-weighted quantities, so those columns are directly comparable: + +- `accept_length` - `1 + sum(accepted_draft_tokens) / sum(verify_steps)` over the category. +- `accept_rate` - `sum(accepted_draft_tokens) / sum(proposed_draft_tokens)` over the category. + +Numerator and denominator are summed separately across the category's requests, then +divided once. Averaging each request's own ratio would weight a 3-step request equally +with a 300-step one and give a different number. + +> [!WARNING] +> **Gauge-based engines are not token-weighted.** SGLang exposes acceptance as a gauge +> (`sglang:spec_accept_length`), as does TensorRT-LLM, and the report reads those as the +> unweighted mean of the scrape samples - a time-average, in which an idle interval counts +> as much as a busy one. The same applies to a vLLM build that publishes +> `vllm:spec_decode_mean_accepted_length` or `vllm:spec_decode_draft_acceptance_rate`: the +> gauges are checked before vLLM's counters, so those cells are time-averaged too. Those cells answer a different question from the token-weighted +> ones and should not be compared value-for-value against them, nor averaged into a single +> `Overall` alongside them. Under `--source auto` the report prints the source it resolved +> for each run to stderr, so mixed provenance is visible rather than silent. + +On an engine with no per-request acceptance reporting, `records` and `summary` both come up empty and `auto` falls through to `server` - identical to the behavior before `--source` existed. On vLLM **with** `--per-request-spec-decode-metrics`, `auto` now prefers the per-request numbers, so a matrix regenerated from an older run directory can shift slightly; pass `--source server` to pin the scraped values. + +Both weight every verify step equally, matching the console's **Token-Weighted Acceptance Length** and **Overall Draft Acceptance Rate** rather than the per-request means beside them. The two coincide when requests run a similar number of verify steps and diverge when step counts vary and correlate with acceptance. Warmup records are excluded. + +Two things to keep in mind when reading the matrix: + +- **Cells are turn-weighted, not row-weighted.** Every qualitative category holds 80 rows, but multi-turn rows expand into several requests, so a category's cell can rest on far more requests than another's (in one reference run: `reasoning` 190 requests, `summarization` 80). Later turns also carry accumulated conversation history, which shifts acceptance relative to a first turn. +- **`Overall` is the macro-average of the category cells**, not the pooled run-level value - it weights each category equally, matching how the SPEED-Bench paper reports an overall figure. The console's **Token-Weighted Acceptance Length** is the pooled number over every verify step in the run. Expect them to differ by a few hundredths; a large gap means your categories differ a lot in both acceptance and request volume. + +> [!TIP] +> On vLLM you can cross-check the whole thing: `--source server` reduces the run to a single +> column computed from the Prometheus counters. It should match the console's +> **Token-Weighted Acceptance Length** closely - both are pooled over the same requests, by +> two independent routes. A mismatch means either another client was hitting the server +> during your run, or the per-request and scraped paths disagree. + + +### What a mixed run does and does not change + +- **Attribution stays exact.** Each acceptance record is produced by the server for that one request, so interleaving categories cannot mix their numbers. +- **Every category sees the same load.** In an eleven-run sweep each category is measured under a load shaped by its own prompt lengths; in one mixed run they all share one steady state. That is usually the fairer comparison, but it is not identical to the paper's isolated per-category methodology - say which one you ran when you publish numbers. +- **Load-adaptive drafting still applies.** vLLM can vary the draft budget with batch size (`num_speculative_tokens_per_batch_size`) or adapt verification per request. Those effects are driven by the mixed load, not by any one category. +- **Throughput cannot be split.** `--metric throughput` is a rate over the whole run, so it stays one column per run even when the records carry categories. For per-category throughput you still need per-category runs. + +### Entropy-tier matrix, also one run + +The throughput splits label rows by entropy tier rather than semantic domain, so the same single-run treatment gives you the tier breakdown at a fixed ISL: ```bash -# Acceptance rate matrix (accepted / draft tokens) -aiperf speed-bench-report ./artifacts/ --metric accept_rate +aiperf profile \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --endpoint-type chat \ + --streaming \ + --url localhost:8000 \ + --custom-dataset-type speed_bench_throughput_1k \ + --input-file ${SPEED_BENCH_DIR}/throughput_1k.jsonl \ + --request-count $(jq -s '[.[].messages | length] | add' ${SPEED_BENCH_DIR}/throughput_1k.jsonl) \ + --extra-inputs temperature:0 \ + --concurrency 64 \ + --output-artifact-dir ./artifacts/speed_bench_throughput_1k -# Throughput matrix (output tokens/sec per category) -aiperf speed-bench-report ./artifacts/ --metric throughput +aiperf speed-bench-report ./artifacts/speed_bench_throughput_1k ``` +The matrix comes back with `low_entropy`, `mixed`, and `high_entropy` columns. Replace `1k` with `2k`, `8k`, `16k`, or `32k` for other input lengths. + --- ## Literature Acceptance-Length Datasets (GSM8K, MT-Bench, MATH-500, HumanEval, MBPP) -The speculative-decoding literature overwhelmingly reports acceptance length against five standard benchmarks. AIPerf registers each as a public dataset that is auto-downloaded from HuggingFace at runtime, so there is no prepare-data step: just select one with `--public-dataset` and run the same `aiperf speed-bench-report` workflow shown above. +The speculative-decoding literature overwhelmingly reports acceptance length against five standard benchmarks. AIPerf registers each as a public dataset that is auto-downloaded from HuggingFace at runtime, so there is no prepare-data step: select one with `--public-dataset`. | Dataset Name | HuggingFace Source | Prompts | Turns | License | |---|---|---|---|---| @@ -221,12 +359,14 @@ The speculative-decoding literature overwhelmingly reports acceptance length aga | `spec_al_mbpp` | `google-research-datasets/mbpp` (`full`, `test`) | 500 | single | CC-BY-4.0 | | `spec_al_mtbench` | `HuggingFaceH4/mt_bench_prompts` (`train`) | 80 | two-turn | Apache-2.0 | +These are five separate datasets, so unlike the SPEED-Bench categories they cannot collapse into one run - a run takes one dataset. What the per-request path changes here is only where each run's number comes from: with `--per-request-spec-decode-metrics` enabled, the report reads it from that run's own requests instead of scraping the server, and `--server-metrics` becomes unnecessary. + Prompts are emitted verbatim (the raw question/problem/prompt field); the served model's chat template wraps them at request time via `--endpoint-type chat`. HumanEval and MBPP are text-completion tasks in the spec-decode literature, so chat-wrapping them keeps the matrix uniform but shifts their acceptance length somewhat from the papers' headline numbers. Acceptance length is correctness-agnostic, so use greedy decoding (`--extra-inputs temperature:0`) to match the headline numbers reported in the literature. Note that `--osl` does not apply to public datasets, so cap generation with `--extra-inputs max_tokens:N` instead. `spec_al_mtbench` is multi-turn: AIPerf dispatches both turns per session and feeds the live assistant reply back as conversation history between them - size it with `--num-conversations` rather than `--request-count` (see below). ### Run All Five with a Matrix Report ```bash -MODEL="meta/llama-3.1-8b-instruct" +MODEL="meta-llama/Llama-3.1-8B-Instruct" ART=./artifacts/spec-al # dedicated root so this matrix never merges with speed_bench_* runs # Single-turn datasets: size each run to the full dataset with --request-count. @@ -239,7 +379,6 @@ for pair in spec_al_gsm8k:1319 spec_al_math500:500 spec_al_humaneval:164 spec_al --streaming \ --url localhost:8000 \ --public-dataset "$ds" \ - --server-metrics http://localhost:8000/metrics \ --request-count "$count" \ --extra-inputs temperature:0 max_tokens:4096 \ --concurrency 16 \ @@ -256,7 +395,6 @@ aiperf profile \ --streaming \ --url localhost:8000 \ --public-dataset spec_al_mtbench \ - --server-metrics http://localhost:8000/metrics \ --num-conversations 80 \ --extra-inputs temperature:0 max_tokens:4096 \ --concurrency 16 \ @@ -266,76 +404,92 @@ aiperf profile \ aiperf speed-bench-report "$ART" --metric accept_length --format both ``` -> Size each run to the full dataset — without an explicit count AIPerf defaults +> Size each run to the full dataset - without an explicit count AIPerf defaults > to 10 requests. Single-turn datasets use `--request-count`; the multi-turn > `spec_al_mtbench` uses `--num-conversations 80` (one run per conversation), > since `--request-count` recycles its 80 sessions to reach the count. Cap > generation with `--extra-inputs max_tokens:N` (`--osl` is ignored for public > datasets), and keep these runs in their own artifacts directory so > `speed-bench-report` does not average them into an unrelated `speed_bench_*` -> matrix. - -The report recognizes these runs the same way it recognizes the `speed_bench_*` runs, producing one matrix column per dataset: - -``` - Acceptance Length Report -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┓ -┃ Model ┃ gsm8k ┃ math500 ┃ mtbench ┃ humaneval ┃ mbpp ┃ Overall ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━┩ -│ meta/llama-3.1-8b-instruct │ 2.40 │ 2.31 │ 1.95 │ 2.62 │ 2.55 │ 2.37 │ -└────────────────────────────┴───────┴─────────┴─────────┴───────────┴──────┴─────────┘ +> matrix. On the server-scrape path, add `--server-metrics http://localhost:8000/metrics` +> to each run. + +The report produces one matrix column per dataset: + +```text + Acceptance Length Report +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┓ +┃ Model ┃ gsm8k ┃ math500 ┃ mtbench ┃ humaneval ┃ mbpp ┃ Overall ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━┩ +│ meta-llama/Llama-3.1-8B-Instruct │ 2.40 │ 2.31 │ 1.95 │ 2.62 │ 2.55 │ 2.37 │ +└──────────────────────────────────┴───────┴─────────┴─────────┴───────────┴──────┴─────────┘ ``` -The `accept_rate` and `throughput` metrics work identically (`aiperf speed-bench-report ./artifacts/ --metric accept_rate`). +The `accept_rate` and `throughput` metrics work identically. --- -## Profile with Aggregate Qualitative Split +## Portable path: server scrape -To run all 880 prompts in a single benchmark (without per-category breakdown): +Use this when the per-request path is unavailable: SGLang, TensorRT-LLM, NIM-LLM containers, or a vLLM behind Dynamo (which strips the custom response field). Acceptance is read from the server's Prometheus endpoint during each run and written to `server_metrics_export.json`; the report computes acceptance length from vLLM's counters (`accepted_tokens / num_drafts + 1`), SGLang's `spec_accept_length` gauge, or a TensorRT-LLM equivalent. -```bash -aiperf profile \ - --model meta/llama-3.1-8b-instruct \ - --endpoint-type chat \ - --streaming \ - --url localhost:8000 \ - --custom-dataset-type speed_bench_qualitative \ - --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ - --server-metrics http://localhost:8000/metrics \ - --concurrency 16 -``` +Because the scrape is a whole-server measurement rather than a per-request one, the matrix needs one run per category. ---- +### Server Metrics Endpoint -## Profile with Throughput Splits +AIPerf auto-discovers the Prometheus endpoint at `{url}/metrics`. If your server uses a different path, pass it explicitly with `--server-metrics`: -The throughput splits benchmark end-to-end performance at fixed input sequence lengths: +| Server Type | Metrics Path | Flag Needed | +|---|---|---| +| Standalone vLLM / SGLang | `/metrics` (default) | None (auto-discovered) | +| NIM-LLM containers | `/v1/metrics` | `--server-metrics http://localhost:8000/v1/metrics` | + +### All 11 Categories ```bash -aiperf profile \ - --model meta/llama-3.1-8b-instruct \ - --endpoint-type chat \ - --streaming \ - --url localhost:8000 \ - --custom-dataset-type speed_bench_throughput_1k \ - --input-file ${SPEED_BENCH_DIR}/throughput_1k.jsonl \ - --server-metrics http://localhost:8000/metrics \ - --concurrency 64 \ - --benchmark-duration 120 +CATEGORIES="coding humanities math multilingual qa rag reasoning roleplay stem summarization writing" +MODEL="meta-llama/Llama-3.1-8B-Instruct" + +for cat in $CATEGORIES; do + echo "=== Running category: $cat ===" + # Size each run to that category's turn count, not its 80 rows. + turns=$(jq -s --arg c "$cat" \ + '[.[] | select(.category == $c) | .messages | length] | add' \ + ${SPEED_BENCH_DIR}/qualitative.jsonl) + aiperf profile \ + --model "$MODEL" \ + --endpoint-type chat \ + --streaming \ + --url localhost:8000 \ + --custom-dataset-type speed_bench_${cat} \ + --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ + --server-metrics http://localhost:8000/metrics \ + --request-count "$turns" \ + --osl 4096 \ + --extra-inputs temperature:0 \ + --concurrency 16 \ + --output-artifact-dir "./artifacts/speed_bench_${cat}" +done + +aiperf speed-bench-report ./artifacts/ --source server --format both ``` -Replace `speed_bench_throughput_1k` with any throughput variant (`_2k`, `_8k`, `_16k`, `_32k`) to test at different input lengths. +`--source server` is optional but worth passing: it stops the report from silently preferring per-request records if some of those runs happen to have them, so every column in the matrix is measured the same way. -### Per-Entropy-Tier Throughput +The same eleven runs are also available as a config-driven sweep template, which drives all eleven variations from a single `aiperf profile --config` invocation: -To isolate entropy effects on acceptance rate at a given ISL: +```bash +aiperf config init --template speed_bench_sweep --output speed_bench_sweep.yaml +aiperf profile --config speed_bench_sweep.yaml +``` + +### Per-entropy-tier scrape ```bash for tier in low_entropy mixed high_entropy; do echo "=== Running throughput_1k tier: $tier ===" aiperf profile \ - --model meta/llama-3.1-8b-instruct \ + --model meta-llama/Llama-3.1-8B-Instruct \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ @@ -347,15 +501,13 @@ for tier in low_entropy mixed high_entropy; do done ``` ---- - -## Disable Server Metrics +### Disable Server Metrics -Server metrics collection is enabled by default. To disable it: +Server metrics collection is enabled by default. On the per-request path it is redundant; turn it off with `--no-server-metrics`: ```bash aiperf profile \ - --model meta/llama-3.1-8b-instruct \ + --model meta-llama/Llama-3.1-8B-Instruct \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ @@ -364,3 +516,47 @@ aiperf profile \ --no-server-metrics \ --concurrency 16 ``` + +--- + +## Throughput per Category + +Output-token throughput is a rate over a whole run, so it cannot be attributed to a category inside a mixed run. `--metric throughput` therefore always reports one column per run directory: + +```bash +aiperf speed-bench-report ./artifacts/ --metric throughput +``` + +To compare throughput across categories, run them separately (the eleven-run loop above) and point the report at the parent directory. + +--- + +## When no acceptance numbers show up + +An empty matrix, or a run with no **Spec Decode** console section, is expected clean degradation rather than an error. Common causes, per path: + +**Per-request path** + +- The server was not started with `--per-request-spec-decode-metrics`. +- The vLLM build predates [PR #48915](https://github.com/vllm-project/vllm/pull/48915). +- The server is behind Dynamo, which strips the custom field - use the server-scrape path. +- The run used `n > 1`; vLLM only reports per-request stats for single-sequence requests. +- The run was exported at `--export-level summary`, so there is no `profile_export.jsonl` to split by category. The run-level number is still reported as one column. + +**Server-scrape path** + +- No `server_metrics_export.json` in the run directory: the run used `--no-server-metrics`, or the endpoint was unreachable. +- The engine exposes acceptance under a metric name the report does not recognize. Inspect `server_metrics_export.json` for a `spec`-flavored metric and file an issue with the name. + +**Either path** + +- Speculative decoding is off on the server, or the requests produced no verify steps. + +--- + +## See also + +- [Per-Request Spec-Decode Metrics](spec-decode-metrics.md) - the per-request path in detail, on any dataset. +- [Speculative Decoding Metrics](../metrics-reference.md#speculative-decoding-metrics) - metric definitions and formulas. +- [Per-Request Speculative-Decoding Acceptance](../reference/spec-decode-acceptance.md) - the engine-neutral record and adapter architecture. +- [SpecBench tutorial](spec-bench.md) - profiling with the SpecBench speculative-decoding dataset. diff --git a/src/aiperf/analysis/speed_bench_report.py b/src/aiperf/analysis/speed_bench_report.py index 8c5035adc1..c1906609e8 100644 --- a/src/aiperf/analysis/speed_bench_report.py +++ b/src/aiperf/analysis/speed_bench_report.py @@ -3,9 +3,41 @@ """Assemble per-category SPEED-Bench results into a matrix report. -Run ``aiperf profile`` once per SPEED-Bench category, then invoke -``aiperf speed-bench-report`` against the output directories to produce an -acceptance-length matrix matching the SPEED-Bench paper format. +Point ``aiperf speed-bench-report`` at one or more ``aiperf profile`` output +directories to produce an acceptance matrix in the SPEED-Bench paper format. + +Acceptance numbers come from one of three sources, tried in this order +(``--source`` pins one explicitly): + +1. **Per-request records** (``profile_export.jsonl``) -- the engine-neutral + ``spec_decode_acceptance`` struct on each request, written at the default + ``--export-level records``. Highest fidelity, and the only source that can + split a *single* run into several columns: SPEED-Bench runs stamp each + record with its row's category via ``metadata.source_kind``, so one run over + an aggregate split (``speed_bench_qualitative``) yields the full matrix. +2. **Summarized per-request metrics** (``profile_export_aiperf.json``) -- the + same per-request data, already reduced by AIPerf to run-level scalars. Same + provenance as ``records``, coarser granularity. One column per run; the only + source available when the run was exported at ``--export-level summary``. +3. **Server scrape** (``server_metrics_export.json``) -- Prometheus counters + sampled from the server during the run. One column per run, whole-server + scope. The portable path: the only one that works for engines with no + per-request acceptance reporting. + +``records``, ``summary``, and the **counter** branch of ``server`` (vLLM's +``vllm:spec_decode_*`` totals) all compute the same token-weighted quantities, +so their columns are directly comparable: acceptance length is +``1 + accepted / steps`` and acceptance rate is ``accepted / drafted``, both +summed over the run rather than averaged per request. + +Engines that expose acceptance as a *gauge* rather than counters (SGLang's +``sglang:spec_accept_length``, the TRT-LLM equivalent, ``vllm:spec_decode_*`` +mean/rate gauges where a build exposes them, and the fuzzy fallback) are read +as the unweighted mean of the scrape samples -- a time-average, in which an +idle interval counts as much as a busy one. Those cells are not volume-weighted +and should not be compared value-for-value against the token-weighted ones. The +gauges are checked first, so on a vLLM build that publishes both, the gauge +wins over the counter branch. """ from __future__ import annotations @@ -13,6 +45,8 @@ import csv import math import sys +from collections.abc import Iterator +from dataclasses import dataclass from pathlib import Path from statistics import mean from typing import Literal @@ -20,6 +54,10 @@ import orjson MetricType = Literal["accept_length", "accept_rate", "throughput"] +AcceptanceMetric = Literal["accept_length", "accept_rate"] +"""The subset of ``MetricType`` derived from acceptance counters; throughput is a +whole-run rate and never routes through the per-request path.""" +MetricSource = Literal["auto", "records", "summary", "server"] OutputFormat = Literal["csv", "table", "both"] @@ -67,8 +105,17 @@ class SpeedBenchReportError(Exception): ] PROFILE_JSON = "profile_export_aiperf.json" +PROFILE_JSONL = "profile_export.jsonl" SERVER_METRICS_JSON = "server_metrics_export.json" +# Run-level metric tags in the summary export that already carry the +# token-weighted per-request acceptance, used when the run was exported at +# --export-level summary (no per-request JSONL to read). +SUMMARY_ACCEPT_METRICS: dict[AcceptanceMetric, str] = { + "accept_length": "spec_decode_token_weighted_acceptance_length", + "accept_rate": "spec_decode_overall_draft_acceptance_rate", +} + def find_run_dirs(paths: list[Path]) -> list[Path]: """Discover aiperf run directories from the given paths. @@ -246,12 +293,166 @@ def extract_throughput(profile: dict) -> float | None: return None +def extract_summary_acceptance( + profile: dict, metric_type: AcceptanceMetric +) -> float | None: + """Extract run-level per-request acceptance from the summary export. + + Reads the token-weighted scalars AIPerf derives from the per-request + records, so the value matches what the records path computes for the same + run. Acceptance rate is stored as a percentage and returned as a 0..1 + fraction to match the server-scrape path. + """ + entry = profile.get(SUMMARY_ACCEPT_METRICS[metric_type]) + if not isinstance(entry, dict): + return None + value = entry.get("avg") + # A hand-edited or third-party export can carry a string or bool here, which + # would reach the "%.2f" formatting path and abort the whole report. + if type(value) not in (int, float): + return None + return value / 100.0 if metric_type == "accept_rate" else value + + +@dataclass +class AcceptanceTotals: + """Summed acceptance counters for one category within one run.""" + + accepted: int = 0 + """Accepted draft tokens, excluding the always-accepted bonus token.""" + + drafted: int = 0 + """Proposed draft tokens counted toward acceptance.""" + + steps: int = 0 + """Speculative verification steps.""" + + def add(self, spec: dict) -> None: + """Fold one request's acceptance record into the totals. + + Validates all three counters before mutating any of them: incrementing + one at a time would let a non-integer second field abort partway and + leave the first already applied, silently inflating the category. + ``type(...) is int`` rather than isinstance so bools -- an int subclass + -- are rejected too, matching the vLLM adapter's histogram check. + """ + accepted = spec["num_accepted_draft_tokens"] + drafted = spec["num_draft_tokens"] + steps = spec["num_spec_steps"] + if not all(type(count) is int for count in (accepted, drafted, steps)): + raise TypeError( + "acceptance counters must be integers, got " + f"{accepted!r}, {drafted!r}, {steps!r}" + ) + # Negative counts yield a negative acceptance length, and accepting more + # drafts than were proposed yields a rate above 1.0 -- both are wrong + # numbers rather than missing ones, so reject the record outright. + if min(accepted, drafted, steps) < 0 or accepted > drafted: + raise ValueError( + "acceptance counters must satisfy 0 <= accepted <= drafted, got " + f"accepted={accepted}, drafted={drafted}, steps={steps}" + ) + self.accepted += accepted + self.drafted += drafted + self.steps += steps + + def value(self, metric_type: AcceptanceMetric) -> float | None: + """Token-weighted acceptance length or rate; None when undefined.""" + if metric_type == "accept_rate": + return self.accepted / self.drafted if self.drafted else None + return 1.0 + self.accepted / self.steps if self.steps else None + + +def iter_records(run_dir: Path) -> Iterator[dict]: + """Yield each parsed line of the per-request records export. + + Silently skips unparseable lines: a truncated final line is normal for a + run that was interrupted, and one bad record should not sink the report. + """ + path = run_dir / PROFILE_JSONL + if not path.exists(): + return + try: + with open(path, "rb") as f: + for line in f: + if not line.strip(): + continue + try: + record = orjson.loads(line) + except orjson.JSONDecodeError: + continue + if isinstance(record, dict): + yield record + except OSError as e: + print(f"Warning: failed to read {path}: {e}", file=sys.stderr) + + +def acceptance_from_records( + run_dir: Path, + metric_type: AcceptanceMetric, + fallback_category: str | None, +) -> dict[str, float]: + """Build a {category: value} mapping from one run's per-request records. + + Records are grouped by ``metadata.source_kind``, which SPEED-Bench runs set + to the row's category -- so a single run over an aggregate split produces + one entry per category. Runs whose loader sets no ``source_kind`` (the + ``spec_al_*`` public datasets, for instance) fall back to the run's own + dataset-derived category and produce a single entry. + + Warmup records and error records are excluded: both are written to the same + JSONL but neither contributes to any other reported number. + """ + totals: dict[str, AcceptanceTotals] = {} + skipped = 0 + for record in iter_records(run_dir): + spec = record.get("spec_decode_acceptance") + if not isinstance(spec, dict): + continue + # Error records are exported with whatever partial acceptance data they + # carried, but MetricsAccumulator excludes them from the spec-decode + # totals (metrics/accumulator.py). Skipping them here keeps `records` + # agreeing with `summary` on any run that had failures. + if record.get("error") is not None: + continue + metadata = record.get("metadata") + if not isinstance(metadata, dict): + continue + if "warmup" in (metadata.get("benchmark_phase"), metadata.get("phase_kind")): + continue + category = metadata.get("source_kind") or fallback_category + if not isinstance(category, str) or not category: + continue + try: + totals.setdefault(category, AcceptanceTotals()).add(spec) + except (KeyError, TypeError, ValueError): + skipped += 1 + continue + + if skipped: + print( + f"Warning: {run_dir}: skipped {skipped} malformed acceptance " + "record(s); reported values cover the remainder", + file=sys.stderr, + ) + + return { + category: value + for category, total in totals.items() + if (value := total.value(metric_type)) is not None + } + + def build_report( run_dirs: list[Path], metric_type: MetricType = "accept_length", + source: MetricSource = "auto", ) -> dict[str, dict[str, float | None]]: """Build a {model: {category: value}} matrix from run directories. + A run directory usually contributes one column, but a SPEED-Bench run read + from per-request records contributes one column per category present in it. + Returns: Nested dict mapping model name -> category -> metric value. """ @@ -263,42 +464,102 @@ def build_report( print(f"Warning: no {PROFILE_JSON} in {run_dir}, skipping", file=sys.stderr) continue - category = extract_category(profile) - if not category: + run_category = extract_category(profile) + if not run_category: print( f"Warning: cannot determine category from {run_dir}, skipping", file=sys.stderr, ) continue - model = extract_model(profile) - if model not in results: - results[model] = {} - - if metric_type in ("accept_length", "accept_rate"): - server_metrics = load_server_metrics(run_dir) - if server_metrics is not None: - if metric_type == "accept_length": - value = extract_accept_length(server_metrics) - else: - value = extract_accept_rate(server_metrics) - else: - value = None - print( - f"Warning: no {SERVER_METRICS_JSON} in {run_dir}", - file=sys.stderr, - ) - elif metric_type == "throughput": - value = extract_throughput(profile) + values: dict[str, float | None] + if metric_type == "throughput": + # Throughput is a rate over the whole run, so it cannot be split + # per category inside a mixed run -- every category shares the same + # window and the same server. One column per run, always. + values = {run_category: extract_throughput(profile)} + elif metric_type in ("accept_length", "accept_rate"): + values = _acceptance_values( + run_dir, + profile, + metric_type=metric_type, + source=source, + run_category=run_category, + ) else: print(f"Unknown metric type: {metric_type}", file=sys.stderr) - value = None + values = {run_category: None} + + model_data = results.setdefault(extract_model(profile), {}) - results[model][category] = value + model_data.update(values) return results +def _acceptance_values( + run_dir: Path, + profile: dict, + *, + metric_type: AcceptanceMetric, + source: MetricSource, + run_category: str, +) -> dict[str, float | None]: + """Resolve one run's acceptance columns from the highest-fidelity source. + + ``auto`` walks per-request records, then the run-level per-request metrics, + then the server scrape, taking the first that yields anything. + """ + values: dict[str, float | None] = {} + resolved = "none" + # build_report gates on a recognized dataset selector before calling this: + # find_run_dirs accepts any directory holding a profile export, so without + # that an unrelated run sharing the parent directory would contribute its + # own source_kind values (weka_main, ...) as SPEED-Bench categories. + if source in ("auto", "records"): + values = dict(acceptance_from_records(run_dir, metric_type, run_category)) + if values: + resolved = "records" + if not values and source in ("auto", "summary"): + summarized = extract_summary_acceptance(profile, metric_type) + if summarized is not None: + values = {run_category: summarized} + resolved = "summary" + if not values and source in ("auto", "server"): + values = _acceptance_from_server(run_dir, metric_type, run_category) + if values: + resolved = "server" + # Under an explicit --source the provenance is already unambiguous; under + # auto it is chosen per run directory, so a matrix assembled from several + # runs can mix sources. Name the winner so that mixing is auditable rather + # than invisible -- gauge-based server cells in particular are not + # comparable value-for-value with token-weighted ones. + if source == "auto": + print(f"{run_dir}: acceptance from {resolved}", file=sys.stderr) + # A run whose category is known but has no acceptance data anywhere still + # contributes an empty cell, so the matrix shows which runs are missing + # numbers instead of dropping them. + return values or {run_category: None} + + +def _acceptance_from_server( + run_dir: Path, + metric_type: AcceptanceMetric, + run_category: str, +) -> dict[str, float | None]: + """Read whole-run acceptance from the scraped server metrics export.""" + server_metrics = load_server_metrics(run_dir) + if server_metrics is None: + print(f"Warning: no {SERVER_METRICS_JSON} in {run_dir}", file=sys.stderr) + return {} + value = ( + extract_accept_length(server_metrics) + if metric_type == "accept_length" + else extract_accept_rate(server_metrics) + ) + return {run_category: value} if value is not None else {} + + def detect_columns(results: dict[str, dict[str, float | None]]) -> list[str]: """Detect which column set to use based on the categories present.""" all_cats: set[str] = set() @@ -314,6 +575,18 @@ def detect_columns(results: dict[str, dict[str, float | None]]) -> list[str]: return sorted(all_cats) +def _overall(data: dict[str, float | None], columns: list[str]) -> float | None: + """Macro-average of the categories present for one model. + + ``Overall`` is computed at render time and is never a key in the per-model + dict, so every consumer -- the CSV, both table renderers, and the width + calculation -- must derive it the same way or the width will not match what + is printed. + """ + values = [value for name in columns if (value := data.get(name)) is not None] + return mean(values) if values else None + + def write_csv( results: dict[str, dict[str, float | None]], columns: list[str], @@ -324,19 +597,41 @@ def write_csv( writer = csv.writer(f) writer.writerow(["Model", *columns, "Overall"]) for model, data in sorted(results.items()): - row = [model] - values = [] - for col in columns: - v = data.get(col) - row.append(f"{v:.2f}" if v is not None else "") - if v is not None: - values.append(v) - overall = mean(values) if values else None - row.append(f"{overall:.2f}" if overall is not None else "") - writer.writerow(row) + cells = [data.get(col) for col in columns] + [_overall(data, columns)] + writer.writerow( + [model, *(f"{v:.2f}" if v is not None else "" for v in cells)] + ) print(f"CSV written to {output}") +def _min_table_width( + results: dict[str, dict[str, float | None]], columns: list[str] +) -> int: + """Width the matrix needs before Rich starts truncating cells to an ellipsis. + + The one-run flow produces a column per SPEED-Bench category, so the + qualitative matrix is 13 columns wide. Rich falls back to an 80-column + console whenever stdout is not a tty -- piping to a file, or the common + ``docker run`` without ``-t`` -- and then shrinks every cell to "..." rather + than overflowing. Sizing the console up front keeps the table readable and + lets the terminal soft-wrap instead. + + Each column costs its widest cell plus two padding spaces and a border, and + the table adds one closing border. + """ + widths = [max([len("Model"), *(len(model) for model in results)])] + for name in (*columns, "Overall"): + cells = [ + f"{value:.2f}" if value is not None else "-" + for data in results.values() + for value in [ + _overall(data, columns) if name == "Overall" else data.get(name) + ] + ] + widths.append(max([len(name), *(len(cell) for cell in cells)])) + return sum(width + 3 for width in widths) + 1 + + def print_table( results: dict[str, dict[str, float | None]], columns: list[str], @@ -363,18 +658,16 @@ def print_table( table.add_column("Overall", justify="right", style="bold green") for model, data in sorted(results.items()): - row = [model] - values = [] - for col in columns: - v = data.get(col) - row.append(f"{v:.2f}" if v is not None else "-") - if v is not None: - values.append(v) - overall = mean(values) if values else None - row.append(f"{overall:.2f}" if overall is not None else "-") - table.add_row(*row) - - Console().print(table) + cells = [data.get(col) for col in columns] + [_overall(data, columns)] + table.add_row(model, *(f"{v:.2f}" if v is not None else "-" for v in cells)) + + # Widen only when the detected console cannot hold the matrix, so a real + # terminal keeps its own width and an oversized table is not forced on it. + console = Console() + required = _min_table_width(results, columns) + if console.width < required: + console = Console(width=required) + console.print(table) except ImportError: header = ["Model", *columns, "Overall"] @@ -384,23 +677,18 @@ def print_table( print(" ".join(h.rjust(w) for h, w in zip(header, widths, strict=True))) print(" ".join("-" * w for w in widths)) for model, data in sorted(results.items()): - values = [] - cells = [model] - for col in columns: - v = data.get(col) - cells.append(f"{v:.2f}" if v is not None else "-") - if v is not None: - values.append(v) - overall = mean(values) if values else None - cells.append(f"{overall:.2f}" if overall is not None else "-") + values = [data.get(col) for col in columns] + [_overall(data, columns)] + cells = [model, *(f"{v:.2f}" if v is not None else "-" for v in values)] print(" ".join(c.rjust(w) for c, w in zip(cells, widths, strict=True))) def generate_report( paths: list[Path], + *, output: Path = Path("speed_bench_report.csv"), output_format: OutputFormat = "both", metric: MetricType = "accept_length", + source: MetricSource = "auto", ) -> None: """Discover run directories, build a SPEED-Bench matrix report, and emit it. @@ -413,7 +701,7 @@ def generate_report( raise SpeedBenchReportError("no aiperf run directories found") print(f"Found {len(run_dirs)} run directories.") - results = build_report(run_dirs, metric_type=metric) + results = build_report(run_dirs, metric_type=metric, source=source) for model_data in results.values(): for cat, v in model_data.items(): diff --git a/src/aiperf/cli_commands/speed_bench_report.py b/src/aiperf/cli_commands/speed_bench_report.py index a2a1dc113a..f9c762c082 100644 --- a/src/aiperf/cli_commands/speed_bench_report.py +++ b/src/aiperf/cli_commands/speed_bench_report.py @@ -16,19 +16,28 @@ @app.default def speed_bench_report( paths: list[Path], + *, output: Path = Path("speed_bench_report.csv"), output_format: Annotated[ Literal["csv", "table", "both"], Parameter(name=["--format"]), ] = "both", metric: Literal["accept_length", "accept_rate", "throughput"] = "accept_length", + source: Literal["auto", "records", "summary", "server"] = "auto", ) -> None: - """Assemble per-category SPEED-Bench aiperf results into a matrix report. + """Assemble SPEED-Bench aiperf results into a matrix report. - Run ``aiperf profile`` once per SPEED-Bench category, then point this command - at the output directories to produce a matrix matching the SPEED-Bench paper format. + Point this command at one or more ``aiperf profile`` output directories. A + single run over an aggregate SPEED-Bench split (``speed_bench_qualitative``) + already carries every category, and each of its per-request records is + stamped with the category it came from, so one run produces the whole + matrix. Runs over a single category still work and contribute one column + each. Examples: + # One run over all categories: matrix comes from its per-request records + aiperf speed-bench-report ./artifacts/speed_bench_qualitative/ + # Scan a parent directory for per-category run subdirectories aiperf speed-bench-report ./artifacts/ @@ -38,15 +47,25 @@ def speed_bench_report( # Acceptance rate matrix (accepted / draft tokens) aiperf speed-bench-report ./artifacts/ --metric accept_rate - # Throughput matrix (output tokens/sec per category) + # Throughput matrix (output tokens/sec per run) aiperf speed-bench-report ./artifacts/ --metric throughput + # Force the Prometheus scrape, ignoring any per-request records + aiperf speed-bench-report ./artifacts/ --source server + Args: paths: Run directories or parent directories containing run subdirectories. output: Output CSV file path. Defaults to ./speed_bench_report.csv. output_format: Output format - 'csv', 'table', or 'both'. Defaults to 'both'. metric: Which metric to report - 'accept_length', 'accept_rate', or 'throughput'. Defaults to 'accept_length'. + source: Where acceptance numbers come from - 'records' (per-request + ``profile_export.jsonl``), 'summary' (the same per-request data reduced + to run-level scalars in ``profile_export_aiperf.json``, the only source + left at ``--export-level summary``), 'server' (Prometheus scrape in + ``server_metrics_export.json``), or 'auto' to try them in that order. + Only 'records' can split one run into per-category columns; only + 'server' reads server-side data. Defaults to 'auto'. """ from aiperf.analysis.speed_bench_report import ( SpeedBenchReportError, @@ -55,7 +74,11 @@ def speed_bench_report( try: generate_report( - paths, output=output, output_format=output_format, metric=metric + paths, + output=output, + output_format=output_format, + metric=metric, + source=source, ) except SpeedBenchReportError as e: print(f"Error: {e}.", file=sys.stderr) diff --git a/src/aiperf/common/models/dataset_models.py b/src/aiperf/common/models/dataset_models.py index 27102ac6f6..bcfbd6e277 100644 --- a/src/aiperf/common/models/dataset_models.py +++ b/src/aiperf/common/models/dataset_models.py @@ -211,8 +211,10 @@ class TurnMetadata(AIPerfBaseModel): source_kind: str | None = Field( default=None, description=( - "Loader-specific source classification for the reconstructed turn " - "(for example weka_main or weka_flat)." + "Loader-specific label for the turn's origin: Weka trace loaders " + "set the reconstruction kind (weka_main, weka_flat), the " + "SPEED-Bench loader sets the row's benchmark category (coding, " + "math, low_entropy)." ), ) replay_predecessors: list["ReplayTurnReference"] = Field( @@ -332,8 +334,10 @@ class Turn(AIPerfBaseModel): source_kind: str | None = Field( default=None, description=( - "Loader-specific source classification for the reconstructed turn " - "(for example weka_main or weka_flat)." + "Loader-specific label for the turn's origin: Weka trace loaders " + "set the reconstruction kind (weka_main, weka_flat), the " + "SPEED-Bench loader sets the row's benchmark category (coding, " + "math, low_entropy)." ), ) replay_predecessors: list["ReplayTurnReference"] = Field( diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index fa8aae0536..1fea04c600 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -212,8 +212,11 @@ class MetricRecordMetadata(AIPerfBaseModel): source_kind: str | None = Field( default=None, description=( - "Loader-specific source classification for this request, when " - "provided by the dataset loader." + "Loader-specific label for the request's origin, when provided by " + "the dataset loader: the Weka reconstruction kind (weka_main, " + "weka_flat) or the SPEED-Bench row's benchmark category (coding, " + "math, low_entropy). Lets a consumer group records by their source " + "without re-reading the input dataset." ), ) credit_issued_ns: int | None = Field( @@ -918,8 +921,11 @@ class RecordContext(AIPerfBaseModel): source_kind: str | None = Field( default=None, description=( - "Loader-specific source classification for this request, when " - "provided by the dataset loader." + "Loader-specific label for the request's origin, when provided by " + "the dataset loader: the Weka reconstruction kind (weka_main, " + "weka_flat) or the SPEED-Bench row's benchmark category (coding, " + "math, low_entropy). Lets a consumer group records by their source " + "without re-reading the input dataset." ), ) x_request_id: str = Field( diff --git a/src/aiperf/config/templates/README.md b/src/aiperf/config/templates/README.md index e2615a356b..ecd54971bf 100644 --- a/src/aiperf/config/templates/README.md +++ b/src/aiperf/config/templates/README.md @@ -58,6 +58,7 @@ aiperf profile --config benchmark.yaml | [multi_turn_conversation.yaml](multi_turn_conversation.yaml) | Multi-Turn Conversation | Simulate realistic chatbot workloads with multi-turn context accumulation | | [inline_dataset.yaml](inline_dataset.yaml) | Inline Dataset | Embed dataset records directly in the YAML config (no separate JSONL file required) | | [trace_replay.yaml](trace_replay.yaml) | Production Trace Replay | Replay production traffic from a trace file with exact request timestamps | +| [speed_bench_per_request.yaml](speed_bench_per_request.yaml) | SPEED-Bench Per-Category Matrix in One Run | One run over all 11 qualitative SPEED-Bench categories, split per category from per-request spec-decode records | ### Sweep & Multi-Run diff --git a/src/aiperf/config/templates/speed_bench_per_request.yaml b/src/aiperf/config/templates/speed_bench_per_request.yaml new file mode 100644 index 0000000000..fdc7994f5e --- /dev/null +++ b/src/aiperf/config/templates/speed_bench_per_request.yaml @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# yaml-language-server: $schema=https://github.com/ai-dynamo/aiperf/blob/main/src/aiperf/config/schema/aiperf-config.schema.json +# @template +# title: SPEED-Bench Per-Category Matrix in One Run +# description: One run over all 11 qualitative SPEED-Bench categories, split per category from per-request spec-decode records. +# category: Datasets +# tags: speed-bench, speculative-decoding, per-request-metrics, acceptance-length +# difficulty: beginner +# features: aggregate dataset, per-request spec-decode metrics, records export +# +# SPEED-Bench Per-Category Matrix in One Run +# ========================================== +# The recommended SPEED-Bench workflow on vLLM. A single run over the aggregate +# `speed_bench_qualitative` split covers all 11 categories at once: each request +# carries its row's category into the per-request records, and each response +# carries its own speculative-decoding acceptance, so +# `aiperf speed-bench-report` can rebuild the per-category matrix afterwards. +# +# Requires vLLM started with `--per-request-spec-decode-metrics summary` +# (or `detailed`), direct -- not behind Dynamo, which strips the field. On +# SGLang / TensorRT-LLM / NIM-LLM, or behind Dynamo, use the +# `speed_bench_sweep` template instead: acceptance can only be scraped from the +# server there, which takes one run per category. +# +# Prepare the dataset first (see docs/tutorials/speed-bench.md): +# curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ +# | python3 - --output_dir ./datasets/speed-bench +# +# Run: +# aiperf profile --config speed_bench_per_request.yaml +# aiperf speed-bench-report ./artifacts/speed_bench_per_request +# +# For the entropy-tier matrix at a fixed ISL, point the dataset at a throughput +# split instead (`throughput_1k.jsonl` with format `speed_bench_throughput_1k`); +# the report then produces low_entropy / mixed / high_entropy columns. + +schemaVersion: "2.0" + +randomSeed: 42 + +benchmark: + model: ${MODEL_NAME:meta-llama/Llama-3.1-8B-Instruct} + + endpoint: + url: ${INFERENCE_URL:http://localhost:8000/v1/chat/completions} + type: chat + streaming: true + timeout: 600.0 + # Greedy decoding keeps acceptance comparable with published numbers. Drop + # `temperature` for reasoning models and follow the model card instead. + extra: + temperature: 0 + max_completion_tokens: 4096 + + datasets: + - name: main + type: file + path: datasets/speed-bench/qualitative.jsonl + format: speed_bench_qualitative + + phases: + - name: warmup + type: concurrency + excludeFromResults: true + requests: 50 + concurrency: 8 + + - name: profiling + type: concurrency + # Stop on sessions, not requests: the split holds 880 conversations but + # multi-turn rows expand into ~1,140 requests, so a request-count of 880 + # would stop partway through the file and silently drop whole categories. + # Sessions run each conversation exactly once regardless of its turn count. + sessions: 880 + concurrency: 16 + + # Acceptance rides the responses, so the Prometheus scrape adds nothing here. + serverMetrics: + enabled: false + + artifacts: + dir: ./artifacts/speed_bench_per_request + summary: [json] + # Required: the per-request trace is what carries the category labels and + # the acceptance records the report groups by. + records: [jsonl] diff --git a/src/aiperf/config/templates/speed_bench_sweep.yaml b/src/aiperf/config/templates/speed_bench_sweep.yaml index b091b791ba..dd373c3b70 100644 --- a/src/aiperf/config/templates/speed_bench_sweep.yaml +++ b/src/aiperf/config/templates/speed_bench_sweep.yaml @@ -17,6 +17,14 @@ # `server_metrics_export.json` from which the report extracts per-category # acceptance length / acceptance rate / throughput. # +# This is the ENGINE-PORTABLE path, needed when acceptance can only be read +# from the server's Prometheus endpoint (SGLang, TensorRT-LLM, NIM-LLM, or a +# vLLM behind Dynamo). On vLLM started with `--per-request-spec-decode-metrics`, +# a SINGLE run over the aggregate `speed_bench_qualitative` split produces the +# same matrix: every request carries its category into the per-request records +# and `aiperf speed-bench-report` groups them. See +# docs/tutorials/speed-bench.md "Per-Category Matrix in One Run". +# # Output layout (one variation per category): # ./artifacts/speed_bench_sweep/ # datasets.main.dataset=speed_bench_coding/profile_runs/run_0001/ diff --git a/src/aiperf/dataset/loader/speed_bench.py b/src/aiperf/dataset/loader/speed_bench.py index 35a67d9cf6..c0d98983e4 100644 --- a/src/aiperf/dataset/loader/speed_bench.py +++ b/src/aiperf/dataset/loader/speed_bench.py @@ -9,7 +9,7 @@ from pydantic import Field, ValidationError, model_validator -from aiperf.common.models import AIPerfBaseModel +from aiperf.common.models import AIPerfBaseModel, Conversation from aiperf.dataset.loader.models import MultiTurn, SingleTurn from aiperf.dataset.loader.multi_turn import MultiTurnDatasetLoader @@ -133,6 +133,7 @@ def __init__( """ self.category = category self.multi_turn = multi_turn + self._category_by_session: dict[str, str] = {} super().__init__(filename=filename, run=run, **kwargs) @classmethod @@ -183,6 +184,7 @@ def load_dataset(self) -> dict[str, list[MultiTurn]]: ) data[multi_turn_data.session_id].append(multi_turn_data) + self._category_by_session[multi_turn_data.session_id] = loaded_line.category if self.category and not data: self.warning( @@ -195,6 +197,29 @@ def load_dataset(self) -> dict[str, list[MultiTurn]]: return data + def convert_to_conversations( + self, data: dict[str, list[MultiTurn]] + ) -> list[Conversation]: + """Build conversations and stamp each turn with its row's category. + + The category rides ``Turn.source_kind``, which the worker copies onto + every request and the record processor writes to + ``MetricRecordMetadata.source_kind``. That makes each line of + ``profile_export.jsonl`` self-describing, so a single run over an + aggregate split (``speed_bench_qualitative``, ``speed_bench_throughput_1k``) + can be broken down per category afterwards -- by + ``aiperf speed-bench-report`` or any other consumer -- without needing + the source JSONL, which the artifact directory does not contain. + """ + conversations = super().convert_to_conversations(data) + for conversation in conversations: + category = self._category_by_session.get(conversation.session_id) + if category is None: + continue + for turn in conversation.turns: + turn.source_kind = category + return conversations + class SpeedBenchSplitLoader(SpeedBenchLoader): """Base loader for a concrete SPEED-Bench JSONL split.""" diff --git a/src/aiperf/dataset/memory_map_utils.py b/src/aiperf/dataset/memory_map_utils.py index 53b667980b..d9f6181f18 100644 --- a/src/aiperf/dataset/memory_map_utils.py +++ b/src/aiperf/dataset/memory_map_utils.py @@ -187,7 +187,8 @@ async def add_conversation( # Pre-encode each turn's raw_payload and write the bytes directly; # workers replay these verbatim with no deserialization. Persist # turn scalars in the index so metric enrichment can restore - # max_tokens / scheduled_send_ms without the full Conversation. + # max_tokens / scheduled_send_ms / source_kind without the full + # Conversation. turn_offsets: list[PayloadOffset] = [] for turn in conversation.turns: payload_bytes = orjson.dumps(turn.raw_payload) @@ -197,6 +198,7 @@ async def add_conversation( size=len(payload_bytes), max_tokens=_resolve_turn_max_tokens(turn), timestamp=turn.timestamp, + source_kind=turn.source_kind, ) ) self._current_offset += len(payload_bytes) @@ -513,6 +515,15 @@ class PayloadOffset(AIPerfBaseModel): "the PAYLOAD_BYTES path." ), ) + source_kind: str | None = Field( + default=None, + description=( + "Loader-specific origin label from Turn.source_kind (Weka " + "reconstruction kind, SPEED-Bench row category). Restored onto " + "reconstructed Turns so per-record attribution survives the " + "PAYLOAD_BYTES path." + ), + ) # Wire-body keys that encode the same Turn.max_tokens scalar across endpoints. @@ -530,6 +541,7 @@ class PayloadTurnData: payload_bytes: bytes max_tokens: int | None = None timestamp: int | float | None = None + source_kind: str | None = None def max_tokens_from_wire_payload(payload: dict[str, Any] | None) -> int | None: @@ -587,6 +599,7 @@ def turn_from_payload_turn(entry: PayloadTurnData) -> Turn: raw_payload=raw_payload, max_tokens=max_tokens, timestamp=entry.timestamp, + source_kind=entry.source_kind, ) @@ -846,8 +859,8 @@ def get_payload_turn( ) -> PayloadTurnData | None: """Get payload bytes plus turn scalars for a specific turn. - Scalars (``max_tokens``, ``timestamp``) are restored from the index - when present. When ``max_tokens`` is missing (legacy indexes or turns + Scalars (``max_tokens``, ``timestamp``, ``source_kind``) are restored + from the index when present. When ``max_tokens`` is missing (legacy indexes or turns that never set it on the Turn), it is recovered from wire JSON keys ``max_tokens`` / ``max_completion_tokens`` / ``max_output_tokens``. @@ -881,6 +894,7 @@ def get_payload_turn( payload_bytes=payload_bytes, max_tokens=max_tokens, timestamp=offset_info.timestamp, + source_kind=offset_info.source_kind, ) def close(self) -> None: diff --git a/src/aiperf/workers/worker.py b/src/aiperf/workers/worker.py index 15aecc996d..7a545e1ad3 100644 --- a/src/aiperf/workers/worker.py +++ b/src/aiperf/workers/worker.py @@ -1150,12 +1150,13 @@ async def _try_payload_bytes_fast_path( payload_bytes=payload_turn.payload_bytes, # Scalars only — payload_bytes carries the wire body; enrichment # reads max_tokens / timestamp off this turn for OSL-mismatch and - # schedule-lag metrics. + # schedule-lag metrics, and source_kind for per-row attribution. turns=[ Turn( role="user", max_tokens=payload_turn.max_tokens, timestamp=payload_turn.timestamp, + source_kind=payload_turn.source_kind, ) ], ) diff --git a/tests/unit/analysis/test_speed_bench_report.py b/tests/unit/analysis/test_speed_bench_report.py index 7f42186dd2..af2736f850 100644 --- a/tests/unit/analysis/test_speed_bench_report.py +++ b/tests/unit/analysis/test_speed_bench_report.py @@ -8,21 +8,28 @@ import orjson import pytest +from pytest import param from aiperf.analysis.speed_bench_report import ( PROFILE_JSON, + PROFILE_JSONL, + QUALITATIVE_CATEGORIES, SERVER_METRICS_JSON, SpeedBenchReportError, _get_metric_stat, + _min_table_width, + acceptance_from_records, build_report, detect_columns, extract_accept_length, extract_accept_rate, extract_category, extract_model, + extract_summary_acceptance, extract_throughput, find_run_dirs, generate_report, + iter_records, load_profile, load_server_metrics, print_table, @@ -66,11 +73,32 @@ def _public_profile(dataset: str, model: str | None = "test-model") -> dict: return {"input_config": input_config} +def _record( + category: str | None, + accepted: int, + drafted: int, + steps: int, + phase: str = "profiling", +) -> dict: + """Construct one ``profile_export.jsonl`` line carrying an acceptance record.""" + return { + "metadata": {"benchmark_phase": phase, "source_kind": category}, + "metrics": {}, + "spec_decode_acceptance": { + "engine": "vllm", + "num_accepted_draft_tokens": accepted, + "num_draft_tokens": drafted, + "num_spec_steps": steps, + }, + } + + def _write_run_dir( tmp_path: Path, name: str, profile: dict | None, server_metrics: dict | None = None, + records: list[dict] | None = None, ) -> Path: """Materialize a fake run directory on disk and return its path.""" run_dir = tmp_path / name @@ -79,6 +107,10 @@ def _write_run_dir( (run_dir / PROFILE_JSON).write_bytes(orjson.dumps(profile)) if server_metrics is not None: (run_dir / SERVER_METRICS_JSON).write_bytes(orjson.dumps(server_metrics)) + if records is not None: + (run_dir / PROFILE_JSONL).write_bytes( + b"\n".join(orjson.dumps(record) for record in records) + b"\n" + ) return run_dir @@ -461,6 +493,412 @@ def test_build_report_skips_run_dir_without_profile_file(self, tmp_path: Path): assert build_report([empty], metric_type="accept_length") == {} +class TestAcceptanceFromRecords: + def test_groups_records_by_source_kind(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, + "run_qualitative", + _profile(dataset="speed_bench_qualitative", model="m1"), + records=[ + _record("coding", accepted=30, drafted=50, steps=10), + _record("coding", accepted=10, drafted=50, steps=10), + _record("math", accepted=60, drafted=100, steps=20), + ], + ) + + values = acceptance_from_records(run_dir, "accept_length", "qualitative") + + assert values == {"coding": 3.0, "math": 4.0} + + def test_accept_rate_is_token_weighted_across_the_category(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, + "run_qualitative", + _profile(dataset="speed_bench_qualitative", model="m1"), + records=[ + _record("coding", accepted=30, drafted=50, steps=10), + _record("coding", accepted=10, drafted=150, steps=10), + ], + ) + + values = acceptance_from_records(run_dir, "accept_rate", "qualitative") + + # Summed, not averaged per request: 40/200 = 0.2 (per-request mean is 0.33). + assert values == {"coding": 0.2} + + def test_error_records_are_excluded(self, tmp_path: Path): + errored = _record("coding", accepted=90, drafted=100, steps=10) + errored["error"] = {"code": 500, "message": "boom"} + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[errored, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + # MetricsAccumulator drops error records from the spec-decode totals, so + # including them here would make `records` disagree with `summary`. + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_warmup_records_are_excluded(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[ + _record("coding", accepted=90, drafted=100, steps=10, phase="warmup"), + _record("coding", accepted=10, drafted=50, steps=10), + ], + ) + + values = acceptance_from_records(run_dir, "accept_length", "coding") + + assert values == {"coding": 2.0} + + def test_excluded_warmup_phase_kind_is_also_dropped(self, tmp_path: Path): + warmup = _record("coding", accepted=90, drafted=100, steps=10) + warmup["metadata"] = { + "benchmark_phase": "profiling", + "phase_kind": "warmup", + "source_kind": "coding", + } + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[warmup, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_non_int_counter_does_not_half_update_its_category(self, tmp_path: Path): + bad = _record("coding", accepted=999, drafted=50, steps=10) + bad["spec_decode_acceptance"]["num_draft_tokens"] = "x" + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[bad, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + @pytest.mark.parametrize( + "field, value", + [ + param("num_accepted_draft_tokens", -5, id="negative-accepted"), + param("num_spec_steps", -1, id="negative-steps"), + param("num_accepted_draft_tokens", 999, id="accepted-exceeds-drafted"), + ], + ) # fmt: skip + def test_impossible_counter_values_are_rejected( + self, tmp_path: Path, field: str, value: int + ): + bad = _record("coding", accepted=10, drafted=50, steps=10) + bad["spec_decode_acceptance"][field] = value + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[bad, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_bool_counter_is_rejected(self, tmp_path: Path): + # bool is an int subclass; accepting it would let True count as 1. + bad = _record("coding", accepted=10, drafted=50, steps=10) + bad["spec_decode_acceptance"]["num_spec_steps"] = True + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[bad], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == {} + + def test_skipped_records_are_reported_not_silently_dropped( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ): + bad = _record("coding", accepted=999, drafted=50, steps=10) + bad["spec_decode_acceptance"]["num_draft_tokens"] = "x" + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[bad, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + acceptance_from_records(run_dir, "accept_length", "coding") + + assert "skipped 1 malformed acceptance record(s)" in capsys.readouterr().err + + def test_record_missing_a_counter_does_not_skew_its_category(self, tmp_path: Path): + # A half-folded record would inflate accepted without its steps. + partial = _record("coding", accepted=999, drafted=999, steps=0) + del partial["spec_decode_acceptance"]["num_spec_steps"] + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[partial, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_records_without_source_kind_fall_back_to_run_category( + self, tmp_path: Path + ): + # spec_al_* public datasets carry no per-row category. + run_dir = _write_run_dir( + tmp_path, + "run_gsm8k", + _public_profile(dataset="spec_al_gsm8k", model="m1"), + records=[_record(None, accepted=20, drafted=40, steps=10)], + ) + + values = acceptance_from_records(run_dir, "accept_length", "gsm8k") + + assert values == {"gsm8k": 3.0} + + def test_records_without_acceptance_are_ignored(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[ + {"metadata": {"benchmark_phase": "profiling"}, "metrics": {}}, + _record("coding", accepted=10, drafted=50, steps=10), + ], + ) + + values = acceptance_from_records(run_dir, "accept_length", "coding") + + assert values == {"coding": 2.0} + + def test_zero_step_category_is_omitted(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[_record("coding", accepted=0, drafted=0, steps=0)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == {} + + def test_unparseable_line_does_not_sink_the_run(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + with open(run_dir / PROFILE_JSONL, "ab") as f: + f.write(b'{"metadata": {"benchm\n') + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_missing_jsonl_returns_empty(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, "run_coding", _profile(dataset="speed_bench_coding", model="m1") + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == {} + + +class TestExtractSummaryAcceptance: + def test_accept_length_read_verbatim(self): + profile = {"spec_decode_token_weighted_acceptance_length": {"avg": 3.2}} + assert extract_summary_acceptance(profile, "accept_length") == 3.2 + + def test_accept_rate_converted_from_percent_to_fraction(self): + profile = {"spec_decode_overall_draft_acceptance_rate": {"avg": 46.0}} + assert extract_summary_acceptance(profile, "accept_rate") == 0.46 + + def test_missing_metric_returns_none(self): + assert extract_summary_acceptance({}, "accept_length") is None + + +class TestBuildReportSources: + def test_single_run_over_aggregate_split_yields_the_full_matrix( + self, tmp_path: Path + ): + _write_run_dir( + tmp_path, + "run_qualitative", + _profile(dataset="speed_bench_qualitative", model="m1"), + records=[ + _record("coding", accepted=20, drafted=40, steps=10), + _record("math", accepted=30, drafted=40, steps=10), + _record("writing", accepted=10, drafted=40, steps=10), + ], + ) + + report = build_report(find_run_dirs([tmp_path]), metric_type="accept_length") + + assert report == {"m1": {"coding": 3.0, "math": 4.0, "writing": 2.0}} + + def test_auto_announces_which_source_it_resolved( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ): + _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + _write_run_dir( + tmp_path, + "run_math", + _profile(dataset="speed_bench_math", model="m1"), + server_metrics=_server_metric("sglang:spec_accept_length", {"avg": 3.0}), + ) + + build_report(find_run_dirs([tmp_path]), metric_type="accept_length") + + err = capsys.readouterr().err + assert "acceptance from records" in err + assert "acceptance from server" in err + + def test_pinned_source_does_not_announce( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ): + _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + + build_report( + find_run_dirs([tmp_path]), metric_type="accept_length", source="records" + ) + + assert "acceptance from" not in capsys.readouterr().err + + def test_records_take_precedence_over_the_server_scrape(self, tmp_path: Path): + _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + server_metrics=_server_metric("sglang:spec_accept_length", {"avg": 9.9}), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + + report = build_report(find_run_dirs([tmp_path]), metric_type="accept_length") + + assert report == {"m1": {"coding": 2.0}} + + def test_source_server_ignores_available_records(self, tmp_path: Path): + _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + server_metrics=_server_metric("sglang:spec_accept_length", {"avg": 9.9}), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + + report = build_report( + find_run_dirs([tmp_path]), metric_type="accept_length", source="server" + ) + + assert report == {"m1": {"coding": 9.9}} + + def test_source_records_does_not_fall_back_to_the_scrape(self, tmp_path: Path): + _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + server_metrics=_server_metric("sglang:spec_accept_length", {"avg": 9.9}), + ) + + report = build_report( + find_run_dirs([tmp_path]), metric_type="accept_length", source="records" + ) + + assert report == {"m1": {"coding": None}} + + def test_source_summary_reads_only_the_run_level_scalars(self, tmp_path: Path): + profile = _profile(dataset="speed_bench_coding", model="m1") + profile["spec_decode_token_weighted_acceptance_length"] = {"avg": 3.5} + _write_run_dir( + tmp_path, + "run_coding", + profile, + server_metrics=_server_metric("sglang:spec_accept_length", {"avg": 9.9}), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + + report = build_report( + find_run_dirs([tmp_path]), metric_type="accept_length", source="summary" + ) + + assert report == {"m1": {"coding": 3.5}} + + def test_auto_prefers_records_over_the_summary_scalar(self, tmp_path: Path): + """Under ``auto``, per-request records win over the run-level scalar. + + Both are present here and disagree, so the assertion pins the + precedence order rather than merely that some value was found. + """ + profile = _profile(dataset="speed_bench_coding", model="m1") + profile["spec_decode_token_weighted_acceptance_length"] = {"avg": 3.5} + _write_run_dir( + tmp_path, + "run_coding", + profile, + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + + report = build_report(find_run_dirs([tmp_path]), metric_type="accept_length") + + assert report == {"m1": {"coding": 2.0}} + + def test_summary_metrics_used_when_no_records_exist(self, tmp_path: Path): + profile = _profile(dataset="speed_bench_coding", model="m1") + profile["spec_decode_token_weighted_acceptance_length"] = {"avg": 3.5} + _write_run_dir( + tmp_path, + "run_coding", + profile, + server_metrics=_server_metric("sglang:spec_accept_length", {"avg": 9.9}), + ) + + report = build_report(find_run_dirs([tmp_path]), metric_type="accept_length") + + assert report == {"m1": {"coding": 3.5}} + + def test_throughput_stays_one_column_per_run(self, tmp_path: Path): + profile = _profile(dataset="speed_bench_qualitative", model="m1") + profile["output_token_throughput"] = {"avg": 1234.0} + _write_run_dir( + tmp_path, + "run_qualitative", + profile, + records=[ + _record("coding", accepted=20, drafted=40, steps=10), + _record("math", accepted=30, drafted=40, steps=10), + ], + ) + + report = build_report(find_run_dirs([tmp_path]), metric_type="throughput") + + assert report == {"m1": {"qualitative": 1234.0}} + + class TestPrintTable: def test_print_table_rich_branch_renders_model_and_overall( self, capsys: pytest.CaptureFixture[str] @@ -556,6 +994,28 @@ def test_generate_report_writes_csv_end_to_end(self, tmp_path: Path): assert output.exists() assert "m1,2.50" in output.read_text() + def test_generate_report_forwards_the_source_selector(self, tmp_path: Path): + """``--source`` must reach ``build_report``, not just be parsed. + + The fixture carries both a per-request record (2.00) and the run-level + summary scalar (3.50), so the CSV distinguishes the two paths. + """ + profile = _profile(dataset="speed_bench_coding", model="m1") + profile["spec_decode_token_weighted_acceptance_length"] = {"avg": 3.5} + _write_run_dir( + tmp_path, + "run_coding", + profile, + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + output = tmp_path / "report.csv" + + generate_report( + [tmp_path], output=output, output_format="csv", source="summary" + ) + + assert "m1,3.50" in output.read_text() + def test_generate_report_table_format_prints_without_writing_csv( self, tmp_path: Path, @@ -576,3 +1036,163 @@ def test_generate_report_table_format_prints_without_writing_csv( out = capsys.readouterr().out assert "m1" in out assert "2.50" in out + + +class TestReportEdgeCases: + @pytest.mark.parametrize( + "avg", + [ + param("4.2", id="string"), + param(True, id="bool"), + param(None, id="none"), + ], + ) # fmt: skip + def test_summary_entry_with_non_numeric_avg_returns_none(self, avg): + # A non-number here would reach the "%.2f" formatting and abort the report. + profile = {"spec_decode_token_weighted_acceptance_length": {"avg": avg}} + assert extract_summary_acceptance(profile, "accept_length") is None + + def test_iter_records_skips_blank_lines(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, "run", _profile(dataset="speed_bench_coding"), records=[] + ) + (run_dir / PROFILE_JSONL).write_bytes( + b"\n\n" + orjson.dumps({"metadata": {}}) + b"\n \n" + ) + + assert len(list(iter_records(run_dir))) == 1 + + def test_iter_records_skips_non_dict_json(self, tmp_path: Path): + run_dir = _write_run_dir( + tmp_path, "run", _profile(dataset="speed_bench_coding"), records=[] + ) + (run_dir / PROFILE_JSONL).write_bytes( + b'[1, 2]\n"a string"\n' + orjson.dumps({"metadata": {}}) + b"\n" + ) + + assert len(list(iter_records(run_dir))) == 1 + + def test_iter_records_warns_when_the_path_is_unreadable( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ): + # A directory in place of the file raises IsADirectoryError (an OSError) + # regardless of the running user, unlike a chmod-based fixture. + run_dir = _write_run_dir( + tmp_path, "run", _profile(dataset="speed_bench_coding") + ) + (run_dir / PROFILE_JSONL).mkdir() + + assert list(iter_records(run_dir)) == [] + assert "failed to read" in capsys.readouterr().err + + def test_record_with_non_dict_metadata_is_skipped(self, tmp_path: Path): + broken = _record("coding", accepted=10, drafted=50, steps=10) + broken["metadata"] = "not-a-dict" + run_dir = _write_run_dir( + tmp_path, + "run", + _profile(dataset="speed_bench_coding"), + records=[broken, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_record_with_a_non_string_category_is_skipped(self, tmp_path: Path): + broken = _record("coding", accepted=999, drafted=999, steps=10) + broken["metadata"]["source_kind"] = 123 + run_dir = _write_run_dir( + tmp_path, + "run", + _profile(dataset="speed_bench_coding"), + records=[broken, _record("coding", accepted=10, drafted=50, steps=10)], + ) + + assert acceptance_from_records(run_dir, "accept_length", "coding") == { + "coding": 2.0 + } + + def test_throughput_run_without_a_recognizable_category_is_skipped( + self, tmp_path: Path + ): + profile = _profile(dataset="sharegpt", model="m1") + profile["output_token_throughput"] = {"avg": 10.0} + _write_run_dir(tmp_path, "run", profile) + + assert build_report(find_run_dirs([tmp_path]), metric_type="throughput") == {} + + +class TestWideMatrixRendering: + _QUALITATIVE = { + "meta-llama/Llama-3.1-8B-Instruct": dict.fromkeys(QUALITATIVE_CATEGORIES, 4.44) + } + + def test_full_matrix_renders_every_column_at_width_80( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ): + monkeypatch.setenv("COLUMNS", "80") + + print_table(self._QUALITATIVE, QUALITATIVE_CATEGORIES, "accept_length") + + out = capsys.readouterr().out + for category in QUALITATIVE_CATEGORIES: + assert category in out, f"{category} was truncated away" + assert "…" not in out + + def test_narrow_matrix_does_not_force_a_wide_console(self): + narrow = {"m": {"coding": 1.0}} + assert _min_table_width(narrow, ["coding"]) < 80 + + def test_wide_overall_is_not_truncated( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ): + """Overall is derived, not a key, so it must be measured like a cell. + + Throughput cells run to five digits, which makes the rendered Overall + wider than the literal "Overall" header. + """ + monkeypatch.setenv("COLUMNS", "80") + results = { + "meta-llama/Llama-3.1-8B-Instruct": dict.fromkeys( + QUALITATIVE_CATEGORIES, 12345.67 + ) + } + + print_table(results, QUALITATIVE_CATEGORIES, "throughput") + + out = capsys.readouterr().out + assert "12345.67" in out + assert "…" not in out + + def test_short_model_name_still_fits_the_model_header(self): + """The first column holds the header "Model" even when every name is shorter.""" + results = {"m1": {"coding": 4.44}} + + assert _min_table_width(results, ["coding"]) >= ( + len("Model") + 3 + max(len("coding"), 4) + 3 + len("Overall") + 3 + 1 + ) + + +class TestUnrelatedRunsDoNotPolluteTheMatrix: + def test_records_are_ignored_when_the_dataset_selector_is_unrecognized( + self, tmp_path: Path + ): + weka = _record("weka_main", accepted=40, drafted=50, steps=10) + _write_run_dir( + tmp_path, + "run_unrelated", + _profile(dataset="sharegpt", model="m1"), + records=[weka], + ) + _write_run_dir( + tmp_path, + "run_coding", + _profile(dataset="speed_bench_coding", model="m1"), + records=[_record("coding", accepted=10, drafted=50, steps=10)], + ) + + report = build_report(find_run_dirs([tmp_path]), metric_type="accept_length") + + assert report == {"m1": {"coding": 2.0}} + assert "weka_main" not in report["m1"] diff --git a/tests/unit/cli_commands/test_speed_bench_report.py b/tests/unit/cli_commands/test_speed_bench_report.py index 0e9bf04999..cbfe40bf8e 100644 --- a/tests/unit/cli_commands/test_speed_bench_report.py +++ b/tests/unit/cli_commands/test_speed_bench_report.py @@ -38,7 +38,11 @@ def test_forwards_all_arguments(self, mock_generate_report: MagicMock) -> None: ) mock_generate_report.assert_called_once_with( - paths, output=output, output_format="csv", metric="accept_rate" + paths, + output=output, + output_format="csv", + metric="accept_rate", + source="auto", ) def test_default_output_path(self, mock_generate_report: MagicMock) -> None: diff --git a/tests/unit/dataset/loader/test_speed_bench.py b/tests/unit/dataset/loader/test_speed_bench.py index e6a01a4fc5..1f867f0b9e 100644 --- a/tests/unit/dataset/loader/test_speed_bench.py +++ b/tests/unit/dataset/loader/test_speed_bench.py @@ -8,7 +8,7 @@ from aiperf.common.models import Conversation from aiperf.config.flags.cli_config import CLIConfig -from aiperf.dataset.loader.models import MultiTurn +from aiperf.dataset.loader.models import MultiTurn, SingleTurn from aiperf.dataset.loader.speed_bench import SpeedBenchLoader, SpeedBenchRow from aiperf.plugin.enums import DatasetSamplingStrategy from tests.unit.conftest import make_run_from_cli @@ -238,6 +238,92 @@ def test_leading_system_message_is_not_hoisted(self, create_jsonl_file): assert conversation.turns[0].texts[0].contents == ["Answer tersely."] +class TestSpeedBenchLoaderCategoryTagging: + def test_turns_carry_their_row_category_as_source_kind(self, create_jsonl_file): + loader, dataset = _load_speed_bench_file( + create_jsonl_file, + [ + _make_speed_bench_row( + question_id=_qid("speed-coding-1"), category="coding" + ), + _make_speed_bench_row( + question_id=_qid("speed-math-1"), category="math" + ), + ], + ) + + conversations = loader.convert_to_conversations(dataset) + + by_id = {c.session_id: c for c in conversations} + assert by_id[_qid("speed-coding-1")].turns[0].source_kind == "coding" + assert by_id[_qid("speed-math-1")].turns[0].source_kind == "math" + + def test_every_turn_of_a_multi_turn_row_carries_the_category( + self, create_jsonl_file + ): + loader, dataset = _load_speed_bench_file( + create_jsonl_file, + [ + _make_speed_bench_row( + question_id=_qid("speed-chat-1"), + category="roleplay", + messages=[ + {"role": "user", "content": "Play a pirate."}, + {"role": "user", "content": "Now find the treasure."}, + ], + ) + ], + ) + + conversation = loader.convert_to_conversations(dataset)[0] + + assert len(conversation.turns) == 2 + assert [turn.source_kind for turn in conversation.turns] == [ + "roleplay", + "roleplay", + ] + + def test_category_filtered_run_still_tags_its_rows(self, create_jsonl_file): + loader, dataset = _load_speed_bench_file( + create_jsonl_file, + [ + _make_speed_bench_row( + question_id=_qid("speed-coding-1"), category="coding" + ), + _make_speed_bench_row( + question_id=_qid("speed-math-1"), category="math" + ), + ], + category="math", + ) + + conversations = loader.convert_to_conversations(dataset) + + assert len(conversations) == 1 + assert conversations[0].turns[0].source_kind == "math" + + def test_conversation_with_no_recorded_category_is_left_untagged( + self, create_jsonl_file + ): + # convert_to_conversations is only meaningful for sessions load_dataset + # saw; an unmapped one must pass through rather than raise. + loader, _ = _load_speed_bench_file( + create_jsonl_file, [_make_speed_bench_row(category="coding")] + ) + unmapped = { + "never-loaded": [ + MultiTurn( + session_id="never-loaded", + turns=[SingleTurn(text="hi", role="user")], + ) + ] + } + + conversations = loader.convert_to_conversations(unmapped) + + assert conversations[0].turns[0].source_kind is None + + class TestSpeedBenchLoaderCategoryFiltering: def test_no_category_returns_all_rows(self, create_jsonl_file): _, dataset = _load_speed_bench_file( diff --git a/tests/unit/dataset/test_payload_mmap.py b/tests/unit/dataset/test_payload_mmap.py index a38bc39719..65eafecfb4 100644 --- a/tests/unit/dataset/test_payload_mmap.py +++ b/tests/unit/dataset/test_payload_mmap.py @@ -349,11 +349,14 @@ async def test_adopt_existing_files_compress_only_missing_zst_raises( @pytest.mark.asyncio async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): - """PAYLOAD_BYTES index must round-trip max_tokens and timestamp. + """PAYLOAD_BYTES index must round-trip max_tokens, timestamp, and source_kind. Turn scalars live outside the wire body for some loaders (e.g. mooncake - ``output_length`` / ``timestamp``). Persisting them on PayloadOffset keeps - OSL-mismatch and schedule-lag metrics alive on the verbatim path. + ``output_length`` / ``timestamp``, SPEED-Bench and Weka ``source_kind``). + Persisting them on PayloadOffset keeps OSL-mismatch and schedule-lag metrics + alive on the verbatim path, and keeps each record attributable to the + dataset row it came from -- without it a per-category split collapses to a + single run-level label. """ from aiperf.dataset.memory_map_utils import ( PayloadOffset, @@ -378,6 +381,7 @@ async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): raw_payload=payload, max_tokens=128, timestamp=42.5, + source_kind="coding", ) ], ) @@ -394,17 +398,20 @@ async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): assert entry is not None assert entry.max_tokens == 128 assert entry.timestamp == 42.5 + assert entry.source_kind == "coding" assert orjson.loads(entry.payload_bytes) == payload turn = turn_from_payload_turn(entry) assert turn.max_tokens == 128 assert turn.timestamp == 42.5 + assert turn.source_kind == "coding" assert turn.raw_payload == payload # Wire-JSON fallback recovers max_tokens when index scalars are absent # (legacy PayloadOffset with only offset/size). legacy = PayloadOffset(offset=0, size=0) assert legacy.max_tokens is None + assert legacy.source_kind is None assert ( max_tokens_from_wire_payload({"max_completion_tokens": 16, "messages": []}) == 16 diff --git a/tests/unit/workers/test_worker.py b/tests/unit/workers/test_worker.py index a22cc546f8..7f462aaa7d 100644 --- a/tests/unit/workers/test_worker.py +++ b/tests/unit/workers/test_worker.py @@ -932,7 +932,7 @@ async def test_successful_fast_path_sets_record_emitted( async def test_fast_path_turn_carries_max_tokens_and_timestamp( self, mock_worker, sample_credit_context ): - """PAYLOAD_BYTES fast path must hoist turn scalars (max_tokens, timestamp) for metric enrichment, else OSL-mismatch and schedule-lag metrics go silent.""" + """PAYLOAD_BYTES fast path must hoist turn scalars (max_tokens, timestamp, source_kind) for metric enrichment, else OSL-mismatch and schedule-lag metrics go silent and every record is attributed to the run-level label.""" mock_worker._is_payload_bytes = True mock_worker._dataset_client = AsyncMock() payload = b'{"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' @@ -940,6 +940,7 @@ async def test_fast_path_turn_carries_max_tokens_and_timestamp( payload_bytes=payload, max_tokens=64, timestamp=1234.5, + source_kind="coding", ) success_record = RequestRecord( @@ -962,17 +963,20 @@ async def test_fast_path_turn_carries_max_tokens_and_timestamp( assert len(sent_request_info.turns) == 1 assert sent_request_info.turns[0].max_tokens == 64 assert sent_request_info.turns[0].timestamp == 1234.5 + assert sent_request_info.turns[0].source_kind == "coding" + assert sent_request_info.source_kind == "coding" async def test_retrieve_conversation_for_session_restores_scalars( self, mock_worker, sample_credit_context ): - """Session-path reconstruction must restore max_tokens and timestamp.""" + """Session-path reconstruction must restore max_tokens, timestamp, and source_kind.""" mock_worker._is_payload_bytes = True mock_worker._dataset_client = AsyncMock() mock_worker._dataset_client.get_payload_turn.return_value = PayloadTurnData( payload_bytes=b'{"max_completion_tokens":32,"messages":[]}', max_tokens=32, timestamp=99, + source_kind="math", ) mock_worker.session_manager = MagicMock() mock_worker.session_manager.default_context_mode = None @@ -984,6 +988,7 @@ async def test_retrieve_conversation_for_session_restores_scalars( assert len(conversation.turns) == 1 assert conversation.turns[0].max_tokens == 32 assert conversation.turns[0].timestamp == 99 + assert conversation.turns[0].source_kind == "math" assert conversation.turns[0].raw_payload == { "max_completion_tokens": 32, "messages": [], From 465102babf3283ba5a8624e8706e6c8b362d2b9e Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:49:35 -0700 Subject: [PATCH 059/119] feat(tools): add capture_server and compare_recordings scripts (#1257) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 6cd02ea0f62e71f6aa2ca8edd60ef6fae74565af) --- tools/capture_server.py | 258 ++++++++++++++++++ tools/compare_recordings.py | 521 ++++++++++++++++++++++++++++++++++++ 2 files changed, 779 insertions(+) create mode 100755 tools/capture_server.py create mode 100755 tools/compare_recordings.py diff --git a/tools/capture_server.py b/tools/capture_server.py new file mode 100755 index 0000000000..bea3d2ee40 --- /dev/null +++ b/tools/capture_server.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Minimal HTTP server that captures raw prompt content from benchmark clients. + +Accepts OpenAI-compatible /v1/chat/completions requests, writes each raw payload +to a JSONL file, and returns a synthetic streaming response so the client doesn't +error out. + +Usage:: + + uv run python tools/capture_server.py --out /tmp/prompts.jsonl --port 18000 + +Then run your benchmark client against http://localhost:18000. +Each JSONL line contains the full request payload with a ``_seq`` counter added. + +The recording holds raw prompt content, so the output file is created mode 0600. +That guarantee is Unix-only: on Windows the mode bits are not enforced and the +file inherits the directory ACL, so capture to a directory you already trust. +""" + +from __future__ import annotations + +import argparse +import contextlib +import os +import sys +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +import orjson + +from aiperf.common.constants import IS_WINDOWS + +_out_file = None +_count = 0 +_model_id = "capture-model" +_lock = threading.Lock() + +_MAX_BODY = 64 * 1024 * 1024 # 64 MiB +_READ_TIMEOUT = 30.0 + + +def _sse_chunk(completion_id: str, model: str, content: str) -> bytes: + data = orjson.dumps( + { + "id": completion_id, + "object": "chat.completion.chunk", + "model": model, + "choices": [ + {"index": 0, "delta": {"content": content}, "finish_reason": None} + ], + "usage": None, + } + ).decode() + return f"data: {data}\n\n".encode() + + +def _sse_done(completion_id: str, model: str, prompt_tokens: int) -> bytes: + data = orjson.dumps( + { + "id": completion_id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": 4, + "total_tokens": prompt_tokens + 4, + }, + } + ).decode() + return f"data: {data}\n\ndata: [DONE]\n\n".encode() + + +def _message_text(message: dict) -> str: + """Extract the text of one chat message, including multimodal part lists. + + OpenAI allows ``content`` to be either a plain string or a list of typed + parts. Only the text parts carry tokens the synthetic ``prompt_tokens`` + estimate can account for; image and audio parts are skipped. + """ + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join( + part["text"] + for part in content + if isinstance(part, dict) and isinstance(part.get("text"), str) + ) + return "" + + +class CaptureHandler(BaseHTTPRequestHandler): + # Bounds every socket read, so a client that under-delivers its declared + # Content-Length cannot hold a worker thread open indefinitely. + timeout = _READ_TIMEOUT + + def log_message(self, fmt: str, *args: object) -> None: # silence access logs + pass + + def do_GET(self) -> None: + if urlparse(self.path).path == "/v1/models": + resp = orjson.dumps( + {"object": "list", "data": [{"id": _model_id, "object": "model"}]} + ) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(resp) + else: + self.send_response(404) + self.end_headers() + + def do_POST(self) -> None: + global _count + + if urlparse(self.path).path != "/v1/chat/completions": + self.send_response(404) + self.end_headers() + return + + raw_length = self.headers.get("Content-Length") + if raw_length is None: + self.send_response(411) + self.end_headers() + return + try: + length = int(raw_length) + except ValueError: + self.send_response(400) + self.end_headers() + return + if length < 0: + self.send_response(400) + self.end_headers() + return + if length > _MAX_BODY: + self.send_response(413) + self.end_headers() + return + + body = self.rfile.read(length) + + try: + payload = orjson.loads(body) + except orjson.JSONDecodeError: + self.send_response(400) + self.end_headers() + return + + if not isinstance(payload, dict): + self.send_response(400) + self.end_headers() + return + + messages = payload.get("messages", []) + if not isinstance(messages, list) or not all( + isinstance(m, dict) for m in messages + ): + self.send_response(400) + self.end_headers() + return + + stream = payload.get("stream", False) + + with _lock: + if _out_file is not None: + record = {**payload, "_seq": _count} + _out_file.write(orjson.dumps(record).decode() + "\n") + _out_file.flush() + + _count += 1 + count = _count + if count % 1000 == 0: + print(f" captured {count} requests", file=sys.stderr) + + content = " ".join(_message_text(m) for m in messages) + prompt_tokens = max(1, len(content) // 4) + + completion_id = f"chatcmpl-{uuid.uuid4().hex}" + + if stream: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + self.wfile.write(_sse_chunk(completion_id, _model_id, "ok")) + self.wfile.write(_sse_done(completion_id, _model_id, prompt_tokens)) + else: + resp = orjson.dumps( + { + "id": completion_id, + "object": "chat.completion", + "model": _model_id, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": 1, + "total_tokens": prompt_tokens + 1, + }, + } + ) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(resp))) + self.end_headers() + self.wfile.write(resp) + + +def main() -> None: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--out", + required=True, + metavar="FILE", + help="Output JSONL file — one captured request per line", + ) + ap.add_argument("--port", type=int, default=18000, metavar="PORT") + ap.add_argument("--host", default="127.0.0.1", metavar="HOST") + ap.add_argument( + "--model", + default="capture-model", + metavar="MODEL_ID", + help="Model ID advertised by /v1/models and used in synthetic responses (default: capture-model)", + ) + args = ap.parse_args() + + global _out_file, _model_id + _model_id = args.model + server = ThreadingHTTPServer((args.host, args.port), CaptureHandler) + print(f"Capture server listening on {args.host}:{args.port}", file=sys.stderr) + print(f"Writing to {args.out}", file=sys.stderr) + fd = os.open(args.out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + if not IS_WINDOWS: + os.chmod( + fd, 0o600 + ) # enforce on existing files; O_CREAT mode is ignored when file exists + with open(fd, "w") as f: + _out_file = f + with contextlib.suppress(KeyboardInterrupt): + server.serve_forever() + print(f"\nCaptured {_count} requests → {args.out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tools/compare_recordings.py b/tools/compare_recordings.py new file mode 100755 index 0000000000..28a4bfd1e5 --- /dev/null +++ b/tools/compare_recordings.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compare two mock server recording JSONL files and produce an HTML report. + +Each input file is the per-request JSONL produced by +``aiperf-mock-server --record-requests ``. The companion +``.summary.json`` file (same base path with ``.summary.json`` suffix) is +read automatically when present to include vocab-distribution data. + +Usage:: + + uv run python tools/compare_recordings.py \\ + --a artifacts/run_a/recording.jsonl --label-a "vLLM bench" \\ + --b artifacts/run_b/recording.jsonl --label-b "aiperf" \\ + --out comparison.html +""" + +from __future__ import annotations + +import argparse +import collections +import html +import math +import statistics +import sys +from pathlib import Path + +import orjson + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + + +def load_jsonl(path: Path) -> list[dict]: + rows = [] + with path.open() as f: + for line in f: + line = line.strip() + if line: + rows.append(orjson.loads(line)) + return rows + + +def load_summary(path: Path) -> dict | None: + # Prefer the exact companion the mock server writes (.summary.json), + # then fall back to the stem-replacement form for legacy files. + for candidate in ( + Path(str(path) + ".summary.json"), + path.with_suffix("").with_suffix(".summary.json"), + ): + if candidate.exists(): + return orjson.loads(candidate.read_bytes()) + return None + + +# --------------------------------------------------------------------------- +# Statistics helpers +# --------------------------------------------------------------------------- + + +def _finite_vals(rows: list[dict], key: str) -> list[float]: + return [ + v + for r in rows + if (v := r.get(key)) is not None + and isinstance(v, (int, float)) + and math.isfinite(v) + ] + + +def field_stats(rows: list[dict], key: str) -> dict: + vals = _finite_vals(rows, key) + if not vals: + return {} + n = len(vals) + if n >= 2: + qs = statistics.quantiles(vals, n=100, method="inclusive") + p5, p95 = qs[4], qs[94] + else: + p5 = p95 = vals[0] + return { + "n": n, + "min": min(vals), + "p5": p5, + "mean": round(statistics.mean(vals), 2), + "median": statistics.median(vals), + "p95": p95, + "max": max(vals), + "std": round(statistics.stdev(vals) if n > 1 else 0, 2), + } + + +def histogram(vals: list[int | float], lo: int, hi: int, step: int = 1) -> list[float]: + n = len(vals) + if n == 0: + return [0.0] * ((hi - lo) // step) + c: dict[int, int] = collections.Counter() + for v in vals: + bucket = int((v - lo) // step) * step + lo + c[bucket] += 1 + bins = range(lo, hi, step) + return [round(c.get(b, 0) / n * 100, 4) for b in bins] + + +def vocab_top_diffs( + summary_a: dict | None, + summary_b: dict | None, + top_n: int = 20, +) -> list[dict]: + """Return top tokens by absolute count difference across the two summaries.""" + if not summary_a or not summary_b: + return [] + + # Flatten per-endpoint vocab_counts + def flatten(summary: dict) -> dict[int, int]: + out: dict[int, int] = {} + for ep_data in summary.get("per_endpoint", {}).values(): + vc = ep_data.get("vocab_distribution", {}) + if not vc: + continue + for tok_id_str, cnt in vc.get("frequencies", vc.get("counts", {})).items(): + out[int(tok_id_str)] = out.get(int(tok_id_str), 0) + cnt + return out + + vc_a = flatten(summary_a) + vc_b = flatten(summary_b) + all_ids = set(vc_a) | set(vc_b) + diffs = [] + for tok_id in all_ids: + ca = vc_a.get(tok_id, 0) + cb = vc_b.get(tok_id, 0) + diffs.append( + {"id": tok_id, "a": ca, "b": cb, "delta": cb - ca, "abs": abs(cb - ca)} + ) + diffs.sort(key=lambda x: -x["abs"]) + return diffs[:top_n] + + +# --------------------------------------------------------------------------- +# HTML generation +# --------------------------------------------------------------------------- + +_CSS = """ +body{font-family:sans-serif;margin:32px;background:#f9f9f9;color:#222} +h1{font-size:20px;margin-bottom:4px} +h2{font-size:16px;margin-top:32px;margin-bottom:12px;border-bottom:2px solid #e0e0e0;padding-bottom:6px} +.subtitle{color:#666;font-size:13px;margin-bottom:24px} +.stats-row{display:flex;gap:16px;margin-bottom:24px;flex-wrap:wrap} +.stat-card{background:white;border:1px solid #e0e0e0;border-radius:8px;padding:14px 18px;min-width:140px} +.stat-card h3{margin:0 0 6px;font-size:11px;color:#666;text-transform:uppercase;letter-spacing:.05em} +.stat-value{font-size:22px;font-weight:bold;margin:0} +.stat-sub{font-size:11px;color:#888;margin-top:4px} +.a-color{color:#2563eb}.b-color{color:#dc2626}.match{color:#059669}.warn{color:#d97706} +.chart-box{background:white;border:1px solid #e0e0e0;border-radius:8px;padding:24px;margin-bottom:24px} +.chart-title{font-size:14px;font-weight:600;margin-bottom:4px} +.chart-sub{font-size:12px;color:#888;margin-bottom:14px} +table{border-collapse:collapse;font-size:12px;width:100%} +th{background:#f4f4f4;text-align:left;padding:6px 10px;border-bottom:2px solid #ddd} +td{padding:5px 10px;border-bottom:1px solid #eee} +.pos{color:#dc2626}.neg{color:#2563eb}.zero{color:#059669} +""" + + +def _js(value: object) -> str: + """Serialize a value as a JavaScript literal safe to embed in a ```` would close the + enclosing block. Escaping the three HTML-significant characters as unicode + escapes keeps the literal inert to the HTML parser while leaving the runtime + string value unchanged, so labels reach the chart verbatim rather than as + HTML entities. + """ + return ( + orjson.dumps(value) + .decode() + .replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + ) + + +def _js_hist( + svg_id: str, + bins: list[int], + a_pct: list[float], + b_pct: list[float], + x_label: str, + label_a: str, + label_b: str, +) -> str: + return f""" +""" + + +def stat_card(title: str, value: str, sub: str, color_class: str = "") -> str: + cls = f"stat-value {color_class}" if color_class else "stat-value" + return ( + f'

{title}

' + f'

{value}

' + f'
{sub}
' + ) + + +def delta_class(d: float) -> str: + if abs(d) < 0.5: + return "zero" + return "pos" if d > 0 else "neg" + + +def render_stat_section( + stats_a: dict, stats_b: dict, label_a: str, label_b: str, field: str +) -> str: + if not stats_a or not stats_b: + return "" + mean_d = round(stats_b["mean"] - stats_a["mean"], 2) + std_d = round(stats_b["std"] - stats_a["std"], 2) + dc = "match" if abs(mean_d) < 1 else "warn" if abs(mean_d) < 5 else "warn" + cards = [ + stat_card( + f"{label_a} mean", + str(stats_a["mean"]), + f"std={stats_a['std']} p5={stats_a['p5']}, p95={stats_a['p95']}", + "a-color", + ), + stat_card( + f"{label_b} mean", + str(stats_b["mean"]), + f"std={stats_b['std']} p5={stats_b['p5']}, p95={stats_b['p95']}", + "b-color", + ), + stat_card("Mean delta", f"{mean_d:+.2f}", f"{label_b} − {label_a}", dc), + stat_card("Std delta", f"{std_d:+.2f}", f"{label_b} − {label_a}", dc), + stat_card( + "p5 / p95", + f"{stats_a['p5']} / {stats_a['p95']}", + f"{label_a}: {stats_a['min']}–{stats_a['max']}", + "", + ), + stat_card( + "n", f"{stats_a['n']:,} / {stats_b['n']:,}", f"{label_a} / {label_b}", "" + ), + ] + row2 = [ + stat_card( + "Min", f"{stats_a['min']} / {stats_b['min']}", f"{label_a} / {label_b}" + ), + stat_card( + "Max", f"{stats_a['max']} / {stats_b['max']}", f"{label_a} / {label_b}" + ), + stat_card("p5", f"{stats_a['p5']} / {stats_b['p5']}", f"{label_a} / {label_b}"), + stat_card( + "p95", f"{stats_a['p95']} / {stats_b['p95']}", f"{label_a} / {label_b}" + ), + ] + return ( + f'
{"".join(cards)}
' + f'
{"".join(row2)}
' + ) + + +def render_vocab_table(diffs: list[dict], label_a: str, label_b: str) -> str: + if not diffs: + return ( + "

No summary.json found — vocab distribution not available.

" + ) + rows = [] + for d in diffs: + dc = delta_class(d["delta"]) + rows.append( + f"{d['id']}{d['a']:,}{d['b']:,}" + f"{d['delta']:+,}" + ) + return ( + f"" + f"" + f"{''.join(rows)}
Token ID{label_a}{label_b}Delta (B−A)
" + ) + + +def render_tokenization_mode_table( + rows_a: list[dict], rows_b: list[dict], label_a: str, label_b: str +) -> str: + def mode_counts(rows: list[dict]) -> dict[str, int]: + c: dict[str, int] = collections.Counter() + for r in rows: + raw = r.get("tokenization_mode") + c[raw if isinstance(raw, str) else "unknown"] += 1 + return dict(c) + + ca, cb = mode_counts(rows_a), mode_counts(rows_b) + modes = sorted(set(ca) | set(cb)) + rows_html = [] + for m in modes: + a_pct = ca.get(m, 0) / len(rows_a) * 100 if rows_a else 0 + b_pct = cb.get(m, 0) / len(rows_b) * 100 if rows_b else 0 + rows_html.append( + f"{html.escape(m)}{ca.get(m, 0):,} ({a_pct:.1f}%)" + f"{cb.get(m, 0):,} ({b_pct:.1f}%)" + ) + return ( + f"" + f"{''.join(rows_html)}
Mode{label_a}{label_b}
" + ) + + +def build_html( + rows_a: list[dict], + rows_b: list[dict], + summary_a: dict | None, + summary_b: dict | None, + label_a: str, + label_b: str, + path_a: str, + path_b: str, +) -> str: + isl_a = _finite_vals(rows_a, "isl") + isl_b = _finite_vals(rows_b, "isl") + osl_a = _finite_vals(rows_a, "requested_osl") + osl_b = _finite_vals(rows_b, "requested_osl") + + st_isl_a = field_stats(rows_a, "isl") + st_isl_b = field_stats(rows_b, "isl") + st_osl_a = field_stats(rows_a, "requested_osl") + st_osl_b = field_stats(rows_b, "requested_osl") + + _MAX_BINS = 500 + + def hist_bounds(va: list, vb: list, base_step: int) -> tuple[int, int, int]: + # Recorded values may be floats (e.g. "max_tokens": 100.0); range() needs ints. + lo = int(min(va + vb) // base_step) * base_step + hi = int(max(va + vb) // base_step + 2) * base_step + n_bins = (hi - lo) // base_step + step = base_step * max(1, math.ceil(n_bins / _MAX_BINS)) + hi = lo + step * math.ceil((hi - lo) / step) + return lo, hi, step + + isl_lo, isl_hi, isl_step = hist_bounds(isl_a or [0], isl_b or [0], 2) + osl_lo, osl_hi, osl_step = hist_bounds(osl_a or [0], osl_b or [0], 2) + + isl_bins = list(range(isl_lo, isl_hi, isl_step)) + osl_bins = list(range(osl_lo, osl_hi, osl_step)) + + isl_a_pct = histogram(isl_a, isl_lo, isl_hi, isl_step) + isl_b_pct = histogram(isl_b, isl_lo, isl_hi, isl_step) + osl_a_pct = histogram(osl_a, osl_lo, osl_hi, osl_step) + osl_b_pct = histogram(osl_b, osl_lo, osl_hi, osl_step) + + vocab_diffs = vocab_top_diffs(summary_a, summary_b) + + la = html.escape(label_a) + lb = html.escape(label_b) + pa = html.escape(path_a) + pb = html.escape(path_b) + + return f""" + + + +Recording comparison: {la} vs {lb} + + + +

Recording comparison: {la} vs {lb}

+
+ A: {pa}
+ B: {pb} +
+ +

Input Sequence Length (ISL)

+{render_stat_section(st_isl_a, st_isl_b, la, lb, "isl")} +
+
ISL Distribution (2-token bins)
+
+ Blue = {la}  |  Red = {lb}  |  Overlapping bars share color +
+ +
+{_js_hist("hist-isl", isl_bins, isl_a_pct, isl_b_pct, "ISL (tokens)", label_a, label_b)} + +

Requested Output Sequence Length (OSL)

+{render_stat_section(st_osl_a, st_osl_b, la, lb, "requested_osl")} +
+
Requested OSL Distribution (2-token bins)
+
+ Blue = {la}  |  Red = {lb}  |  Overlapping bars share color +
+ +
+{_js_hist("hist-osl", osl_bins, osl_a_pct, osl_b_pct, "Requested OSL (tokens)", label_a, label_b)} + +

Tokenization Mode

+{render_tokenization_mode_table(rows_a, rows_b, la, lb)} + +

Vocab Distribution — Top {len(vocab_diffs)} token-count differences

+{render_vocab_table(vocab_diffs, la, lb)} + + +""" + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--a", required=True, metavar="FILE", help="First recording JSONL (A)" + ) + ap.add_argument( + "--b", required=True, metavar="FILE", help="Second recording JSONL (B)" + ) + ap.add_argument( + "--label-a", + default="A", + metavar="LABEL", + help="Display label for file A (default: A)", + ) + ap.add_argument( + "--label-b", + default="B", + metavar="LABEL", + help="Display label for file B (default: B)", + ) + ap.add_argument( + "--out", + default="comparison.html", + metavar="FILE", + help="Output HTML file (default: comparison.html)", + ) + args = ap.parse_args() + + path_a, path_b = Path(args.a), Path(args.b) + for p in (path_a, path_b): + if not p.exists(): + sys.exit(f"File not found: {p}") + + out = Path(args.out) + protected = {p.resolve() for p in (path_a, path_b)} + for p in (path_a, path_b): + protected.add(Path(str(p) + ".summary.json").resolve()) + protected.add(p.with_suffix("").with_suffix(".summary.json").resolve()) + if out.resolve() in protected: + sys.exit(f"--out must not point at an input or companion summary file: {out}") + + print(f"Loading {path_a} ...", file=sys.stderr) + rows_a = load_jsonl(path_a) + print(f"Loading {path_b} ...", file=sys.stderr) + rows_b = load_jsonl(path_b) + summary_a = load_summary(path_a) + summary_b = load_summary(path_b) + + if summary_a: + print(f" summary found: {path_a}.summary.json", file=sys.stderr) + if summary_b: + print(f" summary found: {path_b}.summary.json", file=sys.stderr) + + print(f"Rows: A={len(rows_a)}, B={len(rows_b)}", file=sys.stderr) + + html = build_html( + rows_a, + rows_b, + summary_a, + summary_b, + args.label_a, + args.label_b, + str(path_a), + str(path_b), + ) + + out.write_text(html) + print(f"Written: {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() From ec7fc376efdb21ff8c8234ce589af05d547cd694 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 17 Sep 2026 15:14:10 -0700 Subject: [PATCH 060/119] fix(ci): fetch nightly job logs via REST instead of gh api (#1402) Signed-off-by: Anthony Casagrande (cherry picked from commit 5b7f283988aae3c5e87920db6a6d977a17f6a4b2) --- tools/suggest_test_docs_weights.py | 171 +++++++++++++++++++++++++++-- 1 file changed, 161 insertions(+), 10 deletions(-) diff --git a/tools/suggest_test_docs_weights.py b/tools/suggest_test_docs_weights.py index bb4fbdc834..ecb32a9cb4 100755 --- a/tools/suggest_test_docs_weights.py +++ b/tools/suggest_test_docs_weights.py @@ -40,15 +40,21 @@ from __future__ import annotations import argparse +import http.client import json import logging import os import re import subprocess import sys +import urllib.error +import urllib.request from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass +from functools import lru_cache from pathlib import Path +from typing import IO # Reach the existing parser/data_types so we don't duplicate the bash-block # extraction logic. Both modules import each other by bare name, so we have @@ -193,24 +199,153 @@ def fetch_test_docs_jobs(run_id: int) -> list[tuple[int, str]]: return out -def fetch_job_log(job_id: int) -> str: - return _gh("api", f"/repos/{_repo_slug()}/actions/jobs/{job_id}/logs") +HTTP_TIMEOUT_SECONDS = 30 +class _StripAuthOnRedirect(urllib.request.HTTPRedirectHandler): + """Follow the 302 to the log blob, but drop the GitHub bearer token. + + ``urlopen`` replays the Authorization header on every hop, and the + pre-signed blob-storage URL holding the log rejects a GitHub token + with 401. + """ + + def redirect_request( + self, + req: urllib.request.Request, + fp: IO[bytes], + code: int, + msg: str, + headers: http.client.HTTPMessage, + newurl: str, + ) -> urllib.request.Request | None: + new = super().redirect_request(req, fp, code, msg, headers, newurl) + if new is not None: + new.remove_header("Authorization") + return new + + +def _host_from_url(url: str) -> str | None: + """Host portion of a git remote or server URL, without scheme, user, or port.""" + m = re.match(r"(?:[\w+.-]+://)?(?:[^@/]+@)?([^/:]+)", url.strip()) + return m.group(1) if m else None + + +def _gh_host() -> str: + """Host owning the target repo, enterprise-aware, ``github.com`` by default.""" + host = os.environ.get("GH_HOST") + if host: + return host + for url in (os.environ.get("GITHUB_SERVER_URL"), _git_remote_url()): + host = _host_from_url(url) if url else None + if host: + return host + return "github.com" + + +def _api_base(host: str) -> str: + base = os.environ.get("GITHUB_API_URL") + if base: + return base.rstrip("/") + return ( + "https://api.github.com" if host == "github.com" else f"https://{host}/api/v3" + ) + + +def _gh_token(host: str) -> str: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + return token.strip() + return _gh("auth", "token", "--hostname", host).strip() + + +@dataclass(frozen=True) +class GhContext: + """Per-run constants for the REST log fetch, resolved once before any request.""" + + api_base: str + repo_slug: str + token: str + opener: urllib.request.OpenerDirector + + +def resolve_gh_context() -> GhContext: + host = _gh_host() + return GhContext( + api_base=_api_base(host), + repo_slug=_repo_slug(), + token=_gh_token(host), + opener=urllib.request.build_opener(_StripAuthOnRedirect), + ) + + +def _decode_log(raw: bytes, job_id: int) -> str: + text = raw.decode("utf-8", errors="replace") + if "\ufffd" in text: + log.warning( + "Job %d log contained undecodable bytes; substituted U+FFFD. " + "Test-run counts for this shard may be incomplete.", + job_id, + ) + return text + + +def fetch_job_log(job_id: int, ctx: GhContext) -> str: + """Fetch one job's raw text log over REST rather than ``gh api``. + + Job logs contain ANSI escape sequences, which gh refuses to emit + without ``--allow-escape-sequences`` (gh 2.97+); going direct keeps + this working on whatever gh a contributor has installed. + """ + url = f"{ctx.api_base}/repos/{ctx.repo_slug}/actions/jobs/{job_id}/logs" + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {ctx.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with ctx.opener.open(req, timeout=HTTP_TIMEOUT_SECONDS) as resp: + return _decode_log(resp.read(), job_id) + except urllib.error.HTTPError as e: + # The log endpoint 302s to blob storage; report whichever hop failed. + failed_url = e.geturl() or url + try: + with e: + detail = e.read().decode("utf-8", errors="replace").strip() + except OSError as read_err: + detail = f"" + raise RuntimeError( + f"GET {failed_url} failed for job {job_id}: " + f"HTTP {e.code} {e.reason}: {detail}" + ) from e + except (urllib.error.URLError, TimeoutError) as e: + raise RuntimeError(f"GET {url} failed for job {job_id}: {e}") from e + + +@lru_cache(maxsize=1) +def _git_remote_url() -> str | None: + try: + return subprocess.check_output( + ["git", "config", "--get", "remote.origin.url"], text=True + ).strip() + except (subprocess.CalledProcessError, OSError): + return None + + +@lru_cache(maxsize=1) def _repo_slug() -> str: slug = os.environ.get("GITHUB_REPOSITORY") if slug: return slug # Best-effort: parse from git remote - try: - url = subprocess.check_output( - ["git", "config", "--get", "remote.origin.url"], text=True - ).strip() + url = _git_remote_url() + if url: m = re.search(r"[:/]([\w-]+/[\w-]+?)(?:\.git)?$", url) if m: return m.group(1) - except subprocess.CalledProcessError: - pass raise RuntimeError( "Cannot determine repo slug; set $GITHUB_REPOSITORY or run inside a git checkout." ) @@ -417,9 +552,25 @@ def main(argv: list[str] | None = None) -> int: return 1 runs: list[TestRun] = [] - for job_id, job_name in jobs: + log.info("Fetching logs for %d shards...", len(jobs)) + ctx = resolve_gh_context() + + def _fetch(job: tuple[int, str]) -> str | None: + job_id, job_name = job + log.info("Fetching log for shard %s (job %d)...", job_name, job_id) + try: + return fetch_job_log(job_id, ctx) + except (RuntimeError, OSError) as e: + # Informational report: one unreachable shard must not discard the rest. + log.warning("Skipping shard %s (job %d): %s", job_name, job_id, e) + return None + + with ThreadPoolExecutor(max_workers=min(8, len(jobs))) as pool: + logs = list(pool.map(_fetch, jobs)) + for (job_id, job_name), log_text in zip(jobs, logs, strict=True): + if log_text is None: + continue log.info("Parsing log for shard %s (job %d)...", job_name, job_id) - log_text = fetch_job_log(job_id) runs.extend(parse_shard_log(log_text, job_id=job_id, job_name=job_name)) log.info("Collected %d test-run records.", len(runs)) From 7e89d33f77a0e5590d8f50fd52eb0ed0daf72610 Mon Sep 17 00:00:00 2001 From: Nate Mailhot Date: Thu, 17 Sep 2026 15:17:50 -0700 Subject: [PATCH 061/119] fix(security): bump distroless base image to 3.13-v4.1.4-dev (#1441) (cherry picked from commit cf66eaf86c59f581a0b0824a4925f0ac42cc14dd) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eee6984c9a..181e0841ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -286,7 +286,7 @@ ENTRYPOINT ["/bin/bash", "-c"] ############################################ ############# Runtime Image ################ ############################################ -FROM nvcr.io/nvidia/distroless/python:3.13-v4.0.8-dev AS runtime +FROM nvcr.io/nvidia/distroless/python:3.13-v4.1.4-dev AS runtime # Include project license and asset attributions COPY LICENSE ATTRIBUTIONS.md /legal/ From 1b3dd7439d62546a0ebbccdad9a1bdbb58b856c4 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Sat, 19 Sep 2026 11:46:03 -0700 Subject: [PATCH 062/119] feat(dataset): restore SPEED-Bench as a first-class --public-dataset (#1387) Signed-off-by: lkomali (cherry picked from commit e34f7e59c2df5bb20042ed99cda36b5d4c6d6733) --- .gitattributes | 14 + .pre-commit-config.yaml | 8 +- docs/tutorials/speed-bench.md | 106 +- pyproject.toml | 17 +- src/aiperf/cli_runner/__init__.py | 4 + src/aiperf/cli_runner/_preflight.py | 116 ++ src/aiperf/config/dataset/config.py | 3 + src/aiperf/config/dataset/resolver.py | 2 + src/aiperf/config/flags/cli_config.py | 2 + src/aiperf/config/loader/parsing.py | 53 + .../config/templates/speed_bench_sweep.yaml | 9 +- src/aiperf/dataset/composer/custom.py | 2 + src/aiperf/dataset/composer/public.py | 59 +- .../dataset/loader/base_public_dataset.py | 48 +- src/aiperf/dataset/loader/speed_bench.py | 112 +- .../dataset/loader/speed_bench_public.py | 577 +++++++++ src/aiperf/dataset/loader/vendor/README.md | 50 + src/aiperf/dataset/loader/vendor/__init__.py | 3 + .../loader/vendor/speed_bench_prepare.py | 636 +++++++++ src/aiperf/plugin/enums.py | 4 +- src/aiperf/plugin/plugins.py | 13 + src/aiperf/plugin/plugins.yaml | 545 +++++--- src/aiperf/plugin/schema/plugins.schema.json | 2 +- src/aiperf/plugin/schema/schemas.py | 9 + .../unit/cli_runner/test_dataset_preflight.py | 258 ++++ .../dataset/composer/test_custom_composer.py | 15 - .../composer/test_custom_system_prompt.py | 3 +- .../dataset/composer/test_public_composer.py | 52 + tests/unit/dataset/loader/test_can_load.py | 134 +- tests/unit/dataset/loader/test_speed_bench.py | 171 ++- .../dataset/loader/test_speed_bench_public.py | 1132 +++++++++++++++++ .../test_vendored_speed_bench_prepare.py | 77 ++ tools/check_ergonomics.py | 4 +- 33 files changed, 3952 insertions(+), 288 deletions(-) create mode 100644 .gitattributes create mode 100644 src/aiperf/dataset/loader/speed_bench_public.py create mode 100644 src/aiperf/dataset/loader/vendor/README.md create mode 100644 src/aiperf/dataset/loader/vendor/__init__.py create mode 100644 src/aiperf/dataset/loader/vendor/speed_bench_prepare.py create mode 100644 tests/unit/cli_runner/test_dataset_preflight.py create mode 100644 tests/unit/dataset/loader/test_speed_bench_public.py create mode 100644 tests/unit/dataset/loader/test_vendored_speed_bench_prepare.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..00830ba2be --- /dev/null +++ b/.gitattributes @@ -0,0 +1,14 @@ +# Normalize line endings to LF on checkout, including on Windows, where +# git's default core.autocrlf=true otherwise converts text files to CRLF. +# tests/ui/test_operator_*.py string/regex-strips import lines out of +# src/aiperf/operator/ui/lib/*.js before eval'ing them as data: URL modules; +# that stripping assumes LF and silently no-ops on CRLF, leaving the original +# relative import in place and breaking module resolution on Windows CI. +* text=auto eol=lf + +# Vendored third-party sources are checked out byte-for-byte on every platform. +# Their value is being identical to a pinned upstream commit, and CI asserts a +# sha256 over the raw bytes -- newline conversion on Windows checkouts would +# change the digest without changing the content. This must stay last: later +# patterns win, so the blanket rule above would otherwise re-normalize them. +src/aiperf/dataset/loader/vendor/** -text diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed4e3a5893..843d8949a1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,20 +29,25 @@ repos: - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable + # The vendored SPEED-Bench resolver is byte-pinned to upstream by a + # sha256 CI gate. Any whitespace or newline rewrite breaks that pin. - id: end-of-file-fixer types_or: [cuda, proto, textproto, python] + exclude: ^src/aiperf/dataset/loader/vendor/ - id: mixed-line-ending + exclude: ^src/aiperf/dataset/loader/vendor/ - id: no-commit-to-branch args: [--branch, main] - id: requirements-txt-fixer - id: trailing-whitespace + exclude: ^src/aiperf/dataset/loader/vendor/ - repo: https://github.com/codespell-project/codespell rev: v2.4.2 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] - exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$|.*shakespeare.txt|ATTRIBUTIONS-.*\.md) + exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$|.*shakespeare.txt|ATTRIBUTIONS-.*\.md|src/aiperf/dataset/loader/vendor/.*) - repo: local hooks: - id: add-license @@ -50,6 +55,7 @@ repos: entry: make add-copyright language: python require_serial: true + exclude: ^src/aiperf/dataset/loader/vendor/ - id: generate-cli-docs name: generate-cli-docs entry: make generate-cli-docs diff --git a/docs/tutorials/speed-bench.md b/docs/tutorials/speed-bench.md index 02b9594bfc..38c5b641c5 100644 --- a/docs/tutorials/speed-bench.md +++ b/docs/tutorials/speed-bench.md @@ -80,21 +80,57 @@ Where `{ISL}` is one of: `1k`, `2k`, `8k`, `16k`, `32k`. --- -## Prepare the Dataset +## Running SPEED-Bench -NOTICE: This dataset is governed by the [NVIDIA Evaluation Dataset License Agreement](https://huggingface.co/datasets/nvidia/SPEED-Bench/blob/main/License.pdf). For each dataset a user elects to use, the user is responsible for checking if the dataset license is fit for the intended purpose. The prepare data script below automatically fetches data from all the source datasets. - -You should first download and prepare the dataset using the following one liner: +Select any SPEED-Bench dataset with `--public-dataset`; AIPerf fetches and +resolves it for you: ```bash -SPEED_BENCH_DIR="./datasets/speed-bench" -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py | python3 - --output_dir $SPEED_BENCH_DIR +aiperf profile --public-dataset speed_bench_coding --model --url ``` -This will download all splits into the working directory as JSONL files. Other supported options of the prepare script: +### One-time setup + +SPEED-Bench publishes a placeholder instead of prompt text for rows whose +source datasets do not permit redistribution, so AIPerf refetches that text +from the 14 original sources on first use. + +One source, [`cais/hle`](https://huggingface.co/datasets/cais/hle), is gated +and appears in **every** SPEED-Bench config, so access is required: + +1. Open and accept the terms. + Approval is automatic — no reviewer, no waiting period. +2. Run `hf auth login`. + +AIPerf checks this before downloading anything and fails in about a second with +these instructions if access is missing. + +**Access is granted per user, not per organization.** Everyone running +SPEED-Bench needs their own HuggingFace account and their own click; it cannot +be accepted once on a team's behalf. In CI, use a service account onboarded +once by a human and store its token as a secret. -* `--config`: select which config to prepare, can be one of the splits in the dataset (e.g., `qualitative`, `throughput_2k`) or `all` to prepare all of the configs. -* `--output_dir`: select different output directory to download the dataset to. +**Budget disk and time for the first run.** Resolving a config downloads its +source datasets in full -- around 40 GB for `qualitative` -- into the HuggingFace +cache (`~/.cache/huggingface/hub`), not into AIPerf's cache. What AIPerf writes +is only the extracted prompt text, which is far smaller: roughly 2 MB for +`qualitative`, up to ~200 MB for `throughput_32k`. A small resolved file after a +large download is expected, not a sign of a truncated dataset -- AIPerf logs the +row count and file size when it finishes so you can confirm. The result is +cached under +`.cache/aiperf/datasets/speed-bench/.jsonl` and shared by every +category of that config, so only the first run pays for it. Delete the file to +refetch. + +That path is relative to the working directory you launch `aiperf` from, which +every AIPerf public dataset shares. Running from a different directory resolves +the config again from scratch, so run from a consistent directory -- or +pre-stage the file as shown below -- if you want the cache to be reused. + +Resolution runs before AIPerf starts its services, so the download is not part +of any measured run. + +NOTICE: This dataset is governed by the [NVIDIA Evaluation Dataset License Agreement](https://huggingface.co/datasets/nvidia/SPEED-Bench/blob/main/License.pdf). For each dataset a user elects to use, the user is responsible for checking if the dataset license is fit for the intended purpose. Resolution automatically fetches data from all the source datasets. > [!IMPORTANT] > SPEED-Bench stores prompts as references to their upstream source datasets, and the @@ -119,6 +155,25 @@ This will download all splits into the working directory as JSONL files. Other s --- +## Pre-staging the dataset (air-gapped or CI) + +Resolution writes a single JSONL per config under +`.cache/aiperf/datasets/speed-bench/`. To avoid every machine resolving it -- +or to run on a host with no internet and no HuggingFace credentials -- resolve +once somewhere connected and copy the file: + +```bash +# on a connected machine, with cais/hle terms accepted +aiperf profile --public-dataset speed_bench_qa --model --url + +# then copy the cache to the target machine +scp .cache/aiperf/datasets/speed-bench/qualitative.jsonl \ + target:/path/to/workdir/.cache/aiperf/datasets/speed-bench/ +``` + +AIPerf uses a cached config as-is, so the target machine needs neither network +access nor an accepted licence. One file covers every category of that config. + ## Start a Server with Speculative Decoding Launch an inference server with speculative decoding enabled. On vLLM, add `--per-request-spec-decode-metrics summary` so each response carries its own acceptance stats: @@ -172,9 +227,7 @@ aiperf profile \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ - --custom-dataset-type speed_bench_qualitative \ - --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ - --request-count $(jq -s '[.[].messages | length] | add' ${SPEED_BENCH_DIR}/qualitative.jsonl) \ + --public-dataset speed_bench_qualitative \ --osl 4096 \ --extra-inputs temperature:0 \ --concurrency 16 \ @@ -333,9 +386,7 @@ aiperf profile \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ - --custom-dataset-type speed_bench_throughput_1k \ - --input-file ${SPEED_BENCH_DIR}/throughput_1k.jsonl \ - --request-count $(jq -s '[.[].messages | length] | add' ${SPEED_BENCH_DIR}/throughput_1k.jsonl) \ + --public-dataset speed_bench_throughput_1k \ --extra-inputs temperature:0 \ --concurrency 64 \ --output-artifact-dir ./artifacts/speed_bench_throughput_1k @@ -452,19 +503,13 @@ MODEL="meta-llama/Llama-3.1-8B-Instruct" for cat in $CATEGORIES; do echo "=== Running category: $cat ===" - # Size each run to that category's turn count, not its 80 rows. - turns=$(jq -s --arg c "$cat" \ - '[.[] | select(.category == $c) | .messages | length] | add' \ - ${SPEED_BENCH_DIR}/qualitative.jsonl) aiperf profile \ --model "$MODEL" \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ - --custom-dataset-type speed_bench_${cat} \ - --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ + --public-dataset speed_bench_${cat} \ --server-metrics http://localhost:8000/metrics \ - --request-count "$turns" \ --osl 4096 \ --extra-inputs temperature:0 \ --concurrency 16 \ @@ -479,8 +524,15 @@ aiperf speed-bench-report ./artifacts/ --source server --format both The same eleven runs are also available as a config-driven sweep template, which drives all eleven variations from a single `aiperf profile --config` invocation: ```bash -aiperf config init --template speed_bench_sweep --output speed_bench_sweep.yaml -aiperf profile --config speed_bench_sweep.yaml +aiperf profile \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --endpoint-type chat \ + --streaming \ + --url localhost:8000 \ + --public-dataset speed_bench_throughput_1k \ + --server-metrics http://localhost:8000/metrics \ + --concurrency 64 \ + --benchmark-duration 120 ``` ### Per-entropy-tier scrape @@ -493,8 +545,7 @@ for tier in low_entropy mixed high_entropy; do --endpoint-type chat \ --streaming \ --url localhost:8000 \ - --custom-dataset-type "speed_bench_throughput_1k_${tier}" \ - --input-file ${SPEED_BENCH_DIR}/throughput_1k.jsonl \ + --public-dataset "speed_bench_throughput_1k_${tier}" \ --server-metrics http://localhost:8000/metrics \ --concurrency 64 \ --benchmark-duration 60 @@ -511,8 +562,7 @@ aiperf profile \ --endpoint-type chat \ --streaming \ --url localhost:8000 \ - --custom-dataset-type speed_bench_qualitative \ - --input-file ${SPEED_BENCH_DIR}/qualitative.jsonl \ + --public-dataset speed_bench_qualitative \ --no-server-metrics \ --concurrency 16 ``` diff --git a/pyproject.toml b/pyproject.toml index 5dd141a64a..8624ce7e11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,7 +170,13 @@ required-environments = [ # Base settings for ruff line-length = 88 indent-width = 4 -exclude = ["__pycache__", "build", "dist", ".venv", "venv", "notebooks/TCO_calculator.ipynb"] +exclude = ["__pycache__", "build", "dist", ".venv", "venv", "notebooks/TCO_calculator.ipynb", + # Verbatim third-party sources; see src/aiperf/dataset/loader/vendor/README.md. + # Linting them would pressure us to edit files whose value is being unmodified. + "src/aiperf/dataset/loader/vendor"] +# Honor `exclude` even when a path is passed explicitly (pre-commit passes +# changed files by name), so formatting cannot rewrite vendored sources. +force-exclude = true [tool.ruff.lint] select = [ @@ -217,6 +223,15 @@ max-complexity = 10 "json.dump".msg = "Use orjson.dumps instead (no stdlib-style file writing — serialize then write bytes)." "json.load".msg = "Use orjson.loads on file bytes instead." +[tool.coverage.run] +omit = [ + # Verbatim third-party sources: see src/aiperf/dataset/loader/vendor/README.md. + # Measuring coverage of an unmodified upstream file is meaningless -- AIPerf + # exercises a narrow entry point into it, and the number would only ever + # invite edits to a file whose value is being byte-identical to upstream. + "src/aiperf/dataset/loader/vendor/*", +] + [tool.pytest.ini_options] # Basic pytest configuration testpaths = ["tests"] diff --git a/src/aiperf/cli_runner/__init__.py b/src/aiperf/cli_runner/__init__.py index dbabf1479d..4956b6896a 100644 --- a/src/aiperf/cli_runner/__init__.py +++ b/src/aiperf/cli_runner/__init__.py @@ -19,6 +19,8 @@ from aiperf.cli_runner._preflight import ( _preflight_accuracy_deps, _preflight_artifact_dir, + _preflight_dataset_access, + _preflight_dataset_materialize, _preflight_endpoint_ready, _preflight_fd_limit, ) @@ -61,7 +63,9 @@ def run_benchmark(plan: BenchmarkPlan) -> None: _preflight_artifact_dir(plan) _preflight_accuracy_deps(plan) _preflight_fd_limit() + _preflight_dataset_access(plan) _preflight_endpoint_ready(plan) + _preflight_dataset_materialize(plan) callbacks: list[OnComplete] = [] if plan.configs[0].artifacts.auto_plot: diff --git a/src/aiperf/cli_runner/_preflight.py b/src/aiperf/cli_runner/_preflight.py index 7ac8eb78dd..689f24953e 100644 --- a/src/aiperf/cli_runner/_preflight.py +++ b/src/aiperf/cli_runner/_preflight.py @@ -213,3 +213,119 @@ def _preflight_endpoint_ready(plan: BenchmarkPlan) -> None: headers=headers, ) ) + + +def _public_dataset_loaders(plan: BenchmarkPlan): + """Yield ``(LoaderClass, loader_kwargs)`` for each public dataset in the plan.""" + yield from _public_dataset_loaders_for_configs(plan.configs) + + +def _public_dataset_loaders_for_configs(configs): + """Yield ``(LoaderClass, loader_kwargs)`` for each public dataset in ``configs``. + + Reads plugin metadata directly rather than going through the composer, + which needs a live run. Only the fields the preflight hooks act on are + forwarded, and the yielded pairs are deduplicated by those fields rather + than by selector name: a category sweep names 11 distinct selectors that + all resolve the same underlying config, and keying on the selector would + make preflight issue 11 identical auth probes before the run starts. + """ + from aiperf.config.dataset import PublicDataset + from aiperf.plugin import plugins + from aiperf.plugin.enums import PluginType + + seen: set[tuple[type, str | None]] = set() + for cfg in configs: + dataset = cfg.get_default_dataset() + if not isinstance(dataset, PublicDataset): + continue + + LoaderClass = plugins.get_class( + PluginType.PUBLIC_DATASET_LOADER, dataset.dataset + ) + metadata = plugins.get_public_dataset_loader_metadata(dataset.dataset) + hf_subset = dataset.hf_subset or metadata.hf_subset + + key = (LoaderClass, hf_subset) + if key in seen: + continue + seen.add(key) + + yield LoaderClass, ({"hf_subset": hf_subset} if hf_subset else {}) + + +def _call_optional_hook( + loader_class: type, name: str, kwargs: dict[str, object] +) -> None: + """Invoke a preflight hook when the loader defines one. + + ``PublicDatasetLoaderProtocol`` does not require these hooks, so a loader + registered by an external plugin need not implement them. Absence means + "nothing to prepare", matching how ``_preflight_accuracy_deps`` treats the + optional ``check_available`` hook. + """ + hook = getattr(loader_class, name, None) + if callable(hook): + hook(**kwargs) + + +def _preflight_dataset_access(plan: BenchmarkPlan) -> None: + """Verify gated public datasets are reachable before anything expensive. + + Why: access to a gated dataset is granted per user through a browser, so it + cannot be fixed mid-run. Probing first turns a multi-GB download that ends + in a 403 -- or a wait on a model that was never going to be used -- into an + immediate, actionable message. + """ + for LoaderClass, kwargs in _public_dataset_loaders(plan): + _call_optional_hook(LoaderClass, "preflight_access", kwargs) + + +def _preflight_dataset_materialize(plan: BenchmarkPlan) -> None: + """Fetch and cache public datasets that need resolution before services start. + + Why: ``DatasetManager`` performs dataset setup while ``TimingManager`` + blocks on the profiling handshake, so a large download there trips + ``AIPERF_DATASET_CONFIGURATION_TIMEOUT`` (and raising that limit would + weaken hang detection for every run, since ``PROFILE_CONFIGURE_TIMEOUT`` is + constrained to be at least as large). Runs after the endpoint probe so an + unreachable server fails in seconds rather than after the download. + """ + import logging + + # Same reason as _preflight_endpoint_ready: rich logging isn't installed + # yet, and a multi-GB fetch with no output looks like a hang. That check + # returns early when endpoint waiting is disabled, so don't rely on it + # having installed the handler. + if not logging.getLogger().handlers: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + # Root INFO also switches on httpx's per-request logging, and resolution + # makes hundreds of requests -- including the 404s `datasets` expects + # while probing for a legacy loading script before falling back to + # parquet. That output buries the progress message this handler exists + # to show and reads like a failing run. Only narrowed when we installed + # the handler ourselves, so a caller's own logging config still wins. + for noisy in ("httpx", "httpcore", "urllib3", "filelock"): + logging.getLogger(noisy).setLevel(logging.WARNING) + + for LoaderClass, kwargs in _public_dataset_loaders(plan): + _call_optional_hook(LoaderClass, "preflight_materialize", kwargs) + + +def preflight_public_datasets_for_run(run) -> None: + """Run the public-dataset preflights for a single serialized ``BenchmarkRun``. + + The Kubernetes ``aiperf service`` entry point never calls ``run_benchmark``, + so without this the gate probe and the resolution download would both fall + inside ``DatasetManager``'s configure step and trip + ``AIPERF_DATASET_CONFIGURATION_TIMEOUT``. Only the dataset_manager container + calls this: it is the sole service that composes the dataset, and every + other pod would download the same multi-GB resolution into a private + filesystem for nothing. + """ + for LoaderClass, kwargs in _public_dataset_loaders_for_configs([run.cfg]): + _call_optional_hook(LoaderClass, "preflight_access", kwargs) + _call_optional_hook(LoaderClass, "preflight_materialize", kwargs) diff --git a/src/aiperf/config/dataset/config.py b/src/aiperf/config/dataset/config.py index ccb73fff88..52b2c290a0 100644 --- a/src/aiperf/config/dataset/config.py +++ b/src/aiperf/config/dataset/config.py @@ -18,6 +18,7 @@ from typing import Annotated, Any, ClassVar, Literal from pydantic import ( + BeforeValidator, ConfigDict, Discriminator, Field, @@ -49,6 +50,7 @@ VideoConfig, ) from aiperf.config.loader.normalizers import _hoist_synthetic_prompt_fields +from aiperf.config.loader.parsing import reject_moved_custom_dataset_type from aiperf.config.types import SamplingDistribution from aiperf.plugin.enums import DatasetSamplingStrategy, PublicDatasetType @@ -327,6 +329,7 @@ class FileDataset(SystemPromptMixin): format: Annotated[ DatasetFormat, + BeforeValidator(reject_moved_custom_dataset_type), Field( default=DatasetFormat.SINGLE_TURN, description="Dataset file format determining parsing logic and expected file structure. " diff --git a/src/aiperf/config/dataset/resolver.py b/src/aiperf/config/dataset/resolver.py index b3a500e309..74414bc6b4 100644 --- a/src/aiperf/config/dataset/resolver.py +++ b/src/aiperf/config/dataset/resolver.py @@ -359,6 +359,8 @@ def _detect_type( # Structural detection detected = None for entry, LoaderClass in plugins.iter_all(PluginType.CUSTOM_DATASET_LOADER): + if not plugins.is_autodetectable(entry): + continue if LoaderClass.can_load(data, file_path): if detected is not None: _logger.warning( diff --git a/src/aiperf/config/flags/cli_config.py b/src/aiperf/config/flags/cli_config.py index 6051ad1103..fcfff501cf 100644 --- a/src/aiperf/config/flags/cli_config.py +++ b/src/aiperf/config/flags/cli_config.py @@ -70,6 +70,7 @@ parse_str_or_dict_as_tuple_list, parse_str_or_list, parse_str_or_list_of_positive_values, + reject_moved_custom_dataset_type, require_turn_mean_at_least_one, ) from aiperf.config.runtime import ServiceDefaults @@ -608,6 +609,7 @@ def url(self) -> str: custom_dataset_type: Annotated[ CustomDatasetType | None, + BeforeValidator(reject_moved_custom_dataset_type), Field( description="Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. " "Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), " diff --git a/src/aiperf/config/loader/parsing.py b/src/aiperf/config/loader/parsing.py index 20a851c12d..95f0dead63 100644 --- a/src/aiperf/config/loader/parsing.py +++ b/src/aiperf/config/loader/parsing.py @@ -461,3 +461,56 @@ def require_turn_mean_at_least_one(value: Any) -> Any: "use --conversation-turn-mean 1 or omit it for single-turn conversations." ) return value + + +# SPEED-Bench category and entropy-tier selectors moved from +# --custom-dataset-type to --public-dataset, where AIPerf resolves the dataset +# itself instead of requiring a separately prepared file. The names were the +# only documented way to run those subsets, so a bare enum error would strand +# anyone with an existing script. +_MOVED_TO_PUBLIC_DATASET = frozenset( + [ + f"speed_bench_{c}" + for c in ( + "coding", + "humanities", + "math", + "multilingual", + "qa", + "rag", + "reasoning", + "roleplay", + "stem", + "summarization", + "writing", + ) + ] + + [ + f"speed_bench_throughput_{isl}_{tier}" + for isl in ("1k", "2k", "8k", "16k", "32k") + for tier in ("low_entropy", "mixed", "high_entropy") + ] +) + + +def reject_moved_custom_dataset_type(input: Any) -> Any: + """Point relocated dataset-format values at their new flag. + + Runs before enum coercion so the message names the replacement command + rather than listing every remaining valid value. Shared by the + ``--custom-dataset-type`` flag and the YAML ``format:`` key, which are two + spellings of the same choice and must migrate identically. + + Raises: + ValueError: If the value moved to ``--public-dataset``. + """ + if isinstance(input, str) and input in _MOVED_TO_PUBLIC_DATASET: + raise ValueError( + f"'{input}' is no longer a custom dataset format " + f"(--custom-dataset-type / YAML 'format:'). SPEED-Bench " + f"subsets are now selected with --public-dataset, and AIPerf " + f"resolves the dataset for you, so --input-file is not needed:\n" + f" aiperf profile --public-dataset {input} ...\n" + f"See docs/tutorials/speed-bench.md." + ) + return input diff --git a/src/aiperf/config/templates/speed_bench_sweep.yaml b/src/aiperf/config/templates/speed_bench_sweep.yaml index dd373c3b70..72c7076492 100644 --- a/src/aiperf/config/templates/speed_bench_sweep.yaml +++ b/src/aiperf/config/templates/speed_bench_sweep.yaml @@ -56,8 +56,8 @@ sweep: type: grid parameters: # Body-rooted dot-path. Second segment matches the dataset's `name` in the - # `datasets:` list below; the leaf `format` is the dataset name. - datasets.main.format: + # `datasets:` list below; the leaf `dataset` selects the public dataset. + datasets.main.dataset: - speed_bench_coding - speed_bench_humanities - speed_bench_math @@ -87,9 +87,8 @@ benchmark: datasets: - name: main - type: file - path: datasets/speed-bench/qualitative.jsonl - format: speed_bench_coding # overridden by sweep + type: public + dataset: speed_bench_coding # overridden by sweep phases: - name: warmup diff --git a/src/aiperf/dataset/composer/custom.py b/src/aiperf/dataset/composer/custom.py index 2887b57bb2..aa2ded7655 100644 --- a/src/aiperf/dataset/composer/custom.py +++ b/src/aiperf/dataset/composer/custom.py @@ -195,6 +195,8 @@ def _infer_type( detected_type = None for entry, LoaderClass in plugins.iter_all(PluginType.CUSTOM_DATASET_LOADER): + if not plugins.is_autodetectable(entry): + continue if LoaderClass.can_load(data, filename): self.info( f"Loader {LoaderClass.__name__} can handle the input file data format." diff --git a/src/aiperf/dataset/composer/public.py b/src/aiperf/dataset/composer/public.py index 33c88b9d42..93e8c69736 100644 --- a/src/aiperf/dataset/composer/public.py +++ b/src/aiperf/dataset/composer/public.py @@ -111,16 +111,27 @@ def _build_loader_kwargs( kwargs["message_content_key"] = loader_metadata.message_content_key if loader_metadata.multi_turn: - if not self._loader_accepts_kwarg(loader_class, "multi_turn"): - raise ValueError( - f"Loader {loader_class.__name__} for dataset {dataset_type!r} " - "does not support the 'multi_turn' metadata flag. Remove " - "'multi_turn: true' from this loader's plugin metadata, or " - "use a loader that declares 'multi_turn' on its constructor " - "(e.g. HFConversationDatasetLoader, SpeedBenchLoader, " - "SpecBenchLoader)." - ) - kwargs["multi_turn"] = True + self._set_opt_in_kwarg( + kwargs, + loader_class, + dataset_type, + name="multi_turn", + value=True, + supporting_loaders=( + "HFConversationDatasetLoader, SpeedBenchPublicLoader, " + "SpecBenchLoader" + ), + ) + + if loader_metadata.category is not None: + self._set_opt_in_kwarg( + kwargs, + loader_class, + dataset_type, + name="category", + value=loader_metadata.category, + supporting_loaders="SpeedBenchPublicLoader", + ) if loader_metadata.streaming: kwargs["streaming"] = loader_metadata.streaming @@ -137,6 +148,34 @@ def _build_loader_kwargs( return kwargs + def _set_opt_in_kwarg( + self, + kwargs: dict[str, Any], + loader_class: type, + dataset_type: str, + *, + name: str, + value: Any, + supporting_loaders: str, + ) -> None: + """Forward an opt-in metadata field, rejecting loaders that can't take it. + + These fields are declared per-entry in plugins.yaml but only a handful of + loaders accept them, so a typo would otherwise surface as a TypeError + from the loader constructor rather than a message naming the entry. + + Raises: + ValueError: If ``loader_class.__init__`` does not declare ``name``. + """ + if not self._loader_accepts_kwarg(loader_class, name): + raise ValueError( + f"Loader {loader_class.__name__} for dataset {dataset_type!r} does " + f"not support the {name!r} metadata field. Remove {name!r} from " + f"this loader's plugin metadata, or use a loader that declares " + f"{name!r} on its constructor (e.g. {supporting_loaders})." + ) + kwargs[name] = value + def _inject_trace_kwargs( self, loader_metadata: Any, kwargs: dict[str, Any] ) -> None: diff --git a/src/aiperf/dataset/loader/base_public_dataset.py b/src/aiperf/dataset/loader/base_public_dataset.py index a76454d0e3..370d5b2da4 100644 --- a/src/aiperf/dataset/loader/base_public_dataset.py +++ b/src/aiperf/dataset/loader/base_public_dataset.py @@ -51,11 +51,26 @@ class BasePublicDatasetLoader(BaseLoader): def __init__(self, run: BenchmarkRun | None = None, **kwargs): super().__init__(run=run, **kwargs) - self.http_client = AioHttpClient( - timeout=Environment.DATASET.PUBLIC_DATASET_TIMEOUT - ) + self._http_client: AioHttpClient | None = None self.cache_filepath = AIPERF_DATASET_CACHE_DIR / self.filename + @property + def http_client(self) -> AioHttpClient: + """HTTP client for URL-backed datasets, created on first use. + + Built lazily because ``AioHttpClient`` opens a ``TCPConnector``, which + requires a running event loop. Constructing it eagerly meant any + synchronous caller -- notably the preflight phase, which runs before + the event loop exists -- crashed with ``RuntimeError: no running event + loop`` merely by instantiating a loader, even one that never fetches + over HTTP. + """ + if self._http_client is None: + self._http_client = AioHttpClient( + timeout=Environment.DATASET.PUBLIC_DATASET_TIMEOUT + ) + return self._http_client + async def load_dataset(self) -> dict[str, Any]: """Load the dataset and convert it to AIPerf Conversation format. @@ -189,6 +204,33 @@ def is_valid_sequence( prompt_too_short or output_too_short or prompt_too_long or combined_too_long ) + @classmethod + def preflight_access(cls, **loader_kwargs: Any) -> None: + """Cheaply verify the dataset is reachable, before any bulk download. + + Runs in the CLI process before services start. Keep it sub-second: its + purpose is to turn "authenticate first" into an immediate answer rather + than one the user receives after a long fetch. Default is a no-op. + + Raises: + ConfigurationError: If the dataset cannot be accessed. + """ + return None + + @classmethod + def preflight_materialize(cls, **loader_kwargs: Any) -> None: + """Fetch and cache whatever this dataset needs before services start. + + For loaders whose first use triggers a large download, doing it here + keeps it out of ``DatasetManager``, where it would block the profiling + handshake and trip ``AIPERF_DATASET_CONFIGURATION_TIMEOUT``. Must be + idempotent -- a populated cache makes it a no-op. Default is a no-op. + + Raises: + ConfigurationError: If the dataset could not be materialized. + """ + return None + @classmethod def get_preferred_sampling_strategy(cls) -> DatasetSamplingStrategy: """Get the preferred sampling strategy for this dataset. diff --git a/src/aiperf/dataset/loader/speed_bench.py b/src/aiperf/dataset/loader/speed_bench.py index c0d98983e4..4429335611 100644 --- a/src/aiperf/dataset/loader/speed_bench.py +++ b/src/aiperf/dataset/loader/speed_bench.py @@ -9,6 +9,7 @@ from pydantic import Field, ValidationError, model_validator +from aiperf.common.exceptions import DatasetLoaderError from aiperf.common.models import AIPerfBaseModel, Conversation from aiperf.dataset.loader.models import MultiTurn, SingleTurn from aiperf.dataset.loader.multi_turn import MultiTurnDatasetLoader @@ -16,6 +17,11 @@ if TYPE_CHECKING: from aiperf.config.resolution.plan import BenchmarkRun +HLE_ACCESS_URL = "https://huggingface.co/datasets/cais/hle" +SPEED_BENCH_DOCS_URL = ( + "https://github.com/ai-dynamo/aiperf/blob/main/docs/tutorials/speed-bench.md" +) + class SpeedBenchRow(AIPerfBaseModel): """Defines the schema for Speed-Bench row data. @@ -40,21 +46,39 @@ class SpeedBenchRow(AIPerfBaseModel): @model_validator(mode="after") def validate_messages_structure(self) -> SpeedBenchRow: - """Validate the messages field structure.""" + """Validate the messages field structure. + + Deliberately shape-only: placeholder content is a *preparation* problem, + not a format problem. Rejecting it here would make ``can_load`` return + False for an unprepared file, so auto-detection would fail with the + unrelated "No loader can handle the data format" instead of the + placeholder guidance :meth:`SpeedBenchLoader.load_dataset` emits. + """ if not all( isinstance(message, dict) and isinstance(message.get("role"), str) and bool(message["role"].strip()) and isinstance(message.get("content"), str) and bool(message["content"].strip()) - and message["content"] != self.TURNS_PLACEHOLDER for message in self.messages ): raise ValueError( - "messages must be a non-empty list of dictionaries with role and content fields, and the content must not be the placeholder string" + "messages must be a non-empty list of dictionaries with role and content fields" ) return self + def has_placeholder_turn(self) -> bool: + """Return whether any message still holds unresolved placeholder text. + + Uses a prefix match because the upstream dataset appends per-row detail + after the placeholder sentinel; upstream's own ``prepare.py`` tests the + same way. An equality check misses 144 of the 494 placeholder rows. + """ + return any( + message["content"].startswith(self.TURNS_PLACEHOLDER) + for message in self.messages + ) + def is_speed_bench_row(data: object) -> bool: """Return whether data matches the SPEED-Bench JSONL row shape.""" @@ -143,6 +167,35 @@ def can_load( """Return whether a JSON object matches the SPEED-Bench JSONL shape.""" return is_speed_bench_row(data) + def _suggested_public_dataset(self) -> str: + """Name the registered ``--public-dataset`` entry replacing this file. + + The split is authoritative rather than ``self.category``, which is None + for every shipped entry and would always suggest the qualitative split + even for a throughput file. Category names are also not composed + uniformly across splits -- qualitative registers ``speed_bench_coding`` + while throughput registers ``speed_bench_throughput_1k_low_entropy`` -- + so a composed name is only used when the registry actually holds it. + """ + from aiperf.plugin.enums import PublicDatasetType + + registered = {member.value for member in PublicDatasetType} + split = getattr(self, "split_filename", None) + split_name = f"speed_bench_{Path(split).stem}" if split else None + + candidates = [] + if self.category: + if split_name: + candidates.append(f"{split_name}_{self.category}") + candidates.append(f"speed_bench_{self.category}") + if split_name: + candidates.append(split_name) + candidates.append("speed_bench_qualitative") + + return next( + (c for c in candidates if c in registered), "speed_bench_qualitative" + ) + def load_dataset(self) -> dict[str, list[MultiTurn]]: """Load SPEED-Bench multi-turn data from a JSONL file. @@ -152,9 +205,10 @@ def load_dataset(self) -> dict[str, list[MultiTurn]]: into a ``SingleTurn(role=..., text=...)``. When ``self.category`` is set, lines whose ``category`` field does not - match are skipped. If the filter eliminates every row, a warning is - emitted to surface a likely category/file mismatch rather than - silently returning an empty dataset. + match are skipped. If the filter eliminates every row, the mismatch is + raised here: an empty dataset is never survivable downstream, and + failing at load time keeps the cause next to the configuration that + produced it rather than surfacing later from another service. When ``self.multi_turn`` is set, all turns in the row are used, otherwise only the first turn is used. @@ -162,14 +216,32 @@ def load_dataset(self) -> dict[str, list[MultiTurn]]: Returns: A dictionary mapping session_id (the SPEED-Bench question_id) to a list of MultiTurn objects. + + Raises: + DatasetLoaderError: If the file still holds unresolved placeholder + text, or if a category filter matches no rows. """ data: dict[str, list[MultiTurn]] = defaultdict(list) + total = 0 + matched = 0 + placeholders = 0 for row in self._iter_record_dicts(): loaded_line = SpeedBenchRow.model_validate(row) + total += 1 + # Filter before counting placeholders: the counts must describe the + # rows the user asked for. Counting whole-file would refuse a fully + # prepared category because some other category is unprepared. if self.category and loaded_line.category != self.category: continue + matched += 1 + + # Count every unprepared row rather than failing on the first, so the + # message reports the true scale of the problem in one run. + if loaded_line.has_placeholder_turn(): + placeholders += 1 + continue messages = ( loaded_line.messages if self.multi_turn else loaded_line.messages[:1] @@ -186,13 +258,29 @@ def load_dataset(self) -> dict[str, list[MultiTurn]]: data[multi_turn_data.session_id].append(multi_turn_data) self._category_by_session[multi_turn_data.session_id] = loaded_line.category + if placeholders: + scope = f"category {self.category!r}" if self.category else "this file" + raise DatasetLoaderError( + f"{self.filename}: {placeholders} of {matched} SPEED-Bench rows in " + f"{scope} still contain placeholder text instead of prompts, so it " + f"cannot be benchmarked. " + f"SPEED-Bench ships without the prompt text for rows whose source " + f"datasets do not permit redistribution. Let AIPerf fetch and " + f"resolve it instead of pointing at a file: " + f"aiperf profile --public-dataset {self._suggested_public_dataset()} ... " + f"One of the sources, cais/hle, is gated: accept its terms at " + f"{HLE_ACCESS_URL} (approval is automatic, and required per user " + f"-- HuggingFace grants access to individuals, not organizations), " + f"then run 'hf auth login'. " + f"Full instructions: {SPEED_BENCH_DOCS_URL}" + ) + if self.category and not data: - self.warning( - lambda: ( - f"SPEED-Bench category filter {self.category!r} matched no rows " - f"in {self.filename}. Verify the configured category exists in " - f"this dataset." - ) + raise DatasetLoaderError( + f"{self.filename}: SPEED-Bench category filter {self.category!r} " + f"matched none of the {total} rows in this file. Verify the category " + f"exists in this split -- the qualitative and throughput splits have " + f"different category names." ) return data diff --git a/src/aiperf/dataset/loader/speed_bench_public.py b/src/aiperf/dataset/loader/speed_bench_public.py new file mode 100644 index 0000000000..90c1b7c759 --- /dev/null +++ b/src/aiperf/dataset/loader/speed_bench_public.py @@ -0,0 +1,577 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +import orjson + +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.common.models import Conversation, Text, Turn +from aiperf.dataset.loader.base_public_dataset import ( + AIPERF_DATASET_CACHE_DIR, + BasePublicDatasetLoader, +) +from aiperf.dataset.loader.speed_bench import SpeedBenchRow +from aiperf.plugin.enums import DatasetSamplingStrategy + +if TYPE_CHECKING: + from aiperf.config.resolution.plan import BenchmarkRun + +SPEED_BENCH_CACHE_DIR = AIPERF_DATASET_CACHE_DIR / "speed-bench" +# ``--hf-subset`` is a user-facing override that becomes a filename. +_CONFIG_NAME_RE = re.compile(r"[A-Za-z0-9_-]+") +HLE_ACCESS_URL = "https://huggingface.co/datasets/cais/hle" + +# Pinned so the rows -- and therefore the source URLs the vendored resolver +# fetches -- cannot change under a run. An unpinned load would let an upstream +# edit redirect resolution at arbitrary hosts. +SPEED_BENCH_REVISION = "487aa718444e816458d1a0a52bfce7a454285cf4" + +# Hosts the published dataset legitimately sources prompt text from. Row +# ``source`` values drive what the vendored resolver fetches, so anything +# outside this set is refused before a request is made. Enumerated from the +# dataset itself across all six configs at SPEED_BENCH_REVISION, not from the +# docs -- see test_allowlist_covers_every_published_source_host, which refetches +# and fails if a revision bump introduces a host not listed here. +_ALLOWED_SOURCE_HOSTS = frozenset( + { + "huggingface.co", + "raw.githubusercontent.com", + "github.com", + "opencompass.openxlab.space", + "www.gutenberg.org", + } +) + + +def validate_config_name(config: str) -> None: + """Reject a config name that would escape the cache directory. + + ``config`` reaches here from ``--hf-subset``, a documented user-facing + override, and is interpolated into a filesystem path. + + Raises: + ConfigurationError: If ``config`` is not a bare SPEED-Bench config name. + """ + from aiperf.config.loader.errors import ConfigurationError + + if not _CONFIG_NAME_RE.fullmatch(config): + raise ConfigurationError( + f"Invalid SPEED-Bench config {config!r}. Expected a bare config " + f"name such as 'qualitative' or 'throughput_1k' (letters, digits, " + f"underscores and hyphens only)." + ) + + +class SpeedBenchPublicLoader(BasePublicDatasetLoader): + """Auto-downloading loader for ``nvidia/SPEED-Bench``. + + SPEED-Bench publishes a placeholder in place of prompt text whose source + dataset does not permit redistribution. This loader resolves those rows by + running the vendored upstream prepare script (see ``vendor/README.md``), + caching the resolved config, and serving categories as views over it. + + Two invariants make the result trustworthy: + + * **Whole-config resolution only.** Upstream consumes a module-global RNG + while reconstructing HLE throughput prompts, so a row's text depends on + how many rows preceded it. Resolving an entire config in upstream's own + row order reproduces its output exactly; resolving a category subset does + not. Categories are therefore filtered *after* resolution, never before. + * **Complete data or an error.** A partially-resolved config is rejected + rather than benchmarked, so a selector name always means the same rows. + """ + + tag = "SPEED-Bench" + url = "" # Resolved via the vendored prepare script, not a single URL. + + _GATED_SOURCE: ClassVar[str] = "cais/hle" + + def __init__( + self, + run: BenchmarkRun | None = None, + *, + hf_dataset_name: str = "nvidia/SPEED-Bench", + hf_split: str = "test", + hf_subset: str, + category: str | None = None, + multi_turn: bool = True, + **kwargs: Any, + ) -> None: + """Initialize the loader. + + Args: + run: The benchmark run this loader belongs to. + hf_dataset_name: HuggingFace dataset identifier. + hf_split: Split to load; SPEED-Bench publishes only ``test``. + hf_subset: SPEED-Bench config to resolve (e.g. ``qualitative``). + category: When set, only rows whose ``category`` matches are used. + multi_turn: When True all turns in a row are used, else the first. + kwargs: Forwarded to the base loader. + """ + self.hf_dataset_name = hf_dataset_name + self.hf_split = hf_split + self.config = hf_subset + self.category = category + self.multi_turn = multi_turn + super().__init__(run=run, **kwargs) + + @staticmethod + def cache_path_for(config: str) -> Path: + """Where the resolved rows for ``config`` are cached. + + A plain function of the config name so the synchronous preflight phase + can consult it without constructing a loader. + + Raises: + ConfigurationError: If ``config`` is not a bare SPEED-Bench config + name. It reaches here from ``--hf-subset``, a documented + user-facing override, and is interpolated into a filesystem + path -- an absolute path or ``..`` segment would otherwise write + outside the cache directory. + """ + validate_config_name(config) + return SPEED_BENCH_CACHE_DIR / f"{config}.jsonl" + + @property + def cache_path(self) -> Path: + """Where the resolved config for this loader is cached.""" + return self.cache_path_for(self.config) + + @staticmethod + def _required_config(loader_kwargs: dict[str, Any]) -> str: + """Read the config name a preflight hook was invoked for. + + The hooks are called through a getattr-based dispatcher, so a + registration that omits ``hf_subset`` metadata would otherwise surface + as a bare KeyError from inside preflight with no indication of which + plugin entry is wrong. + + Raises: + ConfigurationError: If no config name was supplied. + """ + from aiperf.config.loader.errors import ConfigurationError + + config = loader_kwargs.get("hf_subset") + if not config: + raise ConfigurationError( + "SPEED-Bench preflight requires a config name. Set 'hf_subset' " + "on the public_dataset_loader plugin entry, or pass --hf-subset." + ) + return config + + @classmethod + def preflight_access(cls, **loader_kwargs: Any) -> None: + """Check gated-source access before downloading anything. + + ``cais/hle`` supplies rows in every SPEED-Bench config, so an + unauthorized account can never produce a complete dataset. Failing here + costs one request and turns a multi-GB dead end into an immediate, + actionable message. + + Raises: + ConfigurationError: If the account cannot read the gated source. + """ + from aiperf.config.loader.errors import ConfigurationError + + if cls.cache_path_for(cls._required_config(loader_kwargs)).exists(): + return + + try: + from huggingface_hub import HfApi, get_token + from huggingface_hub.errors import ( + GatedRepoError, + RepositoryNotFoundError, + ) + except ImportError: + # huggingface_hub absent: let the resolve step report the real + # problem rather than inventing one here. + return + + # Distinguish "no credentials" from "credentials without access": they + # need different actions, and reporting the first as the second sends + # someone to accept terms on an account they have not logged into. + if get_token() is None: + raise ConfigurationError(cls._no_credentials_message()) + + try: + HfApi().auth_check(cls._GATED_SOURCE, repo_type="dataset") + except GatedRepoError as e: + # Must precede RepositoryNotFoundError: GatedRepoError subclasses it. + raise ConfigurationError(cls._not_authorized_message()) from e + except RepositoryNotFoundError as e: + # HuggingFace answers an unauthenticated or rejected request with a + # 401 it cannot disambiguate from "no such repo", and the client + # surfaces that as RepositoryNotFoundError. Reaching here means a + # token exists but was refused, so it is a credential problem -- + # letting it fall through to the catch-all below would silently skip + # the fast fail this check exists for. + raise ConfigurationError(cls._rejected_credentials_message()) from e + except Exception: + # Network failure, HF outage, or an unexpected status. "I could not + # tell" must not be reported as "you lack access" -- the resolve + # step will surface the real error. + return + + @classmethod + def _gate_explanation(cls) -> str: + """Why this step exists and why no tool can do it for you.""" + return ( + f"'{cls._GATED_SOURCE}' is the only gated source dataset, and it " + f"appears in every SPEED-Bench config. Access is granted to " + f"individual users rather than organizations, and HuggingFace " + f"provides no API for requesting it, so this cannot be automated." + ) + + @classmethod + def _no_credentials_message(cls) -> str: + """Guidance when no HuggingFace token is configured at all.""" + return ( + f"SPEED-Bench needs '{cls._GATED_SOURCE}', which is gated on " + f"HuggingFace, and no HuggingFace credentials were found.\n\n" + f" Searched: $HF_TOKEN, then the token file under $HF_HOME " + f"(default ~/.cache/huggingface/token)\n\n" + f" 1. Open {HLE_ACCESS_URL} and accept the terms.\n" + f" Approval is automatic -- no reviewer, no waiting period.\n" + f" 2. Authenticate: run 'hf auth login', or set HF_TOKEN in CI.\n\n" + f"{cls._gate_explanation()}" + ) + + @classmethod + def _not_authorized_message(cls) -> str: + """Guidance when authenticated but the terms have not been accepted.""" + return ( + f"SPEED-Bench needs '{cls._GATED_SOURCE}', which is gated on " + f"HuggingFace. You are authenticated, but this account has not " + f"been granted access.\n\n" + f" Open {HLE_ACCESS_URL} and accept the terms. Approval is " + f"automatic -- no reviewer, no waiting period. No re-login is " + f"needed afterwards.\n\n" + f"{cls._gate_explanation()}" + ) + + @classmethod + def _rejected_credentials_message(cls) -> str: + """Guidance when a token exists but HuggingFace refused it.""" + return ( + f"SPEED-Bench needs '{cls._GATED_SOURCE}', and HuggingFace rejected " + f"the credentials found on this machine. The token is most likely " + f"expired, revoked, or lacks read scope.\n\n" + f" 1. Re-authenticate: run 'hf auth login', or refresh HF_TOKEN " + f"in CI.\n" + f" 2. If that succeeds and this persists, accept the terms at " + f"{HLE_ACCESS_URL}.\n\n" + f"{cls._gate_explanation()}" + ) + + @classmethod + def preflight_materialize(cls, **loader_kwargs: Any) -> None: + """Resolve and cache the config before services start. + + Doing this inside ``DatasetManager`` blocks the profiling handshake and + trips ``AIPERF_DATASET_CONFIGURATION_TIMEOUT`` on any real download. + + Raises: + ConfigurationError: If resolution fails. + """ + from aiperf.config.loader.errors import ConfigurationError + + config = cls._required_config(loader_kwargs) + cache_path = cls.cache_path_for(config) + try: + if cache_path.exists(): + # A cache hit is still untrusted input: the documented + # pre-staging workflow copies a file in from elsewhere, so the + # "complete data or an error" invariant has to hold here too, + # not only on the path that produced the file. + cls._reject_unresolved(config, cache_path, delete_on_failure=False) + return + cls.resolve_config(config) + except DatasetLoaderError as e: + raise ConfigurationError(str(e)) from e + + async def load_dataset(self) -> dict[str, Any]: + """Return resolved rows, resolving and caching the config if needed.""" + source = self.cache_path + if not source.exists(): + await asyncio.get_running_loop().run_in_executor( + None, self.resolve_config, self.config + ) + source = self.cache_path + else: + # Validate a pre-existing cache for the same reason preflight does: + # this is the path a pre-staged or hand-copied file arrives on. + self._reject_unresolved(self.config, source, delete_on_failure=False) + + with open(source, encoding="utf-8") as f: + rows = [orjson.loads(line) for line in f if line.strip()] + return {"dataset": rows} + + @classmethod + def resolve_config( + cls, + config: str, + hf_dataset_name: str = "nvidia/SPEED-Bench", + hf_split: str = "test", + ) -> Path: + """Resolve ``config``'s prompt text and cache it; return the cache path. + + A classmethod so the synchronous preflight phase can call it without + constructing a loader. + + Raises: + ConfigurationError: If the prepare script's dependencies are absent. + DatasetLoaderError: If resolution fails or leaves rows unresolved. + """ + import logging + + from aiperf.config.loader.errors import ConfigurationError + + try: + from aiperf.dataset.loader.vendor import speed_bench_prepare + except ImportError as e: + raise ConfigurationError( + "Resolving SPEED-Bench requires the 'datasets', 'pandas', " + "'numpy' and 'tiktoken' packages. They ship with AIPerf, so " + "this usually means a partial install." + ) from e + + cache_path = cls.cache_path_for(config) + logging.getLogger(__name__).info( + f"Resolving SPEED-Bench '{config}': downloading prompt text from " + f"its source datasets. This is several GB and can take tens of " + f"minutes, but it happens once -- the result is cached to " + f"{cache_path} and shared by every {config} category. Delete that " + f"file to refetch." + ) + SPEED_BENCH_CACHE_DIR.mkdir(parents=True, exist_ok=True) + + # Per-config temp file: the name is process- and config-unique so two + # AIPerf invocations resolving concurrently cannot truncate each other's + # partial write before either is published. + tmp_path = cache_path.with_suffix(f".jsonl.{os.getpid()}.partial") + try: + from datasets import load_dataset as hf_load_dataset + + cls._reset_resolver_state(speed_bench_prepare) + dataset = hf_load_dataset( + hf_dataset_name, + config, + split=hf_split, + revision=SPEED_BENCH_REVISION, + ) + cls._reject_untrusted_sources(dataset) + # Whole-config, in upstream's row order -- see the class docstring. + dataset = speed_bench_prepare._resolve_external_data(dataset, config) + dataset = dataset.map( + lambda example: { + "messages": [ + {"role": "user", "content": turn} for turn in example["turns"] + ] + }, + remove_columns=["turns"], + ) + dataset.to_json(tmp_path) + rows = cls._reject_unresolved(config, tmp_path) + # Publish atomically: a half-written cache must never look complete + # to the next run, which would silently benchmark placeholder text. + tmp_path.replace(cache_path) + # Without this the run reports only the multi-GB source downloads + # (which land in the HuggingFace cache, not here) and then a small + # JSONL, leaving no way to tell a complete resolution from a + # truncated one. + logging.getLogger(__name__).info( + f"Resolved SPEED-Bench '{config}': {rows} prompts written to " + f"{cache_path} ({cache_path.stat().st_size / 1e6:.1f} MB). The " + f"source datasets themselves stay in the HuggingFace cache." + ) + except DatasetLoaderError: + raise + except Exception as e: + # Covers the write itself: an ENOSPC or permissions failure in + # to_json would otherwise leave a partial file behind. + raise DatasetLoaderError(cls._resolution_failed_message(config, e)) from e + finally: + # A finally rather than per-handler unlinks so cancellation, which + # is a BaseException and escapes both excepts, cannot strand a + # multi-GB partial in the cache dir. A no-op once replace() ran. + tmp_path.unlink(missing_ok=True) + + return cache_path + + @classmethod + def _reject_untrusted_sources(cls, dataset: Any) -> None: + """Refuse rows whose ``source`` points outside the known dataset hosts. + + The vendored resolver dispatches on each row's ``source`` and fetches it + verbatim, so a row is an input that selects a URL. Screening here keeps + the check outside the vendored file, which must stay byte-identical to + upstream. + + Raises: + DatasetLoaderError: If any row names an unexpected host. + """ + from urllib.parse import urlparse + + offenders: set[str] = set() + for source in dataset["source"] if "source" in dataset.column_names else []: + if not source: + continue + host = urlparse(str(source)).netloc.split("@")[-1].split(":")[0] + # Bare "org/dataset" identifiers have no netloc and are resolved + # through the HuggingFace client rather than fetched directly. + if host and host not in _ALLOWED_SOURCE_HOSTS: + offenders.add(host) + + if offenders: + raise DatasetLoaderError( + f"SPEED-Bench rows reference unexpected source hosts: " + f"{', '.join(sorted(offenders))}. Resolution fetches prompt text " + f"from each row's source, so AIPerf refuses hosts outside the " + f"published set ({', '.join(sorted(_ALLOWED_SOURCE_HOSTS))})." + ) + + @staticmethod + def _reset_resolver_state(speed_bench_prepare: Any) -> None: + """Re-seed the vendored resolver's module-global RNG. + + Upstream seeds ``HLE_RNG`` once at import and consumes it while building + throughput prompts, so resolving a second config in the same process + continues from the first one's stream and produces prompts that differ + from a fresh process. Re-seeding per config restores the property that + a config's output depends only on the config. + """ + import numpy as np + + speed_bench_prepare.HLE_RNG = np.random.default_rng(42) + + @classmethod + def _reject_unresolved( + cls, config: str, path: Path, *, delete_on_failure: bool = True + ) -> int: + """Fail if any row still holds placeholder text; return the row count. + + Upstream's source dispatch has no terminal ``else``, so an unrecognised + source yields placeholder text and exits successfully. Exit status is + therefore not evidence of success; the rows are. + """ + with open(path, encoding="utf-8") as f: + rows = [orjson.loads(line) for line in f if line.strip()] + + unresolved = sum(1 for row in rows if cls._has_placeholder(row)) + if unresolved: + # Only remove a file this process wrote; never delete a cache the + # user pre-staged, or a failed run destroys their input. + if delete_on_failure: + path.unlink(missing_ok=True) + raise DatasetLoaderError( + f"SPEED-Bench '{config}': {unresolved} of {len(rows)} rows " + f"were left unresolved by the prepare step, which reports " + f"success even when a source is unreachable. AIPerf will not " + f"benchmark placeholder text. Re-run to retry; if it persists, " + f"one of the source datasets is unavailable." + ) + + return len(rows) + + @staticmethod + def _has_placeholder(row: dict[str, Any]) -> bool: + return any( + str(message.get("content", "")).startswith(SpeedBenchRow.TURNS_PLACEHOLDER) + for message in row.get("messages", []) + ) + + @classmethod + def _resolution_failed_message(cls, config: str, error: Exception) -> str: + """Build a message that distinguishes the gated source from other faults.""" + text = str(error) + gated = cls._GATED_SOURCE in text or "gated" in text.lower() + if not gated: + return ( + f"Failed to resolve SPEED-Bench '{config}' from its source " + f"datasets: {error}" + ) + # Reuse the preflight wording rather than restating it: this path is + # reached when the gate is hit despite the preflight probe passing + # (access revoked mid-run, or the probe could not reach HuggingFace). + return f"{cls._not_authorized_message()}\n\nUnderlying error: {error}" + + async def convert_to_conversations( + self, data: dict[str, Any] + ) -> list[Conversation]: + """Convert resolved rows into Conversations, applying the category filter. + + Each turn is stamped with its row's category via ``Turn.source_kind``, + which the worker copies onto every request and the record processor + writes to ``MetricRecordMetadata.source_kind``. That lets a single run + over an aggregate split be broken down per category afterwards -- by + ``aiperf speed-bench-report`` or any other consumer -- without the + source JSONL, which the artifact directory does not contain. The + custom-file loader stamps the same field for the same reason. + + Raises: + DatasetLoaderError: If the category matches no rows. + """ + conversations: list[Conversation] = [] + total = 0 + + for row in data["dataset"]: + total += 1 + if self.category and row.get("category") != self.category: + continue + + texts = [ + str(message.get("content", "")) + for message in row.get("messages", []) + if str(message.get("content", "")).strip() + ] + if not texts: + continue + if not self.multi_turn: + texts = texts[:1] + + category = row.get("category") + conversations.append( + Conversation( + session_id=str(question_id) + if (question_id := row.get("question_id")) + else self.session_id_generator.next(), + turns=[ + Turn(texts=[Text(contents=[text])], source_kind=category) + for text in texts + ], + ) + ) + + if not conversations: + # Rows are dropped by two independent filters, and naming the wrong + # one sends the user to check a category they never set. + if self.category: + raise DatasetLoaderError( + f"SPEED-Bench category {self.category!r} matched none of the " + f"{total} rows in config {self.config!r}. Verify the category " + f"exists in this split -- the qualitative and throughput " + f"splits have different category names." + ) + cause = ( + "the cache file holds no rows at all" + if total == 0 + else f"every one of its {total} rows had empty or whitespace messages" + ) + raise DatasetLoaderError( + f"SPEED-Bench config {self.config!r} produced no usable " + f"conversations: {cause}. The cached resolution at " + f"{self.cache_path_for(self.config)} is corrupt -- delete it and " + f"re-run to resolve the dataset again." + ) + return conversations + + @classmethod + def get_preferred_sampling_strategy(cls) -> DatasetSamplingStrategy: + return DatasetSamplingStrategy.SEQUENTIAL diff --git a/src/aiperf/dataset/loader/vendor/README.md b/src/aiperf/dataset/loader/vendor/README.md new file mode 100644 index 0000000000..812c782e04 --- /dev/null +++ b/src/aiperf/dataset/loader/vendor/README.md @@ -0,0 +1,50 @@ +# Vendored third-party sources + +Files here are checked in **verbatim** from upstream. Do not edit them. + +Their job is to produce byte-identical output to upstream. A hand-adaptation +cannot be mechanically verified, so any fix belongs upstream, followed by a +re-vendor and a SHA bump. + +## `speed_bench_prepare.py` + +| | | +|---|---| +| Upstream | [`NVIDIA-NeMo/Skills`](https://github.com/NVIDIA-NeMo/Skills) `nemo_skills/dataset/speed-bench/prepare.py` | +| Source URL | `https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/5ac8609a56ac941540b10c92e68d556e6343cd4c/nemo_skills/dataset/speed-bench/prepare.py` | +| Pinned commit | `5ac8609a56ac941540b10c92e68d556e6343cd4c` (authored 2026-03-04) | +| Retrieved | 2026-09-01 | +| sha256 | `a551be4df541474e54e21b480022b0cbb66c2da068fda61b2a64bd3223bbbed2` | +| Licence | Apache-2.0, Copyright (c) 2026 NVIDIA CORPORATION (header retained verbatim) | + +Unmodified. AIPerf adds no changes; the Apache-2.0 §4(b) modification notice +does not apply. + +Resolves the placeholder rows in `nvidia/SPEED-Bench` by refetching prompt text +from the 14 upstream source datasets. Consumed by +`aiperf.dataset.loader.speed_bench_public`. + +`tests/unit/dataset/loader/test_vendored_speed_bench_prepare.py` holds two +checks. `test_vendored_prepare_script_is_unmodified` pins the sha256 offline, so +an accidental local edit fails CI. `test_vendored_prepare_script_matches_upstream` +is `@pytest.mark.network` and refetches the pinned commit from +`raw.githubusercontent.com`; only that one can establish that the vendored bytes +are upstream's rather than merely unchanged, and it is what backs the +Apache-2.0 §4(b) "Unmodified" claim below. It skips when upstream is +unreachable, so the offline pin remains the contract that always holds. + +### Why this file and not `specdec_bench` + +The SPEED-Bench dataset card designates `specdec_bench/datasets/speed.py` in +NVIDIA/Model-Optimizer as canonical. This is *not* that file: NeMo-Skills +carries an independent flattened implementation that has already diverged. + +The divergence is load-bearing. Canonical writes parquet with a `turns` column; +this writes JSONL with `messages=[{"role": ..., "content": ...}]`, which is the +shape `SpeedBenchRow` validates. Switching to canonical would require changing +our row model, so the two are not interchangeable. + +Both share one flaw worth knowing: the source dispatch in +`_fetch_all_turns_data` ends in `return example` with no terminal `else`, so an +unrecognised source silently yields placeholder text. AIPerf detects this after +the fact by counting unresolved rows rather than trusting the exit status. diff --git a/src/aiperf/dataset/loader/vendor/__init__.py b/src/aiperf/dataset/loader/vendor/__init__.py new file mode 100644 index 0000000000..93fa1c0f9b --- /dev/null +++ b/src/aiperf/dataset/loader/vendor/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Verbatim third-party sources. See README.md; do not edit the vendored files.""" diff --git a/src/aiperf/dataset/loader/vendor/speed_bench_prepare.py b/src/aiperf/dataset/loader/vendor/speed_bench_prepare.py new file mode 100644 index 0000000000..6226f0c909 --- /dev/null +++ b/src/aiperf/dataset/loader/vendor/speed_bench_prepare.py @@ -0,0 +1,636 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import random +import re +from enum import Enum +from pathlib import Path +from typing import Any, Literal, get_args + +import numpy as np +import pandas as pd +import tiktoken +from datasets import Dataset, concatenate_datasets, load_dataset + +DATASET_CONFIG = Literal[ + "qualitative", "throughput_1k", "throughput_2k", "throughput_8k", "throughput_16k", "throughput_32k" +] + +TURNS_PLACEHOLDER = "FULL BENCHMARK DATA SHOULD BE FETCHED FROM THE SOURCE USING SPECDEC_BENCH" +HLE_RNG = np.random.default_rng(42) + + +class BenchmarkDataset(str, Enum): + """Enum for benchmark datasets used in SPEED-Bench. + + Each enum value represents a HuggingFace dataset identifier used for + loading external benchmark datasets. + """ + + BAMBOO = "RUCAIBox/BAMBOO" + CNN_DAILYMAIL = "abisee/cnn_dailymail" + HLE = "cais/hle" + LIVECODEBENCH = "livecodebench/code_generation_lite" + CODE_CONTESTS = "deepmind/code_contests" + MTBENCH_101 = "mtbench101/mt-bench-101" + OPUS100 = "Helsinki-NLP/opus-100" + CHATRAG_BENCH = "nvidia/ChatRAG-Bench" + MMLU_PRO = "TIGER-Lab/MMLU-Pro" + ADALEVAL_STACKSELECT = "AdaLEval/stackselect" + ADALEVAL_TEXTSORT = "AdaLEval/textsort" + ROLEBENCH = "ZenMoore/RoleBench" + ROLEBENCH_ROLES = "ZenMoore/RoleBench/roles" + COSER = "Neph0s/CoSER" + + +DATASETS_AND_LOADERS_FUNCTIONS = { + BenchmarkDataset.BAMBOO.value: lambda dataset_name, config_name: load_dataset( + "json", data_files={"test": config_name}, split="test" + ), + BenchmarkDataset.CNN_DAILYMAIL.value: lambda dataset_name, config_name: load_dataset( + dataset_name, config_name, split="test" + ), + BenchmarkDataset.HLE.value: lambda dataset_name, config_name: load_dataset( + dataset_name, split="test", revision="021a3d71f516a7ac28ceb8d284969902edf1edeb" + ) + if config_name != "train_test_split" + else load_dataset( + dataset_name, split="test", revision="021a3d71f516a7ac28ceb8d284969902edf1edeb" + ).train_test_split(test_size=0.5, shuffle=True, seed=42), + BenchmarkDataset.LIVECODEBENCH.value: lambda dataset_name, config_name: load_dataset( + "json", + data_files={ + "test": [ + f"https://huggingface.co/datasets/livecodebench/code_generation_lite/resolve/0fe84c3912ea0c4d4a78037083943e8f0c4dd505/{file_name}.jsonl" + for file_name in ["test", "test2", "test3", "test4", "test5", "test6"] + ] + }, + split="test", + ), + BenchmarkDataset.CODE_CONTESTS.value: lambda dataset_name, config_name: load_dataset( + dataset_name, split="test", revision="802411c3010cb00d1b05bad57ca77365a3c699d6" + ), + BenchmarkDataset.MTBENCH_101.value: lambda dataset_name, config_name: load_dataset( + "json", data_files={"test": config_name}, split="test" + ), + BenchmarkDataset.OPUS100.value: lambda dataset_name, config_name: load_dataset( + dataset_name, config_name, split="test", revision="805090dc28bf78897da9641cdf08b61287580df9" + ), + BenchmarkDataset.CHATRAG_BENCH.value: lambda dataset_name, config_names: concatenate_datasets( + [ + load_dataset(dataset_name, config_name, split="test", revision="af6c7d420ddddf21f54f8ab3394bbf462aad2577") + for config_name in config_names + ] + ), + BenchmarkDataset.MMLU_PRO.value: lambda dataset_name, config_name: load_dataset( + dataset_name, split="test", revision="30527804ea8854662078e457808040d872ecdf29" + ), + BenchmarkDataset.ADALEVAL_STACKSELECT.value: lambda dataset_name, config_name: load_dataset( + "json", data_files={"test": config_name}, split="test" + ), + BenchmarkDataset.ADALEVAL_TEXTSORT.value: lambda dataset_name, config_name: load_dataset( + "json", data_files={"test": config_name}, split="test" + ), + BenchmarkDataset.ROLEBENCH.value: lambda dataset_name, config_name: pd.read_json(config_name, lines=True), + BenchmarkDataset.ROLEBENCH_ROLES.value: lambda dataset_name, config_name: load_dataset( + "json", data_files={"test": config_name}, split="test" + ), + BenchmarkDataset.COSER.value: lambda dataset_name, config_name: load_dataset( + "json", data_files={"test": config_name.replace("tree", "raw") + "/test/test_set.json"}, split="test" + ), +} + +EXTERNAL_DATASETS = dict() + + +def _get_external_dataset(dataset_name: str, config_name: str = "default"): + full_name = f"{dataset_name}_{config_name}" + if full_name not in EXTERNAL_DATASETS: + EXTERNAL_DATASETS[full_name] = DATASETS_AND_LOADERS_FUNCTIONS[dataset_name](dataset_name, config_name) + if config_name == "train_test_split": + EXTERNAL_DATASETS[full_name] = ( + EXTERNAL_DATASETS[full_name]["train"], + EXTERNAL_DATASETS[full_name]["test"], + ) + return EXTERNAL_DATASETS[full_name] + + +def _generate_stackselect_prompt(question: str, answers: list[str], answer: str, num_tokens: int) -> str: + random.seed(42) + encoder = tiktoken.get_encoding("o200k_base") + # Original prompt as given in Ada-LEval paper: https://arxiv.org/pdf/2404.06480 + prompt = """ +You are an AI assistant. Your job is to find out the most helpful answer to a given question. +Each time, you will be provided with a question and n answers to this question. +Each answer begins with an 'A' and a number(e.g. A4), which represents its designation. +You need to determine which answer is the most helpful one to the question. +The case sample is shown below and you should give me the answer in the format exactly the same as the sample. + +However, you should NOT focus on the content of sample answer. + +Sample Input (format only): + +The question is given below. +XXX(The content of question) +Possible answers are given below. +A1: +XXX(The content of answer 1) +A2: +XXX(The content of answer 2) +. +. +. +An: +XXX(The content of answer n) +Now the answers are over, please decide which answer is the most helpful one to the question. +You must give me the designation of the MOST helpful answer and the reason why you choose this answer. +For every other answer, you must give me the reason why you do not choose this answer. + +Sample Output (format only): + +Answer: The designation of the most helpful answer.(e.g. A4 means answer 4 is the most helpful answer) +Explanation: +A4: The reason why you choose this answer. +A1: The reason why you do not choose this answer. +A2: The reason why you do not choose this answer. +. +. +. +An: The reason why you do not choose this answer. +""" + prompt += "The question is given below.\n" + prompt += question + "\n\n" + prompt += "Possible answers are given below.\n" + tokens_prompt = len(encoder.encode(prompt, disallowed_special=())) + end_prompt = "Now the answers are over, please decide which answer is the most helpful one to the question. \n" + end_prompt += ( + "You must give me the designation of the MOST helpful answer and the reason why you choose this answer.\n" + ) + end_prompt += "For every other answer, you must give me the reason why you do not choose this answer.\n" + end_prompt_tokens = len(encoder.encode(end_prompt, disallowed_special=())) + correct_answer_i = int(answer.strip("A")) - 1 + correct_answer_tokens = len( + encoder.encode(answer + ":\n\n" + answers[correct_answer_i] + "\n\n", disallowed_special=()) + ) + all_tokens = tokens_prompt + end_prompt_tokens + correct_answer_tokens + answers_to_add_stop = 0 + for i, answer in enumerate(answers): + if i == correct_answer_i: + continue + answer_to_add = f"A{i + 1}:\n\n{answer}\n\n" + answer_to_add_tokens = len(encoder.encode(answer_to_add, disallowed_special=())) + if all_tokens + answer_to_add_tokens > num_tokens: + break + answers_to_add_stop = i + all_tokens += answer_to_add_tokens + answers_to_add = ( + answers[: answers_to_add_stop + 1] + if answers_to_add_stop >= correct_answer_i + else [answers[correct_answer_i]] + answers[: answers_to_add_stop + 1] + ) + random.shuffle(answers_to_add) + for i, answer in enumerate(answers_to_add): + prompt += f"A{i + 1}:\n\n{answer}\n\n" + prompt += end_prompt + return prompt + + +def _generate_textsort_prompt(prompt: str) -> str: + # Original prompt as given in Ada-LEval paper: https://arxiv.org/pdf/2404.06480 + original_instruction = "\n You are an AI assistant. Your job is to sort multiple book sections into the correct order.\n Each time, you will be provided with 4 pieces of text.\n These texts form a continuous part of a book, but are provided in random order.\n You need to find the correct order and return the answer in a string.\n For example, if you output [4, 1, 3, 2], that means the correct order is: Part 4 -> Part 1 -> Part 3 -> Part 2.\n You will also be provided with the neighboring paragraphs before and after the 4 pieces of texts. \n\n The case sample is shown below and you should give me the answer in the format exactly the same as the sample. \n\n However, you should NOT focus on the content of sample answer. \n\n Please do NOT output any extra content. \n Sample Input (format only): \n\n Before: XXX (Text before the continuous book part)\n\n\n Part 1: XXX\n\n\n Part 2: XXX\n\n\n Part 3: XXX\n\n\n Part 4: XXX\n\n\n After: XXX (Text after the continuous book part)\n\n\n Sample Output (format only): \n\n Answer: [4, 1, 3, 2] \n\n\n\n" + + new_instruction = """ +You are an AI assistant. Your job is to sort multiple book sections into the correct order. +Each time, you will be provided with 4 pieces of text. +These texts form a continuous part of a book, but are provided in random order. +You need to find the correct order and write the all the parts in the correct order. +For example, if the correct order is: Part 4 -> Part 1 -> Part 3 -> Part 2, you need to answer with a continous text of all the parts in the correct order. +You should NOT change the text, just write it in the order it should appear. +You will also be provided with the neighboring paragraphs before and after the 4 pieces of texts. +You should NOT output the before and after paragraphs, just the text in the correct order. + +The case sample is shown below and you should give me the answer in the format exactly the same as the sample. + +However, you should NOT focus on the content of sample answer. + +Please do NOT output any extra content. + +Sample Input (format only): + +Before: BBB (Text before the continuous book part) + + +Part 1: XXX + + +Part 2: YYY + + +Part 3: ZZZ + + +Part 4: WWW + + +After: AAA (Text after the continuous book part) + +Sample Output (format only): + +Answer: + + +WWW + +XXX + +ZZZ + +YYY +""" + return prompt.replace(original_instruction, new_instruction, 1) + + +def _generate_writing_prompt(contents: list[str]) -> str: + content = "\n\n".join([f"START CONTENT {i + 1}\n\n{content}\n\nEND CONTENT" for i, content in enumerate(contents)]) + # Inspired by the prompt used in BAMBOO paper: https://arxiv.org/pdf/2309.13345 + prompt = f""" +I want you to act as a long dialogue completer. +Given a long dialogue(s), your objectives are: +1. Add one speaker mentioned in the past dialogue(s) at the end of the last sentence of each dialogue (between START CONTENT and END CONTENT) to complete the sentence and ensure its semantic integrity. At here, the added word must be a person's name which appears in the dialogue. +2. Continue the dialogue(s) with one or more speakers who appeared in the dialogue(s) before. Be coherent with the previous dialogue(s) and be creative in your response. +The content of the dialogue(s) is given below. + + +{content} +""" + return prompt + + +def _pad_or_truncate_prompt(prompt: str, target_num_tokens: int, padding: str = "Answer now please.\n") -> str: + encoder = tiktoken.get_encoding("o200k_base") + + tokens = encoder.encode(prompt, disallowed_special=()) + current_num_tokens = len(tokens) + + if current_num_tokens > target_num_tokens: + # Truncate if too long + tokens = encoder.encode(prompt, disallowed_special=()) + return encoder.decode(tokens[:target_num_tokens]) + elif current_num_tokens < target_num_tokens: + # Add padding if too short + padding_tokens = encoder.encode(padding, disallowed_special=()) + tokens_needed = target_num_tokens - current_num_tokens + # Calculate how many full padding sequences we need + num_padding_repeats = (tokens_needed + len(padding_tokens) - 1) // len(padding_tokens) + padded_prompt = prompt + (padding * num_padding_repeats) + # Truncate to exact target length + padded_tokens = encoder.encode(padded_prompt, disallowed_special=()) + return encoder.decode(padded_tokens[:target_num_tokens]) + else: + return prompt + + +def _generate_bamboo_prompt(external_dataset: "Dataset", num_tokens: int) -> str: + prompt = _generate_writing_prompt(external_dataset["content"]) + return _pad_or_truncate_prompt(prompt, num_tokens) + + +def _generate_chatrag_bench_prompt(external_dataset: "Dataset") -> str: + prompt = ( + "Please give a full and complete answer for the questions. \n\nContext:\n{context}\n\nQuestion:\n{question}" + ) + context = "\n\n".join([ctx["text"] for ctx in external_dataset["ctxs"][0]]) + questions = [message["content"] for message in external_dataset["messages"][0] if message["role"] == "user"] + + return [prompt.format(context=context, question=questions[0])] + questions[1:] + + +def _generate_coser_prompt(external_dataset: "Dataset") -> str: + rng = np.random.default_rng(seed=12347) + # Original prompt as given in CoSER paper: https://arxiv.org/pdf/2404.06480 + prompt = """You are {character} from {book_name}. +==={character}'s Profile=== +{character_profile} +===Current Scenario=== +{scenario} +===Information about the other Characters=== +{other_character_profiles_str} +===Your Inner Thoughts=== +{motivation} + +===Requirements=== +Your output should include **thought**, **speech**, and **action**. Use [your thought] +for thoughts, which others can't see, e.g. [I'm terrified, but I must appear strong.]. Use +(your action) for actions, which others can see, such as (watches silently, trying to control +her fear and anger).""" + character = rng.choice(external_dataset["major_characters"][0]) + character_profile = external_dataset["character_profiles"][0][character] + scenario = external_dataset["scenario"][0] + book_name = external_dataset["book"][0] + motivation = next( + ( + key_character["motivation"] + for key_character in external_dataset["key_characters"][0] + if key_character["name"] == character + ), + "No motivation provided", + ) + other_character_profiles_str = "\n\n".join( + [ + f"{character_name}: {character_profile}" + for character_name, character_profile in external_dataset["character_profiles"][0].items() + if character_name != character and character_profile is not None + ] + ) + return prompt.format( + character=character, + character_profile=character_profile, + book_name=book_name, + scenario=scenario, + other_character_profiles_str=other_character_profiles_str, + motivation=motivation, + ) + + +def _generate_mmlu_pro_prompt(external_dataset: "Dataset", subject: str) -> str: + def get_question_and_options(question, options): + options = [(chr(ord("A") + i), a) for i, a in enumerate(options)] + options_str = "\n".join([f"({letter}) {option}" for letter, option in options]) + return f"Question: {question}\n\nOptions: {options_str}\n\n" + + # Original prompt as given in MMLU-Pro paper: https://arxiv.org/pdf/2406.01574 + prompt = 'The following are multiple choice questions (with answers) about {subject}. Think step by step and then finish your answer with "the answer is (X)" where X is the correct letter choice.\n\n' + first_question = prompt.format(subject=subject) + get_question_and_options( + external_dataset["question"][0], external_dataset["options"][0] + ) + return [first_question] + [ + get_question_and_options(question, options) + for question, options in zip(external_dataset["question"][1:], external_dataset["options"][1:]) + ] + + +def _generate_hle_prompt( + example: dict[str, Any], hle_train: "pd.DataFrame", num_tokens: int, rng: "np.random.Generator" +) -> str: + encoder = tiktoken.get_encoding("o200k_base") + prompt = ( + "Please answer the question below.\n\nHere are some examples of question and answer pairs in the category of " + + example["category"] + + ":\n\n" + ) + prompt_tokens = encoder.encode(prompt) + example_tokens = encoder.encode(example["question"]) + current_num_tokens = len(prompt_tokens) + len(example_tokens) + hle_train_category = hle_train[hle_train["category"] == example["category"]] + + while current_num_tokens < num_tokens: + hle_train_category_sample = hle_train_category.sample(1, random_state=rng) + prompt += hle_train_category_sample["demonstration"].iloc[0] + current_num_tokens += len(hle_train_category_sample["tokens"].iloc[0]) + prompt_tokens += list(hle_train_category_sample["tokens"].iloc[0]) + + return encoder.decode(prompt_tokens[: num_tokens - len(example_tokens) + 1] + example_tokens) + + +def _get_num_tokens_from_config(speed_config: DATASET_CONFIG | str) -> int: + match = re.search(r"throughput_(\d+)k", speed_config) + if match: + return int(match.group(1)) * 1000 + else: + raise ValueError(f"Could not determine num_tokens from speed_config: {speed_config}") + + +def _fetch_all_turns_data(example: dict[str, Any], speed_config: DATASET_CONFIG | str) -> dict[str, Any]: + turns = example["turns"] + if not turns[0].startswith(TURNS_PLACEHOLDER): + return example + + if BenchmarkDataset.BAMBOO.value in example["source"]: + num_tokens = _get_num_tokens_from_config(speed_config) + src_ids = [int(match) for match in re.findall(r"_(\d+)", example["src_id"])] + external_dataset = _get_external_dataset(BenchmarkDataset.BAMBOO.value, config_name=example["source"]) + external_dataset = external_dataset.select(src_ids) + example["turns"] = [_generate_bamboo_prompt(external_dataset, num_tokens)] + + elif BenchmarkDataset.CNN_DAILYMAIL.value in example["source"]: + external_dataset = _get_external_dataset(BenchmarkDataset.CNN_DAILYMAIL.value, config_name="3.0.0").to_pandas() + src_id = example["src_id"] + article = external_dataset[external_dataset["id"] == src_id]["article"].iloc[0] + example["turns"] = [example["turns"][0].removeprefix(f"{TURNS_PLACEHOLDER}\n\n").format(article=article)] + + elif BenchmarkDataset.HLE.value in example["source"]: + if "qualitative" in speed_config: + external_dataset = _get_external_dataset(BenchmarkDataset.HLE.value, config_name="test").to_pandas() + src_id = example["src_id"] + example["turns"] = [external_dataset[external_dataset["id"] == src_id]["question"].iloc[0]] + elif "throughput" in speed_config: + num_tokens = _get_num_tokens_from_config(speed_config) + hle_train, hle_test = _get_external_dataset(BenchmarkDataset.HLE.value, config_name="train_test_split") + hle_train = hle_train.to_pandas() + hle_train = hle_train[hle_train["image"] == ""] + hle_train["demonstration"] = hle_train.apply( + lambda e: "Question: " + e["question"] + "\n\nAnswer: " + e["rationale"] + "\n\n", axis=1 + ) + hle_train["tokens"] = hle_train["demonstration"].apply( + lambda e: tiktoken.get_encoding("o200k_base").encode(e, disallowed_special=()) + ) + src_id = example["src_id"] + hle_test = hle_test.to_pandas() + external_dataset_example = hle_test[hle_test["id"] == src_id].iloc[0] + example["turns"] = [_generate_hle_prompt(external_dataset_example, hle_train, num_tokens, HLE_RNG)] + else: + raise ValueError(f"Invalid speed_config: {speed_config}") + + elif BenchmarkDataset.LIVECODEBENCH.value in example["source"]: + external_dataset = _get_external_dataset(BenchmarkDataset.LIVECODEBENCH.value, config_name="test").to_pandas() + src_id = example["src_id"] + external_dataset_example = external_dataset[external_dataset["question_id"] == src_id].iloc[0] + example["turns"] = [ + example["turns"][0] + .removeprefix(f"{TURNS_PLACEHOLDER}\n\n") + .format( + question=external_dataset_example["question_content"], + starter_code=external_dataset_example["starter_code"], + ) + ] + + elif BenchmarkDataset.CODE_CONTESTS.value in example["source"]: + external_dataset = _get_external_dataset(BenchmarkDataset.CODE_CONTESTS.value, config_name="test").to_pandas() + src_id = example["src_id"] + external_dataset_example = external_dataset[external_dataset["name"] == src_id].iloc[0] + example["turns"] = [ + example["turns"][0] + .removeprefix(f"{TURNS_PLACEHOLDER}\n\n") + .format(question=external_dataset_example["description"]) + ] + + elif BenchmarkDataset.MTBENCH_101.value in example["source"]: + external_dataset = _get_external_dataset(BenchmarkDataset.MTBENCH_101.value, config_name=example["source"]) + src_id = example["src_id"].rsplit("_", 1)[1] + external_dataset_example = external_dataset.select([int(src_id)]) + example["turns"] = [entry["user"] for entry in external_dataset_example["history"][0]] + + elif BenchmarkDataset.OPUS100.value in example["source"]: + _, config_name, src_id = example["src_id"].split("_") + external_dataset = _get_external_dataset(BenchmarkDataset.OPUS100.value, config_name=config_name) + external_dataset_example = external_dataset.select([int(src_id)]) + example["turns"] = [ + example["turns"][0] + .removeprefix(f"{TURNS_PLACEHOLDER}\n\n") + .format(question=external_dataset_example["translation"][0]) + ] + + elif BenchmarkDataset.CHATRAG_BENCH.value in example["source"]: + external_dataset = _get_external_dataset( + BenchmarkDataset.CHATRAG_BENCH.value, config_name=["hybridial", "sqa"] + ) + src_id = example["src_id"].rsplit("_", 1)[1] + external_dataset_example = external_dataset.select([int(src_id)]) + example["turns"] = _generate_chatrag_bench_prompt(external_dataset_example) + + elif BenchmarkDataset.MMLU_PRO.value in example["source"]: + external_dataset = _get_external_dataset(BenchmarkDataset.MMLU_PRO.value, config_name="test") + src_id = int(example["src_id"].split("(")[1].split(",")[0]) + external_dataset_example = external_dataset.select(range(src_id, src_id + len(example["turns"]))) + example["turns"] = _generate_mmlu_pro_prompt(external_dataset_example, example["sub_category"]) + + elif BenchmarkDataset.ADALEVAL_STACKSELECT.value in example["source"]: + num_tokens = _get_num_tokens_from_config(speed_config) + external_dataset = _get_external_dataset( + BenchmarkDataset.ADALEVAL_STACKSELECT.value, config_name=example["source"] + ).to_pandas() + src_id = example["src_id"] + external_dataset_example = external_dataset[external_dataset["question_id"] == src_id].iloc[0] + example["turns"] = [ + _pad_or_truncate_prompt( + _generate_stackselect_prompt( + question=external_dataset_example["question"], + answers=external_dataset_example["all_answers"], + answer=external_dataset_example["answer"], + num_tokens=num_tokens, + ), + num_tokens, + ) + ] + + elif BenchmarkDataset.ADALEVAL_TEXTSORT.value in example["source"]: + num_tokens = _get_num_tokens_from_config(speed_config) + external_dataset = _get_external_dataset( + BenchmarkDataset.ADALEVAL_TEXTSORT.value, config_name=example["source"] + ) + src_id = example["src_id"].split("_")[1] + external_dataset_example = external_dataset.select([int(src_id)]) + example["turns"] = [ + _pad_or_truncate_prompt(_generate_textsort_prompt(external_dataset_example["prompt"][0]), num_tokens) + ] + + elif BenchmarkDataset.ROLEBENCH.value in example["source"]: + config_name = example["src_id"].split("_")[1] + external_dataset = _get_external_dataset( + BenchmarkDataset.ROLEBENCH.value, + config_name=example["source"].replace("tree", "raw") + f"/{config_name}/role_specific/test.jsonl", + ) + roles_dataset = _get_external_dataset( + BenchmarkDataset.ROLEBENCH_ROLES.value, + config_name="https://huggingface.co/datasets/ZenMoore/RoleBench/raw/a57ed54f9613921e4a5f1b63601a558cd5acf971/profiles-eng/desc.json", + ) + src_ids = [int(match) for match in re.findall(r"_(\d+)", example["src_id"])][: len(example["turns"])] + external_dataset_example = external_dataset.iloc[src_ids] + role_name = external_dataset_example["role"].iloc[0] + role_description_and_catchphrases = roles_dataset[role_name][0] + example["turns"] = [ + example["turns"][0] + .removeprefix(f"{TURNS_PLACEHOLDER}\n\n") + .format(role_name=role_name, role_description_and_catchphrases=role_description_and_catchphrases) + + "\n" + + external_dataset_example["question"].iloc[0] + ] + [ + question.removeprefix(f"{role_name}, ").removeprefix(f" {role_name},") + for question in external_dataset_example["question"].iloc[1:] + ] + + elif BenchmarkDataset.COSER.value in example["source"]: + external_dataset = _get_external_dataset(BenchmarkDataset.COSER.value, config_name=example["source"]) + src_id = example["src_id"].split("_")[1] + external_dataset_example = external_dataset.select([int(src_id)]) + example["turns"] = [_generate_coser_prompt(external_dataset_example)] + + return example + + +def _resolve_external_data(dataset: Dataset, speed_config: DATASET_CONFIG | str) -> Dataset: + """Resolve all external data references in the dataset. + + Applies ``_fetch_all_turns_data`` to every example so that turn + placeholders are replaced with fully-resolved prompt text. + + Args: + dataset: The HuggingFace dataset with potentially unresolved turns. + speed_config: The SPEED-Bench config name used to determine + token-length parameters for throughput configs. + + Returns: + The dataset with all turns fully resolved. + """ + return dataset.map( + _fetch_all_turns_data, fn_kwargs={"speed_config": speed_config}, desc=f"Preparing config {speed_config}" + ) + + +def prepare_data(args: argparse.Namespace) -> None: + """Prepare and save benchmark data to disk. + + Calls the dataset's ``prepare_data`` classmethod which downloads and + resolves all external data references, then saves the fully-resolved + result as a parquet file so that subsequent benchmark runs can load + directly from disk without re-downloading. + + Args: + args: Parsed CLI arguments containing dataset type, config, + output directory, and optional filtering parameters. + """ + configs = get_args(DATASET_CONFIG) if args.config == "all" else [args.config] + + for config in configs: + dataset = load_dataset("nvidia/SPEED-Bench", config, split="test") + dataset = _resolve_external_data(dataset, config) + dataset = dataset.map( + lambda example: {"messages": [{"role": "user", "content": turn} for turn in example["turns"]]}, + remove_columns=["turns"], + ) + output_path = args.output_dir / f"{config}.jsonl" + dataset.to_json(output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Download and prepare SPEED-Bench dataset for nemo-skills evaluation.", + ) + parser.add_argument( + "--config", + type=str, + default="all", + choices=list(get_args(DATASET_CONFIG)) + ["all"], + help='SPEED-Bench configuration to prepare. Use "all" to prepare all configs. (default: %(default)s)', + ) + parser.add_argument( + "--output_dir", + type=Path, + default=Path(__file__).parent, + help="Directory to save the prepared dataset files (default: %(default)s)", + ) + + args = parser.parse_args() + prepare_data(args) diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index 3c702f8efe..575831520c 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -59,11 +59,11 @@ CustomDatasetTypeStr: TypeAlias = str CustomDatasetType = plugins.create_enum(PluginType.CUSTOM_DATASET_LOADER, "CustomDatasetType", module=__name__) -"""Dynamic enum for custom dataset loader. Example: CustomDatasetType.BAILIAN_TRACE, CustomDatasetType.SPEED_BENCH_THROUGHPUT_16K, CustomDatasetType.WEKA_TRACE""" +"""Dynamic enum for custom dataset loader. Example: CustomDatasetType.BAILIAN_TRACE, CustomDatasetType.SAGEMAKER_DATA_CAPTURE, CustomDatasetType.WEKA_TRACE""" PublicDatasetTypeStr: TypeAlias = str PublicDatasetType = plugins.create_enum(PluginType.PUBLIC_DATASET_LOADER, "PublicDatasetType", module=__name__) -"""Dynamic enum for public dataset loader. Example: PublicDatasetType.AIMO, PublicDatasetType.SEMIANALYSIS_CC_TRACES_WEKA_062126_256K, PublicDatasetType.WEKA_HF""" +"""Dynamic enum for public dataset loader. Example: PublicDatasetType.AIMO, PublicDatasetType.SPEC_BENCH, PublicDatasetType.WEKA_HF""" EndpointTypeStr: TypeAlias = str EndpointType = plugins.create_enum(PluginType.ENDPOINT, "EndpointType", module=__name__) diff --git a/src/aiperf/plugin/plugins.py b/src/aiperf/plugin/plugins.py index c29f426c33..b38704364f 100644 --- a/src/aiperf/plugin/plugins.py +++ b/src/aiperf/plugin/plugins.py @@ -1271,6 +1271,19 @@ def is_trace_dataset(name: str) -> bool: return get_dataset_loader_metadata(name).is_trace +def is_autodetectable(entry: PluginEntry) -> bool: + """Whether a registry entry may claim a file during structural detection. + + An entry that pre-filters rows to a subset (``metadata.category``) loads + part of a file, which cannot be inferred from the file's contents -- it can + only be requested by name. Including such entries makes every valid file + look ambiguous, because the unfiltered entry and each category variant all + match. Both structural-detection loops must apply this, or a file resolves + differently depending on which one runs. + """ + return entry.metadata.get("category") is None + + def get_convergence_criterion_metadata(name: str) -> ConvergenceCriterionMetadata: """Get typed metadata for a convergence criterion plugin. diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 616740cbff..951a611297 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -624,90 +624,7 @@ custom_dataset_loader: SPEED-Bench qualitative split from HuggingFace (nvidia). 880 prompts across 11 semantic domains for measuring speculative decoding acceptance rates. - # --------------------------------------------------------------------------- - # SPEED-Bench: qualitative split per category (80 prompts each) - # --------------------------------------------------------------------------- - - speed_bench_coding: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench coding category (80 prompts). Code generation and programming tasks. - metadata: - category: coding - - speed_bench_humanities: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench humanities category (80 prompts). History, philosophy, and liberal arts. - metadata: - category: humanities - - speed_bench_math: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench math category (80 prompts). Mathematical reasoning and computation. - metadata: - category: math - - speed_bench_multilingual: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench multilingual category (80 prompts). Tasks across 23 languages. - metadata: - category: multilingual - - speed_bench_qa: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench QA category (80 prompts). Question answering tasks. - metadata: - category: qa - - speed_bench_rag: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench RAG category (80 prompts). Retrieval-augmented generation tasks. - metadata: - category: rag - - speed_bench_reasoning: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench reasoning category (80 prompts). Logical and analytical reasoning. - metadata: - category: reasoning - speed_bench_roleplay: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench roleplay category (80 prompts). Creative roleplay and dialogue. - metadata: - category: roleplay - - speed_bench_stem: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench STEM category (80 prompts). Science, technology, engineering tasks. - metadata: - category: stem - - speed_bench_summarization: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench summarization category (80 prompts). Text summarization tasks. - metadata: - category: summarization - - speed_bench_writing: - class: aiperf.dataset.loader.speed_bench:SpeedBenchQualitativeLoader - description: | - SPEED-Bench writing category (80 prompts). Creative and technical writing. - metadata: - category: writing - - # --------------------------------------------------------------------------- - # SPEED-Bench: throughput splits (all entropy tiers combined) - # --------------------------------------------------------------------------- speed_bench_throughput_1k: class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput1KLoader description: | @@ -738,114 +655,6 @@ custom_dataset_loader: SPEED-Bench throughput split at ~32K input tokens from HuggingFace (nvidia). 1536 prompts across 3 entropy tiers for speculative decoding throughput benchmarking. - # --------------------------------------------------------------------------- - # SPEED-Bench: throughput per entropy tier (512 prompts each per ISL bucket) - # --------------------------------------------------------------------------- - speed_bench_throughput_1k_low_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput1KLoader - description: | - SPEED-Bench throughput ~1K ISL, low entropy tier (512 prompts). Code and sorting tasks. - metadata: - category: low_entropy - - speed_bench_throughput_1k_mixed: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput1KLoader - description: | - SPEED-Bench throughput ~1K ISL, mixed entropy tier (512 prompts). Needle-in-a-haystack and exams. - metadata: - category: mixed - - speed_bench_throughput_1k_high_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput1KLoader - description: | - SPEED-Bench throughput ~1K ISL, high entropy tier (512 prompts). Creative writing and dialogue. - metadata: - category: high_entropy - - speed_bench_throughput_2k_low_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput2KLoader - description: | - SPEED-Bench throughput ~2K ISL, low entropy tier (512 prompts). Code and sorting tasks. - metadata: - category: low_entropy - - speed_bench_throughput_2k_mixed: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput2KLoader - description: | - SPEED-Bench throughput ~2K ISL, mixed entropy tier (512 prompts). Needle-in-a-haystack and exams. - metadata: - category: mixed - - speed_bench_throughput_2k_high_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput2KLoader - description: | - SPEED-Bench throughput ~2K ISL, high entropy tier (512 prompts). Creative writing and dialogue. - metadata: - category: high_entropy - - speed_bench_throughput_8k_low_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput8KLoader - description: | - SPEED-Bench throughput ~8K ISL, low entropy tier (512 prompts). Code and sorting tasks. - metadata: - category: low_entropy - - speed_bench_throughput_8k_mixed: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput8KLoader - description: | - SPEED-Bench throughput ~8K ISL, mixed entropy tier (512 prompts). Needle-in-a-haystack and exams. - metadata: - category: mixed - - speed_bench_throughput_8k_high_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput8KLoader - description: | - SPEED-Bench throughput ~8K ISL, high entropy tier (512 prompts). Creative writing and dialogue. - metadata: - category: high_entropy - - speed_bench_throughput_16k_low_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput16KLoader - description: | - SPEED-Bench throughput ~16K ISL, low entropy tier (512 prompts). Code and sorting tasks. - metadata: - category: low_entropy - - speed_bench_throughput_16k_mixed: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput16KLoader - description: | - SPEED-Bench throughput ~16K ISL, mixed entropy tier (512 prompts). Needle-in-a-haystack and exams. - metadata: - category: mixed - - speed_bench_throughput_16k_high_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput16KLoader - description: | - SPEED-Bench throughput ~16K ISL, high entropy tier (512 prompts). Creative writing and dialogue. - metadata: - category: high_entropy - - speed_bench_throughput_32k_low_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput32KLoader - description: | - SPEED-Bench throughput ~32K ISL, low entropy tier (512 prompts). Code and sorting tasks. - metadata: - category: low_entropy - - speed_bench_throughput_32k_mixed: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput32KLoader - description: | - SPEED-Bench throughput ~32K ISL, mixed entropy tier (512 prompts). Needle-in-a-haystack and exams. - metadata: - category: mixed - - speed_bench_throughput_32k_high_entropy: - class: aiperf.dataset.loader.speed_bench:SpeedBenchThroughput32KLoader - description: | - SPEED-Bench throughput ~32K ISL, high entropy tier (512 prompts). Creative writing and dialogue. - metadata: - category: high_entropy - tracelab: class: aiperf.dataset.loader.tracelab_trace:TraceLabTraceDatasetLoader description: | @@ -2513,6 +2322,360 @@ public_dataset_loader: default_block_size: 64 default_prompt_corpus: coding + # --------------------------------------------------------------------------- + # SPEED-Bench. Prompt text for rows whose source datasets forbid + # redistribution is resolved on first use via the vendored upstream prepare + # script, then cached per config and shared by every category of that config. + # Requires accepting the cais/hle terms on HuggingFace; see + # docs/tutorials/speed-bench.md. + # --------------------------------------------------------------------------- + speed_bench_qualitative: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench qualitative split (880 prompts across 11 semantic domains) + for measuring speculative decoding acceptance rates. + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + + speed_bench_coding: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench coding category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: coding + + speed_bench_humanities: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench humanities category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: humanities + + speed_bench_math: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench math category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: math + + speed_bench_multilingual: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench multilingual category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: multilingual + + speed_bench_qa: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench qa category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: qa + + speed_bench_rag: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench rag category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: rag + + speed_bench_reasoning: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench reasoning category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: reasoning + + speed_bench_roleplay: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench roleplay category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: roleplay + + speed_bench_stem: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench stem category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: stem + + speed_bench_summarization: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench summarization category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: summarization + + speed_bench_writing: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench writing category (80 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: qualitative + multi_turn: true + category: writing + + speed_bench_throughput_1k: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 1K split (1,536 prompts, ~1k input tokens). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_1k + multi_turn: true + + speed_bench_throughput_1k_low_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 1K low_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_1k + multi_turn: true + category: low_entropy + + speed_bench_throughput_1k_mixed: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 1K mixed tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_1k + multi_turn: true + category: mixed + + speed_bench_throughput_1k_high_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 1K high_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_1k + multi_turn: true + category: high_entropy + + speed_bench_throughput_2k: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 2K split (1,536 prompts, ~2k input tokens). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_2k + multi_turn: true + + speed_bench_throughput_2k_low_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 2K low_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_2k + multi_turn: true + category: low_entropy + + speed_bench_throughput_2k_mixed: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 2K mixed tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_2k + multi_turn: true + category: mixed + + speed_bench_throughput_2k_high_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 2K high_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_2k + multi_turn: true + category: high_entropy + + speed_bench_throughput_8k: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 8K split (1,536 prompts, ~8k input tokens). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_8k + multi_turn: true + + speed_bench_throughput_8k_low_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 8K low_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_8k + multi_turn: true + category: low_entropy + + speed_bench_throughput_8k_mixed: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 8K mixed tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_8k + multi_turn: true + category: mixed + + speed_bench_throughput_8k_high_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 8K high_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_8k + multi_turn: true + category: high_entropy + + speed_bench_throughput_16k: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 16K split (1,536 prompts, ~16k input tokens). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_16k + multi_turn: true + + speed_bench_throughput_16k_low_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 16K low_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_16k + multi_turn: true + category: low_entropy + + speed_bench_throughput_16k_mixed: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 16K mixed tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_16k + multi_turn: true + category: mixed + + speed_bench_throughput_16k_high_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 16K high_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_16k + multi_turn: true + category: high_entropy + + speed_bench_throughput_32k: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 32K split (1,536 prompts, ~32k input tokens). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_32k + multi_turn: true + + speed_bench_throughput_32k_low_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 32K low_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_32k + multi_turn: true + category: low_entropy + + speed_bench_throughput_32k_mixed: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 32K mixed tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_32k + multi_turn: true + category: mixed + + speed_bench_throughput_32k_high_entropy: + class: aiperf.dataset.loader.speed_bench_public:SpeedBenchPublicLoader + description: | + SPEED-Bench throughput 32K high_entropy tier (512 prompts). + metadata: + hf_dataset_name: nvidia/SPEED-Bench + hf_split: test + hf_subset: throughput_32k + multi_turn: true + category: high_entropy + # ============================================================================= # ZMQ Proxies # ============================================================================= diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 0b68712e6a..44c99acc85 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"plugins.schema.json","additionalProperties":false,"description":"Root model for plugins.yaml file.\n\nThis file registers plugin implementations for AIPerf. Each top-level section\ncorresponds to a category from categories.yaml and maps type names to their\nimplementing classes.\n\nNote: Package metadata (name, version, author) comes from pyproject.toml\nvia importlib.metadata, not from this file.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n my_custom:\n class: my_package.endpoints.my_custom:MyCustomEndpoint\n description: Custom endpoint for my API.\n metadata:\n endpoint_path: /v1/generate\n supports_streaming: true\n produces_tokens: true\n tokenizes_input: true\n metrics_title: My Custom Metrics\n\ncustom_dataset_loader:\n my_jsonl:\n class: my_package.dataset_loaders.my_jsonl:MyJSONLDatasetLoader\n description: Custom dataset loader for my API.\n```","properties":{"schema_version":{"default":"1.0","description":"Version of the plugins.yaml schema format. Use '1.0' for current format.","title":"Schema Version","type":"string"},"api_router":{"title":"API Router Plugins","type":"object","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded.","additionalProperties":{"$ref":"#/$defs/ApiRouterPlugin"}},"timing_strategy":{"title":"Timing Strategy Plugins","type":"object","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/TimingStrategyPlugin"}},"arrival_pattern":{"title":"Arrival Pattern Plugins","type":"object","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode.","additionalProperties":{"$ref":"#/$defs/ArrivalPatternPlugin"}},"ramp":{"title":"Ramp Plugins","type":"object","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns.","additionalProperties":{"$ref":"#/$defs/RampPlugin"}},"dataset_backing_store":{"title":"Dataset Backing Store Plugins","type":"object","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access.","additionalProperties":{"$ref":"#/$defs/DatasetBackingStorePlugin"}},"dataset_client_store":{"title":"Dataset Client Store Plugins","type":"object","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance.","additionalProperties":{"$ref":"#/$defs/DatasetClientStorePlugin"}},"dataset_sampler":{"title":"Dataset Sampler Plugins","type":"object","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration.","additionalProperties":{"$ref":"#/$defs/DatasetSamplerPlugin"}},"dataset_composer":{"title":"Dataset Composer Plugins","type":"object","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration.","additionalProperties":{"$ref":"#/$defs/DatasetComposerPlugin"}},"custom_dataset_loader":{"title":"Custom Dataset Loader Plugins","type":"object","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load().","additionalProperties":{"$ref":"#/$defs/CustomDatasetLoaderPlugin"}},"public_dataset_loader":{"title":"Public Dataset Loader Plugins","type":"object","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration.","additionalProperties":{"$ref":"#/$defs/PublicDatasetLoaderPlugin"}},"endpoint":{"title":"Endpoint Plugins","type":"object","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration.","additionalProperties":{"$ref":"#/$defs/EndpointPlugin"}},"transport":{"title":"Transport Plugins","type":"object","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration.","additionalProperties":{"$ref":"#/$defs/TransportPlugin"}},"record_processor":{"title":"Record Processor Plugins","type":"object","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordProcessorPlugin"}},"record_observer":{"title":"Record Observer Plugins","type":"object","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordObserverPlugin"}},"accumulator":{"title":"Accumulator Plugins","type":"object","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AccumulatorPlugin"}},"stream_exporter":{"title":"Stream Exporter Plugins","type":"object","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/StreamExporterPlugin"}},"analyzer":{"title":"Analyzer Plugins","type":"object","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AnalyzerPlugin"}},"spec_decode_adapter":{"title":"Spec Decode Adapter Plugins","type":"object","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins.","additionalProperties":{"$ref":"#/$defs/SpecDecodeAdapterPlugin"}},"accuracy_grader":{"title":"Accuracy Grader Plugins","type":"object","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default.","additionalProperties":{"$ref":"#/$defs/AccuracyGraderPlugin"}},"accuracy_benchmark":{"title":"Accuracy Benchmark Plugins","type":"object","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration.","additionalProperties":{"$ref":"#/$defs/AccuracyBenchmarkPlugin"}},"data_exporter":{"title":"Data Exporter Plugins","type":"object","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/DataExporterPlugin"}},"console_exporter":{"title":"Console Exporter Plugins","type":"object","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/ConsoleExporterPlugin"}},"ui":{"title":"UI Plugins","type":"object","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration.","additionalProperties":{"$ref":"#/$defs/UiPlugin"}},"url_selection_strategy":{"title":"Url Selection Strategy Plugins","type":"object","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration.","additionalProperties":{"$ref":"#/$defs/UrlSelectionStrategyPlugin"}},"service":{"title":"Service Plugins","type":"object","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services.","additionalProperties":{"$ref":"#/$defs/ServicePlugin"}},"service_manager":{"title":"Service Manager Plugins","type":"object","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration.","additionalProperties":{"$ref":"#/$defs/ServiceManagerPlugin"}},"communication":{"title":"Communication Plugins","type":"object","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration.","additionalProperties":{"$ref":"#/$defs/CommunicationPlugin"}},"communication_client":{"title":"Communication Client Plugins","type":"object","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage.","additionalProperties":{"$ref":"#/$defs/CommunicationClientPlugin"}},"zmq_proxy":{"title":"ZMQ Proxy Plugins","type":"object","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration.","additionalProperties":{"$ref":"#/$defs/ZmqProxyPlugin"}},"plot":{"title":"Plot Plugins","type":"object","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection.","additionalProperties":{"$ref":"#/$defs/PlotPlugin"}},"gpu_telemetry_collector":{"title":"GPU Telemetry Collector Plugins","type":"object","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration.","additionalProperties":{"$ref":"#/$defs/GpuTelemetryCollectorPlugin"}},"search_recipe":{"title":"Search Recipe Plugins","type":"object","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchRecipePlugin"}},"search_recipe_post_process":{"title":"Search Recipe Post Process Plugins","type":"object","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec.","additionalProperties":{"$ref":"#/$defs/SearchRecipePostProcessPlugin"}},"convergence_criterion":{"title":"Convergence Criterion Plugins","type":"object","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/ConvergenceCriterionPlugin"}},"search_planner":{"title":"Search Planner Plugins","type":"object","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchPlannerPlugin"}}},"title":"PluginsManifest","type":"object","$defs":{"ApiRouterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"API Router Plugin","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded."},"TimingStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Timing Strategy Plugin","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run."},"ArrivalPatternPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Arrival Pattern Plugin","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode."},"RampPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Ramp Plugin","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns."},"DatasetBackingStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Backing Store Plugin","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access."},"DatasetClientStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Client Store Plugin","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance."},"DatasetSamplerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Sampler Plugin","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration."},"DatasetComposerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Composer Plugin","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration."},"CustomDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for custom dataset loader plugins.\n\nDefines format-specific defaults for dataset loaders. When a loader specifies\n``block_size``, it overrides the user's ``--isl-block-size`` config default,\nensuring hash-based prompt generation uses the correct token block size for the\ntrace format (e.g. 16 for Bailian, 512 for Mooncake).\n\nReferenced by: categories.yaml custom_dataset_loader.metadata_class\nUsed in: plugins.yaml custom_dataset_loader entries","properties":{"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace datasets use hash_ids-based prompt generation, support synthesis options, and prefer sequential sampling with fixed_schedule timing.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids (e.g. 16 for Bailian, 512 for Mooncake).","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces (e.g. weka_trace) override to 'coding' so reconstructed prompts resemble real tool-use content.","title":"Default Prompt Corpus","type":"string"},"category":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Filter dataset rows to a specific category value. Used by custom loaders that support per-category subsets (e.g. SPEED-Bench).","title":"Category"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects.","title":"Multi Turn","type":"boolean"}},"title":"CustomDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Custom Dataset Loader Plugin","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load()."},"PublicDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for public dataset loader plugins.\n\nReferenced by: categories.yaml public_dataset_loader.metadata_class\nUsed in: plugins.yaml public_dataset_loader entries","properties":{"hf_dataset_name":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset identifier (e.g. 'AI-MO/NuminaMath-TIR'). Required for HF-backed loaders.","title":"Hf Dataset Name"},"hf_split":{"default":"train","description":"HuggingFace dataset split to load (e.g. 'train', 'test', 'validation').","title":"Hf Split","type":"string"},"hf_subset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name. Only needed for datasets with multiple configs.","title":"Hf Subset"},"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace public datasets reuse hash_ids-based prompt generation, require a tokenizer, and prefer sequential sampling. Mirrors the field of the same name on CustomDatasetLoaderMetadata so trace loaders can live in either pipeline.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids.","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces override to 'coding'.","title":"Default Prompt Corpus","type":"string"},"prompt_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the prompt/instruction text. Required for HFInstructionResponseDatasetLoader.","title":"Prompt Column"},"image_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the image data (PIL Image). Used for multimodal datasets.","title":"Image Column"},"video_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the video data (URL string or bytes dict). Used for video multimodal datasets.","title":"Video Column"},"audio_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing audio data. Depending on loader/dataset, this can be an HF Audio dict with array/sampling_rate (decoded) or bytes/path (decode=False). Used for ASR/speech datasets.","title":"Audio Column"},"conversation_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the conversation messages array. Required for HFConversationDatasetLoader.","title":"Conversation Column"},"message_content_key":{"default":"content","description":"Key inside each message dict for the text content. Used with conversation_column (e.g. 'content', 'value').","title":"Message Content Key","type":"string"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects. For HFConversationDatasetLoader: user→assistant (OpenAI roles) or human→gpt (ShareGPT-style ``from`` fields) pairs.","title":"Multi Turn","type":"boolean"},"streaming":{"default":false,"description":"Whether to load the HuggingFace dataset in streaming mode. Use true for large datasets (>10 GB) to avoid downloading the full dataset. Use false (default) for small datasets to leverage HF caching and len() support.","title":"Streaming","type":"boolean"},"has_timing_data":{"default":false,"description":"Whether the loader emits timestamps suitable for fixed-schedule replay.","title":"Has Timing Data","type":"boolean"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.","title":"Prompt Template"}},"title":"PublicDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Public Dataset Loader Plugin","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration."},"EndpointPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for endpoint plugins.\n\nDefines API capabilities, paths, and multimodal support for endpoint implementations.\nUsed by the framework to route requests, configure streaming, and enable\nmultimodal inputs/outputs (images, audio, video).\n\nReferenced by: categories.yaml endpoint.metadata_class\nUsed in: plugins.yaml endpoint entries","properties":{"metrics_title":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"Display title for metrics dashboard.","title":"Metrics Title"},"endpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"API path (e.g., /v1/chat/completions).","title":"Endpoint Path"},"streaming_path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Streaming API path if different from the endpoint path (e.g., /generate_stream).","title":"Streaming Path"},"service_kind":{"default":"openai","description":"The service kind of the endpoint (used for artifact naming).","title":"Service Kind","type":"string"},"supports_streaming":{"description":"Whether endpoint supports streaming responses.","title":"Supports Streaming","type":"boolean"},"tokenizes_input":{"description":"Whether endpoint tokenizes text inputs.","title":"Tokenizes Input","type":"boolean"},"produces_tokens":{"description":"Whether endpoint produces token-based output.","title":"Produces Tokens","type":"boolean"},"supports_cache_bust":{"default":false,"description":"Whether the endpoint formatter can apply a cache-bust marker to the structured request payload.","title":"Supports Cache Bust","type":"boolean"},"supports_audio":{"default":false,"description":"Whether endpoint accepts audio input.","title":"Supports Audio","type":"boolean"},"supports_images":{"default":false,"description":"Whether endpoint accepts image input.","title":"Supports Images","type":"boolean"},"supports_videos":{"default":false,"description":"Whether endpoint accepts video input.","title":"Supports Videos","type":"boolean"},"produces_audio":{"default":false,"description":"Whether endpoint produces audio-based outputs.","title":"Produces Audio","type":"boolean"},"produces_images":{"default":false,"description":"Whether endpoint produces image-based outputs.","title":"Produces Images","type":"boolean"},"produces_videos":{"default":false,"description":"Whether endpoint produces video-based outputs.","title":"Produces Videos","type":"boolean"},"requires_polling":{"default":false,"description":"Whether endpoint uses async job polling (submit job, poll for status, retrieve result).","title":"Requires Polling","type":"boolean"},"requires_form_data":{"default":false,"description":"True for endpoints that require multipart/form-data (binary uploads), e.g., video_generation, image_edit. EndpointConfig consumes this flag to auto-select the request encoding.","title":"Requires Form Data","type":"boolean"},"requires_inline_media":{"default":false,"description":"Whether endpoint requires media URLs to be downloaded and inlined as base64 data URLs.","title":"Requires Inline Media","type":"boolean"},"consumes_system_message":{"default":false,"description":"Whether endpoint sends RequestInfo.system_message on the wire (e.g. a leading system role for chat, top-level instructions for responses). When False, the multi-turn loader does not hoist a leading system turn into the conversation system_message, since it would be silently dropped.","title":"Consumes System Message","type":"boolean"}},"required":["metrics_title","endpoint_path","supports_streaming","tokenizes_input","produces_tokens"],"title":"EndpointMetadata","type":"object"}},"required":["class"],"title":"Endpoint Plugin","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration."},"TransportPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for transport plugins.\n\nDefines network layer configuration including transport type and URL schemes.\nUsed by the framework to auto-detect appropriate transport based on URL scheme.\n\nReferenced by: categories.yaml transport.metadata_class\nUsed in: plugins.yaml transport entries","properties":{"transport_type":{"description":"Transport type identifier for this transport","title":"Transport Type","type":"string"},"url_schemes":{"description":"URL schemes this transport handles (for auto-detection and validation).","items":{"type":"string"},"title":"Url Schemes","type":"array"}},"required":["transport_type"],"title":"TransportMetadata","type":"object"}},"required":["class"],"title":"Transport Plugin","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration."},"RecordProcessorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries","properties":{"record_type":{"description":"The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.","title":"Record Type","type":"string"}},"required":["record_type"],"title":"RecordProducerMetadata","type":"object"}},"required":["class"],"title":"Record Processor Plugin","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously."},"RecordObserverPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Record Observer Plugin","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously."},"AccumulatorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Accumulator Plugin","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously."},"StreamExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Stream Exporter Plugin","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously."},"AnalyzerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary — e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) — e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries","properties":{"required_accumulators":{"description":"AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.","items":{"type":"string"},"title":"Required Accumulators","type":"array"},"required_summaries":{"description":"AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.","items":{"type":"string"},"title":"Required Summaries","type":"array"}},"title":"AnalyzerMetadata","type":"object"}},"required":["class"],"title":"Analyzer Plugin","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously."},"SpecDecodeAdapterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Spec Decode Adapter Plugin","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins."},"AccuracyGraderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Grader Plugin","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default."},"AccuracyBenchmarkPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Benchmark Plugin","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration."},"DataExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Data Exporter Plugin","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously."},"ConsoleExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Console Exporter Plugin","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously."},"UiPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"UI Plugin","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration."},"UrlSelectionStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Url Selection Strategy Plugin","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration."},"ServicePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for service plugins.\n\nDefines lifecycle and runtime configuration for AIPerf distributed services.\nUsed by SystemController to manage service startup order and optimize\nlatency-sensitive services (timing, workers) by disabling garbage collection.\n\nReferenced by: categories.yaml service.metadata_class\nUsed in: plugins.yaml service entries","properties":{"required":{"description":"Whether the service is required for benchmark execution.","title":"Required","type":"boolean"},"auto_start":{"description":"Whether the service is automatically started by the system controller.","title":"Auto Start","type":"boolean"},"disable_gc":{"default":false,"description":"Whether to disable garbage collection in the service for timing-critical operations.","title":"Disable Gc","type":"boolean"},"replicable":{"default":false,"description":"Whether the service can have multiple instances running in parallel.","title":"Replicable","type":"boolean"}},"required":["required","auto_start"],"title":"ServiceMetadata","type":"object"}},"required":["class"],"title":"Service Plugin","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services."},"ServiceManagerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Service Manager Plugin","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration."},"CommunicationPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Plugin","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration."},"CommunicationClientPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Client Plugin","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage."},"ZmqProxyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"ZMQ Proxy Plugin","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration."},"PlotPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for plot plugins.\n\nDefines display properties and categorization for visualization handlers.\nUsed by the framework to group plots and generate selection interfaces.\n\nReferenced by: categories.yaml plot.metadata_class\nUsed in: plugins.yaml plot entries","properties":{"display_name":{"description":"Human-readable name for UI display.","title":"Display Name","type":"string"},"category":{"description":"Plot category (per_request, aggregated, combined, comparison).","title":"Category","type":"string"}},"required":["display_name","category"],"title":"PlotMetadata","type":"object"}},"required":["class"],"title":"Plot Plugin","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection."},"GpuTelemetryCollectorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Config-time metadata for GPU telemetry collector plugins.\n\nLocal collectors set ``is_local`` so ``--gpu-telemetry `` is treated\nas an in-process collector keyword instead of a DCGM URL. Native binding\nvalidation lives on the collector class itself via ``validate_environment``\nso each implementation owns its own dependency check.\n\nReferenced by: categories.yaml gpu_telemetry_collector.metadata_class\nUsed in: plugins.yaml gpu_telemetry_collector entries","properties":{"is_local":{"default":false,"description":"Whether this collector runs in-process against the local host.","title":"Is Local","type":"boolean"}},"title":"GPUTelemetryCollectorMetadata","type":"object"}},"required":["class"],"title":"GPU Telemetry Collector Plugin","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration."},"SearchRecipePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Plugin","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run."},"SearchRecipePostProcessPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Post Process Plugin","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec."},"ConvergenceCriterionPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for convergence criterion plugins.\n\nDeclares statistical-method capabilities so the CLI/config layer can\nvalidate `--convergence-metric` / `--convergence-stat` against the chosen\ncriterion before the plugin is imported.\n\nReferenced by: categories.yaml convergence_criterion.metadata_class\nUsed in: plugins.yaml convergence_criterion entries","properties":{"min_samples":{"description":"Minimum number of successful runs required before convergence can trigger.","minimum":1,"title":"Min Samples","type":"integer"},"requires_confidence_level":{"default":false,"description":"Whether the criterion consumes plan.confidence_level (e.g. CI-width does, CV doesn't).","title":"Requires Confidence Level","type":"boolean"},"requires_jsonl_export":{"default":false,"description":"Whether the criterion reads per-request metrics from JSONL exports (e.g. distribution does).","title":"Requires Jsonl Export","type":"boolean"},"metric_kinds":{"description":"Kinds of metrics this criterion handles. One or more of 'continuous', 'counts', 'categorical'.","items":{"type":"string"},"title":"Metric Kinds","type":"array"}},"required":["min_samples"],"title":"ConvergenceCriterionMetadata","type":"object"}},"required":["class"],"title":"Convergence Criterion Plugin","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run."},"SearchPlannerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for search planner plugins.\n\nDeclares dimension-type and objective-direction support so the CLI/config\nlayer can validate `--search-space` shape against the chosen planner\nbefore the planner (and its heavy soft-dep imports) is loaded.\n\nReferenced by: categories.yaml search_planner.metadata_class\nUsed in: plugins.yaml search_planner entries","properties":{"supports_continuous":{"description":"Whether the planner accepts Real-valued search-space dimensions.","title":"Supports Continuous","type":"boolean"},"supports_discrete":{"description":"Whether the planner accepts Integer-valued search-space dimensions.","title":"Supports Discrete","type":"boolean"},"supports_categorical":{"default":false,"description":"Whether the planner accepts Categorical search-space dimensions.","title":"Supports Categorical","type":"boolean"},"requires_initial_samples":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Minimum number of initial random/Sobol samples required before the planner's model is fit. None when the planner has no warm-up phase.","title":"Requires Initial Samples"},"compatible_objective_directions":{"description":"Objective directions the planner can optimize. Lower-case strings.","items":{"type":"string"},"title":"Compatible Objective Directions","type":"array"},"requires_extras":{"description":"Names of pyproject.toml extras (e.g. ['optuna']) required to install this planner's heavy dependencies. Informational; the planner class itself owns the ImportError surface.","items":{"type":"string"},"title":"Requires Extras","type":"array"}},"required":["supports_continuous","supports_discrete"],"title":"SearchPlannerMetadata","type":"object"}},"required":["class"],"title":"Search Planner Plugin","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run."}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"plugins.schema.json","additionalProperties":false,"description":"Root model for plugins.yaml file.\n\nThis file registers plugin implementations for AIPerf. Each top-level section\ncorresponds to a category from categories.yaml and maps type names to their\nimplementing classes.\n\nNote: Package metadata (name, version, author) comes from pyproject.toml\nvia importlib.metadata, not from this file.\n\nExample:\n```yaml\nschema_version: \"1.0\"\n\nendpoint:\n my_custom:\n class: my_package.endpoints.my_custom:MyCustomEndpoint\n description: Custom endpoint for my API.\n metadata:\n endpoint_path: /v1/generate\n supports_streaming: true\n produces_tokens: true\n tokenizes_input: true\n metrics_title: My Custom Metrics\n\ncustom_dataset_loader:\n my_jsonl:\n class: my_package.dataset_loaders.my_jsonl:MyJSONLDatasetLoader\n description: Custom dataset loader for my API.\n```","properties":{"schema_version":{"default":"1.0","description":"Version of the plugins.yaml schema format. Use '1.0' for current format.","title":"Schema Version","type":"string"},"api_router":{"title":"API Router Plugins","type":"object","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded.","additionalProperties":{"$ref":"#/$defs/ApiRouterPlugin"}},"timing_strategy":{"title":"Timing Strategy Plugins","type":"object","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/TimingStrategyPlugin"}},"arrival_pattern":{"title":"Arrival Pattern Plugins","type":"object","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode.","additionalProperties":{"$ref":"#/$defs/ArrivalPatternPlugin"}},"ramp":{"title":"Ramp Plugins","type":"object","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns.","additionalProperties":{"$ref":"#/$defs/RampPlugin"}},"dataset_backing_store":{"title":"Dataset Backing Store Plugins","type":"object","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access.","additionalProperties":{"$ref":"#/$defs/DatasetBackingStorePlugin"}},"dataset_client_store":{"title":"Dataset Client Store Plugins","type":"object","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance.","additionalProperties":{"$ref":"#/$defs/DatasetClientStorePlugin"}},"dataset_sampler":{"title":"Dataset Sampler Plugins","type":"object","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration.","additionalProperties":{"$ref":"#/$defs/DatasetSamplerPlugin"}},"dataset_composer":{"title":"Dataset Composer Plugins","type":"object","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration.","additionalProperties":{"$ref":"#/$defs/DatasetComposerPlugin"}},"custom_dataset_loader":{"title":"Custom Dataset Loader Plugins","type":"object","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load().","additionalProperties":{"$ref":"#/$defs/CustomDatasetLoaderPlugin"}},"public_dataset_loader":{"title":"Public Dataset Loader Plugins","type":"object","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration.","additionalProperties":{"$ref":"#/$defs/PublicDatasetLoaderPlugin"}},"endpoint":{"title":"Endpoint Plugins","type":"object","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration.","additionalProperties":{"$ref":"#/$defs/EndpointPlugin"}},"transport":{"title":"Transport Plugins","type":"object","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration.","additionalProperties":{"$ref":"#/$defs/TransportPlugin"}},"record_processor":{"title":"Record Processor Plugins","type":"object","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordProcessorPlugin"}},"record_observer":{"title":"Record Observer Plugins","type":"object","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/RecordObserverPlugin"}},"accumulator":{"title":"Accumulator Plugins","type":"object","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AccumulatorPlugin"}},"stream_exporter":{"title":"Stream Exporter Plugins","type":"object","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/StreamExporterPlugin"}},"analyzer":{"title":"Analyzer Plugins","type":"object","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/AnalyzerPlugin"}},"spec_decode_adapter":{"title":"Spec Decode Adapter Plugins","type":"object","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins.","additionalProperties":{"$ref":"#/$defs/SpecDecodeAdapterPlugin"}},"accuracy_grader":{"title":"Accuracy Grader Plugins","type":"object","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default.","additionalProperties":{"$ref":"#/$defs/AccuracyGraderPlugin"}},"accuracy_benchmark":{"title":"Accuracy Benchmark Plugins","type":"object","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration.","additionalProperties":{"$ref":"#/$defs/AccuracyBenchmarkPlugin"}},"data_exporter":{"title":"Data Exporter Plugins","type":"object","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/DataExporterPlugin"}},"console_exporter":{"title":"Console Exporter Plugins","type":"object","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously.","additionalProperties":{"$ref":"#/$defs/ConsoleExporterPlugin"}},"ui":{"title":"UI Plugins","type":"object","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration.","additionalProperties":{"$ref":"#/$defs/UiPlugin"}},"url_selection_strategy":{"title":"Url Selection Strategy Plugins","type":"object","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration.","additionalProperties":{"$ref":"#/$defs/UrlSelectionStrategyPlugin"}},"service":{"title":"Service Plugins","type":"object","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services.","additionalProperties":{"$ref":"#/$defs/ServicePlugin"}},"service_manager":{"title":"Service Manager Plugins","type":"object","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration.","additionalProperties":{"$ref":"#/$defs/ServiceManagerPlugin"}},"communication":{"title":"Communication Plugins","type":"object","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration.","additionalProperties":{"$ref":"#/$defs/CommunicationPlugin"}},"communication_client":{"title":"Communication Client Plugins","type":"object","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage.","additionalProperties":{"$ref":"#/$defs/CommunicationClientPlugin"}},"zmq_proxy":{"title":"ZMQ Proxy Plugins","type":"object","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration.","additionalProperties":{"$ref":"#/$defs/ZmqProxyPlugin"}},"plot":{"title":"Plot Plugins","type":"object","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection.","additionalProperties":{"$ref":"#/$defs/PlotPlugin"}},"gpu_telemetry_collector":{"title":"GPU Telemetry Collector Plugins","type":"object","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration.","additionalProperties":{"$ref":"#/$defs/GpuTelemetryCollectorPlugin"}},"search_recipe":{"title":"Search Recipe Plugins","type":"object","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchRecipePlugin"}},"search_recipe_post_process":{"title":"Search Recipe Post Process Plugins","type":"object","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec.","additionalProperties":{"$ref":"#/$defs/SearchRecipePostProcessPlugin"}},"convergence_criterion":{"title":"Convergence Criterion Plugins","type":"object","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/ConvergenceCriterionPlugin"}},"search_planner":{"title":"Search Planner Plugins","type":"object","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run.","additionalProperties":{"$ref":"#/$defs/SearchPlannerPlugin"}}},"title":"PluginsManifest","type":"object","$defs":{"ApiRouterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"API Router Plugin","description":"API routers are lifecycle-managed components that extend BaseRouter.\nEach router participates in the full service lifecycle, with access\nto ZMQ communication, configuration, and async initialization.\nRouters expose HTTP/WebSocket endpoints via get_router() -> APIRouter.\nOne-to-many mapping: all registered routers are loaded."},"TimingStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Timing Strategy Plugin","description":"Timing strategies control request scheduling and credit issuance.\nDetermines when requests are sent based on fixed schedules, request rates,\nor user-centric patterns. One-to-one mapping per benchmark run."},"ArrivalPatternPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Arrival Pattern Plugin","description":"Interval generators determine inter-arrival times for request rate strategy.\nControls the distribution of request timing (constant, Poisson, gamma, etc.).\nOne-to-one mapping when using request_rate timing mode."},"RampPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Ramp Plugin","description":"Ramp strategies control how values are gradually transitioned over time.\nUsed for ramping concurrency limits, request rates, and other numeric parameters.\nSupports linear, exponential, and stochastic ramping patterns."},"DatasetBackingStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Backing Store Plugin","description":"Dataset backing stores manage conversation data on the DatasetManager side.\nProvides efficient storage for dataset distribution to workers.\nHandles streaming writes and finalization for client access."},"DatasetClientStorePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Client Store Plugin","description":"Dataset client stores read conversation data on the Worker side.\nProvides efficient access to datasets from backing stores.\nSupports zero-copy reads and O(1) lookup performance."},"DatasetSamplerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Sampler Plugin","description":"Dataset samplers control how conversations are selected from the dataset.\nSupports random, sequential, and shuffle sampling strategies.\nOne-to-one mapping based on sampling strategy configuration."},"DatasetComposerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Dataset Composer Plugin","description":"Dataset composers create conversation datasets from various sources.\nHandles synthetic generation, custom file loading, and specialized formats.\nOne-to-one mapping based on composer_type configuration."},"CustomDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for custom dataset loader plugins.\n\nDefines format-specific defaults for dataset loaders. When a loader specifies\n``block_size``, it overrides the user's ``--isl-block-size`` config default,\nensuring hash-based prompt generation uses the correct token block size for the\ntrace format (e.g. 16 for Bailian, 512 for Mooncake).\n\nReferenced by: categories.yaml custom_dataset_loader.metadata_class\nUsed in: plugins.yaml custom_dataset_loader entries","properties":{"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace datasets use hash_ids-based prompt generation, support synthesis options, and prefer sequential sampling with fixed_schedule timing.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids (e.g. 16 for Bailian, 512 for Mooncake).","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces (e.g. weka_trace) override to 'coding' so reconstructed prompts resemble real tool-use content.","title":"Default Prompt Corpus","type":"string"},"category":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Filter dataset rows to a specific category value. Used by custom loaders that support per-category subsets (e.g. SPEED-Bench).","title":"Category"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects.","title":"Multi Turn","type":"boolean"}},"title":"CustomDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Custom Dataset Loader Plugin","description":"Custom dataset loaders parse different JSONL file formats into conversations.\nSupports single-turn, multi-turn, random-pool, and trace formats.\nAuto-detection: loaders are tried in priority order based on can_load()."},"PublicDatasetLoaderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for public dataset loader plugins.\n\nReferenced by: categories.yaml public_dataset_loader.metadata_class\nUsed in: plugins.yaml public_dataset_loader entries","properties":{"hf_dataset_name":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset identifier (e.g. 'AI-MO/NuminaMath-TIR'). Required for HF-backed loaders.","title":"Hf Dataset Name"},"hf_split":{"default":"train","description":"HuggingFace dataset split to load (e.g. 'train', 'test', 'validation').","title":"Hf Split","type":"string"},"hf_subset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name. Only needed for datasets with multiple configs.","title":"Hf Subset"},"is_trace":{"default":false,"description":"Whether this loader handles trace-format datasets. Trace public datasets reuse hash_ids-based prompt generation, require a tokenizer, and prefer sequential sampling. Mirrors the field of the same name on CustomDatasetLoaderMetadata so trace loaders can live in either pipeline.","title":"Is Trace","type":"boolean"},"default_block_size":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Default token block size for hash-based prompt caching. Used when the user does not explicitly set --isl-block-size. Must match the block size used to generate the trace's hash_ids.","title":"Default Block Size"},"default_prompt_corpus":{"default":"sonnet","description":"Default synthetic prompt corpus for this loader. Applied when the user does not explicitly pass --prompt-corpus. Loaders for coding agent traces override to 'coding'.","title":"Default Prompt Corpus","type":"string"},"prompt_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the prompt/instruction text. Required for HFInstructionResponseDatasetLoader.","title":"Prompt Column"},"image_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the image data (PIL Image). Used for multimodal datasets.","title":"Image Column"},"video_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the video data (URL string or bytes dict). Used for video multimodal datasets.","title":"Video Column"},"audio_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing audio data. Depending on loader/dataset, this can be an HF Audio dict with array/sampling_rate (decoded) or bytes/path (decode=False). Used for ASR/speech datasets.","title":"Audio Column"},"conversation_column":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Column name containing the conversation messages array. Required for HFConversationDatasetLoader.","title":"Conversation Column"},"message_content_key":{"default":"content","description":"Key inside each message dict for the text content. Used with conversation_column (e.g. 'content', 'value').","title":"Message Content Key","type":"string"},"multi_turn":{"default":false,"description":"When true, each row becomes one Conversation with multiple Turn objects. For HFConversationDatasetLoader: user→assistant (OpenAI roles) or human→gpt (ShareGPT-style ``from`` fields) pairs.","title":"Multi Turn","type":"boolean"},"streaming":{"default":false,"description":"Whether to load the HuggingFace dataset in streaming mode. Use true for large datasets (>10 GB) to avoid downloading the full dataset. Use false (default) for small datasets to leverage HF caching and len() support.","title":"Streaming","type":"boolean"},"has_timing_data":{"default":false,"description":"Whether the loader emits timestamps suitable for fixed-schedule replay.","title":"Has Timing Data","type":"boolean"},"category":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Filter dataset rows to a specific category value. Used by loaders that expose per-category subsets as separate selectable datasets (e.g. SPEED-Bench). Mirrors the field of the same name on CustomDatasetLoaderMetadata.","title":"Category"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.","title":"Prompt Template"}},"title":"PublicDatasetLoaderMetadata","type":"object"}},"required":["class"],"title":"Public Dataset Loader Plugin","description":"Public dataset loaders fetch shared benchmark datasets without requiring a local file.\nSupports HTTP-downloaded and HuggingFace-hosted datasets.\nOne-to-one mapping: selected based on --public-dataset configuration."},"EndpointPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for endpoint plugins.\n\nDefines API capabilities, paths, and multimodal support for endpoint implementations.\nUsed by the framework to route requests, configure streaming, and enable\nmultimodal inputs/outputs (images, audio, video).\n\nReferenced by: categories.yaml endpoint.metadata_class\nUsed in: plugins.yaml endpoint entries","properties":{"metrics_title":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"Display title for metrics dashboard.","title":"Metrics Title"},"endpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"API path (e.g., /v1/chat/completions).","title":"Endpoint Path"},"streaming_path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Streaming API path if different from the endpoint path (e.g., /generate_stream).","title":"Streaming Path"},"service_kind":{"default":"openai","description":"The service kind of the endpoint (used for artifact naming).","title":"Service Kind","type":"string"},"supports_streaming":{"description":"Whether endpoint supports streaming responses.","title":"Supports Streaming","type":"boolean"},"tokenizes_input":{"description":"Whether endpoint tokenizes text inputs.","title":"Tokenizes Input","type":"boolean"},"produces_tokens":{"description":"Whether endpoint produces token-based output.","title":"Produces Tokens","type":"boolean"},"supports_cache_bust":{"default":false,"description":"Whether the endpoint formatter can apply a cache-bust marker to the structured request payload.","title":"Supports Cache Bust","type":"boolean"},"supports_audio":{"default":false,"description":"Whether endpoint accepts audio input.","title":"Supports Audio","type":"boolean"},"supports_images":{"default":false,"description":"Whether endpoint accepts image input.","title":"Supports Images","type":"boolean"},"supports_videos":{"default":false,"description":"Whether endpoint accepts video input.","title":"Supports Videos","type":"boolean"},"produces_audio":{"default":false,"description":"Whether endpoint produces audio-based outputs.","title":"Produces Audio","type":"boolean"},"produces_images":{"default":false,"description":"Whether endpoint produces image-based outputs.","title":"Produces Images","type":"boolean"},"produces_videos":{"default":false,"description":"Whether endpoint produces video-based outputs.","title":"Produces Videos","type":"boolean"},"requires_polling":{"default":false,"description":"Whether endpoint uses async job polling (submit job, poll for status, retrieve result).","title":"Requires Polling","type":"boolean"},"requires_form_data":{"default":false,"description":"True for endpoints that require multipart/form-data (binary uploads), e.g., video_generation, image_edit. EndpointConfig consumes this flag to auto-select the request encoding.","title":"Requires Form Data","type":"boolean"},"requires_inline_media":{"default":false,"description":"Whether endpoint requires media URLs to be downloaded and inlined as base64 data URLs.","title":"Requires Inline Media","type":"boolean"},"consumes_system_message":{"default":false,"description":"Whether endpoint sends RequestInfo.system_message on the wire (e.g. a leading system role for chat, top-level instructions for responses). When False, the multi-turn loader does not hoist a leading system turn into the conversation system_message, since it would be silently dropped.","title":"Consumes System Message","type":"boolean"}},"required":["metrics_title","endpoint_path","supports_streaming","tokenizes_input","produces_tokens"],"title":"EndpointMetadata","type":"object"}},"required":["class"],"title":"Endpoint Plugin","description":"Endpoints define how to format requests and parse responses for different APIs.\nSupports OpenAI-compatible, HuggingFace, Cohere, NIM, and custom API formats.\nOne-to-one mapping based on endpoint_type configuration."},"TransportPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for transport plugins.\n\nDefines network layer configuration including transport type and URL schemes.\nUsed by the framework to auto-detect appropriate transport based on URL scheme.\n\nReferenced by: categories.yaml transport.metadata_class\nUsed in: plugins.yaml transport entries","properties":{"transport_type":{"description":"Transport type identifier for this transport","title":"Transport Type","type":"string"},"url_schemes":{"description":"URL schemes this transport handles (for auto-detection and validation).","items":{"type":"string"},"title":"Url Schemes","type":"array"}},"required":["transport_type"],"title":"TransportMetadata","type":"object"}},"required":["class"],"title":"Transport Plugin","description":"Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration."},"RecordProcessorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries","properties":{"record_type":{"description":"The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.","title":"Record Type","type":"string"}},"required":["record_type"],"title":"RecordProducerMetadata","type":"object"}},"required":["class"],"title":"Record Processor Plugin","description":"Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously."},"RecordObserverPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Record Observer Plugin","description":"Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously."},"AccumulatorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Accumulator Plugin","description":"Accumulators ingest records, support time-range queries, and produce summaries.\nPrimary data stores in the records pipeline. Each accumulator declares which\nrecord types it handles. One-to-many mapping: multiple accumulators loaded simultaneously."},"StreamExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries","properties":{"record_types":{"description":"Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.","items":{"type":"string"},"title":"Record Types","type":"array"}},"required":["record_types"],"title":"RecordRoutingMetadata","type":"object"}},"required":["class"],"title":"Stream Exporter Plugin","description":"Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously."},"AnalyzerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary — e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) — e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries","properties":{"required_accumulators":{"description":"AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.","items":{"type":"string"},"title":"Required Accumulators","type":"array"},"required_summaries":{"description":"AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.","items":{"type":"string"},"title":"Required Summaries","type":"array"}},"title":"AnalyzerMetadata","type":"object"}},"required":["class"],"title":"Analyzer Plugin","description":"Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously."},"SpecDecodeAdapterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Spec Decode Adapter Plugin","description":"Spec-decode adapters read an engine's raw per-request speculative-decoding\npayload off the parsed responses and fill the engine-neutral\nSpecDecodeAcceptanceRecord, so nothing engine-specific leaks into the\nmetrics layer. Auto-detected: adapters are tried in priority order and the\nfirst whose can_adapt() recognizes the payload wins."},"AccuracyGraderPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Grader Plugin","description":"Accuracy graders evaluate LLM responses against ground truth answers.\nSupports exact match, math expression, multiple choice, and code execution grading.\nOne-to-one mapping based on grader configuration or benchmark default."},"AccuracyBenchmarkPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Accuracy Benchmark Plugin","description":"Accuracy benchmarks provide standardized problem sets for evaluating LLM correctness.\nSupports MMLU, AIME, HellaSwag, BigBench, Math500, GPQA Diamond, and more.\nOne-to-one mapping based on accuracy benchmark configuration."},"DataExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Data Exporter Plugin","description":"Data exporters write benchmark results to files in various formats.\nSupports CSV, JSON, Parquet, and specialized export formats.\nOne-to-many mapping: multiple exporters can be loaded simultaneously."},"ConsoleExporterPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Console Exporter Plugin","description":"Console exporters display benchmark results and diagnostics to stdout.\nProvides formatted output for metrics, errors, telemetry, and traces.\nOne-to-many mapping: multiple console exporters can be loaded simultaneously."},"UiPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"UI Plugin","description":"UI components provide progress tracking and visualization during benchmark execution.\nSupports rich terminal dashboards, simple progress bars, or headless execution.\nOne-to-one mapping based on ui configuration."},"UrlSelectionStrategyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Url Selection Strategy Plugin","description":"URL selection strategies control how requests are distributed across multiple endpoints.\nSupports round-robin, random, and custom distribution patterns.\nOne-to-one mapping based on url_selection_strategy configuration."},"ServicePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for service plugins.\n\nDefines lifecycle and runtime configuration for AIPerf distributed services.\nUsed by SystemController to manage service startup order and optimize\nlatency-sensitive services (timing, workers) by disabling garbage collection.\n\nReferenced by: categories.yaml service.metadata_class\nUsed in: plugins.yaml service entries","properties":{"required":{"description":"Whether the service is required for benchmark execution.","title":"Required","type":"boolean"},"auto_start":{"description":"Whether the service is automatically started by the system controller.","title":"Auto Start","type":"boolean"},"disable_gc":{"default":false,"description":"Whether to disable garbage collection in the service for timing-critical operations.","title":"Disable Gc","type":"boolean"},"replicable":{"default":false,"description":"Whether the service can have multiple instances running in parallel.","title":"Replicable","type":"boolean"}},"required":["required","auto_start"],"title":"ServiceMetadata","type":"object"}},"required":["class"],"title":"Service Plugin","description":"Services are the core processes that make up the AIPerf distributed system.\nEach service runs in a separate process and communicates via ZMQ message bus.\nIncludes SystemController, Workers, Managers, and processing services."},"ServiceManagerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Service Manager Plugin","description":"Service managers orchestrate how services are launched and managed.\nSupports multiprocessing (single-node) and Kubernetes (multi-node) deployments.\nOne-to-one mapping based on run_mode configuration."},"CommunicationPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Plugin","description":"Communication backends provide the underlying transport for inter-service messaging.\nSupports ZMQ IPC (single-node) and ZMQ TCP (multi-node) communication.\nOne-to-one mapping based on deployment configuration."},"CommunicationClientPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Communication Client Plugin","description":"Communication clients implement different ZMQ socket patterns for messaging.\nIncludes PUB/SUB, PUSH/PULL, REQUEST/REPLY, and streaming patterns.\nInternal infrastructure: automatically created by framework based on usage."},"ZmqProxyPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"ZMQ Proxy Plugin","description":"ZMQ proxies provide message routing between different socket patterns.\nIncludes XPUB/XSUB, DEALER/ROUTER, and PUSH/PULL proxy types.\nInternal infrastructure: automatically created by framework based on configuration."},"PlotPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for plot plugins.\n\nDefines display properties and categorization for visualization handlers.\nUsed by the framework to group plots and generate selection interfaces.\n\nReferenced by: categories.yaml plot.metadata_class\nUsed in: plugins.yaml plot entries","properties":{"display_name":{"description":"Human-readable name for UI display.","title":"Display Name","type":"string"},"category":{"description":"Plot category (per_request, aggregated, combined, comparison).","title":"Category","type":"string"}},"required":["display_name","category"],"title":"PlotMetadata","type":"object"}},"required":["class"],"title":"Plot Plugin","description":"Plot handlers create different types of visualizations from benchmark data.\nSupports scatter, histogram, timeline, percentile bands, and multi-run comparisons.\nOne-to-one mapping based on plot type selection."},"GpuTelemetryCollectorPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Config-time metadata for GPU telemetry collector plugins.\n\nLocal collectors set ``is_local`` so ``--gpu-telemetry `` is treated\nas an in-process collector keyword instead of a DCGM URL. Native binding\nvalidation lives on the collector class itself via ``validate_environment``\nso each implementation owns its own dependency check.\n\nReferenced by: categories.yaml gpu_telemetry_collector.metadata_class\nUsed in: plugins.yaml gpu_telemetry_collector entries","properties":{"is_local":{"default":false,"description":"Whether this collector runs in-process against the local host.","title":"Is Local","type":"boolean"}},"title":"GPUTelemetryCollectorMetadata","type":"object"}},"required":["class"],"title":"GPU Telemetry Collector Plugin","description":"GPU telemetry collectors gather GPU metrics from various sources and deliver them via callbacks.\nSupports DCGM HTTP endpoints and plugin-defined local collectors.\nOne-to-one mapping based on gpu_telemetry_collector configuration."},"SearchRecipePlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Plugin","description":"Search recipes are named, plugin-registered presets that compile down to\nAdaptiveSearchSweep (Bayesian Optimization) or sweep.parameters (grid) in\nthe CLI converter (`aiperf.config.flags._converter_optionals.expand_search_recipe`,\nlifted onto the top-level sweep envelope by `aiperf.config.flags.converter\n._apply_recipe_sweep_parameters`). Selected via `--search-recipe ` on\nthe CLI. One-to-one mapping per benchmark run."},"SearchRecipePostProcessPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"anyOf":[{"additionalProperties":true,"type":"object"},{"type":"null"}],"default":null,"description":"Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.","title":"Metadata"}},"required":["class"],"title":"Search Recipe Post Process Plugin","description":"Post-process handlers run after `SweepAnalyzer.compute()` in\n`aggregate_sweep_and_export` and emit derived artifacts (curves, knee\npoints) into `sweep_aggregate/`. Handlers are stateless and one instance\nis reused per recipe. Selected indirectly via a recipe's PostProcessSpec.\nOne-to-one mapping per recipe-emitted PostProcessSpec."},"ConvergenceCriterionPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for convergence criterion plugins.\n\nDeclares statistical-method capabilities so the CLI/config layer can\nvalidate `--convergence-metric` / `--convergence-stat` against the chosen\ncriterion before the plugin is imported.\n\nReferenced by: categories.yaml convergence_criterion.metadata_class\nUsed in: plugins.yaml convergence_criterion entries","properties":{"min_samples":{"description":"Minimum number of successful runs required before convergence can trigger.","minimum":1,"title":"Min Samples","type":"integer"},"requires_confidence_level":{"default":false,"description":"Whether the criterion consumes plan.confidence_level (e.g. CI-width does, CV doesn't).","title":"Requires Confidence Level","type":"boolean"},"requires_jsonl_export":{"default":false,"description":"Whether the criterion reads per-request metrics from JSONL exports (e.g. distribution does).","title":"Requires Jsonl Export","type":"boolean"},"metric_kinds":{"description":"Kinds of metrics this criterion handles. One or more of 'continuous', 'counts', 'categorical'.","items":{"type":"string"},"title":"Metric Kinds","type":"array"}},"required":["min_samples"],"title":"ConvergenceCriterionMetadata","type":"object"}},"required":["class"],"title":"Convergence Criterion Plugin","description":"Convergence criteria decide when benchmark metrics have stabilized across\nrepeated runs and the adaptive trial loop can stop. Each criterion declares\nits statistical assumptions (min samples, confidence-level usage, JSONL\nconsumption) via metadata so the CLI can validate `--convergence-*` flags\nbefore importing scipy/statsmodels. Selected via `--convergence-mode `.\nOne-to-one mapping per benchmark run."},"SearchPlannerPlugin":{"type":"object","properties":{"class":{"description":"Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.","title":"Class","type":"string"},"description":{"default":"","description":"Brief explanation of what this plugin type does and when to use it.","title":"Description","type":"string"},"priority":{"default":0,"description":"Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.","title":"Priority","type":"integer"},"metadata":{"description":"Metadata schema for search planner plugins.\n\nDeclares dimension-type and objective-direction support so the CLI/config\nlayer can validate `--search-space` shape against the chosen planner\nbefore the planner (and its heavy soft-dep imports) is loaded.\n\nReferenced by: categories.yaml search_planner.metadata_class\nUsed in: plugins.yaml search_planner entries","properties":{"supports_continuous":{"description":"Whether the planner accepts Real-valued search-space dimensions.","title":"Supports Continuous","type":"boolean"},"supports_discrete":{"description":"Whether the planner accepts Integer-valued search-space dimensions.","title":"Supports Discrete","type":"boolean"},"supports_categorical":{"default":false,"description":"Whether the planner accepts Categorical search-space dimensions.","title":"Supports Categorical","type":"boolean"},"requires_initial_samples":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Minimum number of initial random/Sobol samples required before the planner's model is fit. None when the planner has no warm-up phase.","title":"Requires Initial Samples"},"compatible_objective_directions":{"description":"Objective directions the planner can optimize. Lower-case strings.","items":{"type":"string"},"title":"Compatible Objective Directions","type":"array"},"requires_extras":{"description":"Names of pyproject.toml extras (e.g. ['optuna']) required to install this planner's heavy dependencies. Informational; the planner class itself owns the ImportError surface.","items":{"type":"string"},"title":"Requires Extras","type":"array"}},"required":["supports_continuous","supports_discrete"],"title":"SearchPlannerMetadata","type":"object"}},"required":["class"],"title":"Search Planner Plugin","description":"Search planners drive the adaptive outer loop, proposing the next variation\nvia `ask()` and absorbing observed metrics via `tell()`. Each declares\ndimension-type and objective-direction support via metadata so the CLI can\nvalidate `--search-space` shape before importing the planner's heavy\nsoft-dep tree (optuna, botorch, scipy, etc.). Selected via `--search-planner `.\nOne-to-one mapping per benchmark run."}}} diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index 462374529e..7658471e44 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -492,6 +492,15 @@ class PublicDatasetLoaderMetadata(BaseModel): default=False, description="Whether the loader emits timestamps suitable for fixed-schedule replay.", ) + category: str | None = Field( + default=None, + description=( + "Filter dataset rows to a specific category value. Used by loaders " + "that expose per-category subsets as separate selectable datasets " + "(e.g. SPEED-Bench). Mirrors the field of the same name on " + "CustomDatasetLoaderMetadata." + ), + ) prompt_template: str | None = Field( default=None, description="Python str.format() template for constructing the prompt from multiple columns (e.g. '{code}\\n\\n{change_request}'). When set, overrides prompt_column. All referenced column names must exist in the dataset.", diff --git a/tests/unit/cli_runner/test_dataset_preflight.py b/tests/unit/cli_runner/test_dataset_preflight.py new file mode 100644 index 0000000000..8146079709 --- /dev/null +++ b/tests/unit/cli_runner/test_dataset_preflight.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Dataset preflight: enumeration and hook dispatch. + +These run in the CLI process before service bootstrap, so they must work with +no event loop and must not construct loaders (whose base class opens an +aiohttp client). +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from aiperf.cli_runner._preflight import ( + _preflight_dataset_access, + _preflight_dataset_materialize, + _public_dataset_loaders, + preflight_public_datasets_for_run, +) +from aiperf.config.dataset import FileDataset, PublicDataset + + +def _plan(dataset): + """Minimal stand-in for BenchmarkPlan; only get_default_dataset is read.""" + return SimpleNamespace( + configs=[SimpleNamespace(get_default_dataset=lambda: dataset)] + ) + + +def _public(name: str, hf_subset: str | None = None): + return PublicDataset(name="main", type="public", dataset=name, hf_subset=hf_subset) + + +class TestPublicDatasetEnumeration: + def test_yields_loader_and_kwargs_for_a_public_dataset(self) -> None: + plan = _plan(_public("speed_bench_qa")) + + found = list(_public_dataset_loaders(plan)) + + assert len(found) == 1 + loader_class, kwargs = found[0] + assert loader_class.__name__ == "SpeedBenchPublicLoader" + assert kwargs == {"hf_subset": "qualitative"} + + def test_only_hook_relevant_kwargs_are_forwarded(self) -> None: + """category does not affect access or materialization, so it is not sent.""" + plan = _plan(_public("speed_bench_qualitative")) + + _, kwargs = next(iter(_public_dataset_loaders(plan))) + + assert kwargs == {"hf_subset": "qualitative"} + + def test_selectors_sharing_a_config_are_probed_once(self) -> None: + """A category sweep names many selectors backed by one config. + + Keying dedup on the selector made preflight issue one auth probe per + category before the run started -- 11 identical round trips for the + qualitative sweep, and a rate-limit risk on the preflight path. + """ + plan = SimpleNamespace( + configs=[ + SimpleNamespace(get_default_dataset=lambda d=d: d) + for d in ( + _public("speed_bench_qa"), + _public("speed_bench_coding"), + _public("speed_bench_math"), + ) + ] + ) + + assert len(list(_public_dataset_loaders(plan))) == 1 + + def test_different_configs_are_probed_separately(self) -> None: + plan = SimpleNamespace( + configs=[ + SimpleNamespace(get_default_dataset=lambda d=d: d) + for d in ( + _public("speed_bench_qa"), + _public("speed_bench_throughput_1k"), + ) + ] + ) + + assert len(list(_public_dataset_loaders(plan))) == 2 + + def test_file_datasets_are_skipped(self, tmp_path) -> None: + f = tmp_path / "d.jsonl" + f.write_text('{"text": "hi"}\n') + plan = _plan(FileDataset(name="main", type="file", path=f)) + + assert list(_public_dataset_loaders(plan)) == [] + + +class TestPreflightDispatch: + def test_access_hook_is_invoked_with_the_loader_kwargs(self) -> None: + plan = _plan(_public("speed_bench_qa")) + + with patch( + "aiperf.dataset.loader.speed_bench_public.SpeedBenchPublicLoader" + ".preflight_access" + ) as hook: + _preflight_dataset_access(plan) + + hook.assert_called_once_with(hf_subset="qualitative") + + def test_materialize_hook_is_invoked_with_the_loader_kwargs(self) -> None: + plan = _plan(_public("speed_bench_qa")) + + with patch( + "aiperf.dataset.loader.speed_bench_public.SpeedBenchPublicLoader" + ".preflight_materialize" + ) as hook: + _preflight_dataset_materialize(plan) + + hook.assert_called_once_with(hf_subset="qualitative") + + def test_hook_failures_propagate(self) -> None: + """A preflight must not swallow the error it exists to surface.""" + from aiperf.config.loader.errors import ConfigurationError + + plan = _plan(_public("speed_bench_qa")) + + with ( + patch( + "aiperf.dataset.loader.speed_bench_public.SpeedBenchPublicLoader" + ".preflight_access", + side_effect=ConfigurationError("gated"), + ), + pytest.raises(ConfigurationError, match="gated"), + ): + _preflight_dataset_access(plan) + + def test_datasets_without_hooks_are_a_no_op(self) -> None: + """Most public datasets inherit the base no-op hooks.""" + plan = _plan(_public("sharegpt")) + + _preflight_dataset_access(plan) + _preflight_dataset_materialize(plan) + + +class TestOptionalHooks: + """The hooks are not part of PublicDatasetLoaderProtocol. + + An externally registered loader need not implement them, so absence must + mean "nothing to prepare" rather than AttributeError mid-preflight. + """ + + def test_loader_without_hooks_is_skipped(self) -> None: + class BareLoader: + """Implements the protocol surface only.""" + + plan = _plan(_public("speed_bench_qa")) + + with patch("aiperf.plugin.plugins.get_class", return_value=BareLoader): + _preflight_dataset_access(plan) + _preflight_dataset_materialize(plan) + + def test_non_callable_attribute_is_not_invoked(self) -> None: + class OddLoader: + preflight_access = "not callable" + + plan = _plan(_public("speed_bench_qa")) + + with patch("aiperf.plugin.plugins.get_class", return_value=OddLoader): + _preflight_dataset_access(plan) + + +class TestKubernetesServiceEntryPoint: + """`aiperf service` never calls run_benchmark. + + Without an explicit call the gate probe and the resolution download both + land inside DatasetManager's configure step, which is bounded by + AIPERF_DATASET_CONFIGURATION_TIMEOUT. + """ + + def test_both_hooks_run_for_a_serialized_run(self) -> None: + run = SimpleNamespace( + cfg=SimpleNamespace(get_default_dataset=lambda: _public("speed_bench_qa")) + ) + calls: list[str] = [] + + class RecordingLoader: + @staticmethod + def preflight_access(**kwargs): + calls.append("access") + + @staticmethod + def preflight_materialize(**kwargs): + calls.append("materialize") + + with patch("aiperf.plugin.plugins.get_class", return_value=RecordingLoader): + preflight_public_datasets_for_run(run) + + assert calls == ["access", "materialize"] + + def test_gate_failure_surfaces_before_the_handshake(self) -> None: + from aiperf.config.loader.errors import ConfigurationError + + run = SimpleNamespace( + cfg=SimpleNamespace(get_default_dataset=lambda: _public("speed_bench_qa")) + ) + + with ( + patch( + "aiperf.dataset.loader.speed_bench_public.SpeedBenchPublicLoader" + ".preflight_access", + side_effect=ConfigurationError("gated"), + ), + pytest.raises(ConfigurationError, match="gated"), + ): + preflight_public_datasets_for_run(run) + + def test_non_public_dataset_is_a_no_op(self) -> None: + run = SimpleNamespace( + cfg=SimpleNamespace( + get_default_dataset=lambda: FileDataset( + name="main", type="file", path="x.jsonl" + ) + ) + ) + + preflight_public_datasets_for_run(run) + + +class TestResolutionOutputStaysReadable: + """Root INFO switches on httpx's per-request logging. + + Resolution makes hundreds of requests, including the 404s `datasets` emits + while probing for a legacy loading script before falling back to parquet. A + reviewer reported that flood as a suspected failure, so the progress message + this handler exists to show must not be buried by it. + """ + + def test_noisy_third_party_loggers_are_quieted(self, monkeypatch) -> None: + import logging + + root = logging.getLogger() + monkeypatch.setattr(root, "handlers", []) + for name in ("httpx", "httpcore", "urllib3", "filelock"): + logging.getLogger(name).setLevel(logging.NOTSET) + + _preflight_dataset_materialize(_plan(_public("sharegpt"))) + + for name in ("httpx", "httpcore", "urllib3", "filelock"): + assert logging.getLogger(name).level == logging.WARNING, name + + def test_an_existing_logging_setup_is_left_alone(self, monkeypatch) -> None: + """A caller who configured logging deliberately must keep their levels.""" + import logging + + root = logging.getLogger() + monkeypatch.setattr(root, "handlers", [logging.NullHandler()]) + logging.getLogger("httpx").setLevel(logging.DEBUG) + + _preflight_dataset_materialize(_plan(_public("sharegpt"))) + + assert logging.getLogger("httpx").level == logging.DEBUG diff --git a/tests/unit/dataset/composer/test_custom_composer.py b/tests/unit/dataset/composer/test_custom_composer.py index 564757f941..492f4ecb49 100644 --- a/tests/unit/dataset/composer/test_custom_composer.py +++ b/tests/unit/dataset/composer/test_custom_composer.py @@ -14,7 +14,6 @@ RandomPoolDatasetLoader, SingleTurnDatasetLoader, ) -from aiperf.dataset.loader.speed_bench import SpeedBenchLoader from aiperf.plugin.enums import CustomDatasetType, DatasetSamplingStrategy from tests.unit.dataset.composer.conftest import make_run @@ -89,20 +88,6 @@ def test_create_loader_instance_dataset_types( composer._create_loader_instance(dataset_type) assert isinstance(composer.loader, expected_instance) - def test_create_loader_instance_passes_speed_bench_category_metadata( - self, custom_config, mock_tokenizer - ): - dataset_type = CustomDatasetType("speed_bench_coding") - custom_config.custom_dataset_type = dataset_type - - composer = CustomDatasetComposer( - run=make_run(custom_config), tokenizer=mock_tokenizer - ) - composer._create_loader_instance(dataset_type) - - assert isinstance(composer.loader, SpeedBenchLoader) - assert composer.loader.category == "coding" - def test_no_category_in_kwargs_when_none(self, custom_config, mock_tokenizer): from aiperf.plugin.schema.schemas import CustomDatasetLoaderMetadata diff --git a/tests/unit/dataset/composer/test_custom_system_prompt.py b/tests/unit/dataset/composer/test_custom_system_prompt.py index c71233b0a1..cb2efb3cc8 100644 --- a/tests/unit/dataset/composer/test_custom_system_prompt.py +++ b/tests/unit/dataset/composer/test_custom_system_prompt.py @@ -202,7 +202,7 @@ def _compose( CLIConfig( model_names=["test-model"], input_file=str(speed_bench_file), - custom_dataset_type="speed_bench_coding", + custom_dataset_type="speed_bench_qualitative", **kwargs, ) ) @@ -314,6 +314,7 @@ async def _compose( multi_turn=False, streaming=False, is_trace=False, + category=None, ), ), ): diff --git a/tests/unit/dataset/composer/test_public_composer.py b/tests/unit/dataset/composer/test_public_composer.py index c973936d66..5b426067d2 100644 --- a/tests/unit/dataset/composer/test_public_composer.py +++ b/tests/unit/dataset/composer/test_public_composer.py @@ -121,6 +121,56 @@ def test_multi_turn_raises_for_unsupported_loader(self, aimo_config): ): composer._build_loader_kwargs(PublicDatasetType.AIMO, ShareGPTLoader) + def test_category_forwarded_to_supporting_loader(self, aimo_config): + from aiperf.dataset.loader.speed_bench_public import SpeedBenchPublicLoader + from aiperf.plugin.schema.schemas import PublicDatasetLoaderMetadata + + composer = PublicDatasetComposer(run=make_run(aimo_config), tokenizer=None) + with patch( + "aiperf.dataset.composer.public.plugins.get_public_dataset_loader_metadata", + return_value=PublicDatasetLoaderMetadata( + hf_dataset_name="nvidia/SPEED-Bench", + hf_split="test", + hf_subset="qualitative", + category="coding", + ), + ): + kwargs = composer._build_loader_kwargs( + PublicDatasetType.AIMO, SpeedBenchPublicLoader + ) + + assert kwargs["category"] == "coding" + assert kwargs["hf_subset"] == "qualitative" + + def test_category_raises_for_unsupported_loader(self, aimo_config): + """A loader without `category` must not swallow it via **kwargs.""" + from aiperf.plugin.schema.schemas import PublicDatasetLoaderMetadata + + composer = PublicDatasetComposer(run=make_run(aimo_config), tokenizer=None) + with ( + patch( + "aiperf.dataset.composer.public.plugins.get_public_dataset_loader_metadata", + return_value=PublicDatasetLoaderMetadata(category="coding"), + ), + pytest.raises(ValueError, match="does not support the 'category'"), + ): + composer._build_loader_kwargs(PublicDatasetType.AIMO, ShareGPTLoader) + + def test_category_none_does_not_validate_loader_support(self, aimo_config): + """category=None is the default for all but a handful of entries.""" + from aiperf.plugin.schema.schemas import PublicDatasetLoaderMetadata + + composer = PublicDatasetComposer(run=make_run(aimo_config), tokenizer=None) + with patch( + "aiperf.dataset.composer.public.plugins.get_public_dataset_loader_metadata", + return_value=PublicDatasetLoaderMetadata(category=None), + ): + kwargs = composer._build_loader_kwargs( + PublicDatasetType.AIMO, ShareGPTLoader + ) + + assert "category" not in kwargs + def test_multi_turn_false_does_not_validate_loader_support(self, aimo_config): """multi_turn=False is the default; no need to gate it on loader support.""" from aiperf.plugin.schema.schemas import PublicDatasetLoaderMetadata @@ -166,6 +216,7 @@ async def test_returns_conversations_with_finalized_turns(self, aimo_config): multi_turn=False, streaming=False, is_trace=False, + category=None, ), ), ): @@ -210,6 +261,7 @@ async def test_sets_sampling_strategy_from_loader(self, aimo_config): multi_turn=False, streaming=False, is_trace=False, + category=None, ), ), ): diff --git a/tests/unit/dataset/loader/test_can_load.py b/tests/unit/dataset/loader/test_can_load.py index 729d5811b9..7a2045402c 100644 --- a/tests/unit/dataset/loader/test_can_load.py +++ b/tests/unit/dataset/loader/test_can_load.py @@ -1,8 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json import tempfile from pathlib import Path +import orjson import pytest from pytest import param @@ -155,7 +157,11 @@ class TestSpeedBenchCanLoad: {"role": "user", "content": SpeedBenchRow.TURNS_PLACEHOLDER} ], }, - False, + # Shape is valid; the content is merely unprepared. Rejecting it + # here would make auto-detection fail with "No loader can handle + # the data format" instead of the placeholder guidance the loader + # raises. + True, id="turns_placeholder", ), param( @@ -583,3 +589,129 @@ def test_unrecognized_type_field_does_not_raise( } result = composer._infer_type(data) assert result == CustomDatasetType.BAILIAN_TRACE + + +class TestSpeedBenchAutoDetection: + """Auto-detection must not be defeated by category-filtered registry entries. + + SPEED-Bench registers 28 category-filtered plugin entries across 6 loader + classes (e.g. ``speed_bench_coding`` -> ``SpeedBenchQualitativeLoader``). + Because ``_infer_type`` counted matching *entries* rather than classes, a + correctly prepared file matched 12 of them and auto-detection died with + "Multiple loaders can handle the data format" -- failing the user who did + the preparation step correctly. + """ + + @staticmethod + def _valid_row() -> dict: + return { + "question_id": "a" * 32, + "category": "coding", + "messages": [{"role": "user", "content": "Implement binary search."}], + } + + def test_prepared_file_infers_unfiltered_split_loader( + self, create_cfg_and_composer + ): + _, composer = create_cfg_and_composer() + + result = composer._infer_type(self._valid_row(), filename="qualitative.jsonl") + + assert result == CustomDatasetType.SPEED_BENCH_QUALITATIVE + + def test_resolver_also_infers_the_unfiltered_split_loader(self, tmp_path): + """Both production detection loops must agree. + + ``CustomDatasetComposer._infer_type`` and ``DatasetResolver._detect_type`` + each walk the registry independently; if only one excludes + category-filtered entries, a file resolves differently depending on + which path runs. + """ + from aiperf.config.dataset.resolver import DatasetResolver + + f = tmp_path / "qualitative.jsonl" + f.write_text(json.dumps(self._valid_row()) + "\n") + + detected, _ = DatasetResolver._detect_type(str(f)) + + assert detected == CustomDatasetType.SPEED_BENCH_QUALITATIVE + + def test_predicate_excludes_category_filtered_entries(self): + """``is_autodetectable`` is the shared rule both loops apply. + + Exercised against synthetic entries rather than the registry: the + SPEED-Bench category selectors moved to ``--public-dataset``, so no + shipped custom entry carries a category today. The rule still guards + the invariant -- re-adding one would otherwise make every valid file + look ambiguous again. + """ + from types import SimpleNamespace + + from aiperf.plugin import plugins + + unfiltered = SimpleNamespace(name="speed_bench_qualitative", metadata={}) + filtered = SimpleNamespace( + name="speed_bench_coding", metadata={"category": "coding"} + ) + + assert plugins.is_autodetectable(unfiltered) is True + assert plugins.is_autodetectable(filtered) is False + + def test_no_shipped_custom_entry_is_category_filtered(self): + """Pins the migration: category selectors live under --public-dataset. + + If one reappears here, `_infer_type` sees two loaders claiming the same + file and auto-detection breaks for correctly prepared files. + """ + from aiperf.plugin import plugins + from aiperf.plugin.enums import PluginType + + offenders = [ + entry.name + for entry, _ in plugins.iter_all(PluginType.CUSTOM_DATASET_LOADER) + if entry.metadata.get("category") is not None + ] + + assert offenders == [] + + +class TestSpeedBenchAutoDetectionThroughProduction: + """Exercises the real `_infer_type` loop, not a reimplementation of it. + + `test_no_shipped_custom_entry_is_category_bearing` pins the registry, but it + reimplements the exclusion in its own comprehension, so it stays green if + `_infer_type` stopped consulting `plugins.is_autodetectable` entirely. These + drive a prepared SPEED-Bench row through the production detection path. + """ + + def _row(self) -> dict: + return { + "question_id": "a" * 32, + "category": "coding", + "messages": [{"role": "user", "content": "Write a function."}], + } + + def test_prepared_qualitative_file_resolves_to_exactly_one_loader( + self, create_cfg_and_composer, tmp_path + ): + _, composer = create_cfg_and_composer() + path = tmp_path / "qualitative.jsonl" + path.write_text(orjson.dumps(self._row()).decode() + "\n", encoding="utf-8") + + assert ( + composer._infer_type(self._row(), filename=path) + == CustomDatasetType.SPEED_BENCH_QUALITATIVE + ) + + def test_prepared_throughput_file_resolves_to_its_own_split_loader( + self, create_cfg_and_composer, tmp_path + ): + """The split loaders are filename-gated; detection must honor that.""" + _, composer = create_cfg_and_composer() + path = tmp_path / "throughput_1k.jsonl" + path.write_text(orjson.dumps(self._row()).decode() + "\n", encoding="utf-8") + + assert ( + composer._infer_type(self._row(), filename=path) + == CustomDatasetType.SPEED_BENCH_THROUGHPUT_1K + ) diff --git a/tests/unit/dataset/loader/test_speed_bench.py b/tests/unit/dataset/loader/test_speed_bench.py index 1f867f0b9e..a401ae8c2d 100644 --- a/tests/unit/dataset/loader/test_speed_bench.py +++ b/tests/unit/dataset/loader/test_speed_bench.py @@ -5,11 +5,18 @@ import pytest from pydantic import ValidationError +from pytest import param +from aiperf.common.exceptions import DatasetLoaderError from aiperf.common.models import Conversation from aiperf.config.flags.cli_config import CLIConfig from aiperf.dataset.loader.models import MultiTurn, SingleTurn -from aiperf.dataset.loader.speed_bench import SpeedBenchLoader, SpeedBenchRow +from aiperf.dataset.loader.speed_bench import ( + SpeedBenchLoader, + SpeedBenchQualitativeLoader, + SpeedBenchRow, + SpeedBenchThroughput1KLoader, +) from aiperf.plugin.enums import DatasetSamplingStrategy from tests.unit.conftest import make_run_from_cli @@ -361,21 +368,26 @@ def test_category_filter_returns_matching_rows(self, create_jsonl_file): == "Implement binary search." ) - def test_category_filter_no_matches_returns_empty(self, create_jsonl_file): - _, dataset = _load_speed_bench_file( - create_jsonl_file, - [ - _make_speed_bench_row( - question_id=_qid("speed-math-1"), category="math" - ), - _make_speed_bench_row( - question_id=_qid("speed-stem-1"), category="stem" - ), - ], - category="coding", - ) + def test_category_filter_no_matches_raises(self, create_jsonl_file): + """An empty dataset is never survivable downstream. - assert dict(dataset) == {} + It previously warned and returned ``{}``, then died ~26s later inside + timing_manager with ``conversation_ids cannot be empty`` and a SIGKILL, + long after the cause had scrolled off screen. + """ + with pytest.raises(DatasetLoaderError, match="matched none of the 2 rows"): + _load_speed_bench_file( + create_jsonl_file, + [ + _make_speed_bench_row( + question_id=_qid("speed-math-1"), category="math" + ), + _make_speed_bench_row( + question_id=_qid("speed-stem-1"), category="stem" + ), + ], + category="coding", + ) def test_category_stored_on_loader(self, create_jsonl_file): unfiltered_loader, _ = _load_speed_bench_file( @@ -448,7 +460,103 @@ def test_load_dataset_rejects_placeholder_content(self, create_jsonl_file): filename = _write_speed_bench_file(create_jsonl_file, [placeholder_row]) loader = SpeedBenchLoader(filename=filename, run=_make_run()) - with pytest.raises(ValidationError, match="placeholder"): + with pytest.raises(DatasetLoaderError, match="placeholder text"): + loader.load_dataset() + + def test_load_dataset_rejects_prefixed_placeholder_content(self, create_jsonl_file): + """Upstream appends per-row detail after the sentinel. + + An equality check misses these rows; 144 of SPEED-Bench's 494 unprepared + rows take this prefixed form. + """ + prefixed_row = _make_speed_bench_row( + question_id="0123456789abcdef0123456789abcdef", + category="coding", + messages=[ + { + "role": "user", + "content": f"{SpeedBenchRow.TURNS_PLACEHOLDER} (hle: question 4718)", + } + ], + ) + filename = _write_speed_bench_file(create_jsonl_file, [prefixed_row]) + loader = SpeedBenchLoader(filename=filename, run=_make_run()) + + with pytest.raises(DatasetLoaderError, match="placeholder text"): + loader.load_dataset() + + def test_load_dataset_placeholder_error_counts_every_bad_row( + self, create_jsonl_file + ): + """The message must report scale, not just fail on the first bad row.""" + rows = [ + _make_speed_bench_row( + question_id=_qid(f"speed-ok-{i}"), + messages=[{"role": "user", "content": f"Real prompt {i}"}], + ) + for i in range(2) + ] + [ + _make_speed_bench_row( + question_id=_qid(f"speed-bad-{i}"), + messages=[{"role": "user", "content": SpeedBenchRow.TURNS_PLACEHOLDER}], + ) + for i in range(3) + ] + filename = _write_speed_bench_file(create_jsonl_file, rows) + loader = SpeedBenchLoader(filename=filename, run=_make_run()) + + with pytest.raises(DatasetLoaderError, match=r"3 of 5 SPEED-Bench rows"): + loader.load_dataset() + + def test_complete_category_loads_from_partially_prepared_file( + self, create_jsonl_file + ): + """Placeholder counts must describe the request, not the whole file. + + `qa` is complete in the published dataset while `summarization` is + entirely placeholder. Counting whole-file refused a request that was + fully satisfiable, with a message describing rows the user never asked + for. + """ + rows = [ + _make_speed_bench_row( + question_id=_qid("speed-qa-1"), + category="qa", + messages=[{"role": "user", "content": "A real question."}], + ), + _make_speed_bench_row( + question_id=_qid("speed-sum-1"), + category="summarization", + messages=[{"role": "user", "content": SpeedBenchRow.TURNS_PLACEHOLDER}], + ), + ] + _, dataset = _load_speed_bench_file(create_jsonl_file, rows, category="qa") + + assert set(dataset) == {_qid("speed-qa-1")} + + def test_placeholder_error_counts_within_the_requested_category( + self, create_jsonl_file + ): + rows = [ + _make_speed_bench_row( + question_id=_qid("speed-sum-1"), + category="summarization", + messages=[{"role": "user", "content": SpeedBenchRow.TURNS_PLACEHOLDER}], + ), + _make_speed_bench_row( + question_id=_qid("speed-qa-1"), + category="qa", + messages=[{"role": "user", "content": "A real question."}], + ), + ] + filename = _write_speed_bench_file(create_jsonl_file, rows) + loader = SpeedBenchLoader( + filename=filename, run=_make_run(), category="summarization" + ) + + with pytest.raises( + DatasetLoaderError, match=r"1 of 1 SPEED-Bench rows in category" + ): loader.load_dataset() @@ -524,3 +632,34 @@ def test_multi_turn_with_category_filter(self, create_jsonl_file): assert set(dataset) == {_qid("speed-coding-1")} assert len(turns) == 2 assert [turn.text for turn in turns] == ["Code Q1", "Code Q2"] + + +class TestSuggestedPublicDataset: + """The placeholder error is the main migration path off custom files. + + A suggestion naming an unregistered loader turns one clear failure into + two, so every rendered name must exist in the registry. + """ + + @pytest.mark.parametrize( + ("loader_cls", "category", "expected"), + [ + param(SpeedBenchQualitativeLoader, None, "speed_bench_qualitative", id="qualitative_no_category"), + param(SpeedBenchQualitativeLoader, "coding", "speed_bench_coding", id="qualitative_category"), + param(SpeedBenchThroughput1KLoader, None, "speed_bench_throughput_1k", id="throughput_no_category"), + param(SpeedBenchThroughput1KLoader, "low_entropy", "speed_bench_throughput_1k_low_entropy", id="throughput_category"), + param(SpeedBenchThroughput1KLoader, "not_a_category", "speed_bench_throughput_1k", id="unknown_category_falls_back_to_split"), + ], + ) # fmt: skip + def test_suggestion_is_a_registered_public_dataset( + self, loader_cls, category, expected + ): + from aiperf.plugin.enums import PublicDatasetType + + loader = loader_cls.__new__(loader_cls) + loader.category = category + + suggested = loader._suggested_public_dataset() + + assert suggested == expected + assert suggested in {member.value for member in PublicDatasetType} diff --git a/tests/unit/dataset/loader/test_speed_bench_public.py b/tests/unit/dataset/loader/test_speed_bench_public.py new file mode 100644 index 0000000000..53ee6a146a --- /dev/null +++ b/tests/unit/dataset/loader/test_speed_bench_public.py @@ -0,0 +1,1132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib +from pathlib import Path +from unittest.mock import Mock, patch + +import orjson +import pytest +from pytest import param + +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.loader.errors import ConfigurationError +from aiperf.config.loader.parsing import _MOVED_TO_PUBLIC_DATASET +from aiperf.dataset.loader.speed_bench import SpeedBenchRow +from aiperf.dataset.loader.speed_bench_public import ( + HLE_ACCESS_URL, + SpeedBenchPublicLoader, + validate_config_name, +) +from aiperf.plugin.enums import DatasetSamplingStrategy +from tests.unit.conftest import make_run_from_cli + +PLACEHOLDER = SpeedBenchRow.TURNS_PLACEHOLDER + + +def _make_run(): + return make_run_from_cli(CLIConfig(model_names=["test-model"])) + + +def _row(question_id: str, category: str, *contents: str) -> dict: + return { + "question_id": question_id, + "category": category, + "messages": [{"role": "user", "content": c} for c in contents], + } + + +def _loader(category: str | None = None, multi_turn: bool = True, **kw): + return SpeedBenchPublicLoader( + run=_make_run(), + hf_subset="qualitative", + category=category, + multi_turn=multi_turn, + **kw, + ) + + +@pytest.fixture(autouse=True) +def _isolate_from_machine_state(tmp_path_factory): + """Keep every test in this file independent of the developer's machine. + + Two pieces of ambient state leak into these tests otherwise, and both hide + failures locally while breaking CI: + + * a real resolved cache under ``.cache/aiperf/datasets`` makes + ``preflight_access`` return before it checks credentials at all; + * a cached HuggingFace token makes the no-credentials branch unreachable. + + Tests that assert on a specific credential state patch over these. + """ + cache_dir = tmp_path_factory.mktemp("speed-bench-cache") + with ( + patch.object( + SpeedBenchPublicLoader, + "cache_path_for", + staticmethod(lambda config: cache_dir / f"{config}.jsonl"), + ), + patch("huggingface_hub.get_token", return_value="hf_isolated"), + ): + yield + + +class TestSpeedBenchPublicLoader: + def test_preferred_sampling_strategy_is_sequential(self) -> None: + assert ( + SpeedBenchPublicLoader.get_preferred_sampling_strategy() + == DatasetSamplingStrategy.SEQUENTIAL + ) + + async def test_category_filter_selects_only_matching_rows(self) -> None: + data = { + "dataset": [ + _row("a" * 32, "qa", "A question."), + _row("b" * 32, "coding", "Implement binary search."), + _row("c" * 32, "qa", "Another question."), + ] + } + + conversations = await _loader(category="qa").convert_to_conversations(data) + + assert len(conversations) == 2 + + async def test_multi_turn_false_keeps_only_first_turn(self) -> None: + data = {"dataset": [_row("a" * 32, "qa", "First.", "Second.")]} + + conversations = await _loader(multi_turn=False).convert_to_conversations(data) + + assert len(conversations[0].turns) == 1 + + async def test_unknown_category_reports_the_mismatch(self) -> None: + data = {"dataset": [_row("a" * 32, "qa", "A question.")]} + + with pytest.raises(DatasetLoaderError, match="matched none of the 1 rows"): + await _loader(category="nope").convert_to_conversations(data) + + async def test_session_id_comes_from_question_id(self) -> None: + data = {"dataset": [_row("a" * 32, "qa", "A question.")]} + + conversations = await _loader().convert_to_conversations(data) + + assert conversations[0].session_id == "a" * 32 + + async def test_cached_config_is_reused_without_resolving(self, tmp_path) -> None: + """A populated cache must not trigger a multi-GB refetch. + + This is also the documented pre-staging path: drop a resolved JSONL at + the cache location and an offline machine can benchmark from it. + """ + cached = tmp_path / "qualitative.jsonl" + cached.write_text( + orjson.dumps(_row("a" * 32, "qa", "Cached prompt.")).decode() + "\n" + ) + + with ( + patch.object(SpeedBenchPublicLoader, "cache_path_for", return_value=cached), + patch.object( + SpeedBenchPublicLoader, + "resolve_config", + side_effect=AssertionError("resolved despite a populated cache"), + ), + ): + data = await _loader().load_dataset() + + assert data["dataset"][0]["messages"][0]["content"] == "Cached prompt." + + +class TestUnresolvedRowRejection: + """Exit status is not evidence the prompts are real; the rows are.""" + + def test_rejects_and_removes_a_partially_resolved_cache(self, tmp_path) -> None: + partial = tmp_path / "qualitative.jsonl.partial" + partial.write_text( + "\n".join( + orjson.dumps(r).decode() + for r in [ + _row("a" * 32, "qa", "Real prompt."), + _row("b" * 32, "stem", PLACEHOLDER), + ] + ) + ) + + with pytest.raises(DatasetLoaderError, match=r"1 of 2 rows"): + SpeedBenchPublicLoader._reject_unresolved("qualitative", partial) + + assert not partial.exists(), "a rejected cache must not be left behind" + + def test_accepts_a_fully_resolved_cache(self, tmp_path) -> None: + partial = tmp_path / "qualitative.jsonl.partial" + partial.write_text(orjson.dumps(_row("a" * 32, "qa", "Real prompt.")).decode()) + + SpeedBenchPublicLoader._reject_unresolved("qualitative", partial) + + assert partial.exists() + + def test_prefixed_placeholder_counts_as_unresolved(self, tmp_path) -> None: + partial = tmp_path / "qualitative.jsonl.partial" + partial.write_text( + orjson.dumps(_row("a" * 32, "qa", f"{PLACEHOLDER}\n\nx")).decode() + ) + + with pytest.raises(DatasetLoaderError, match=r"1 of 1 rows"): + SpeedBenchPublicLoader._reject_unresolved("qualitative", partial) + + +class TestGatePreflight: + """The gate check must distinguish credential states. + + "No token" and "token without access" need different actions: telling an + unauthenticated user their account lacks access sends them to accept terms + on an account they never logged into. + """ + + @pytest.fixture(autouse=True) + def _uncached(self, tmp_path): + """Point the cache at an empty dir. + + preflight_access short-circuits when the config is already cached, so + without this the test passes or fails depending on whether the machine + happens to have resolved SPEED-Bench. + """ + with patch.object( + SpeedBenchPublicLoader, + "cache_path_for", + staticmethod(lambda config: tmp_path / f"{config}.jsonl"), + ): + yield + + def test_missing_credentials_says_so_and_lists_where_it_looked(self) -> None: + with ( + patch("huggingface_hub.get_token", return_value=None), + pytest.raises(ConfigurationError) as excinfo, + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + message = str(excinfo.value) + assert "no HuggingFace credentials were found" in message + assert "$HF_TOKEN" in message + assert "hf auth login" in message + + def test_authenticated_but_unauthorized_says_accept_the_terms(self) -> None: + from huggingface_hub.errors import GatedRepoError + + with ( + patch("huggingface_hub.get_token", return_value="hf_fake"), + patch( + "huggingface_hub.HfApi.auth_check", + side_effect=GatedRepoError("gated", response=Mock()), + ), + pytest.raises(ConfigurationError) as excinfo, + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + message = str(excinfo.value) + assert "You are authenticated" in message + assert HLE_ACCESS_URL in message + # Must not send an already-logged-in user back through login. + assert "hf auth login" not in message + + def test_rejected_credentials_are_reported_not_swallowed(self) -> None: + """`auth_check` raises RepositoryNotFoundError for a refused 401. + + GatedRepoError subclasses RepositoryNotFoundError, not the reverse, so + catching only the former let a bad token fall into the catch-all and + skip the fast fail this check exists for. + """ + from huggingface_hub.errors import RepositoryNotFoundError + + with ( + patch("huggingface_hub.get_token", return_value="hf_expired"), + patch( + "huggingface_hub.HfApi.auth_check", + side_effect=RepositoryNotFoundError("401", response=Mock()), + ), + pytest.raises(ConfigurationError) as excinfo, + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + message = str(excinfo.value) + assert "rejected the credentials" in message + assert "hf auth login" in message + + def test_network_failure_is_not_reported_as_missing_access(self) -> None: + with ( + patch("huggingface_hub.get_token", return_value="hf_fake"), + patch("huggingface_hub.HfApi.auth_check", side_effect=OSError("no route")), + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + def test_authorized_account_passes(self) -> None: + with ( + patch("huggingface_hub.get_token", return_value="hf_fake"), + patch("huggingface_hub.HfApi.auth_check", return_value=None), + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + def test_cached_config_skips_the_probe_entirely(self, tmp_path) -> None: + """An already-resolved config must not need credentials at all. + + This is what makes the pre-staging workflow work on an air-gapped box. + """ + cached = tmp_path / "qualitative.jsonl" + cached.write_text("{}\n") + + with ( + patch.object( + SpeedBenchPublicLoader, "cache_path_for", staticmethod(lambda c: cached) + ), + patch("huggingface_hub.get_token", side_effect=AssertionError("probed")), + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + +class TestResolveConfig: + """resolve_config drives the vendored script and publishes the cache.""" + + @staticmethod + def _fake_hf_dataset(rows): + """Stand in for a datasets.Dataset through the calls resolve_config makes.""" + + class _FakeDataset: + column_names = ["source"] + + def __init__(self, rows): + self._rows = rows + + def __contains__(self, key): + return key in self.column_names + + def __getitem__(self, key): + return [r.get("source", "") for r in self._rows] + + def map(self, fn, remove_columns=None): + return _FakeDataset([{**r, **fn(r)} for r in self._rows]) + + def to_json(self, path): + Path(path).write_text( + "\n".join( + orjson.dumps( + {k: v for k, v in r.items() if k != "turns"} + ).decode() + for r in self._rows + ) + ) + + return _FakeDataset(rows) + + def _patched_resolve(self, monkeypatch, tmp_path, rows): + """Point the cache at tmp_path and stub out every network call. + + Patches attributes on the real modules rather than replacing them in + ``sys.modules``: the vendored script does ``from datasets import + Dataset, ...`` at import time, so a stub module breaks its import + before the code under test is reached. + + Skips where ``datasets`` is unavailable. It has no Windows-on-ARM + wheel, which is why the production code imports it lazily; resolution + is unreachable on that platform, so there is nothing to exercise. + """ + datasets_mod = pytest.importorskip( + "datasets", reason="no Windows-on-ARM wheel; resolution is unreachable" + ) + + from aiperf.dataset.loader.vendor import speed_bench_prepare + + monkeypatch.setattr( + SpeedBenchPublicLoader, + "cache_path_for", + staticmethod(lambda config: tmp_path / f"{config}.jsonl"), + ) + fake = self._fake_hf_dataset(rows) + monkeypatch.setattr(datasets_mod, "load_dataset", lambda *a, **k: fake) + monkeypatch.setattr( + speed_bench_prepare, "_resolve_external_data", lambda ds, config: ds + ) + return fake + + def test_publishes_a_cache_for_a_fully_resolved_config( + self, monkeypatch, tmp_path + ) -> None: + rows = [{"question_id": "a" * 32, "category": "qa", "turns": ["A prompt."]}] + self._patched_resolve(monkeypatch, tmp_path, rows) + + path = SpeedBenchPublicLoader.resolve_config("qualitative") + + assert path.exists() + written = orjson.loads(path.read_text().splitlines()[0]) + assert written["messages"] == [{"role": "user", "content": "A prompt."}] + + def test_leaves_no_partial_file_when_rows_are_unresolved( + self, monkeypatch, tmp_path + ) -> None: + """A rejected resolve must not leave a cache the next run would trust.""" + rows = [{"question_id": "a" * 32, "category": "qa", "turns": [PLACEHOLDER]}] + self._patched_resolve(monkeypatch, tmp_path, rows) + + with pytest.raises(DatasetLoaderError, match="unresolved"): + SpeedBenchPublicLoader.resolve_config("qualitative") + + assert list(tmp_path.iterdir()) == [], "no cache or .partial may survive" + + def test_wraps_a_resolution_failure_with_context( + self, monkeypatch, tmp_path + ) -> None: + self._patched_resolve(monkeypatch, tmp_path, []) + from aiperf.dataset.loader.vendor import speed_bench_prepare + + def _boom(dataset, config): + raise RuntimeError("source unreachable") + + monkeypatch.setattr(speed_bench_prepare, "_resolve_external_data", _boom) + + with pytest.raises(DatasetLoaderError, match="Failed to resolve"): + SpeedBenchPublicLoader.resolve_config("qualitative") + + def test_gated_failure_is_reported_as_a_gate_problem( + self, monkeypatch, tmp_path + ) -> None: + """A 403 surfacing during resolve must still name the browser step.""" + self._patched_resolve(monkeypatch, tmp_path, []) + from aiperf.dataset.loader.vendor import speed_bench_prepare + + def _gated(dataset, config): + raise RuntimeError("cais/hle is a gated dataset on the Hub") + + monkeypatch.setattr(speed_bench_prepare, "_resolve_external_data", _gated) + + with pytest.raises(DatasetLoaderError) as excinfo: + SpeedBenchPublicLoader.resolve_config("qualitative") + + assert "https://huggingface.co/datasets/cais/hle" in str(excinfo.value) + + +class TestPreflightMaterialize: + def test_resolves_when_the_cache_is_cold(self, monkeypatch, tmp_path) -> None: + called = [] + monkeypatch.setattr( + SpeedBenchPublicLoader, + "cache_path_for", + staticmethod(lambda config: tmp_path / f"{config}.jsonl"), + ) + monkeypatch.setattr( + SpeedBenchPublicLoader, + "resolve_config", + classmethod(lambda cls, config: called.append(config)), + ) + + SpeedBenchPublicLoader.preflight_materialize(hf_subset="qualitative") + + assert called == ["qualitative"] + + def test_surfaces_resolution_failure_as_a_configuration_error( + self, monkeypatch, tmp_path + ) -> None: + """Preflight failures must render as a clean panel, not a traceback.""" + + def _fail(cls, config): + raise DatasetLoaderError("resolution blew up") + + monkeypatch.setattr( + SpeedBenchPublicLoader, + "cache_path_for", + staticmethod(lambda config: tmp_path / f"{config}.jsonl"), + ) + monkeypatch.setattr( + SpeedBenchPublicLoader, "resolve_config", classmethod(_fail) + ) + + with pytest.raises(ConfigurationError, match="resolution blew up"): + SpeedBenchPublicLoader.preflight_materialize(hf_subset="qualitative") + + +class TestConvertEdgeCases: + async def test_rows_with_only_blank_content_are_skipped(self) -> None: + data = { + "dataset": [ + _row("a" * 32, "qa", " "), + _row("b" * 32, "qa", "Real."), + ] + } + + conversations = await _loader().convert_to_conversations(data) + + assert len(conversations) == 1 + + +class TestMovedNameMigration: + """The 26 category selectors moved to --public-dataset. + + They were the only documented way to run those subsets, so a bare enum + error would strand anyone with an existing script or tutorial command. + """ + + @pytest.mark.parametrize("name", sorted(_MOVED_TO_PUBLIC_DATASET)) + def test_moved_name_points_at_the_new_flag(self, name) -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError) as excinfo: + CLIConfig(model_names=["m"], custom_dataset_type=name) + + message = str(excinfo.value) + assert "--public-dataset" in message + assert f"--public-dataset {name}" in message + + def test_every_moved_name_is_a_registered_public_dataset(self) -> None: + """The error names a replacement command, so it must actually exist. + + The set is built by comprehension over ISL buckets and entropy tiers, + so a typo there yields a plausible-looking name that resolves to + nothing. Spot-checking a few members cannot catch that; this walks all + of them against the registry. + """ + from aiperf.plugin.enums import PublicDatasetType + + registered = {member.value for member in PublicDatasetType} + + assert registered >= _MOVED_TO_PUBLIC_DATASET + + @pytest.mark.parametrize("name", sorted(_MOVED_TO_PUBLIC_DATASET)) + def test_moved_name_in_yaml_format_points_at_the_new_flag(self, name) -> None: + """`--custom-dataset-type` and YAML `format:` are the same choice. + + A user with an existing config file never types the flag, so guarding + only the CLI spelling leaves them with a bare enum error naming every + remaining value instead of the command that replaced theirs. + """ + from pydantic import ValidationError + + from aiperf.config.dataset import FileDataset + + with pytest.raises(ValidationError) as excinfo: + FileDataset(name="main", type="file", path="x.jsonl", format=name) + + message = str(excinfo.value) + assert "--public-dataset" in message + assert f"--public-dataset {name}" in message + + @pytest.mark.parametrize( + "name", ["speed_bench_qualitative", "speed_bench_throughput_1k"] + ) + def test_surviving_names_still_load_from_yaml(self, name) -> None: + """The 6 base entries still address a prepared file via `format:`.""" + from aiperf.config.dataset import FileDataset + + dataset = FileDataset(name="main", type="file", path="x.jsonl", format=name) + + assert dataset.format == name + + @pytest.mark.parametrize( + "name", ["speed_bench_qualitative", "speed_bench_throughput_1k"] + ) + def test_surviving_names_still_accepted(self, name) -> None: + """The 6 base entries remain: they let AIPerf read a prepared file.""" + cfg = CLIConfig(model_names=["m"], custom_dataset_type=name) + + assert cfg.custom_dataset_type == name + + +class TestPreflightRunsWithoutAnEventLoop: + """Preflight runs synchronously, so it must not construct a loader. + + Unit tests run under pytest-asyncio and always have a loop, so this class + deliberately exercises the no-loop path they would otherwise hide. + """ + + def test_materialize_does_not_construct_a_loader(self, tmp_path) -> None: + cached = tmp_path / "qualitative.jsonl" + cached.write_text(orjson.dumps(_row("a" * 32, "qa", "Cached.")).decode() + "\n") + + with patch.object( + SpeedBenchPublicLoader, "cache_path_for", return_value=cached + ): + SpeedBenchPublicLoader.preflight_materialize(hf_subset="qualitative") + + def test_access_check_does_not_construct_a_loader(self) -> None: + with ( + patch("huggingface_hub.get_token", return_value="hf_fake"), + patch("huggingface_hub.HfApi.auth_check", return_value=None), + ): + SpeedBenchPublicLoader.preflight_access(hf_subset="qualitative") + + def test_cache_path_is_resolvable_without_an_instance(self) -> None: + path = SpeedBenchPublicLoader.cache_path_for("throughput_1k") + + assert path.name == "throughput_1k.jsonl" + + +def test_public_loaders_construct_outside_an_event_loop() -> None: + loader = SpeedBenchPublicLoader(hf_subset="qualitative") + + assert loader.config == "qualitative" + + +class TestDeterminism: + """The composed dataset must not vary between runs. + + Benchmark results are only comparable if the prompts are identical, so + conversion is order-preserving and does not sample: the same cached config + yields the same conversations regardless of seed. Resolution is deterministic + in the same sense -- same rows, same order, same prompt text as upstream's + prepare step. The cache file is not guaranteed byte-stable: ``datasets`` + serializes the nested message structs with different key ordering depending + on whether it recomputes or reuses an Arrow cache, so compare resolved + content, never the file's checksum. + """ + + @staticmethod + def _data(): + return {"dataset": [_row(f"{i:032d}", "qa", f"Prompt {i}.") for i in range(25)]} + + async def test_repeated_conversion_yields_identical_conversations(self) -> None: + first = await _loader().convert_to_conversations(self._data()) + second = await _loader().convert_to_conversations(self._data()) + + assert [c.session_id for c in first] == [c.session_id for c in second] + assert [c.turns[0].texts[0].contents for c in first] == [ + c.turns[0].texts[0].contents for c in second + ] + + async def test_conversion_preserves_source_order(self) -> None: + """Order-preserving, so a truncated run is a prefix rather than a sample.""" + conversations = await _loader().convert_to_conversations(self._data()) + + assert [c.session_id for c in conversations] == [f"{i:032d}" for i in range(25)] + + +class TestResolverStateIsolation: + """Each config must resolve independently of what ran before it.""" + + @pytest.fixture(autouse=True) + def _requires_datasets(self): + """The vendored resolver imports ``datasets`` at module load. + + It has no Windows-on-ARM wheel, which is why production imports it + lazily; resolution is unreachable there, so there is nothing to pin. + """ + pytest.importorskip( + "datasets", reason="no Windows-on-ARM wheel; resolution is unreachable" + ) + + def test_rng_is_reseeded_per_config(self) -> None: + """Upstream seeds HLE_RNG once at import and consumes it building + throughput prompts, so a second config resolved in the same process + would continue from the first one's stream and produce prompts that + differ from a fresh process.""" + import numpy as np + + from aiperf.dataset.loader.vendor import speed_bench_prepare + + SpeedBenchPublicLoader._reset_resolver_state(speed_bench_prepare) + first = [int(speed_bench_prepare.HLE_RNG.integers(0, 10_000)) for _ in range(5)] + + # Simulate a prior config having drawn from the shared generator. + for _ in range(17): + speed_bench_prepare.HLE_RNG.integers(0, 10_000) + + SpeedBenchPublicLoader._reset_resolver_state(speed_bench_prepare) + second = [ + int(speed_bench_prepare.HLE_RNG.integers(0, 10_000)) for _ in range(5) + ] + + assert first == second + assert isinstance(speed_bench_prepare.HLE_RNG, np.random.Generator) + + def test_resolve_config_reseeds_before_resolving( + self, monkeypatch, tmp_path + ) -> None: + """The helper is useless unless resolve_config actually calls it.""" + rows = [{"question_id": "a" * 32, "category": "qa", "turns": ["A prompt."]}] + TestResolveConfig()._patched_resolve(monkeypatch, tmp_path, rows) + + calls: list[bool] = [] + monkeypatch.setattr( + SpeedBenchPublicLoader, + "_reset_resolver_state", + staticmethod(lambda mod: calls.append(True)), + ) + + SpeedBenchPublicLoader.resolve_config("qualitative") + + assert calls, "resolve_config did not re-seed the vendored resolver" + + +class TestSessionIdFallback: + """``str(None)`` is the truthy string "None", so a naive ``or`` fallback + never fires and every row missing ``question_id`` collides on one id.""" + + async def test_rows_without_question_id_get_distinct_ids(self) -> None: + data = { + "dataset": [ + {"category": "qa", "messages": [{"role": "user", "content": "One."}]}, + {"category": "qa", "messages": [{"role": "user", "content": "Two."}]}, + ] + } + + conversations = await _loader().convert_to_conversations(data) + + ids = [c.session_id for c in conversations] + assert len(set(ids)) == 2, f"session ids collided: {ids}" + assert "None" not in ids + + async def test_present_question_id_is_used_verbatim(self) -> None: + data = {"dataset": [_row("a" * 32, "qa", "A prompt.")]} + + conversations = await _loader().convert_to_conversations(data) + + assert conversations[0].session_id == "a" * 32 + + +class TestConfigNameValidation: + """``--hf-subset`` is a user-facing override interpolated into a filepath.""" + + @pytest.mark.parametrize( + "config", + [ + param("../../../etc/passwd", id="parent_traversal"), + param("/tmp/evil", id="absolute_path"), + param("a/b", id="nested_path"), + param("", id="empty"), + ], + ) # fmt: skip + def test_path_escaping_config_names_are_rejected(self, config: str) -> None: + with pytest.raises(ConfigurationError, match="Invalid SPEED-Bench config"): + validate_config_name(config) + + @pytest.mark.parametrize( + "config", [param("qualitative", id="qualitative"), param("throughput_1k", id="throughput")] + ) # fmt: skip + def test_published_config_names_are_accepted(self, config: str) -> None: + validate_config_name(config) + + +class TestUntrustedSourceRejection: + """Row ``source`` values select what the vendored resolver fetches. + + The dispatch happens inside the vendored file, which must stay byte-identical + to upstream, so the screen lives here instead. + """ + + class _FakeDataset: + def __init__(self, sources: list[str]) -> None: + self._sources = sources + self.column_names = ["source"] + + def __contains__(self, key: str) -> bool: + return key in self.column_names + + def __getitem__(self, key: str) -> list[str]: + return self._sources + + def test_rows_pointing_at_unexpected_hosts_are_refused(self) -> None: + dataset = self._FakeDataset( + ["http://127.0.0.1:8123/private.jsonl?src=RUCAIBox/BAMBOO"] + ) + + with pytest.raises(DatasetLoaderError, match="unexpected source hosts"): + SpeedBenchPublicLoader._reject_untrusted_sources(dataset) + + def test_published_sources_are_accepted(self) -> None: + dataset = self._FakeDataset( + [ + "https://huggingface.co/datasets/cais/hle/tree/021a3d71", + "https://raw.githubusercontent.com/RUCAIBox/BAMBOO/f230f206/x.jsonl", + "http://opencompass.openxlab.space/utils/AdaLEval/stackselect_1k.json", + "", + ] + ) + + SpeedBenchPublicLoader._reject_untrusted_sources(dataset) + + +class TestCachedInputIsValidated: + """A cache hit is untrusted: the documented pre-staging workflow copies a + file in from elsewhere, so the completeness invariant must hold on read.""" + + def _cache_with(self, tmp_path, rows: list[dict]): + cached = tmp_path / "qualitative.jsonl" + cached.write_text("\n".join(orjson.dumps(r).decode() for r in rows) + "\n") + return cached + + def test_preflight_rejects_a_pre_staged_cache_holding_placeholders( + self, tmp_path + ) -> None: + cached = self._cache_with(tmp_path, [_row("a" * 32, "qa", PLACEHOLDER)]) + + with ( + patch.object( + SpeedBenchPublicLoader, "cache_path_for", staticmethod(lambda c: cached) + ), + pytest.raises(ConfigurationError, match="unresolved"), + ): + SpeedBenchPublicLoader.preflight_materialize(hf_subset="qualitative") + + assert cached.exists(), "a user-supplied cache must never be deleted" + + async def test_load_dataset_rejects_a_pre_staged_cache_holding_placeholders( + self, tmp_path + ) -> None: + cached = self._cache_with(tmp_path, [_row("a" * 32, "qa", PLACEHOLDER)]) + + with ( + patch.object( + SpeedBenchPublicLoader, "cache_path_for", staticmethod(lambda c: cached) + ), + pytest.raises(DatasetLoaderError, match="unresolved"), + ): + await _loader().load_dataset() + + assert cached.exists() + + +class TestMissingDependencyPanel: + def test_missing_datasets_raises_the_error_profile_quiets_on( + self, monkeypatch, tmp_path + ) -> None: + """profile.py passes quiet_for=(config.loader.errors.ConfigurationError,). + + Raising the same-named class from aiperf.common.exceptions instead + prints a raw traceback -- the exact outcome the preflight exists to + avoid. + """ + import sys + + monkeypatch.setattr( + SpeedBenchPublicLoader, + "cache_path_for", + staticmethod(lambda c: tmp_path / f"{c}.jsonl"), + ) + monkeypatch.setitem(sys.modules, "aiperf.dataset.loader.vendor", None) + + with pytest.raises(ConfigurationError, match="partial install"): + SpeedBenchPublicLoader.resolve_config("qualitative") + + +class TestPlaceholderSentinelStaysPinned: + """The sentinel exists in two independently-versioned files. + + ``vendor/speed_bench_prepare.py`` is upstream verbatim and must not be + edited, so it cannot import the AIPerf copy. If a vendor bump changes the + string, every placeholder check in AIPerf silently stops matching and + unresolved rows would be benchmarked as real prompts. This test is the only + thing holding the two equal. + """ + + def test_aiperf_copy_matches_the_vendored_copy(self): + pytest.importorskip("datasets") + pytest.importorskip("tiktoken") + + from aiperf.dataset.loader.vendor import speed_bench_prepare + + assert SpeedBenchRow.TURNS_PLACEHOLDER == speed_bench_prepare.TURNS_PLACEHOLDER + + +class TestResolutionReportsItsResult: + """A reviewer could not tell a complete resolution from a truncated one. + + The run logs GBs of source downloads (which land in the HuggingFace cache) + and then produces a small JSONL, so the completion line has to state the + row count and where the bytes actually went. + """ + + def test_row_count_is_reported_on_success(self, tmp_path, caplog) -> None: + import logging + + rows = [ + { + "question_id": "a" * 32, + "category": "coding", + "messages": [{"role": "user", "content": "hi"}], + } + ] + path = tmp_path / "qualitative.jsonl" + path.write_text( + "\n".join(orjson.dumps(r).decode() for r in rows), encoding="utf-8" + ) + + with caplog.at_level(logging.INFO): + counted = SpeedBenchPublicLoader._reject_unresolved( + "qualitative", path, delete_on_failure=False + ) + + assert counted == len(rows) + + def test_unresolved_rows_still_raise_rather_than_report(self, tmp_path) -> None: + path = tmp_path / "qualitative.jsonl" + path.write_text( + orjson.dumps( + { + "question_id": "a" * 32, + "category": "coding", + "messages": [ + {"role": "user", "content": SpeedBenchRow.TURNS_PLACEHOLDER} + ], + } + ).decode(), + encoding="utf-8", + ) + + with pytest.raises(DatasetLoaderError, match="left unresolved"): + SpeedBenchPublicLoader._reject_unresolved( + "qualitative", path, delete_on_failure=False + ) + + +@pytest.fixture +def _allow_hub_access(monkeypatch): + """Let a `@pytest.mark.network` test actually reach the HuggingFace Hub. + + `tests/unit/conftest.py` sets `HF_HUB_OFFLINE=1` session-wide so no unit + test downloads a tokenizer by accident. Without opting out, a test that + reads the published dataset raises `OfflineModeIsEnabled`, the + `pytest.skip` below converts that to a skip, and the check silently never + runs on a clean CI machine -- it only appears to pass on a developer box + whose HuggingFace cache already holds the dataset. + """ + monkeypatch.delenv("HF_HUB_OFFLINE", raising=False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False) + + # huggingface_hub freezes HF_HUB_OFFLINE into a module constant at import + # time, and the session fixture sets it before this module is imported, so + # clearing the variable alone leaves the frozen True in place. + for module in ("huggingface_hub.constants", "datasets.config"): + try: + mod = importlib.import_module(module) + except ImportError: # pragma: no cover - datasets is an optional extra + continue + for attr in ("HF_HUB_OFFLINE", "HF_DATASETS_OFFLINE"): + if hasattr(mod, attr): + monkeypatch.setattr(mod, attr, False, raising=False) + + +@pytest.mark.network +class TestAllowlistMatchesThePublishedDataset: + """The allowlist is a denylist for everything else, so a gap breaks resolution. + + It was originally written from the documented source list and omitted + `github.com` and `www.gutenberg.org`, which rejected every config on a cold + cache -- invisible to every offline test, because they all supply their own + fake rows. Only reading the real dataset can catch this. + """ + + CONFIGS = ( + "qualitative", + "throughput_1k", + "throughput_2k", + "throughput_8k", + "throughput_16k", + "throughput_32k", + ) + + def test_allowlist_covers_every_published_source_host( + self, _allow_hub_access + ) -> None: + from urllib.parse import urlparse + + datasets = pytest.importorskip("datasets") + + from aiperf.dataset.loader.speed_bench_public import ( + _ALLOWED_SOURCE_HOSTS, + SPEED_BENCH_REVISION, + ) + + found: set[str] = set() + for config in self.CONFIGS: + try: + rows = datasets.load_dataset( + "nvidia/SPEED-Bench", + config, + split="test", + revision=SPEED_BENCH_REVISION, + ) + except Exception as e: + pytest.skip(f"SPEED-Bench unreachable: {e}") + + if "source" not in rows.column_names: + continue + for source in rows["source"]: + if not source: + continue + host = urlparse(str(source)).netloc.split("@")[-1].split(":")[0] + if host: + found.add(host) + + assert found <= _ALLOWED_SOURCE_HOSTS, ( + f"SPEED-Bench rows reference hosts missing from the allowlist: " + f"{sorted(found - _ALLOWED_SOURCE_HOSTS)}. Resolution rejects every " + f"affected config until they are added or the fetch is rerouted." + ) + + +class TestEmptyCacheNamesTheRealCause: + """A message that blames the wrong filter sends users to the wrong fix. + + The zero-row case previously read "from 0 rows: every row's messages were + empty", which asserts something about rows that do not exist. + """ + + def _loader(self, path): + loader = SpeedBenchPublicLoader.__new__(SpeedBenchPublicLoader) + loader.config = "qualitative" + loader.category = None + loader.multi_turn = False + return loader + + @pytest.mark.asyncio + async def test_no_rows_at_all_says_so(self, tmp_path) -> None: + path = tmp_path / "qualitative.jsonl" + path.write_text("", encoding="utf-8") + + with patch.object( + SpeedBenchPublicLoader, "cache_path_for", staticmethod(lambda c: path) + ): + loader = self._loader(path) + with pytest.raises(DatasetLoaderError, match="holds no rows at all"): + await loader.convert_to_conversations(await loader.load_dataset()) + + @pytest.mark.asyncio + async def test_blank_rows_report_their_count(self, tmp_path) -> None: + path = tmp_path / "qualitative.jsonl" + path.write_text( + "\n".join( + orjson.dumps( + { + "question_id": "a" * 32, + "category": "coding", + "messages": [{"role": "user", "content": " "}], + } + ).decode() + for _ in range(3) + ), + encoding="utf-8", + ) + + with patch.object( + SpeedBenchPublicLoader, "cache_path_for", staticmethod(lambda c: path) + ): + loader = self._loader(path) + with pytest.raises(DatasetLoaderError, match="every one of its 3 rows"): + await loader.convert_to_conversations(await loader.load_dataset()) + + +@pytest.mark.network +class TestRegisteredEntriesMatchThePublishedDataset: + """Every registered selector must actually address rows that exist. + + `plugins.yaml` declares an `hf_subset` and `category` per entry and + advertises a row count in its description. Nothing offline can confirm the + pair resolves to anything: a renamed upstream category yields zero rows and + a re-split yields silently fewer. Resolution preserves row count -- it only + fills in prompt text -- so the raw dataset is enough to check this, and + reading it costs one small parquet per config. + """ + + def _expected(self, hf_subset: str, category: str | None) -> int: + if hf_subset == "qualitative": + return 880 if category is None else 80 + return 1536 if category is None else 512 + + def test_every_entry_yields_its_documented_row_count( + self, _allow_hub_access + ) -> None: + import collections + + datasets = pytest.importorskip("datasets") + + from aiperf.dataset.loader.speed_bench_public import SPEED_BENCH_REVISION + from aiperf.plugin import plugins + from aiperf.plugin.enums import PluginType + + entries = [ + (entry.name, plugins.get_public_dataset_loader_metadata(entry.name)) + for entry, _ in plugins.iter_all(PluginType.PUBLIC_DATASET_LOADER) + if entry.name.startswith("speed_bench") + ] + assert entries, "no SPEED-Bench public dataset entries are registered" + + counts: dict[str, collections.Counter] = {} + mismatches = [] + for name, metadata in sorted(entries): + if metadata.hf_subset not in counts: + try: + rows = datasets.load_dataset( + "nvidia/SPEED-Bench", + metadata.hf_subset, + split="test", + revision=SPEED_BENCH_REVISION, + ) + except Exception as e: + pytest.skip(f"SPEED-Bench unreachable: {e}") + counts[metadata.hf_subset] = collections.Counter(rows["category"]) + + tally = counts[metadata.hf_subset] + actual = ( + sum(tally.values()) + if metadata.category is None + else tally[metadata.category] + ) + expected = self._expected(metadata.hf_subset, metadata.category) + if actual != expected: + mismatches.append(f"{name}: {actual} rows, expected {expected}") + + assert not mismatches, ( + "Registered SPEED-Bench entries no longer match the published " + "dataset:\n " + "\n ".join(mismatches) + ) + + +class TestCategoryTagging: + """A single run over an aggregate split must carry its own breakdown. + + `aiperf speed-bench-report --source records` groups per-request records by + `metadata.source_kind`. Without the stamp every record falls back to the + run-level category and an 11-column matrix silently collapses to one, so + the public path has to tag exactly as the custom-file loader does. + """ + + @pytest.mark.asyncio + async def test_every_turn_carries_its_row_category(self) -> None: + loader = SpeedBenchPublicLoader.__new__(SpeedBenchPublicLoader) + loader.config = "qualitative" + loader.category = None + loader.multi_turn = True + loader.session_id_generator = type( + "G", (), {"next": staticmethod(lambda: "generated")} + )() + + conversations = await loader.convert_to_conversations( + { + "dataset": [ + _row("a" * 32, "coding", "Write a function."), + _row("b" * 32, "math", "Solve for x."), + ] + } + ) + + assert [t.source_kind for c in conversations for t in c.turns] == [ + "coding", + "math", + ] + + @pytest.mark.asyncio + async def test_filtered_run_still_tags_its_single_category(self) -> None: + """A per-category selector yields one column, not an untagged run.""" + loader = SpeedBenchPublicLoader.__new__(SpeedBenchPublicLoader) + loader.config = "qualitative" + loader.category = "coding" + loader.multi_turn = True + loader.session_id_generator = type( + "G", (), {"next": staticmethod(lambda: "generated")} + )() + + conversations = await loader.convert_to_conversations( + { + "dataset": [ + _row("a" * 32, "coding", "Write a function."), + _row("b" * 32, "math", "Solve for x."), + ] + } + ) + + assert {t.source_kind for c in conversations for t in c.turns} == {"coding"} diff --git a/tests/unit/dataset/loader/test_vendored_speed_bench_prepare.py b/tests/unit/dataset/loader/test_vendored_speed_bench_prepare.py new file mode 100644 index 0000000000..81205a4d41 --- /dev/null +++ b/tests/unit/dataset/loader/test_vendored_speed_bench_prepare.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Guards the vendored SPEED-Bench prepare script against local edits. + +The file's value is being byte-identical to a known upstream commit: benchmark +prompts are reproducible only if the resolution code is. A local "small fix" +silently changes published numbers, so pin the hash and route fixes upstream. +""" + +import hashlib +from pathlib import Path + +import pytest + +VENDORED = ( + Path(__file__).resolve().parents[4] + / "src/aiperf/dataset/loader/vendor/speed_bench_prepare.py" +) + +UPSTREAM_COMMIT = "5ac8609a56ac941540b10c92e68d556e6343cd4c" +UPSTREAM_SHA256 = "a551be4df541474e54e21b480022b0cbb66c2da068fda61b2a64bd3223bbbed2" + + +def test_vendored_prepare_script_is_unmodified(): + digest = hashlib.sha256(VENDORED.read_bytes()).hexdigest() + + normalized = hashlib.sha256( + VENDORED.read_bytes().replace(b"\r\n", b"\n") + ).hexdigest() + if digest != UPSTREAM_SHA256 and normalized == UPSTREAM_SHA256: + raise AssertionError( + f"{VENDORED.name} content is intact but its line endings were " + "rewritten on checkout. .gitattributes marks the vendor directory " + "`-text` to prevent exactly this; confirm it is present and that " + "the working copy was checked out after it was added." + ) + + assert digest == UPSTREAM_SHA256, ( + f"{VENDORED.name} no longer matches upstream commit {UPSTREAM_COMMIT}. " + "Do not edit vendored files: fix it upstream in NVIDIA-NeMo/Skills, " + "then re-vendor and update UPSTREAM_COMMIT/UPSTREAM_SHA256 together." + ) + + +UPSTREAM_URL = ( + "https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/" + f"{UPSTREAM_COMMIT}/nemo_skills/dataset/speed-bench/prepare.py" +) + + +@pytest.mark.network +def test_vendored_prepare_script_matches_upstream(): + """Pin the vendored bytes to upstream, not just to themselves. + + The offline gate above proves only that nobody edited the file locally -- + ``UPSTREAM_SHA256`` was derived from the checked-in bytes, so it can never + establish that what was vendored is what upstream published. Only a fetch + can, which is why this is separate and network-marked. The vendor README's + Apache-2.0 section 4(b) "unmodified" claim rests on this test. + """ + import urllib.error + import urllib.request + + try: + with urllib.request.urlopen(UPSTREAM_URL, timeout=30) as response: + upstream = response.read() + except (urllib.error.URLError, TimeoutError) as e: + pytest.skip(f"upstream unreachable: {e}") + + upstream_digest = hashlib.sha256(upstream.replace(b"\r\n", b"\n")).hexdigest() + + assert upstream_digest == UPSTREAM_SHA256, ( + f"The vendored file does not match {UPSTREAM_URL}. Either the pinned " + "commit was force-pushed, or what was vendored was never upstream's " + "bytes. Re-vendor from the URL above and update UPSTREAM_COMMIT and " + "UPSTREAM_SHA256 together." + ) diff --git a/tools/check_ergonomics.py b/tools/check_ergonomics.py index aa9468bdfa..6778229e0b 100755 --- a/tools/check_ergonomics.py +++ b/tools/check_ergonomics.py @@ -610,7 +610,9 @@ def _iter_py_files(paths: list[Path]) -> list[Path]: out.append(p) elif p.is_dir(): out.extend(sorted(p.rglob("*.py"))) - return [f for f in out if "__pycache__" not in f.parts] + # "vendor" holds verbatim third-party sources whose value is being byte + # identical to upstream, so AIPerf's ergonomics rules cannot apply to them. + return [f for f in out if "__pycache__" not in f.parts and "vendor" not in f.parts] def _run_per_file( From 747de0d3b4c6b4c13a0f99f8812724cd5a964a57 Mon Sep 17 00:00:00 2001 From: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:46:27 -0700 Subject: [PATCH 063/119] feat: group post-run MLflow diagnostic metrics under system/ (#1447) Signed-off-by: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> (cherry picked from commit 31a8089712bde563224b4c9a46d09f5b2ce96b3f) --- docs/tutorials/otel-mlflow.md | 25 ++ src/aiperf/exporters/mlflow_data_exporter.py | 116 ++++++++- .../exporters/test_mlflow_data_exporter.py | 220 ++++++++++++++++++ 3 files changed, 353 insertions(+), 8 deletions(-) diff --git a/docs/tutorials/otel-mlflow.md b/docs/tutorials/otel-mlflow.md index f20cfd5bdd..66877fae97 100644 --- a/docs/tutorials/otel-mlflow.md +++ b/docs/tutorials/otel-mlflow.md @@ -138,6 +138,31 @@ When the benchmark finishes, AIPerf performs a deferred export: 2. The MLflow data exporter detects the live run via `mlflow_export.json` (written during the run). 3. All artifacts (JSON export, CSV export, GPU telemetry, metadata) are uploaded to the same MLflow run. +The deferred export also logs summary metrics. Inference performance and quality +metrics keep their top-level names, such as `time_to_first_token.p95`, +`request_throughput`, and `accuracy.correct`. Diagnostic and hardware metrics use +MLflow's `system/` namespace so they appear in the System Metrics section: + +| Category | Example MLflow key | +| --- | --- | +| HTTP transport timings and counters | `system/http_req_waiting.p95` | +| Internal diagnostics and intermediate counts | `system/credit_drop_latency`, `system/num_images` | +| Server-reported usage, usage totals, and cache statistics | `system/usage_completion_tokens`, `system/total_usage_prompt_tokens` | +| Token-count discrepancies, output-length targets, and mismatches | `system/usage_prompt_tokens_diff_pct`, `system/requested_osl`, `system/osl_mismatch_count` | +| NVIDIA and AMD power, energy, and efficiency metrics | `system/nvidia_total_gpu_power`, `system/amd_energy_delay_product` | + +The prefix applies to each exported statistic, including percentiles, counts, +and sums; values and units are unchanged. Benchmark duration, request timestamps, +and the `aiperf.completed_requests` / `aiperf.total_expected_requests` bookkeeping +keys remain top-level. Custom metrics retain their names unless their exact tag +is one of AIPerf's classified built-in metrics. + +**Migration:** dashboards and API queries for these post-run diagnostic metrics +must use the new keys (for example, replace `http_req_waiting.p95` with +`system/http_req_waiting.p95`). Existing MLflow runs keep their original keys. +The `live.*` namespace and metric names inside uploaded JSON/CSV artifacts are +unchanged. + The `mlflow_export.json` file records the mapping between the local run and the MLflow run: ```json diff --git a/src/aiperf/exporters/mlflow_data_exporter.py b/src/aiperf/exporters/mlflow_data_exporter.py index ab7e3b2be7..554941a6a7 100644 --- a/src/aiperf/exporters/mlflow_data_exporter.py +++ b/src/aiperf/exporters/mlflow_data_exporter.py @@ -30,6 +30,90 @@ class MLflowDataExporter(AIPerfLoggerMixin): _PLOT_SUFFIXES: frozenset[str] = frozenset( {".png", ".jpg", ".jpeg", ".svg", ".gif", ".webp", ".html"} ) + # MLflow's grouping is an export naming policy, independent of console + # visibility and INTERNAL flags (timestamps remain top-level metrics). + _SYSTEM_METRIC_TAGS: frozenset[str] = frozenset( + { + "http_req_blocked", + "http_req_dns_lookup", + "http_req_connecting", + "http_req_sending", + "http_req_waiting", + "http_req_receiving", + "http_req_duration", + "http_req_connection_overhead", + "http_req_total", + "http_req_data_sent", + "http_req_data_received", + "http_req_chunks_sent", + "http_req_chunks_received", + "http_req_connection_reused", + "credit_drop_latency", + "inter_chunk_latency", + "error_isl", + "total_error_isl", + "num_images", + "video_peak_memory", + "usage_prompt_tokens", + "usage_completion_tokens", + "usage_total_tokens", + "usage_reasoning_tokens", + "usage_prompt_cache_read_tokens", + "usage_prompt_cache_write_tokens", + "usage_prompt_cache_miss_tokens", + "usage_prompt_audio_tokens", + "usage_completion_audio_tokens", + "usage_prompt_audio_seconds", + "usage_tool_use_prompt_tokens", + "usage_accepted_prediction_tokens", + "usage_rejected_prediction_tokens", + "total_usage_prompt_tokens", + "total_usage_completion_tokens", + "total_usage_total_tokens", + "total_usage_reasoning_tokens", + "total_usage_prompt_cache_read_tokens", + "total_usage_prompt_cache_write_tokens", + "total_usage_prompt_cache_miss_tokens", + "total_usage_prompt_audio_tokens", + "total_usage_completion_audio_tokens", + "total_usage_prompt_audio_seconds", + "total_usage_tool_use_prompt_tokens", + "total_usage_accepted_prediction_tokens", + "total_usage_rejected_prediction_tokens", + "overall_usage_prompt_cache_read_pct", + "usage_prompt_tokens_diff_pct", + "usage_completion_tokens_diff_pct", + "usage_reasoning_tokens_diff_pct", + "usage_discrepancy_count", + "requested_osl", + "osl_mismatch_diff_pct", + "osl_mismatch_count", + "nvidia_energy_delay_product", + "nvidia_performance_per_watt", + "nvidia_output_tps_per_watt", + "nvidia_goodput_per_watt", + "nvidia_average_gpu_power", + "nvidia_total_gpu_energy", + "nvidia_total_gpu_power", + "nvidia_energy_per_total_token", + "nvidia_energy_per_output_token", + "nvidia_energy_per_request", + "nvidia_output_tokens_per_joule", + "nvidia_energy_per_user", + "amd_energy_delay_product", + "amd_performance_per_watt", + "amd_output_tps_per_watt", + "amd_goodput_per_watt", + "amd_average_gpu_power", + "amd_total_gpu_energy", + "amd_total_gpu_power", + "amd_energy_per_total_token", + "amd_energy_per_output_token", + "amd_energy_per_request", + "amd_output_tokens_per_joule", + "amd_energy_per_user", + } + ) is_deferred = True # runs after all local exporters write their files def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: @@ -333,25 +417,41 @@ def _derive_default_run_name(self) -> str: _STAT_FIELDS = ("avg", "p1", "p5", "p10", "p25", "p50", "p75", "p90", "p95", "p99", "min", "max", "std", "count", "sum") # fmt: skip def _build_metric_payload(self) -> dict[str, float]: - payload: dict[str, float] = {} + payload: dict[str, float] = { + "aiperf.completed_requests": float(self._results.completed) + } + if self._results.total_expected is not None: + payload["aiperf.total_expected_requests"] = float( + self._results.total_expected + ) + sources: dict[str, str] = dict.fromkeys(payload, "benchmark bookkeeping") for metric in self._results.records or []: + tag = ( + f"system/{metric.tag}" + if metric.tag in self._SYSTEM_METRIC_TAGS + else metric.tag + ) for field in self._STAT_FIELDS: value = getattr(metric, field, None) if value is None: continue try: - key = metric.tag if field == "avg" else f"{metric.tag}.{field}" - payload[key] = float(value) + key = tag if field == "avg" else f"{tag}.{field}" + numeric_value = float(value) except (TypeError, ValueError): self.debug( f"Skipping non-numeric metric for MLflow export: " f"{metric.tag}.{field}" ) - payload["aiperf.completed_requests"] = float(self._results.completed) - if self._results.total_expected is not None: - payload["aiperf.total_expected_requests"] = float( - self._results.total_expected - ) + continue + source = f"{metric.tag}.{field}" + if previous_source := sources.get(key): + raise ValueError( + f"Duplicate MLflow metric key {key!r} after namespace " + f"normalization from {previous_source!r} and {source!r}." + ) + payload[key] = numeric_value + sources[key] = source return payload def _build_param_payload(self) -> dict[str, str]: diff --git a/tests/unit/exporters/test_mlflow_data_exporter.py b/tests/unit/exporters/test_mlflow_data_exporter.py index 6cca066cb8..cc2c6a09a3 100644 --- a/tests/unit/exporters/test_mlflow_data_exporter.py +++ b/tests/unit/exporters/test_mlflow_data_exporter.py @@ -213,6 +213,226 @@ def mlflow_cfg(tmp_path: Path) -> BenchmarkConfig: class TestMLflowDataExporter: + @pytest.mark.parametrize( + ("tag", "expected_key"), + [ + param("http_req_waiting", "system/http_req_waiting", id="http-timing"), + param("http_req_connection_reused", "system/http_req_connection_reused", id="http-counter"), + param("credit_drop_latency", "system/credit_drop_latency", id="internal"), + param("inter_chunk_latency", "system/inter_chunk_latency", id="stream-diagnostic"), + param("error_isl", "system/error_isl", id="error-input"), + param("total_error_isl", "system/total_error_isl", id="total-error-input"), + param("num_images", "system/num_images", id="image-count"), + param("video_peak_memory", "system/video_peak_memory", id="video-memory"), + param("usage_prompt_cache_read_tokens", "system/usage_prompt_cache_read_tokens", id="usage-cache"), + param("total_usage_completion_tokens", "system/total_usage_completion_tokens", id="total-usage"), + param("overall_usage_prompt_cache_read_pct", "system/overall_usage_prompt_cache_read_pct", id="cache-percent"), + param("usage_prompt_tokens_diff_pct", "system/usage_prompt_tokens_diff_pct", id="usage-discrepancy"), + param("requested_osl", "system/requested_osl", id="output-target"), + param("osl_mismatch_count", "system/osl_mismatch_count", id="output-mismatch"), + param("nvidia_total_gpu_power", "system/nvidia_total_gpu_power", id="nvidia-power"), + param("amd_energy_per_user", "system/amd_energy_per_user", id="amd-energy"), + param("nvidia_energy_delay_product", "system/nvidia_energy_delay_product", id="nvidia-efficiency"), + param("amd_goodput_per_watt", "system/amd_goodput_per_watt", id="amd-efficiency"), + param("time_to_first_token", "time_to_first_token", id="primary-latency"), + param("request_throughput", "request_throughput", id="primary-throughput"), + param("error_request_count", "error_request_count", id="primary-error-count"), + param("benchmark_duration", "benchmark_duration", id="hidden-primary"), + param("min_request_timestamp", "min_request_timestamp", id="internal-primary-start"), + param("max_response_timestamp", "max_response_timestamp", id="internal-primary-end"), + param("accuracy.correct", "accuracy.correct", id="model-quality"), + param("video_inference_time", "video_inference_time", id="video-latency"), + param("custom_latency", "custom_latency", id="custom"), + param("http_req_custom", "http_req_custom", id="custom-http-prefix"), + param("usage_custom", "usage_custom", id="custom-usage-prefix"), + param("nvidia_custom_power", "nvidia_custom_power", id="custom-vendor-prefix"), + param("system/http_req_waiting", "system/http_req_waiting", id="already-prefixed"), + ], + ) # fmt: skip + def test_build_metric_payload_uses_system_namespace_for_diagnostics( + self, + tag: str, + expected_key: str, + sample_results: ProfileResults, + mlflow_cfg: BenchmarkConfig, + ) -> None: + sample_results.records = [MetricResult(tag=tag, header=tag, unit="ms", avg=2.5)] + exporter = MLflowDataExporter( + ExporterConfig( + results=sample_results, cfg=mlflow_cfg, telemetry_results=None + ) + ) + + assert exporter._build_metric_payload() == { + expected_key: 2.5, + "aiperf.completed_requests": 10.0, + "aiperf.total_expected_requests": 12.0, + } + + @pytest.mark.asyncio + async def test_export_categorizes_metrics_without_changing_records( + self, + monkeypatch: pytest.MonkeyPatch, + sample_results: ProfileResults, + mlflow_cfg: BenchmarkConfig, + ) -> None: + sample_results.records = [ + MetricResult( + tag="http_req_waiting", + header="HTTP waiting", + unit="ms", + avg=2.5, + p95=4.0, + count=2, + sum=5.0, + ), + MetricResult( + tag="time_to_first_token", + header="TTFT", + unit="ms", + avg=12.0, + ), + MetricResult( + tag="usage_completion_tokens", + header="Usage", + unit="tokens", + avg=3.0, + ), + MetricResult( + tag="amd_total_gpu_energy", + header="Energy", + unit="J", + avg=9.0, + ), + ] + original_results = sample_results.model_dump() + state = _install_fake_mlflow_modules(monkeypatch) + exporter = MLflowDataExporter( + ExporterConfig( + results=sample_results, cfg=mlflow_cfg, telemetry_results=None + ) + ) + + await asyncio.to_thread(exporter._export_sync) + + assert len(state["log_batch_calls"]) == 1 + metrics = state["log_batch_calls"][0]["metrics"] + expected = { + "system/http_req_waiting": 2.5, + "system/http_req_waiting.p95": 4.0, + "system/http_req_waiting.count": 2.0, + "system/http_req_waiting.sum": 5.0, + "time_to_first_token": 12.0, + "system/usage_completion_tokens": 3.0, + "system/amd_total_gpu_energy": 9.0, + "aiperf.completed_requests": 10.0, + "aiperf.total_expected_requests": 12.0, + } + assert len(metrics) == len(expected) + assert {metric.key: metric.value for metric in metrics} == expected + assert sample_results.model_dump() == original_results + + @pytest.mark.parametrize( + ("field", "reverse"), + [ + param("avg", False, id="avg-generated-first"), + param("avg", True, id="avg-prefixed-first"), + param("p95", False, id="p95-generated-first"), + param("p95", True, id="p95-prefixed-first"), + ], + ) + def test_build_metric_payload_rejects_normalized_key_collisions( + self, + field: str, + reverse: bool, + sample_results: ProfileResults, + mlflow_cfg: BenchmarkConfig, + ) -> None: + values = {field: 2.5} + records = [ + MetricResult( + tag="http_req_waiting", + header="HTTP waiting", + unit="ms", + **values, + ), + MetricResult( + tag="system/http_req_waiting", + header="Already prefixed HTTP waiting", + unit="ms", + **values, + ), + ] + sample_results.records = list(reversed(records)) if reverse else records + exporter = MLflowDataExporter( + ExporterConfig( + results=sample_results, cfg=mlflow_cfg, telemetry_results=None + ) + ) + + expected_key = ( + "system/http_req_waiting" + if field == "avg" + else f"system/http_req_waiting.{field}" + ) + with pytest.raises( + ValueError, + match=rf"Duplicate MLflow metric key '{expected_key}'", + ): + exporter._build_metric_payload() + + @pytest.mark.parametrize( + ("tag", "total_expected"), + [ + param("aiperf.completed_requests", None, id="completed"), + param("aiperf.total_expected_requests", 12, id="total-expected"), + ], + ) + def test_build_metric_payload_rejects_bookkeeping_key_collisions( + self, + tag: str, + total_expected: int | None, + sample_results: ProfileResults, + mlflow_cfg: BenchmarkConfig, + ) -> None: + sample_results.total_expected = total_expected + sample_results.records = [ + MetricResult(tag=tag, header="Custom count", unit="requests", avg=42.0) + ] + exporter = MLflowDataExporter( + ExporterConfig( + results=sample_results, cfg=mlflow_cfg, telemetry_results=None + ) + ) + + with pytest.raises(ValueError, match=rf"Duplicate MLflow metric key '{tag}'"): + exporter._build_metric_payload() + + def test_build_metric_payload_allows_absent_bookkeeping_key( + self, + sample_results: ProfileResults, + mlflow_cfg: BenchmarkConfig, + ) -> None: + sample_results.total_expected = None + sample_results.records = [ + MetricResult( + tag="aiperf.total_expected_requests", + header="Custom count", + unit="requests", + avg=42.0, + ) + ] + exporter = MLflowDataExporter( + ExporterConfig( + results=sample_results, cfg=mlflow_cfg, telemetry_results=None + ) + ) + + assert exporter._build_metric_payload() == { + "aiperf.completed_requests": 10.0, + "aiperf.total_expected_requests": 42.0, + } + def test_disabled_without_tracking_uri( self, tmp_path: Path, sample_results: ProfileResults ) -> None: From 49a6577a85f495181e57da8f53eaa84cf3aac3dc Mon Sep 17 00:00:00 2001 From: Manu Nicholas Jacob <118115267+manunicholasjacob@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:02:51 -0500 Subject: [PATCH 064/119] feat(telemetry): probe the NVML energy counter once instead of failing silently (#1350) Signed-off-by: Manu Nicholas Jacob Signed-off-by: manunicholasjacob Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Anthony Casagrande Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit c78644090ff5a3b61b756c0e8073dea0c11018f3) --- docs/metrics-reference.md | 4 +- docs/tutorials/gpu-telemetry.md | 10 +- src/aiperf/gpu_telemetry/pynvml_collector.py | 50 +++++++- .../gpu_telemetry/test_pynvml_collector.py | 113 +++++++++++++++++- 4 files changed, 168 insertions(+), 9 deletions(-) diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index 3699a5f877..1cb355cec4 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -2211,7 +2211,9 @@ http_req_chunks_received = trace.response_chunks_count > [!NOTE] > Each vendor's metrics render in their own console section — `GPU Power Efficiency (NVIDIA)` (`console_group = MetricConsoleGroup.GPU_POWER_EFFICIENCY_NVIDIA`) and `GPU Power Efficiency (AMD)` (`MetricConsoleGroup.GPU_POWER_EFFICIENCY_AMD`) — separate from the main metrics table. A section is omitted entirely when no GPU of that vendor reported. -**Energy source.** Per-vendor total GPU energy is taken from the vendor's energy counter delta when available (`source = dcgm_counter` for NVIDIA; `source = power_integration` fallback for amdsmi when only the power gauge is exposed), and the time-averaged fleet power is derived from it as `total_gpu_energy / profiling_duration_s`. If neither signal is present the family is skipped for that vendor. +**Energy source.** Per-vendor total GPU energy is taken from that vendor's energy counter delta when available (`source = dcgm_counter`), and the time-averaged fleet power is derived from it as `total_gpu_energy / profiling_duration_s`. When only the power gauge is exposed, energy is integrated from it instead (`source = power_integration`, energy = fleet power * duration). If neither signal is present the family is skipped for that vendor. + +The `power_integration` fallback is vendor-agnostic: `EnergyEfficiencyAnalyzer._resolve_energy` selects on whether the energy counter reported, not on which vendor asked. It therefore covers AMD GPUs exposing only the power gauge and NVIDIA GPUs without the NVML total-energy counter alike, so a pre-Volta card still reports power efficiency metrics, integrated from power rather than counted, provided no GPU on that platform reported the counter. The choice is fleet-wide, not per GPU: in a mixed-generation fleet one reporting GPU sends the whole platform down the counter branch, and the GPUs without a counter then contribute power but no energy, so `total_gpu_energy` is short by their share while still labelled `dcgm_counter`. The family is skipped entirely when GPU telemetry is not collected. Each tag is independently omitted when its underlying signal is unavailable (e.g. `nvidia_energy_per_user` only appears for concurrency runs; `nvidia_goodput_per_watt` only when goodput SLOs are configured). NVIDIA tags: `nvidia_energy_delay_product`, `nvidia_performance_per_watt`, `nvidia_output_tps_per_watt`, `nvidia_goodput_per_watt`, `nvidia_average_gpu_power`, `nvidia_total_gpu_energy`, `nvidia_total_gpu_power`, `nvidia_energy_per_total_token`, `nvidia_energy_per_output_token`, `nvidia_energy_per_request`, `nvidia_output_tokens_per_joule`, `nvidia_energy_per_user`. AMD tags follow the same pattern with the `amd_` prefix. diff --git a/docs/tutorials/gpu-telemetry.md b/docs/tutorials/gpu-telemetry.md index b217a55319..33c25ebddb 100644 --- a/docs/tutorials/gpu-telemetry.md +++ b/docs/tutorials/gpu-telemetry.md @@ -415,7 +415,7 @@ The nvidia-ml-py library (pynvml) collects the following metrics directly from t | Metric | Description | Unit | |--------|-------------|------| | `nvidia_power_usage` | Current power draw | W | -| `nvidia_energy_consumption` | Total energy since boot | MJ | +| `nvidia_energy_consumption` | Total energy since boot (see note below) | MJ | | `nvidia_gpu_utilization` | GPU compute utilization | % | | `nvidia_memory_utilization` | Memory controller utilization | % | | `nvidia_memory_used` | Framebuffer memory in use | GB | @@ -429,6 +429,14 @@ The nvidia-ml-py library (pynvml) collects the following metrics directly from t > [!NOTE] > Not all metrics are available on all GPU models. AIPerf gracefully handles missing metrics and reports only what the hardware supports. +> [!NOTE] +> `nvidia_energy_consumption` needs the NVML total-energy counter, which requires Volta or +> newer and a driver that exports `nvmlDeviceGetTotalEnergyConsumption`. AIPerf probes it +> once at startup; where it is absent the field is omitted for that GPU and a warning names +> the requirement, rather than the run finishing silently without energy. Power efficiency +> metrics are still produced, derived from the power gauge instead. See +> [Energy source](../metrics-reference.md#gpu-power-efficiency-metrics). + ## Comparing DCGM vs pynvml | Feature | DCGM | pynvml | diff --git a/src/aiperf/gpu_telemetry/pynvml_collector.py b/src/aiperf/gpu_telemetry/pynvml_collector.py index 7da7b5e787..621e11e797 100644 --- a/src/aiperf/gpu_telemetry/pynvml_collector.py +++ b/src/aiperf/gpu_telemetry/pynvml_collector.py @@ -62,11 +62,15 @@ class GpuDeviceState: handle: NVML device handle metadata: GPU metadata gpm_samples: GPM samples (prev, curr) if GPM supported, else None + energy_counter_supported: whether nvmlDeviceGetTotalEnergyConsumption + works on this device. Probed once at init; see + _init_energy_for_device. """ handle: object metadata: GpuMetadata gpm_samples: tuple[object, object] | None = None + energy_counter_supported: bool = True class PyNVMLTelemetryCollector(AIPerfLifecycleMixin): @@ -257,6 +261,7 @@ def _create_gpu_for_device_index(self, index: int) -> GpuDeviceState | None: # Check GPM support and allocate samples for efficient SM utilization self._init_gpm_for_device(gpu) + self._init_energy_for_device(gpu) return gpu def _init_gpm_for_device(self, gpu: GpuDeviceState) -> None: @@ -274,6 +279,35 @@ def _init_gpm_for_device(self, gpu: GpuDeviceState) -> None: # GPM unavailable, will use process API fallback self.debug(lambda: f"GPM not supported for GPU {gpu.metadata.gpu_index}") + def _init_energy_for_device(self, gpu: GpuDeviceState) -> None: + """Probe the energy counter once, warning and disabling it where the + device cannot provide it, so the loop stops querying a counter that + will never answer.""" + try: + pynvml.nvmlDeviceGetTotalEnergyConsumption(gpu.handle) + except ( + pynvml.NVMLError_NotSupported, + pynvml.NVMLError_FunctionNotFound, + ) as e: + # Neither a pre-Volta device nor a missing driver symbol can start + # working later in this process. + gpu.energy_counter_supported = False + self.warning( + f"GPU {gpu.metadata.gpu_index} ({gpu.metadata.gpu_model_name}): " + f"NVML total energy counter not available ({e}). Energy metrics " + f"will be absent for this GPU; the counter requires Volta or " + f"newer and a driver that exports " + f"nvmlDeviceGetTotalEnergyConsumption." + ) + except pynvml.NVMLError as e: + # Other NVML failures may be transient, so energy stays enabled for + # the collection loop to retry. + self.warning( + f"GPU {gpu.metadata.gpu_index} ({gpu.metadata.gpu_model_name}): " + f"energy counter probe failed ({e}); keeping it enabled in case " + f"the failure is transient." + ) + def _free_gpm_samples(self) -> None: """Free all allocated GPM sample buffers.""" for gpu in self._gpus: @@ -398,12 +432,16 @@ def _collect_gpu_metrics(self) -> list[TelemetryRecord]: power_mw * ScalingFactors.nvidia_power_usage ) - # Total energy consumption (millijoules -> megajoules) - with contextlib.suppress(NVMLError): - energy_mj = pynvml.nvmlDeviceGetTotalEnergyConsumption(handle) - telemetry_data.nvidia_energy_consumption = ( - energy_mj * ScalingFactors.nvidia_energy_consumption - ) + # Total energy consumption (millijoules -> megajoules). + # Skipped entirely on devices that do not implement the counter, + # which was established once at init rather than rediscovered by + # raising and suppressing on every sample. + if gpu.energy_counter_supported: + with contextlib.suppress(NVMLError): + energy_mj = pynvml.nvmlDeviceGetTotalEnergyConsumption(handle) + telemetry_data.nvidia_energy_consumption = ( + energy_mj * ScalingFactors.nvidia_energy_consumption + ) # GPU and memory utilization (percent) with contextlib.suppress(NVMLError): diff --git a/tests/unit/gpu_telemetry/test_pynvml_collector.py b/tests/unit/gpu_telemetry/test_pynvml_collector.py index 039381ca5e..579d82c210 100644 --- a/tests/unit/gpu_telemetry/test_pynvml_collector.py +++ b/tests/unit/gpu_telemetry/test_pynvml_collector.py @@ -8,6 +8,7 @@ """ import asyncio +import logging from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -28,8 +29,19 @@ def mock_pynvml(): """Create a mock pynvml module with typical NVML responses.""" mock_module = MagicMock() - # NVML error class and constants + # NVML error class and constants. NVMLError_NotSupported is a real + # subclass in pynvml and code distinguishes it from transient failures, so + # the mock mirrors that rather than collapsing both into one type. mock_module.NVMLError = Exception + + class _NVMLErrorNotSupported(Exception): + pass + + class _NVMLErrorFunctionNotFound(Exception): + pass + + mock_module.NVMLError_NotSupported = _NVMLErrorNotSupported + mock_module.NVMLError_FunctionNotFound = _NVMLErrorFunctionNotFound mock_module.NVML_TEMPERATURE_GPU = 0 mock_module.NVML_PERF_POLICY_POWER = 0 @@ -342,6 +354,105 @@ async def test_init_skips_failed_device_handles(self, patch_pynvml): # --------------------------------------------------------------------------- +def _energy_warnings(caplog) -> list[str]: + """Warning-level records from the energy-counter probe.""" + return [ + record.message + for record in caplog.records + if record.levelno == logging.WARNING and "energy counter" in record.message + ] + + +class TestPyNVMLEnergyCounterProbe: + """One-time probing of the NVML energy counter capability.""" + + @pytest.mark.asyncio + async def test_unsupported_device_warns_then_is_not_queried_again( + self, patch_pynvml, caplog + ): + """A pre-Volta device warns once at init, then the loop stops asking. + + The warning is what the probe is for: without it the run reports no + energy and says nothing about why. Asserting only on + ``energy_counter_supported`` would stay green if it were downgraded to + debug. + """ + mock_pynvml, PyNVMLTelemetryCollector = patch_pynvml + mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = ( + mock_pynvml.NVMLError_NotSupported("Not Supported") + ) + + collector = PyNVMLTelemetryCollector() + with caplog.at_level(logging.WARNING): + await collector.initialize() + + warnings = _energy_warnings(caplog) + assert len(warnings) == 2, "one per GPU in the fixture" + assert all("Volta" in message for message in warnings) + + assert all(not gpu.energy_counter_supported for gpu in collector._gpus) + + calls_after_init = mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.call_count + collector._collect_gpu_metrics() + collector._collect_gpu_metrics() + + assert ( + mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.call_count + == calls_after_init + ) + + @pytest.mark.asyncio + async def test_function_not_found_disables_energy_permanently( + self, patch_pynvml, caplog + ): + """A missing NVML function disables energy collection permanently.""" + mock_pynvml, PyNVMLTelemetryCollector = patch_pynvml + mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = ( + mock_pynvml.NVMLError_FunctionNotFound("Function Not Found") + ) + + collector = PyNVMLTelemetryCollector() + with caplog.at_level(logging.WARNING): + await collector.initialize() + + assert len(_energy_warnings(caplog)) == 2 + assert all(not gpu.energy_counter_supported for gpu in collector._gpus) + + calls_after_init = mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.call_count + collector._collect_gpu_metrics() + collector._collect_gpu_metrics() + assert ( + mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.call_count + == calls_after_init + ) + + @pytest.mark.asyncio + async def test_transient_probe_error_does_not_disable_energy(self, patch_pynvml): + """A non-NotSupported error must not cost the run its energy metrics. + + Only NVMLError_NotSupported means the device lacks the counter. Any + other NVML failure at init may be transient, so the capability stays + enabled and the collection loop tries again. + """ + mock_pynvml, PyNVMLTelemetryCollector = patch_pynvml + mock_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = [ + mock_pynvml.NVMLError("Unknown Error"), # GPU 0 probe + mock_pynvml.NVMLError("Unknown Error"), # GPU 1 probe + 5_000_000, # collection succeeds afterwards + 6_000_000, + ] + + collector = PyNVMLTelemetryCollector() + await collector.initialize() + + assert all(gpu.energy_counter_supported for gpu in collector._gpus) + + records = collector._collect_gpu_metrics() + assert all( + r.telemetry_data.nvidia_energy_consumption is not None for r in records + ) + + class TestPyNVMLMetricsCollection: """Test GPU metrics collection.""" From 4a0b929572f9b423bce4e99d416dad922650f140 Mon Sep 17 00:00:00 2001 From: Matthias Brunel Date: Tue, 29 Sep 2026 01:52:58 +0200 Subject: [PATCH 065/119] feat(dataset): add the H CUA Perf public dataset loader (#1449) Signed-off-by: Matthias Brunel Co-authored-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> (cherry picked from commit a8e3dc7c8c69ae65a3d18e0b72705bea8206d6fc) --- README.md | 1 + docs/benchmark-datasets.md | 5 + docs/index.yml | 2 + docs/tutorials/h-cua-perf.md | 80 +++++ src/aiperf/dataset/loader/h_cua_perf.py | 203 +++++++++++++ .../dataset/loader/h_cua_perf_processing.py | 274 ++++++++++++++++++ src/aiperf/plugin/enums.py | 2 +- src/aiperf/plugin/plugins.yaml | 17 ++ .../dataset/loader/_h_cua_perf_records.py | 113 ++++++++ tests/unit/dataset/loader/test_h_cua_perf.py | 153 ++++++++++ .../loader/test_h_cua_perf_processing.py | 198 +++++++++++++ 11 files changed, 1047 insertions(+), 1 deletion(-) create mode 100644 docs/tutorials/h-cua-perf.md create mode 100644 src/aiperf/dataset/loader/h_cua_perf.py create mode 100644 src/aiperf/dataset/loader/h_cua_perf_processing.py create mode 100644 tests/unit/dataset/loader/_h_cua_perf_records.py create mode 100644 tests/unit/dataset/loader/test_h_cua_perf.py create mode 100644 tests/unit/dataset/loader/test_h_cua_perf_processing.py diff --git a/README.md b/README.md index 2a4c8a1ca3..30a3acf632 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,7 @@ Log File: /home/user/Code/aiperf/artifacts/granite4:350m-openai-chat-concurrency - [Agentic Code Generator](docs/tutorials/agentic-code-generator.md) - Generate multi-turn coding-agent traces for KV cache benchmarking - [Weka Traces](docs/tutorials/weka-trace.md) - Replay real agentic coding sessions with KV-cache-aware Weka traces - [TraceLab Traces](docs/tutorials/tracelab-trace.md) - Replay the public TraceLab corpus of real agentic coding sessions +- [H CUA Perf Traces](docs/tutorials/h-cua-perf.md) - Replay H Company's computer-use agent traces (tool calling, screenshots, recorded think time) from HuggingFace - [AgentX scenarios](docs/tutorials/agentx-mvp.md) - InferenceX defaults (`--scenario agentx`) and the original MVP benchmark (`--scenario inferencex-agentx-mvp`) - [Fixed Schedule](docs/tutorials/fixed-schedule.md) - Precise timestamp-based execution - [Time-based Benchmarking](docs/tutorials/time-based-benchmarking.md) - Duration-based testing diff --git a/docs/benchmark-datasets.md b/docs/benchmark-datasets.md index c2f5f50228..0c7ad5ac92 100644 --- a/docs/benchmark-datasets.md +++ b/docs/benchmark-datasets.md @@ -67,6 +67,11 @@ This document describes datasets that AIPerf can use to generate stimulus. Addit ✅ Synthetic multi-turn coding-agent traces with shared prompt layers, repository context, and cache-aware turn growth. Generated via aiperf synthesize agentic-code and replayed as a Mooncake trace. + + H CUA Perf + ✅ + Real computer-use agent traces (tool calling, one screenshot per request, recorded think time) from H Company, replayed as a Mooncake trace --public-dataset h_cua_perf + TraceLab ✅ diff --git a/docs/index.yml b/docs/index.yml index b3a8b534c6..5eded2cce7 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -108,6 +108,8 @@ navigation: path: tutorials/weka-trace.md - page: Replay TraceLab Agentic Coding Traces path: tutorials/tracelab-trace.md + - page: Replay H CUA Perf Computer-Use Agent Traces + path: tutorials/h-cua-perf.md - page: InferenceX AgentX MVP Benchmark path: tutorials/agentx-mvp.md - section: Load Patterns & Scheduling diff --git a/docs/tutorials/h-cua-perf.md b/docs/tutorials/h-cua-perf.md new file mode 100644 index 0000000000..54bb693f76 --- /dev/null +++ b/docs/tutorials/h-cua-perf.md @@ -0,0 +1,80 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +sidebar-title: Replay H CUA Perf Computer-Use Agent Traces +--- + +# Replay H CUA Perf Computer-Use Agent Traces + +[H CUA Perf](https://huggingface.co/datasets/Hcompany/h_cua_perf) is 477 sessions (18,224 requests) recorded from [H Company](https://www.hcompany.ai)'s computer-use agent on [CUA-Gym](https://github.com/xlang-ai/CUA-Gym) desktop tasks. License: CC BY 4.0. + +Each record is one real chat-completion request: + +- the full text history of the session so far +- 10 tool definitions +- the latest screenshot +- the recorded completion length +- the time the agent waited before sending it, tool execution included + +Prompts grow with every turn and a sliding window bounds the screenshots per request, so the replay exercises prefix caching, multimodal prefill and tool-call parsing the way a real agent does. The [dataset card](https://huggingface.co/datasets/Hcompany/h_cua_perf) describes the record schema. + +## Server + +The server needs a vision model, a tool parser, a context window as large as the model allows and an image limit at least as large as the screenshot window. + +```bash +docker run --gpus all -p 8000:8000 -e HF_TOKEN vllm/vllm-openai:latest \ + Qwen/Qwen2.5-VL-7B-Instruct --max-model-len 128000 \ + --enable-auto-tool-choice --tool-call-parser hermes \ + --limit-mm-per-prompt '{"image":5}' \ + --enable-prompt-tokens-details +``` + +`--enable-prompt-tokens-details` is what lets AIPerf report prompt-cache hits. + +## Replay + +```bash +AIPERF_DATASET_CONFIGURATION_TIMEOUT=3600 \ +AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=3600 \ +aiperf profile \ + --model Qwen/Qwen2.5-VL-7B-Instruct \ + --url localhost:8000 \ + --endpoint-type chat \ + --streaming \ + --use-server-token-count \ + --extra-inputs ignore_eos:true \ + --public-dataset h_cua_perf \ + --num-dataset-entries 20 \ + --dataset-filter n_screenshots=3 \ + --dataset-filter max_trace_length=40 \ + --num-conversations 20 \ + --concurrency 4 +``` + +| Flag | Effect | +| --- | --- | +| `--concurrency N` | Sessions in flight at once. The turns of a session are sequential, so this is the number of agents working at the same time. Always set it, the default is 1. | +| `--inter-turn-delay-cap-seconds` | Caps the recorded wait between two turns of a session. Unset, the waits are replayed as recorded; `0` sends each session's requests back to back and the run becomes a plain concurrency test. | +| `--num-conversations N` | Stops after N whole sessions. | +| `--request-count N` | Stops after N requests, cutting the last sessions short. | +| `--use-server-token-count` | Takes token counts from the server's usage. AIPerf's own count tokenizes text only and ignores images, so without it the input sequence length misses every screenshot. | +| `--extra-inputs ignore_eos:true` | Makes the model generate exactly the recorded completion length, which the loader passes as `max_tokens`. Without it a model that did not produce the traces stops early on most requests and AIPerf prints an output-length mismatch warning. | + +Some requests are larger than the 128k-token context of the example server; `max_trace_length=40` and the small screenshot window keep the example's requests under it. + +Each session is one multi-turn conversation: a request is sent, the response awaited, the recorded wait slept, then the next recorded request is sent. There are no timestamps, so `--fixed-schedule` does not apply. + +The screenshot window and the session selection are `--dataset-filter` options: + +| Filter | Effect | +| --- | --- | +| `n_screenshots` | Sliding window of the N latest screenshots per request; the published records carry one. Wider windows both increase the input length and lower the KV-Cache hit rate. See the [dataset card](https://huggingface.co/datasets/Hcompany/h_cua_perf) for why. | +| `min_trace_length` | Drop shorter sessions; truncations never go below it (default 1). | +| `max_trace_length` | Keep each session's first N turns. | +| `avg_trace_length` | Scale every session's length by the same factor until the mean reaches the target. | + +## Related Tutorials + +- [Trace Replay with Mooncake Traces](../benchmark-modes/trace-replay.md) +- [Multi-Turn Conversations](multi-turn.md) diff --git a/src/aiperf/dataset/loader/h_cua_perf.py b/src/aiperf/dataset/loader/h_cua_perf.py new file mode 100644 index 0000000000..490306526b --- /dev/null +++ b/src/aiperf/dataset/loader/h_cua_perf.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import hashlib +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +import orjson +import psutil +import zstandard +from pydantic import ValidationError + +from aiperf.common.enums import ConversationContextMode +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.common.models import Conversation +from aiperf.dataset.generator.prompt import PromptGenerator +from aiperf.dataset.loader.base_hf_dataset import BaseHFDatasetLoader +from aiperf.dataset.loader.h_cua_perf_processing import ( + HCuaPerfFilters, + iter_selected_records, + open_dataset, + select_trace_lengths, + supported_filter_keys, +) +from aiperf.dataset.loader.models import MooncakeTrace +from aiperf.dataset.loader.mooncake_trace import MooncakeTraceDatasetLoader +from aiperf.plugin.enums import DatasetSamplingStrategy + +if TYPE_CHECKING: + from aiperf.config.resolution.plan import BenchmarkRun + +LIVE_OBJECT_FACTOR = 2.0 +"""Live bytes per on-disk byte once a record is parsed; measured at 1.4 to 1.7, rounded up for text-heavy sessions.""" + + +class HCuaPerfDatasetLoader(BaseHFDatasetLoader): + """Download ``Hcompany/h_cua_perf`` and replay it as a Mooncake trace.""" + + tag: ClassVar[str] = "HCuaPerf" + hf_filename: ClassVar[str] = "h_cua.jsonl.zst" + hf_manifest_filename: ClassVar[str] = "h_cua.meta.json" + + def __init__( + self, + *, + run: BenchmarkRun | None = None, + hf_dataset_name: str, + hf_split: str = "train", + hf_subset: str | None = None, + prompt_generator: PromptGenerator, + default_block_size: int | None = None, + filters: dict[str, str] | None = None, + **kwargs: Any, + ) -> None: + # The corpus is one file fetched whole; HF streaming mode does not apply. + kwargs.pop("streaming", None) + super().__init__( + run=run, + hf_dataset_name=hf_dataset_name, + hf_split=hf_split, + hf_subset=hf_subset, + streaming=False, + **kwargs, + ) + if hf_subset is not None: + raise DatasetLoaderError( + f"{self.tag}: {hf_dataset_name} has no subsets; drop --hf-subset" + ) + dataset = run.cfg.get_default_dataset() if run is not None else None + if getattr(dataset, "ignore_trace_delays", False): + raise DatasetLoaderError( + f"{self.tag}: --ignore-trace-delays applies to the Weka loaders only; " + "use --inter-turn-delay-cap-seconds 0 to send each session's turns " + "back to back" + ) + try: + self.filters = HCuaPerfFilters.model_validate(filters or {}) + except ValidationError as e: + raise DatasetLoaderError( + f"Invalid h_cua_perf dataset filters: {e}; supported keys: " + f"{supported_filter_keys()}" + ) from e + self._mooncake_kwargs = { + "run": self.run, + "prompt_generator": prompt_generator, + "default_block_size": default_block_size, + } + self._mooncake: MooncakeTraceDatasetLoader | None = None + + def _load_hf_dataset(self) -> tuple[Path, Path]: + """Fetch the manifest and the trace file into the Hub cache.""" + # hf_hub_download caches the 2.5 GB file across runs; datasets.load_dataset + # would re-stream it every run (streaming) or write a 9 GB Arrow copy. + from huggingface_hub import hf_hub_download + + manifest, trace = ( + Path( + hf_hub_download( + repo_id=self.hf_dataset_name, + filename=filename, + repo_type="dataset", + revision=self.hf_revision, + ) + ) + for filename in (self.hf_manifest_filename, self.hf_filename) + ) + return manifest, trace + + def _explicit_entries(self) -> int | None: + """``--num-dataset-entries`` when set explicitly; request-count fallbacks must not cap.""" + dataset = self.run.cfg.get_default_dataset() + if not getattr(dataset, "entries_explicit", False): + return None + return getattr(dataset, "entries", None) + + def _read_records(self, trace: Path, plan: dict[str, int]) -> list[dict[str, Any]]: + with open_dataset(trace) as lines: + records = (orjson.loads(line) for line in lines if line.strip()) + return list( + iter_selected_records(records, plan, self.filters.n_screenshots) + ) + + async def load_dataset(self) -> dict[str, list[MooncakeTrace]]: + """Download, plan the trajectory selection, read that far, delegate the rest.""" + manifest, trace = (await super().load_dataset())["dataset"] + loop = asyncio.get_running_loop() + try: + meta = orjson.loads(manifest.read_bytes()) + session_turns = meta["session_turns"] + plan = select_trace_lengths( + self.filters, session_turns, first_n=self._explicit_entries() + ) + self._warn_if_selection_exceeds_memory(meta, plan) + await loop.run_in_executor(None, self._verify_trace, meta, trace) + records = await loop.run_in_executor(None, self._read_records, trace, plan) + except (KeyError, ValueError, zstandard.ZstdError) as e: + raise DatasetLoaderError(f"{self.tag}: {e}") from e + + self._mooncake = MooncakeTraceDatasetLoader( + inline_records=records, **self._mooncake_kwargs + ) + data = await loop.run_in_executor(None, self._mooncake.load_dataset) + self.info( + f"Selected {len(data)}/{len(session_turns)} trajectories " + f"({len(records):,} requests) from {self.hf_dataset_name}" + ) + return data + + def _verify_trace(self, meta: dict[str, Any], trace: Path) -> None: + """The manifest names the sha256 of the trace it describes; any other file is refused.""" + digests = meta.get("sha256") + expected = digests.get(self.hf_filename) if isinstance(digests, dict) else None + if not expected: + raise DatasetLoaderError( + f"{self.tag}: the manifest carries no sha256 for {self.hf_filename}" + ) + with open(trace, "rb") as f: + actual = hashlib.file_digest(f, "sha256").hexdigest() + if actual != expected: + raise DatasetLoaderError( + f"{self.tag}: {self.hf_filename} does not match the manifest " + f"(sha256 {actual[:12]}, manifest says {expected[:12]}); " + "delete it from the Hub cache and download again" + ) + + def _warn_if_selection_exceeds_memory( + self, meta: dict[str, Any], plan: dict[str, int] + ) -> None: + """The whole corpus is parsed into memory; say so before the read when it will not fit.""" + if not meta.get("size_mb") or not meta.get("num_turns"): + return + planned = sum(plan.values()) + estimate = ( + meta["size_mb"] * 1e6 * planned / meta["num_turns"] * LIVE_OBJECT_FACTOR + ) + available = psutil.virtual_memory().available + if estimate > available: + self.warning( + f"{self.tag}: the {planned:,} selected requests need about " + f"{estimate / 1e9:.1f} GB of RAM once parsed and {available / 1e9:.1f} GB " + "is available; select fewer with --num-dataset-entries or --dataset-filter" + ) + + async def convert_to_conversations( + self, data: dict[str, list[MooncakeTrace]] + ) -> list[Conversation]: + if self._mooncake is None: + raise DatasetLoaderError(f"{self.tag}: load_dataset() must run first") + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, self._mooncake.convert_to_conversations, data + ) + + @classmethod + def get_default_context_mode(cls) -> ConversationContextMode | None: + return MooncakeTraceDatasetLoader.get_default_context_mode() + + @classmethod + def get_preferred_sampling_strategy(cls) -> DatasetSamplingStrategy: + return DatasetSamplingStrategy.SEQUENTIAL diff --git a/src/aiperf/dataset/loader/h_cua_perf_processing.py b/src/aiperf/dataset/loader/h_cua_perf_processing.py new file mode 100644 index 0000000000..4c91f71635 --- /dev/null +++ b/src/aiperf/dataset/loader/h_cua_perf_processing.py @@ -0,0 +1,274 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import statistics +from collections.abc import Iterator +from itertools import groupby, islice +from operator import itemgetter +from pathlib import Path +from typing import Any, TextIO + +import zstandard +from pydantic import ConfigDict, Field, model_validator + +from aiperf.common.aiperf_logger import AIPerfLogger +from aiperf.common.models import AIPerfBaseModel + +_logger = AIPerfLogger(__name__) + +# Mirrors the dataset's trace_processor.py (published beside the data); keep the two in sync. + +IMAGE_PLACEHOLDER = "[Image omitted by context cleaning]" +"""Marker the agent leaves in place of a screenshot it dropped from its context.""" + +IMAGE_OMITTED_TEXT = f"\n\n{IMAGE_PLACEHOLDER}\n\n\n\n" + +SCREENSHOT_ROLES = frozenset({"user", "tool"}) +"""Observations carry the screenshots: user messages for GUI agents, tool results for tool-calling ones.""" + +_BISECTION_STEPS = 32 + + +class HCuaPerfFilters(AIPerfBaseModel): + """``--dataset-filter`` keys accepted by ``h_cua_perf``. + + Names and defaults match the options of the dataset's ``trace_processor.py``; + its ``--num-traces`` / ``--seed`` sampling is replaced by + ``--num-dataset-entries`` (first N eligible trajectories). + """ + + model_config = ConfigDict(extra="forbid") + + n_screenshots: int | None = Field( + default=None, + ge=1, + description="Screenshots each request keeps: its own plus the preceding " + "ones of the trajectory, restored from the records that carry them; " + "older ones stay as the agent's placeholder text. None leaves the " + "published single screenshot per request.", + ) + min_trace_length: int = Field( + default=1, + ge=1, + description="Drop trajectories with fewer requests; also the floor of every truncation.", + ) + max_trace_length: int | None = Field( + default=None, + ge=1, + description="Truncate trajectories to their first N requests.", + ) + avg_trace_length: float | None = Field( + default=None, + gt=0, + description="Target mean requests per trajectory, reached by truncating " + "every trajectory by the same factor.", + ) + + @model_validator(mode="after") + def _max_not_below_min(self) -> HCuaPerfFilters: + if ( + self.max_trace_length is not None + and self.max_trace_length < self.min_trace_length + ): + raise ValueError( + f"max_trace_length={self.max_trace_length} is below " + f"min_trace_length={self.min_trace_length}" + ) + return self + + +def supported_filter_keys() -> str: + """Comma-separated filter names, for error messages.""" + return ", ".join(HCuaPerfFilters.model_fields) + + +def select_trace_lengths( + filters: HCuaPerfFilters, + turns: dict[str, int], + *, + first_n: int | None = None, +) -> dict[str, int]: + """Requests to keep per selected trajectory, in file order. + + Filtered on ``min_trace_length``, capped at ``max_trace_length``, cut to the + first ``first_n`` eligible trajectories, then fitted to ``avg_trace_length``. + """ + selected = { + session_id: n + if filters.max_trace_length is None + else min(n, filters.max_trace_length) + for session_id, n in turns.items() + if n >= filters.min_trace_length + } + if first_n is not None: + selected = dict(islice(selected.items(), first_n)) + if not selected: + raise ValueError(f"No trajectory has {filters.min_trace_length}+ requests") + if filters.avg_trace_length is not None: + lengths = fit_lengths( + list(selected.values()), + filters.avg_trace_length, + filters.min_trace_length, + ) + selected = dict(zip(selected, lengths, strict=True)) + return selected + + +def fit_lengths(lengths: list[int], avg: float, floor: int) -> list[int]: + """Scale every length by the smallest common factor, never below floor, whose mean reaches avg.""" + if statistics.mean(lengths) <= avg: + _logger.warning( + f"Mean trajectory length {statistics.mean(lengths):.1f} is already " + f"below {avg}; nothing truncated" + ) + return lengths + low, high = 0.0, 1.0 + for _ in range(_BISECTION_STEPS): + mid = (low + high) / 2 + low, high = ( + (mid, high) + if statistics.mean(_scaled(lengths, mid, floor)) < avg + else (low, mid) + ) + return _scaled(lengths, high, floor) + + +def _scaled(lengths: list[int], factor: float, floor: int) -> list[int]: + return [max(floor, round(factor * length)) for length in lengths] + + +def open_dataset(path: Path) -> TextIO: + """Open a dataset as text, decompressing on the fly when it is stored as ``.zst``.""" + if path.suffix != ".zst": + return open(path, encoding="utf-8") + return zstandard.open(path, "rt", encoding="utf-8") + + +def iter_selected_records( + records: Iterator[dict[str, Any]], + plan: dict[str, int], + n_screenshots: int | None, +) -> Iterator[dict[str, Any]]: + """Yield the planned prefix of every selected trajectory, re-windowed, in file order. + + The plan comes from the manifest and the file is verified against the + manifest's sha256 before this runs, so the plan is trusted. Trajectories + are contiguous in the source, so each is consumed as a group and iteration + stops after the last planned one; the rest of the file is never read. + """ + remaining = dict(plan) + for session_id, session in groupby(records, key=itemgetter("session_id")): + kept_n = remaining.pop(session_id, None) + if kept_n is None: + continue + kept = list(islice(session, kept_n)) + if n_screenshots is not None: + apply_screenshot_window(kept, n_screenshots) + yield from kept + if not remaining: + return + + +def apply_screenshot_window(records: list[dict[str, Any]], n_screenshots: int) -> None: + """Resize the screenshot window of every record of one trajectory, in place, to its last n_screenshots slots.""" + slots_per_record = [screenshot_slots(record["messages"]) for record in records] + for start, stop in _history_segments(records, slots_per_record): + _window_segment(slots_per_record[start:stop], n_screenshots) + + +def _history_segments( + records: list[dict[str, Any]], + slots_per_record: list[list[tuple[list[dict[str, Any]], int]]], +) -> Iterator[tuple[int, int]]: + """Spans of records whose slot indexes refer to the same observations. + + Message and slot counts only grow while a trajectory's history is + append-only, so a drop in either means the history was reset: the agent + compacted its context, or an auxiliary model was invoked under the same + session id. Slot indexes restart there, so windowing across the boundary + would restore a screenshot from before the reset into a slot that now + stands for a different observation. + """ + start = 0 + for idx in range(1, len(records)): + if len(slots_per_record[idx]) < len(slots_per_record[idx - 1]) or len( + records[idx]["messages"] + ) < len(records[idx - 1]["messages"]): + yield start, idx + start = idx + yield start, len(records) + + +def _window_segment( + slots_per_record: list[list[tuple[list[dict[str, Any]], int]]], + n_screenshots: int, +) -> None: + images_by_slot: dict[int, dict[str, Any]] = {} + for slots in slots_per_record: + for slot_idx, (parts, part_idx) in enumerate(slots): + if _is_image(parts[part_idx]): + images_by_slot.setdefault(slot_idx, parts[part_idx]) + + for slots in slots_per_record: + first_kept = max(0, len(slots) - n_screenshots) + for slot_idx, (parts, part_idx) in enumerate(slots): + if _is_image(parts[part_idx]) and slot_idx < first_kept: + drop_screenshot(parts, part_idx) + elif ( + not _is_image(parts[part_idx]) + and slot_idx >= first_kept + and slot_idx in images_by_slot + ): + _restore_screenshot(parts, part_idx, images_by_slot[slot_idx]) + + +def screenshot_slots( + messages: list[dict[str, Any]], +) -> list[tuple[list[dict[str, Any]], int]]: + """A slot is a screenshot part, or the placeholder standing for a dropped one, in any observation message.""" + return [ + (message["content"], part_idx) + for message in messages + if message.get("role") in SCREENSHOT_ROLES + and isinstance(message.get("content"), list) + for part_idx, part in enumerate(message["content"]) + if _is_image(part) or IMAGE_PLACEHOLDER in part.get("text", "") + ] + + +def drop_screenshot(parts: list[dict[str, Any]], part_idx: int) -> None: + """Replace a screenshot by the placeholder, merged into the text parts surrounding it.""" + before, after = _text(parts, part_idx - 1), _text(parts, part_idx + 1) + start = part_idx if before is None else part_idx - 1 + end = part_idx + 1 if after is None else part_idx + 2 + parts[start:end] = [ + {"type": "text", "text": (before or "") + IMAGE_OMITTED_TEXT + (after or "")} + ] + + +def _restore_screenshot( + parts: list[dict[str, Any]], part_idx: int, image: dict[str, Any] +) -> None: + """Split a placeholder text part back into text, screenshot and text.""" + text = parts[part_idx]["text"] + marker = IMAGE_OMITTED_TEXT if IMAGE_OMITTED_TEXT in text else IMAGE_PLACEHOLDER + before, _, after = text.partition(marker) + # Empty sides are dropped: the placeholder often begins or ends its text + # part, and a zero-length text part is a malformed content part on the wire. + parts[part_idx : part_idx + 1] = [ + *([{"type": "text", "text": before}] if before else []), + image, + *([{"type": "text", "text": after}] if after else []), + ] + + +def _is_image(part: dict[str, Any]) -> bool: + return part.get("type") == "image_url" + + +def _text(parts: list[dict[str, Any]], part_idx: int) -> str | None: + if not 0 <= part_idx < len(parts): + return None + return parts[part_idx]["text"] if parts[part_idx].get("type") == "text" else None diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index 575831520c..fb38d66b03 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -63,7 +63,7 @@ PublicDatasetTypeStr: TypeAlias = str PublicDatasetType = plugins.create_enum(PluginType.PUBLIC_DATASET_LOADER, "PublicDatasetType", module=__name__) -"""Dynamic enum for public dataset loader. Example: PublicDatasetType.AIMO, PublicDatasetType.SPEC_BENCH, PublicDatasetType.WEKA_HF""" +"""Dynamic enum for public dataset loader. Example: PublicDatasetType.AIMO, PublicDatasetType.SPEC_AL_MTBENCH, PublicDatasetType.WEKA_HF""" EndpointTypeStr: TypeAlias = str EndpointType = plugins.create_enum(PluginType.ENDPOINT, "EndpointType", module=__name__) diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 951a611297..c97b71307e 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -2676,6 +2676,23 @@ public_dataset_loader: multi_turn: true category: high_entropy + h_cua_perf: + class: aiperf.dataset.loader.h_cua_perf:HCuaPerfDatasetLoader + description: | + H CUA Perf computer-use agent traces — hosted on HuggingFace as + Hcompany/h_cua_perf (477 trajectories, 18,224 requests, CC BY 4.0): + https://huggingface.co/datasets/Hcompany/h_cua_perf + One Mooncake record per step of a tool-calling, screenshot-driven agent + on CUA-Gym desktop tasks: full text history, 10 tool definitions and one + screenshot per request. Delegates to the mooncake_trace loader, replaying + the recorded inter-request delays. --dataset-filter reshapes the corpus in + place: n_screenshots, min_trace_length, max_trace_length, avg_trace_length. + Usage: --public-dataset h_cua_perf --dataset-filter n_screenshots=3 + metadata: + hf_dataset_name: Hcompany/h_cua_perf + hf_split: train + is_trace: true + # ============================================================================= # ZMQ Proxies # ============================================================================= diff --git a/tests/unit/dataset/loader/_h_cua_perf_records.py b/tests/unit/dataset/loader/_h_cua_perf_records.py new file mode 100644 index 0000000000..87e2b4b42c --- /dev/null +++ b/tests/unit/dataset/loader/_h_cua_perf_records.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any + +from aiperf.dataset.loader.h_cua_perf_processing import ( + IMAGE_OMITTED_TEXT, + screenshot_slots, +) + +TOOLS = [ + { + "type": "function", + "function": { + "name": "shell", + "description": "Run a shell command", + "parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}}, + }, + } +] + + +def image(session_id: str, slot: int) -> dict[str, Any]: + return { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{session_id}-{slot}"}, + "uuid": f"{slot:016x}", + } + + +def observation( + session_id: str, slot: int, *, with_image: bool +) -> list[dict[str, Any]]: + if with_image: + return [ + {"type": "text", "text": f"obs {slot} before"}, + image(session_id, slot), + {"type": "text", "text": f"obs {slot} after"}, + ] + return [ + { + "type": "text", + "text": f"obs {slot} before{IMAGE_OMITTED_TEXT}obs {slot} after", + } + ] + + +def record( + session_id: str, + step: int, + *, + delay: int | None = None, + image_slots: set[int] | None = None, +) -> dict[str, Any]: + """Request at ``step`` with observation slots 0..step; the source keeps only the latest screenshot.""" + image_slots = {step} if image_slots is None else image_slots + messages: list[dict[str, Any]] = [ + {"role": "system", "content": "You are a computer-use agent."}, + { + "role": "user", + "content": observation(session_id, 0, with_image=0 in image_slots), + }, + ] + for slot in range(1, step + 1): + messages.append( + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": f"call_{slot}", + "type": "function", + "function": {"name": "shell", "arguments": '{"cmd": "ls"}'}, + } + ], + } + ) + messages.append( + { + "role": "tool", + "tool_call_id": f"call_{slot}", + "content": observation( + session_id, slot, with_image=slot in image_slots + ), + } + ) + out: dict[str, Any] = { + "session_id": session_id, + "messages": messages, + "tools": TOOLS, + "output_length": 10 + step, + "extra": {"tool_choice": "auto"}, + } + if delay is not None: + out["delay"] = delay + return out + + +def trajectory(session_id: str, steps: int) -> list[dict[str, Any]]: + """Requests of one trajectory; the first carries no delay, the rest the agent's think time.""" + return [ + record(session_id, step, delay=None if step == 0 else 2000) + for step in range(steps) + ] + + +def image_slots(messages: list[dict[str, Any]]) -> list[int]: + """Slot indexes that currently hold a screenshot rather than the placeholder.""" + return [ + idx + for idx, (parts, part_idx) in enumerate(screenshot_slots(messages)) + if parts[part_idx].get("type") == "image_url" + ] diff --git a/tests/unit/dataset/loader/test_h_cua_perf.py b/tests/unit/dataset/loader/test_h_cua_perf.py new file mode 100644 index 0000000000..874026f332 --- /dev/null +++ b/tests/unit/dataset/loader/test_h_cua_perf.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import orjson +import pytest +import zstandard + +from aiperf.common.enums import ConversationContextMode +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.dataset.loader.h_cua_perf import HCuaPerfDatasetLoader +from aiperf.plugin import plugins +from aiperf.plugin.enums import PluginType, PublicDatasetType +from tests.unit.conftest import make_run_from_cli +from tests.unit.dataset.loader._h_cua_perf_records import ( + TOOLS, + image_slots, + trajectory, +) + +HF_DATASET_NAME = "Hcompany/h_cua_perf" +SESSION_TURNS = {"traj-a": 3, "traj-b": 1, "traj-c": 2} +RECORDS = [r for sid, n in SESSION_TURNS.items() for r in trajectory(sid, n)] + + +@pytest.fixture +def hub(tmp_path: Path): + """Serve the fixture manifest and trace through a patched ``hf_hub_download``.""" + files = { + "h_cua.meta.json": tmp_path / "h_cua.meta.json", + "h_cua.jsonl.zst": tmp_path / "h_cua.jsonl.zst", + } + with zstandard.open(files["h_cua.jsonl.zst"], "wb") as f: + for r in RECORDS: + f.write(orjson.dumps(r) + b"\n") + digest = hashlib.sha256(files["h_cua.jsonl.zst"].read_bytes()).hexdigest() + files["h_cua.meta.json"].write_bytes( + orjson.dumps( + {"session_turns": SESSION_TURNS, "sha256": {"h_cua.jsonl.zst": digest}} + ) + ) + with patch( + "huggingface_hub.hf_hub_download", + side_effect=lambda repo_id, filename, repo_type, revision: str(files[filename]), + ) as download: + yield download + + +def _loader(filters: dict[str, str] | None = None, **cli: Any) -> HCuaPerfDatasetLoader: + run = make_run_from_cli( + CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + public_dataset=PublicDatasetType.H_CUA_PERF, + **cli, + ) + ) + return HCuaPerfDatasetLoader( + run=run, + hf_dataset_name=HF_DATASET_NAME, + prompt_generator=MagicMock(), + filters=filters, + ) + + +class TestRegistry: + def test_plugin_resolves_to_loader_class(self) -> None: + cls = plugins.get_class( + PluginType.PUBLIC_DATASET_LOADER, PublicDatasetType.H_CUA_PERF + ) + assert cls is HCuaPerfDatasetLoader + + def test_metadata_declares_trace_with_timing(self) -> None: + metadata = plugins.get_public_dataset_loader_metadata( + PublicDatasetType.H_CUA_PERF + ) + assert metadata.hf_dataset_name == HF_DATASET_NAME + assert (metadata.is_trace, metadata.has_timing_data) == (True, False) + + +@pytest.mark.asyncio +class TestLoader: + """Async: the base class opens an aiohttp client at construction.""" + + async def test_invalid_filters_are_rejected(self) -> None: + with pytest.raises(DatasetLoaderError, match="supported keys: n_screenshots"): + _loader({"screenshots": "3"}) + + async def test_end_to_end_replays_selected_trajectories_verbatim( + self, hub: MagicMock + ) -> None: + loader = _loader( + {"max_trace_length": "2", "n_screenshots": "2"}, + conversation_num_dataset_entries=2, + ) + + data = await loader.load_dataset() + conversations = await loader.convert_to_conversations(data) + + assert [c.kwargs["filename"] for c in hub.call_args_list] == [ + "h_cua.meta.json", + "h_cua.jsonl.zst", + ] + assert {c.kwargs["repo_id"] for c in hub.call_args_list} == {HF_DATASET_NAME} + assert {sid: len(traces) for sid, traces in data.items()} == { + "traj-a": 2, + "traj-b": 1, + } + + by_id = {c.session_id: c for c in conversations} + assert ( + by_id["traj-a"].context_mode + == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + ) + first, second = by_id["traj-a"].turns + assert first.raw_messages == RECORDS[0]["messages"] + assert first.raw_tools == TOOLS + assert first.extra_body == {"tool_choice": "auto"} + assert (first.max_tokens, first.timestamp, first.delay) == (10, None, None) + assert (second.max_tokens, second.timestamp, second.delay) == (11, None, 2000) + assert image_slots(second.raw_messages) == [0, 1] + + async def test_request_count_fallback_does_not_cap_trajectories( + self, hub: MagicMock + ) -> None: + data = await _loader(request_count=1).load_dataset() + assert {sid: len(t) for sid, t in data.items()} == SESSION_TURNS + + async def test_manifest_without_session_turns_is_rejected( + self, hub: MagicMock, tmp_path: Path + ) -> None: + (tmp_path / "h_cua.meta.json").write_bytes(orjson.dumps({"num_traces": 3})) + with pytest.raises(DatasetLoaderError, match="session_turns"): + await _loader().load_dataset() + + async def test_trace_not_matching_the_manifest_hash_is_rejected( + self, hub: MagicMock, tmp_path: Path + ) -> None: + (tmp_path / "h_cua.meta.json").write_bytes( + orjson.dumps( + { + "session_turns": SESSION_TURNS, + "sha256": {"h_cua.jsonl.zst": "0" * 64}, + } + ) + ) + with pytest.raises(DatasetLoaderError, match="does not match the manifest"): + await _loader().load_dataset() diff --git a/tests/unit/dataset/loader/test_h_cua_perf_processing.py b/tests/unit/dataset/loader/test_h_cua_perf_processing.py new file mode 100644 index 0000000000..2ba4f4b9d0 --- /dev/null +++ b/tests/unit/dataset/loader/test_h_cua_perf_processing.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import copy +from typing import Any + +import pytest +from pydantic import ValidationError +from pytest import param + +from aiperf.dataset.loader.h_cua_perf_processing import ( + IMAGE_OMITTED_TEXT, + IMAGE_PLACEHOLDER, + HCuaPerfFilters, + apply_screenshot_window, + drop_screenshot, + fit_lengths, + iter_selected_records, + screenshot_slots, + select_trace_lengths, +) +from tests.unit.dataset.loader._h_cua_perf_records import ( + image, + image_slots, + record, + trajectory, +) + +TURNS = {"traj-a": 3, "traj-b": 1, "traj-c": 2, "traj-d": 4} +RECORDS = [r for sid, n in TURNS.items() for r in trajectory(sid, n)] + + +def _image(slot: int) -> dict[str, Any]: + return image("traj", slot) + + +def _record(step: int, *, image_slots: set[int] | None = None) -> dict[str, Any]: + return record("traj", step, image_slots=image_slots) + + +def _image_slots(rec: dict[str, Any]) -> list[int]: + return image_slots(rec["messages"]) + + +class TestSelectTraceLengths: + def test_defaults_keep_everything_in_file_order(self) -> None: + assert select_trace_lengths(HCuaPerfFilters(), TURNS) == TURNS + + def test_min_length_drops_short_trajectories(self) -> None: + plan = select_trace_lengths(HCuaPerfFilters(min_trace_length=2), TURNS) + assert plan == {"traj-a": 3, "traj-c": 2, "traj-d": 4} + + def test_max_length_truncates(self) -> None: + plan = select_trace_lengths(HCuaPerfFilters(max_trace_length=2), TURNS) + assert plan == {"traj-a": 2, "traj-b": 1, "traj-c": 2, "traj-d": 2} + + def test_first_n_applies_after_eligibility(self) -> None: + plan = select_trace_lengths( + HCuaPerfFilters(min_trace_length=2), TURNS, first_n=2 + ) + assert plan == {"traj-a": 3, "traj-c": 2} + + def test_nothing_eligible_raises(self) -> None: + with pytest.raises(ValueError, match="No trajectory has 10\\+"): + select_trace_lengths(HCuaPerfFilters(min_trace_length=10), TURNS) + + def test_avg_length_scales_every_trajectory_by_one_factor(self) -> None: + # Lengths [3, 1, 2, 4] scaled by 0.75 round to [2, 1, 2, 3] (mean 2.0); + # any smaller factor rounds 2 -> 1 and drops the mean below 2. + plan = select_trace_lengths(HCuaPerfFilters(avg_trace_length=2), TURNS) + assert plan == {"traj-a": 2, "traj-b": 1, "traj-c": 2, "traj-d": 3} + + +class TestFitLengths: + def test_mean_already_below_target_is_untouched(self) -> None: + assert fit_lengths([1, 2, 3], avg=5, floor=1) == [1, 2, 3] + + def test_floor_is_never_crossed(self) -> None: + lengths = fit_lengths([10, 10, 10], avg=1, floor=3) + assert lengths == [3, 3, 3] + + +class TestIterSelectedRecords: + def test_yields_planned_prefixes_in_file_order(self) -> None: + plan = {"traj-a": 2, "traj-c": 2} + out = list(iter_selected_records(iter(RECORDS), plan, None)) + assert [(r["session_id"], r["output_length"]) for r in out] == [ + ("traj-a", 10), ("traj-a", 11), ("traj-c", 10), ("traj-c", 11), + ] # fmt: skip + + def test_stops_reading_after_the_last_planned_trajectory(self) -> None: + consumed: list[str] = [] + + def records(): + for r in RECORDS: + consumed.append(r["session_id"]) + yield r + + list(iter_selected_records(records(), {"traj-a": 3, "traj-b": 1}, None)) + assert consumed == ["traj-a"] * 3 + ["traj-b"] + + +class TestScreenshotWindow: + def test_slots_span_user_and_tool_observations(self) -> None: + messages = _record(2)["messages"] + by_content = {id(m["content"]): m["role"] for m in messages} + slots = screenshot_slots(messages) + assert [by_content[id(parts)] for parts, _ in slots] == ["user", "tool", "tool"] + + def test_window_of_one_is_a_no_op_on_the_source(self) -> None: + records = [_record(0), _record(1), _record(2)] + untouched = copy.deepcopy(records) + apply_screenshot_window(records, 1) + assert records == untouched + + def test_wider_window_restores_earlier_screenshots(self) -> None: + records = [_record(0), _record(1), _record(2)] + apply_screenshot_window(records, 2) + + assert [_image_slots(r) for r in records] == [[0], [0, 1], [1, 2]] + slots = screenshot_slots(records[2]["messages"]) + parts, idx = slots[1] + assert parts[idx] == _image(1) + assert parts[idx - 1] == {"type": "text", "text": "obs 1 before"} + assert parts[idx + 1] == {"type": "text", "text": "obs 1 after"} + + def test_window_does_not_restore_across_a_history_reset(self) -> None: + before_reset = [_record(0), _record(1), _record(2)] + after_reset = [record("traj-reset", 0), record("traj-reset", 1)] + records = before_reset + after_reset + apply_screenshot_window(records, 3) + + assert [_image_slots(r) for r in records] == [ + [0], + [0, 1], + [0, 1, 2], + [0], + [0, 1], + ] + parts, idx = screenshot_slots(records[4]["messages"])[0] + assert parts[idx] == image("traj-reset", 0) + + def test_restored_screenshot_leaves_no_empty_text_parts(self) -> None: + system = {"role": "system", "content": "sys"} + placeholder = {"type": "text", "text": IMAGE_OMITTED_TEXT} + first = {"messages": [system, {"role": "user", "content": [_image(0)]}]} + second = { + "messages": [ + system, + {"role": "user", "content": [placeholder]}, + {"role": "user", "content": [_image(1)]}, + ] + } + apply_screenshot_window([first, second], 2) + + assert second["messages"][1]["content"] == [_image(0)] + + def test_narrower_window_drops_screenshots(self) -> None: + records = [_record(2, image_slots={0, 1, 2})] + apply_screenshot_window(records, 1) + assert _image_slots(records[0]) == [2] + first_user = records[0]["messages"][1]["content"] + assert first_user == [ + {"type": "text", "text": f"obs 0 before{IMAGE_OMITTED_TEXT}obs 0 after"} + ] + + @pytest.mark.parametrize( + "parts, expected_text", + [ + param( + [{"type": "text", "text": "a"}, _image(0), {"type": "text", "text": "b"}], + f"a{IMAGE_OMITTED_TEXT}b", + id="text_both_sides", + ), + param([_image(0)], IMAGE_OMITTED_TEXT, id="image_alone"), + ], + ) # fmt: skip + def test_drop_screenshot_merges_neighbouring_text( + self, parts: list[dict[str, Any]], expected_text: str + ) -> None: + drop_screenshot(parts, next(i for i, p in enumerate(parts) if "uuid" in p)) + assert parts == [{"type": "text", "text": expected_text}] + assert IMAGE_PLACEHOLDER in parts[0]["text"] + + +class TestFilters: + def test_string_values_from_the_cli_are_coerced(self) -> None: + filters = HCuaPerfFilters.model_validate( + {"n_screenshots": "3", "avg_trace_length": "20"} + ) + assert (filters.n_screenshots, filters.avg_trace_length) == (3, 20.0) + + def test_max_length_below_min_length_is_rejected(self) -> None: + with pytest.raises(ValidationError, match="below min_trace_length"): + HCuaPerfFilters(min_trace_length=3, max_trace_length=2, avg_trace_length=1) + + def test_defaults_match_trace_processor(self) -> None: + assert HCuaPerfFilters().min_trace_length == 1 From d7182e69891e4ed38dc6f6ff38de690f217be9b2 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:16:09 -0700 Subject: [PATCH 066/119] ci: install from uv.lock instead of re-resolving dependencies (#1487) Signed-off-by: Elias Bermudez (cherry picked from commit 0968f7e609e1c5ccf7d80841bde774a6f30575ad) --- .github/workflows/run-unit-tests.yml | 2 +- CONTRIBUTING.md | 1 + Makefile | 14 +++++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index f90a6e0d43..8fd35f8c76 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -72,7 +72,7 @@ jobs: PYTHONIOENCODING: utf-8 run: | uv venv .venv --python ${{ matrix.python-version }} - uv pip install -e ".[dev]" + uv sync --locked --extra dev --no-dev uv pip install -e "tests/aiperf_mock_server[dev]" uv run python -c "import aiperf_mock_server" uv run python tools/generate_plugin_artifacts.py --enums diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5174b0bb9..8986d9e0c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,6 +67,7 @@ pre-commit install # Install pre-commit hooks | `make first-time-setup` | Full environment setup (venv + install + hooks) | | `make install` | Install project, mock server, and fake amdsmi bindings in editable mode | | `make install-app` | Install project only | +| `make install-app-locked` | Install project only, at the exact versions in `uv.lock` (what CI's `make ci-install` uses) | | `make install-mock-server` | Install mock server only | | `make install-mock-amdsmi` | Install fake `amdsmi` bindings to exercise the AMD telemetry path on non-AMD hardware (see [Mocking a ROCm Environment](docs/reference/mock-amdsmi.md)) | | `make test` | Unit tests (parallel, excludes integration) | diff --git a/Makefile b/Makefile index 8eeb81b23c..a131fa3561 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ .PHONY: ruff lint ruff-fix lint-fix format fmt check-format check-fmt \ - test coverage clean install install-app docker docker-run first-time-setup \ + test coverage clean install install-app install-app-locked precompile docker docker-run first-time-setup \ ci-install check-mock-server-install \ test-verbose setup-venv install-mock-server install-mock-amdsmi test-ci test-all \ integration-tests integration-tests-ci integration-tests-verbose integration-tests-ci-macos \ @@ -156,6 +156,12 @@ install: install-app install-mock-server install-mock-amdsmi #? install the proj install-app: #? install the project in editable mode. $(activate_venv) && uv pip install -e ".[dev]" +install-app-locked: #? install the project and its dev extra at the exact versions in uv.lock (CI). + $(activate_venv) && uv sync --locked --active --extra dev --no-dev + +precompile: #? precompile source bytecode so parallel test subprocesses never race to write __pycache__ on first import. + $(activate_venv) && python -m compileall -q -x '/(\.venv|\.ruff_cache|.*\.egg-info)/' src/aiperf tests/aiperf_mock_server tests/aiperf_mock_amdsmi + docker: #? build the docker image. docker build -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) $(args) . @@ -230,8 +236,10 @@ first-time-setup: #? convenience command to setup the environment for the first ci-install: #? CI-only environment setup: venv + project + plugin artifacts. No pre-commit hooks, no redundant mock-server install. $(MAKE) setup-venv --no-print-directory - @printf "$(bold)$(green)Installing project (+ mock server)...$(reset)\n" - @PATH="$(UV_PATH):$(PATH)" $(MAKE) --no-print-directory install + @# From uv.lock, not a fresh resolve: an unpinned install takes whatever was + @# released minutes ago, so an upstream release could fail every PR at once. + @printf "$(bold)$(green)Installing project from uv.lock (+ mock server)...$(reset)\n" + @PATH="$(UV_PATH):$(PATH)" $(MAKE) --no-print-directory install-app-locked install-mock-server install-mock-amdsmi precompile @printf "$(bold)$(green)Verifying mock server install...$(reset)\n" @PATH="$(UV_PATH):$(PATH)" $(MAKE) --no-print-directory check-mock-server-install From 7db73c7f057f0d92e1802caf165b3bcf8f49c794 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:09:45 -0700 Subject: [PATCH 067/119] fix(server-metrics): probe endpoint reachability with GET instead of HEAD (#1486) Signed-off-by: lkomali (cherry picked from commit cc8769a47d8a71c6e894c84e432f11d60e1c0d56) --- .../mixins/base_metrics_collector_mixin.py | 30 ++-- .../test_base_metrics_collector_mixin.py | 132 ++++++++++++++++++ .../test_telemetry_data_collector.py | 45 +----- 3 files changed, 156 insertions(+), 51 deletions(-) diff --git a/src/aiperf/common/mixins/base_metrics_collector_mixin.py b/src/aiperf/common/mixins/base_metrics_collector_mixin.py index 721508e6c1..e71dd87e20 100644 --- a/src/aiperf/common/mixins/base_metrics_collector_mixin.py +++ b/src/aiperf/common/mixins/base_metrics_collector_mixin.py @@ -398,9 +398,8 @@ async def _cleanup_http_client(self) -> None: async def is_url_reachable(self) -> bool: """Check if metrics endpoint is accessible before starting collection. - Tests endpoint reachability using a two-phase approach: - 1. HEAD request (lightweight, doesn't fetch content) - 2. GET request fallback if HEAD not supported (some servers disable HEAD) + Probes with GET; see `_check_reachability_with_session` for why a HEAD + pre-flight is unsafe here. Uses existing session if available (during lifecycle), otherwise creates a temporary session for pre-initialization testing. This allows reachability @@ -435,6 +434,15 @@ async def _check_reachability_with_session( ) -> bool: """Check reachability using a specific session. + Uses GET rather than a cheaper HEAD pre-flight. Servers are free to + reject HEAD, and some answer it with a body, which RFC 9110 forbids -- + Triton's /metrics frontend routes every non-GET method through an error + path that writes `{"error":"Method Not Allowed"}` to the response + buffer. aiohttp >= 3.14 reports those stray bytes as a bad status line, + either on the HEAD itself or on whatever request next reuses the + keep-alive connection, so a HEAD pre-flight could report a perfectly + healthy endpoint as unreachable. + Args: session: aiohttp session to use for the check @@ -442,16 +450,16 @@ async def _check_reachability_with_session( True if endpoint is reachable with HTTP 200 """ try: - # Try HEAD first for efficiency - async with session.head( - self._endpoint_url, allow_redirects=False - ) as response: - if response.status == 200: - return True - # Fall back to GET if HEAD is not supported async with session.get(self._endpoint_url) as response: return response.status == 200 - except (TimeoutError, aiohttp.ClientError): + except (TimeoutError, aiohttp.ClientError) as exc: + # Both halves are redacted: the endpoint may embed userinfo, and + # some aiohttp errors (InvalidUrlClientError) render the requested + # URL verbatim in their repr, credentials included. + self.debug( + f"Reachability probe failed for {self._display_url}: " + f"{redact_url(repr(exc))}" + ) return False @background_task(immediate=True, interval=lambda self: self.collection_interval) diff --git a/tests/unit/common/mixins/test_base_metrics_collector_mixin.py b/tests/unit/common/mixins/test_base_metrics_collector_mixin.py index 7f5086174a..659557347d 100644 --- a/tests/unit/common/mixins/test_base_metrics_collector_mixin.py +++ b/tests/unit/common/mixins/test_base_metrics_collector_mixin.py @@ -206,3 +206,135 @@ async def test_application_json_response_raises_incompatible(self) -> None: # The body should never have been read — Content-Type rejection is # cheaper than full parse and avoids any work on a bad endpoint. mock_response.text.assert_not_awaited() + + +class BodyOnHeadServer: + """Serves a HEAD reply that illegally carries a body, as Triton does. + + Raw sockets because no compliant HTTP framework will emit a body on a HEAD + reply, which is the behavior under test. + """ + + GET_BODY = b"# HELP up Server is up\n# TYPE up gauge\nup 1\n" + HEAD_ERROR_BODY = b'{"error":"Method Not Allowed"}' + + def __init__(self) -> None: + self.methods_seen: list[str] = [] + self._server: asyncio.AbstractServer | None = None + + async def __aenter__(self) -> "BodyOnHeadServer": + self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0) + return self + + async def __aexit__(self, *exc_info: object) -> None: + if self._server is not None: + self._server.close() + await self._server.wait_closed() + + @property + def url(self) -> str: + assert self._server is not None + return f"http://127.0.0.1:{self._server.sockets[0].getsockname()[1]}/metrics" + + async def _handle( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + try: + while True: + request_line = await reader.readline() + if not request_line: + return + while True: + header = await reader.readline() + if header in (b"\r\n", b"\n", b""): + break + + method = request_line.split(b" ")[0].decode() + self.methods_seen.append(method) + + if method == "HEAD": + body = self.HEAD_ERROR_BODY + writer.write( + b"HTTP/1.1 405 Method Not Allowed\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + else: + writer.write( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: " + + str(len(self.GET_BODY)).encode() + + b"\r\n\r\n" + + self.GET_BODY + ) + await writer.drain() + except (ConnectionResetError, BrokenPipeError): + pass + finally: + writer.close() + + +class TestReachabilityAgainstMalformedHeadReplies: + """Reachability must hold for a server that mishandles HEAD. + + Drives the real aiohttp client against a real socket server. The + pre-existing reachability tests mock `_check_reachability_with_session` + outright, so they cannot catch a regression in the probe itself. + """ + + @pytest.mark.asyncio + async def test_body_on_head_reply_does_not_block_reachability(self) -> None: + """Triton's RFC-violating HEAD reply must not hide a GET-serving endpoint.""" + async with BodyOnHeadServer() as server: + collector = ConcreteCollector( + endpoint_url=server.url, + collection_interval=1.0, + reachability_timeout=5.0, + ) + + assert await collector.is_url_reachable() is True + assert server.methods_seen == ["GET"], ( + "the probe must reach the server with GET and never send HEAD" + ) + + @pytest.mark.asyncio + async def test_unreachable_endpoint_reports_false(self) -> None: + collector = ConcreteCollector( + endpoint_url="http://127.0.0.1:1/metrics", + collection_interval=1.0, + reachability_timeout=2.0, + ) + + assert await collector.is_url_reachable() is False + + +class TestReachabilityProbeRedactsCredentials: + """The reachability probe must never log endpoint credentials. + + Two separate leak paths: the endpoint URL itself may embed userinfo, and + some aiohttp errors render the requested URL verbatim in their repr -- + InvalidUrlClientError is an aiohttp.ClientError subclass, so it reaches the + probe's except clause with credentials attached. + """ + + SECRET = "sup3rs3cret" # noqa: S105 - test fixture, not a real credential + + @pytest.mark.asyncio + async def test_probe_failure_never_logs_credentials(self, caplog) -> None: + endpoint_url = f"http://admin:{self.SECRET}@/metrics" + collector = ConcreteCollector( + endpoint_url=endpoint_url, + collection_interval=1.0, + reachability_timeout=2.0, + ) + + with caplog.at_level("DEBUG"): + assert await collector.is_url_reachable() is False + + assert self.SECRET not in caplog.text, ( + "endpoint credentials leaked into the reachability probe log" + ) diff --git a/tests/unit/gpu_telemetry/test_telemetry_data_collector.py b/tests/unit/gpu_telemetry/test_telemetry_data_collector.py index 360a9d6ad6..d7f408f1e7 100644 --- a/tests/unit/gpu_telemetry/test_telemetry_data_collector.py +++ b/tests/unit/gpu_telemetry/test_telemetry_data_collector.py @@ -153,39 +153,12 @@ def test_empty_response_handling(self): class TestHttpCommunication: """Test HTTP communication with DCGM endpoints using aiohttp.""" - @pytest.mark.asyncio - async def test_endpoint_reachability_success(self): - """Test DCGM endpoint reachability check with successful HTTP response.""" - collector = DCGMTelemetryCollector("http://localhost:9401/metrics") - - with patch("aiohttp.ClientSession.head") as mock_head: - # Mock successful HEAD response with Prometheus content-type - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.headers = { - "Content-Type": "text/plain; version=0.0.4; charset=utf-8" - } - mock_response.raise_for_status = AsyncMock() - mock_head.return_value.__aenter__.return_value = mock_response - - # Initialize the collector to set up the session - await collector.initialize() - - # Test reachability - result = await collector.is_url_reachable() - assert result is True - - await collector.stop() - @pytest.mark.asyncio async def test_endpoint_reachability_failures(self, time_traveler): """Test DCGM endpoint reachability check with various failure scenarios.""" collector = DCGMTelemetryCollector("http://nonexistent:9401/metrics") - with ( - patch("aiohttp.ClientSession.head") as mock_head, - patch("aiohttp.ClientSession.get") as mock_get, - ): + with patch("aiohttp.ClientSession.get") as mock_get: # Mock different failure scenarios failure_scenarios = [ aiohttp.ClientError("Connection failed"), @@ -195,7 +168,6 @@ async def test_endpoint_reachability_failures(self, time_traveler): await collector.initialize() for exception in failure_scenarios: - mock_head.side_effect = exception mock_get.side_effect = exception result = await collector.is_url_reachable() assert result is False @@ -203,20 +175,14 @@ async def test_endpoint_reachability_failures(self, time_traveler): await collector.stop() @pytest.mark.asyncio - async def test_endpoint_reachability_head_fallback(self): - """Test that HEAD request falls back to GET when HEAD returns non-200.""" + async def test_endpoint_reachability_never_probes_with_head(self): + """Reachability must rely on GET alone, never on a HEAD pre-flight.""" collector = DCGMTelemetryCollector("http://localhost:9401/metrics") with ( patch("aiohttp.ClientSession.head") as mock_head, patch("aiohttp.ClientSession.get") as mock_get, ): - # Mock HEAD returning 405 (Method Not Allowed) - mock_head_response = AsyncMock() - mock_head_response.status = 405 - mock_head.return_value.__aenter__.return_value = mock_head_response - - # Mock GET returning 200 mock_get_response = AsyncMock() mock_get_response.status = 200 mock_get.return_value.__aenter__.return_value = mock_get_response @@ -225,9 +191,8 @@ async def test_endpoint_reachability_head_fallback(self): result = await collector.is_url_reachable() assert result is True - # Both HEAD and GET should have been called - mock_head.assert_called_once() mock_get.assert_called_once() + mock_head.assert_not_called() await collector.stop() @@ -249,7 +214,7 @@ async def test_endpoint_reachability_without_session(self): # Create mock session mock_session = MagicMock() - mock_session.head = MagicMock(return_value=mock_response_cm) + mock_session.get = MagicMock(return_value=mock_response_cm) # Make ClientSession() return an async context manager that yields the mock_session mock_context_manager = MagicMock() From 69246c722582c50c9ffb0333e48e471e2e9d1eeb Mon Sep 17 00:00:00 2001 From: Dang Hung Do <63043543+danghungdo@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:34:41 +0200 Subject: [PATCH 068/119] fix(dataset): replay bailian sessions as self-contained turns (#1393) Signed-off-by: Dang Hung Do Signed-off-by: Dang Hung Do <63043543+danghungdo@users.noreply.github.com> Signed-off-by: Dang Hung Do Co-authored-by: Dang Hung Do (cherry picked from commit 5054ce829b4e2f9deed8c78949c788f166996609) --- docs/benchmark-modes/trace-replay.md | 1 + docs/reference/conversation-context-mode.md | 3 +++ docs/tutorials/bailian-trace.md | 4 ++-- src/aiperf/dataset/loader/bailian_trace.py | 19 +++++++++++++++++++ src/aiperf/dataset/loader/models.py | 6 +++++- .../unit/dataset/loader/test_bailian_trace.py | 2 ++ 6 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/benchmark-modes/trace-replay.md b/docs/benchmark-modes/trace-replay.md index 1faad5275c..647184e22f 100644 --- a/docs/benchmark-modes/trace-replay.md +++ b/docs/benchmark-modes/trace-replay.md @@ -20,6 +20,7 @@ For other use cases: - **Custom prompts without timing**: See [Custom Prompt Benchmarking](../tutorials/custom-prompt-benchmarking.md) - **Precise timestamp control for any dataset**: See [Fixed Schedule](../tutorials/fixed-schedule.md) - **Multi-turn conversations from files**: See [Multi-Turn Conversations](../tutorials/multi-turn.md) +- **Alibaba Bailian traces**: See [Bailian Traces](../tutorials/bailian-trace.md). Unlike Mooncake sessions, each Bailian row already covers its full request context, so turns are replayed as self-contained prompts ([`message_array_with_responses`](../reference/conversation-context-mode.md)) rather than accumulating history ## Start a vLLM Server diff --git a/docs/reference/conversation-context-mode.md b/docs/reference/conversation-context-mode.md index 58bf901a1b..f8f5d6a73a 100644 --- a/docs/reference/conversation-context-mode.md +++ b/docs/reference/conversation-context-mode.md @@ -91,7 +91,10 @@ Request 3: sends Turn 3 as-is Each turn is sent exactly as it appears in the dataset. Default for: + - Mooncake traces with pre-built `messages` arrays +- Bailian traces: a follow-up row's `input_length` and `hash_ids` already cover the history the service retained plus the new text, so each row is sent as its own generated prompt + ### `message_array_without_responses` diff --git a/docs/tutorials/bailian-trace.md b/docs/tutorials/bailian-trace.md index 3de24afe18..8a1e3ce19a 100644 --- a/docs/tutorials/bailian-trace.md +++ b/docs/tutorials/bailian-trace.md @@ -47,10 +47,10 @@ Example entries: ```json {"chat_id": 159, "parent_chat_id": -1, "timestamp": 61.114, "input_length": 521, "output_length": 132, "type": "text", "turn": 1} -{"chat_id": 160, "parent_chat_id": 159, "timestamp": 62.5, "input_length": 400, "output_length": 80, "type": "text", "turn": 2} +{"chat_id": 160, "parent_chat_id": 159, "timestamp": 62.5, "input_length": 676, "output_length": 80, "type": "text", "turn": 2} ``` -Entries with the same root `chat_id` form a session and are replayed in `turn` order. +Entries with the same root `chat_id` form a session and are replayed in `turn` order. Each entry is a complete request as the service received it: a follow-up's `input_length` counts the history the service retained plus the new text, and its `hash_ids` cover that whole span. Sessions therefore run in [`message_array_with_responses`](../reference/conversation-context-mode.md) mode and every turn is sent as its own generated prompt, rather than accumulating earlier turns and live responses in front of it. --- diff --git a/src/aiperf/dataset/loader/bailian_trace.py b/src/aiperf/dataset/loader/bailian_trace.py index 536797f9d3..449d0a89db 100644 --- a/src/aiperf/dataset/loader/bailian_trace.py +++ b/src/aiperf/dataset/loader/bailian_trace.py @@ -7,6 +7,7 @@ from pydantic import ValidationError +from aiperf.common.enums import ConversationContextMode from aiperf.dataset.loader.base_trace_loader import BaseTraceDatasetLoader from aiperf.dataset.loader.models import BailianTrace @@ -24,6 +25,11 @@ class BailianTraceDatasetLoader(BaseTraceDatasetLoader[BailianTrace]): Timestamps are **seconds since request arrival** and are converted to **milliseconds** internally. + Every row is a self-contained request: a follow-up's `input_length` and + `hash_ids` describe the whole request the service received, history + included, so sessions replay in `MESSAGE_ARRAY_WITH_RESPONSES` mode and + each turn is sent as its own generated prompt. + The 16-token SipHash block size is declared in `plugins.yaml` metadata and applied automatically—no need to pass `--isl-block-size`. @@ -66,6 +72,19 @@ def _preprocess_trace(self, trace: BailianTrace) -> None: def _group_traces(self, items: list[BailianTrace]) -> dict[str, list[BailianTrace]]: return self._group_into_sessions(items) + def _infer_context_mode( + self, traces: list[BailianTrace] + ) -> ConversationContextMode | None: + """Bailian rows already carry their full context. + + A follow-up's `input_length` is the history the service retained plus + the new text, and its `hash_ids` cover that whole span. + Accumulating prior turns and live responses in front of it (the + `DELTAS_WITHOUT_RESPONSES` default) would send roughly twice the + recorded tokens at turn 2 and more at each later turn. + """ + return ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + def _group_into_sessions( self, items: list[BailianTrace] ) -> dict[str, list[BailianTrace]]: diff --git a/src/aiperf/dataset/loader/models.py b/src/aiperf/dataset/loader/models.py index 6d5d099d31..4399139e5b 100644 --- a/src/aiperf/dataset/loader/models.py +++ b/src/aiperf/dataset/loader/models.py @@ -375,7 +375,11 @@ class BailianTrace(AIPerfBaseModel): Examples: - Root request: ``{"chat_id": 159, "parent_chat_id": -1, "timestamp": 61.114, "input_length": 521, "output_length": 132, "type": "text", "turn": 1, "hash_ids": [1089, 1090, 1091]}`` - - Follow-up: ``{"chat_id": 160, "parent_chat_id": 159, "timestamp": 62.5, "input_length": 400, "output_length": 80, "type": "text", "turn": 2, "hash_ids": [1089, 1090]}`` + - Follow-up: ``{"chat_id": 160, "parent_chat_id": 159, "timestamp": 62.5, "input_length": 676, "output_length": 80, "type": "text", "turn": 2, "hash_ids": [1089, 1090, 1091, 1092]}`` + + The ``hash_ids`` lists above are abbreviated. A real row carries one id per + 16-token block, so 521 and 676 input tokens give 33 and 43 ids respectively, + and a follow-up's list typically starts with its parent's full blocks. Note: The ``type`` field in Bailian JSONL is the request type (text/search/image/file), diff --git a/tests/unit/dataset/loader/test_bailian_trace.py b/tests/unit/dataset/loader/test_bailian_trace.py index 60fcf670d5..44b9c1e7b8 100644 --- a/tests/unit/dataset/loader/test_bailian_trace.py +++ b/tests/unit/dataset/loader/test_bailian_trace.py @@ -7,6 +7,7 @@ from pydantic import ValidationError from pytest import param +from aiperf.common.enums import ConversationContextMode from aiperf.config.flags.cli_config import CLIConfig from aiperf.dataset.loader.bailian_trace import BailianTraceDatasetLoader from aiperf.dataset.loader.models import BailianTrace @@ -570,6 +571,7 @@ def test_multi_turn_conversation_ordering( assert len(conversations) == 1 conv = conversations[0] + assert conv.context_mode == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES assert len(conv.turns) == 3 assert conv.turns[0].timestamp == 1000.0 assert conv.turns[1].timestamp == 2000.0 From 40e99afe1f7c8f6c4631782ae6abb60e4c46e47b Mon Sep 17 00:00:00 2001 From: jxj Date: Sat, 3 Oct 2026 01:27:28 +0800 Subject: [PATCH 069/119] fix(dataset): load path-backed HuggingFace ASR audio (#1412) Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> (cherry picked from commit eed41d3e95e7e26abf76344d6343169fe6b50401) --- docs/tutorials/asr.md | 5 + src/aiperf/dataset/loader/hf_asr.py | 26 +++- .../unit/dataset/loader/test_hf_asr_loader.py | 121 ++++++++++++++++++ 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/asr.md b/docs/tutorials/asr.md index 2fad14c38e..93a1ce1184 100644 --- a/docs/tutorials/asr.md +++ b/docs/tutorials/asr.md @@ -20,6 +20,11 @@ Five ASR datasets are built in: | AMI | `ami` | No | Meeting recordings with individual headset microphone audio | | SPGISpeech | `spgispeech` | Yes | Financial earnings call recordings (Kensho) | +Audio rows can contain embedded audio bytes or a path to a local/cache file. +Both forms are decoded to WAV for requests and checked against the duration limit. +Path-backed files with unreadable duration metadata are skipped without loading +their full contents. + Clips longer than 30 seconds are automatically skipped to stay within typical ASR model context limits. diff --git a/src/aiperf/dataset/loader/hf_asr.py b/src/aiperf/dataset/loader/hf_asr.py index 83f02b54c5..1dd64e97ab 100644 --- a/src/aiperf/dataset/loader/hf_asr.py +++ b/src/aiperf/dataset/loader/hf_asr.py @@ -3,10 +3,13 @@ from __future__ import annotations +import asyncio import base64 import io from typing import TYPE_CHECKING, Any, TypedDict +import aiofiles + from aiperf.common.models import Audio, Conversation, Text, Turn from aiperf.dataset.loader.base_hf_dataset import BaseHFDatasetLoader @@ -79,14 +82,14 @@ def _audio_from_bytes(self, audio_value: _HFAudioBytesRow) -> list[Audio]: return [] def _duration_seconds(self, audio_value: _HFAudioBytesRow) -> float | None: - """Estimate clip duration in seconds from raw bytes via soundfile.""" raw_bytes = audio_value.get("bytes") - if not raw_bytes: + source = io.BytesIO(raw_bytes) if raw_bytes else audio_value.get("path") + if not source: return None try: import soundfile as sf - info = sf.info(io.BytesIO(raw_bytes)) + info = sf.info(source) return info.duration except (ImportError, OSError, ValueError, RuntimeError) as e: self.debug( @@ -121,11 +124,24 @@ async def convert_to_conversations( skipped += 1 continue - duration = self._duration_seconds(audio_value) - if duration is not None and duration > _MAX_DURATION_SECONDS: + duration = await asyncio.to_thread(self._duration_seconds, audio_value) + if ( + duration is None + and not audio_value.get("bytes") + and audio_value.get("path") + ) or (duration is not None and duration > _MAX_DURATION_SECONDS): skipped += 1 continue + if not audio_value.get("bytes") and audio_value.get("path"): + try: + async with aiofiles.open(audio_value["path"], "rb") as audio_file: + audio_value = {**audio_value, "bytes": await audio_file.read()} + except (OSError, ValueError) as exc: + self.debug(lambda exc=exc: f"Failed to read audio file: {exc}") + skipped += 1 + continue + audios = self._audio_from_bytes(audio_value) if not audios: skipped += 1 diff --git a/tests/unit/dataset/loader/test_hf_asr_loader.py b/tests/unit/dataset/loader/test_hf_asr_loader.py index 65bb96e9b8..22e36ac559 100644 --- a/tests/unit/dataset/loader/test_hf_asr_loader.py +++ b/tests/unit/dataset/loader/test_hf_asr_loader.py @@ -3,10 +3,15 @@ import base64 import io +from pathlib import Path +from unittest.mock import patch +import aiofiles import numpy as np import pytest import soundfile as sf +from datasets import Audio as HFAudio +from datasets import Dataset from aiperf.common.models import Conversation from aiperf.config.flags.cli_config import CLIConfig @@ -171,3 +176,119 @@ async def test_each_row_becomes_single_turn(self, loader): data = {"dataset": [_make_audio_row(1.0)]} conversations = await loader.convert_to_conversations(data) assert len(conversations[0].turns) == 1 + + +@pytest.mark.asyncio +async def test_path_backed_hf_audio_is_loaded( + loader: HFASRDatasetLoader, tmp_path: Path +) -> None: + """HF keeps cached/local audio as paths when decoding is disabled.""" + audio_path = tmp_path / "clip.wav" + audio_path.write_bytes(_make_audio_bytes(0.5)) + dataset = Dataset.from_dict({"audio": [str(audio_path)]}).cast_column( + "audio", HFAudio(decode=False) + ) + assert dataset[0]["audio"]["bytes"] is None + conversations = await loader.convert_to_conversations({"dataset": dataset}) + assert len(conversations) == 1 + turn = conversations[0].turns[0] + assert turn.audio_duration_seconds == pytest.approx(0.5) + encoded = turn.audios[0].contents[0].split(",", 1)[1] + samples, rate = sf.read(io.BytesIO(base64.b64decode(encoded))) + assert rate == 16000 + assert len(samples) == 8000 + + +@pytest.mark.asyncio +async def test_corrupt_path_backed_audio_is_skipped_without_reading( + loader: HFASRDatasetLoader, tmp_path: Path +) -> None: + corrupt_path = tmp_path / "corrupt.wav" + corrupt_path.write_bytes(b"\x00" * (9 * 1024 * 1024)) + valid_path = tmp_path / "valid.wav" + valid_path.write_bytes(_make_audio_bytes(0.5)) + dataset = Dataset.from_dict( + {"audio": [str(corrupt_path), str(valid_path)]} + ).cast_column("audio", HFAudio(decode=False)) + assert dataset[0]["audio"]["bytes"] is None + + with patch( + "aiperf.dataset.loader.hf_asr.aiofiles.open", wraps=aiofiles.open + ) as open_audio: + conversations = await loader.convert_to_conversations({"dataset": dataset}) + + assert [call.args[0] for call in open_audio.call_args_list] == [str(valid_path)] + assert len(conversations) == 1 + turn = conversations[0].turns[0] + assert turn.audio_duration_seconds == pytest.approx(0.5) + encoded = turn.audios[0].contents[0].split(",", 1)[1] + samples, rate = sf.read(io.BytesIO(base64.b64decode(encoded))) + assert rate == 16000 + assert len(samples) == 8000 + + +@pytest.mark.asyncio +async def test_embedded_audio_bytes_are_decoded_when_duration_is_unknown( + loader: HFASRDatasetLoader, +) -> None: + row = _make_audio_row(0.5) + with ( + patch.object(loader, "_duration_seconds", return_value=None), + patch("aiperf.dataset.loader.hf_asr.aiofiles.open") as open_audio, + ): + conversations = await loader.convert_to_conversations({"dataset": [row]}) + + open_audio.assert_not_called() + assert len(conversations) == 1 + turn = conversations[0].turns[0] + assert turn.audio_duration_seconds is None + encoded = turn.audios[0].contents[0].split(",", 1)[1] + samples, rate = sf.read(io.BytesIO(base64.b64decode(encoded))) + assert rate == 16000 + assert len(samples) == 8000 + + +@pytest.mark.asyncio +async def test_path_backed_long_and_missing_audio_are_skipped( + loader: HFASRDatasetLoader, tmp_path: Path +) -> None: + long_path = tmp_path / "long.wav" + long_path.write_bytes(_make_audio_bytes(_MAX_DURATION_SECONDS + 1)) + rows = [ + {"audio": {"bytes": None, "path": str(long_path)}}, + {"audio": {"bytes": None, "path": str(tmp_path / "missing.wav")}}, + _make_audio_row(1.0), + ] + with patch( + "aiperf.dataset.loader.hf_asr.aiofiles.open", wraps=aiofiles.open + ) as open_audio: + conversations = await loader.convert_to_conversations({"dataset": rows}) + assert str(long_path) not in [call.args[0] for call in open_audio.call_args_list] + assert len(conversations) == 1 + assert conversations[0].turns[0].audio_duration_seconds == pytest.approx(1.0) + + +@pytest.mark.asyncio +async def test_invalid_audio_path_does_not_abort_conversion( + loader: HFASRDatasetLoader, +) -> None: + rows = [ + {"audio": {"bytes": None, "path": "invalid\x00.wav"}}, + _make_audio_row(1.0), + ] + conversations = await loader.convert_to_conversations({"dataset": rows}) + assert len(conversations) == 1 + assert conversations[0].turns[0].audio_duration_seconds == pytest.approx(1.0) + + +@pytest.mark.asyncio +async def test_embedded_audio_bytes_take_precedence_over_path( + loader: HFASRDatasetLoader, tmp_path: Path +) -> None: + """A metadata path must not replace an embedded recording.""" + audio_path = tmp_path / "different.wav" + audio_path.write_bytes(_make_audio_bytes(2.0)) + row = {"audio": {"bytes": _make_audio_bytes(0.5), "path": str(audio_path)}} + conversations = await loader.convert_to_conversations({"dataset": [row]}) + assert len(conversations) == 1 + assert conversations[0].turns[0].audio_duration_seconds == pytest.approx(0.5) From 64b5a6aad84420b9fa09cd6aa79510f405803927 Mon Sep 17 00:00:00 2001 From: jxj Date: Sat, 3 Oct 2026 05:10:33 +0800 Subject: [PATCH 070/119] docs: add profile to reproducibility examples (#1498) Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> (cherry picked from commit 5b3d008510ea7dfec98632c7eedcb26b75f2c23f) --- docs/reproducibility.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reproducibility.md b/docs/reproducibility.md index b1f781877b..4b1771c0c8 100644 --- a/docs/reproducibility.md +++ b/docs/reproducibility.md @@ -88,10 +88,10 @@ Reproducibility is enforced by automated tests on every commit: ```bash # Reproducible dataset -aiperf --random-seed 42 [options...] +aiperf profile --random-seed 42 [options...] # Non-reproducible (default) -aiperf [options...] +aiperf profile [options...] ``` Same seed + same config = identical dataset content. Performance metrics always vary. @@ -100,23 +100,23 @@ Same seed + same config = identical dataset content. Performance metrics always **Debugging:** Reproduce exact prompts across runs to isolate prompt-related vs. network/timing issues ```bash -aiperf --random-seed 42 [...] --profile-export-file run1.json -aiperf --random-seed 42 [...] --profile-export-file run2.json +aiperf profile --random-seed 42 [...] --profile-export-file run1.json +aiperf profile --random-seed 42 [...] --profile-export-file run2.json # Prompts identical; metrics may vary ``` **Performance Testing:** Compare metrics with same dataset ```bash -aiperf --random-seed 42 [...] --profile-export-file baseline.json +aiperf profile --random-seed 42 [...] --profile-export-file baseline.json # After optimization... -aiperf --random-seed 42 [...] --profile-export-file optimized.json +aiperf profile --random-seed 42 [...] --profile-export-file optimized.json # Use statistical analysis (median, p95, p99) ``` **Stress Testing:** Vary patterns by omitting seed ```bash for i in {1..10}; do - aiperf [...] --profile-export-file run_$i.json + aiperf profile [...] --profile-export-file run_$i.json done ``` From 3ec534146042c45c3206ea54222fa5260d190262 Mon Sep 17 00:00:00 2001 From: jxj Date: Sat, 3 Oct 2026 12:10:32 +0800 Subject: [PATCH 071/119] fix(transport): serialize multipart lists as repeated fields (#1411) Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> Co-authored-by: Anthony Casagrande (cherry picked from commit fd38d2454e826dffa539c49dc64eab2accf911c6) --- docs/tutorials/asr.md | 13 ++++++++++++ src/aiperf/transports/aiohttp_transport.py | 9 +++++++-- tests/unit/transports/test_form_data.py | 23 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/asr.md b/docs/tutorials/asr.md index 93a1ce1184..3ac6166611 100644 --- a/docs/tutorials/asr.md +++ b/docs/tutorials/asr.md @@ -186,3 +186,16 @@ aiperf profile \ --request-count 10 \ --concurrency 4 ``` + +## Multipart Transcription Parameters + +For servers exposing `/v1/audio/transcriptions`, select +`--endpoint-type audio_transcription`. Array-valued multipart parameters are +sent as repeated fields with the exact supplied name. For example, to request +word and segment timestamps from a Whisper-compatible server, add: + +```bash +--extra-inputs '{"response_format":"verbose_json","timestamp_granularities[]":["word","segment"]}' +``` + +Use the field name expected by your server, including `[]` when required. diff --git a/src/aiperf/transports/aiohttp_transport.py b/src/aiperf/transports/aiohttp_transport.py index 9ccd4b48e7..3413c67532 100644 --- a/src/aiperf/transports/aiohttp_transport.py +++ b/src/aiperf/transports/aiohttp_transport.py @@ -467,6 +467,9 @@ def _build_form_data(payload: dict[str, Any]) -> aiohttp.FormData: "content_type": }``. Keeping bytes base64-encoded in the payload lets it stay JSON-serialisable upstream; decoding happens here. + List values become repeated fields using the supplied key verbatim + (e.g., ``timestamp_granularities[]`` for transcription timestamps). + ``default_to_multipart=True`` forces multipart/form-data even when the payload happens to be text-only (e.g., image_edit with a `url` field instead of an inline image), so the wire format always matches the @@ -491,8 +494,10 @@ def _build_form_data(payload: dict[str, Any]) -> aiohttp.FormData: or "application/octet-stream", ) continue - str_value = str(value).lower() if isinstance(value, bool) else str(value) - form_data.add_field(key, str_value) + values = value if isinstance(value, list) else [value] + for item in values: + str_value = str(item).lower() if isinstance(item, bool) else str(item) + form_data.add_field(key, str_value) return form_data async def _submit_video_job( diff --git a/tests/unit/transports/test_form_data.py b/tests/unit/transports/test_form_data.py index ab1e6b84de..447c2113af 100644 --- a/tests/unit/transports/test_form_data.py +++ b/tests/unit/transports/test_form_data.py @@ -6,6 +6,7 @@ import aiohttp import pytest +from pytest import param from aiperf.transports.aiohttp_transport import AioHttpTransport @@ -103,3 +104,25 @@ def test_text_only_payload_still_encodes_as_multipart(self): assert content_type.startswith("multipart/form-data"), ( f"text-only FormData must be multipart, got {content_type!r}" ) + + @pytest.mark.parametrize( + "values, expected", + [ + param(["word", "segment"], ["word", "segment"], id="multiple"), + param([], [], id="empty"), + param([True, False, 2], ["true", "false", "2"], id="scalar-types"), + ], + ) # fmt: skip + def test_list_fields_emit_repeated_parts( + self, values: list[str | bool | int], expected: list[str] + ) -> None: + form = AioHttpTransport._build_form_data( + {"timestamp_granularities[]": values, "model": "whisper-1"} + ) + actual = [ + value + for options, _, value in form._fields + if options["name"] == "timestamp_granularities[]" + ] + assert actual == expected + assert form._fields[-1][2] == "whisper-1" From a0ef8a68eee59f9ad1924e584e43045959162ebd Mon Sep 17 00:00:00 2001 From: jxj Date: Sat, 3 Oct 2026 23:53:14 +0800 Subject: [PATCH 072/119] fix: preserve signal amplitude in 24-bit synthetic audio (#1395) Signed-off-by: xinjun.jiang Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> (cherry picked from commit f35146396f34d10789e74948a3b6759fec98025f) --- docs/tutorials/synthetic-dataset.md | 4 +-- docs/tutorials/synthetic-video.md | 4 ++- src/aiperf/dataset/generator/audio.py | 7 ++--- src/aiperf/dataset/generator/video.py | 7 ++--- .../dataset/generator/test_audio_generator.py | 30 +++++++++++++++++++ .../dataset/generator/test_video_generator.py | 20 +++++++++++++ 6 files changed, 61 insertions(+), 11 deletions(-) diff --git a/docs/tutorials/synthetic-dataset.md b/docs/tutorials/synthetic-dataset.md index 86410db5aa..121412c09c 100644 --- a/docs/tutorials/synthetic-dataset.md +++ b/docs/tutorials/synthetic-dataset.md @@ -136,8 +136,8 @@ Audio files are generated as synthetic Gaussian noise: 1. **Parameter Selection**: Random selection of sample rate and bit depth from configured lists 2. **Duration Sampling**: Duration follows normal distribution (with rejection sampling for ≥0.01s) 3. **Noise Generation**: Gaussian noise generated as a NumPy array -4. **Scaling**: Clipped to [-1, 1] and scaled to the target bit depth range -5. **Encoding**: Written as WAV or MP3 using soundfile library +4. **Scaling**: Clipped to [-1, 1] and scaled to the integer sample type's full range to preserve signal amplitude +5. **Encoding**: Written using soundfile as WAV at the requested PCM bit depth, or as lossy MP3 (which ignores the PCM bit depth) 6. **Base64 Encoding**: Encoded as `,` string **Audio Characteristics**: diff --git a/docs/tutorials/synthetic-video.md b/docs/tutorials/synthetic-video.md index b2f06bfe04..35c2cf07ec 100644 --- a/docs/tutorials/synthetic-video.md +++ b/docs/tutorials/synthetic-video.md @@ -295,6 +295,8 @@ This generates videos with a mono audio track using an auto-selected codec (libv | `--video-audio-codec` | `string` | auto | Audio codec (`libvorbis`, `libopus`, `aac`; `aac` is not in the AIPerf container) | | `--video-audio-depth` | `int` | `16` | Bit depth per sample (8, 16, 24, or 32) | +Audio samples are clipped to [-1, 1] and scaled to the integer sample type's full range to preserve signal amplitude, including at 24-bit depth. `--video-audio-depth` controls the intermediate WAV's PCM bit depth; the selected audio codec then encodes that WAV into the video track. + #### Audio Codec Selection When `--video-audio-codec` is not specified, the codec is automatically selected based on the video format: @@ -343,7 +345,7 @@ aiperf profile \ #### How It Works 1. A Gaussian noise audio signal is generated matching the video duration -2. The audio is encoded as 16-bit PCM WAV +2. The audio is encoded as PCM WAV at the configured `--video-audio-depth` (16-bit by default) 3. FFmpeg muxes the video and audio streams together using `-shortest` to ensure duration alignment 4. The audio codec converts the WAV data to the target format (Vorbis or Opus) 5. The resulting video+audio file is base64-encoded for API requests diff --git a/src/aiperf/dataset/generator/audio.py b/src/aiperf/dataset/generator/audio.py index f96ffc7d6d..48917f3d93 100644 --- a/src/aiperf/dataset/generator/audio.py +++ b/src/aiperf/dataset/generator/audio.py @@ -168,11 +168,10 @@ def generate(self, *args, **kwargs) -> str: self._data_rng, num_samples, self.config.channels ) - # Scale to the appropriate bit depth range - # Note: For 8-bit, we use int16 input and let soundfile convert to PCM_U8 + # soundfile scales integer samples by their dtype, including int32 + # input for PCM_24 and int16 input for PCM_U8. numpy_type, _ = SUPPORTED_BIT_DEPTHS[bit_depth] - scale_depth = 16 if bit_depth == 8 else bit_depth - max_val = 2 ** (scale_depth - 1) - 1 + max_val = np.iinfo(numpy_type).max audio_data = (signal * max_val).astype(numpy_type) # Write audio using soundfile diff --git a/src/aiperf/dataset/generator/video.py b/src/aiperf/dataset/generator/video.py index ec512f1e05..98594e484d 100644 --- a/src/aiperf/dataset/generator/video.py +++ b/src/aiperf/dataset/generator/video.py @@ -321,12 +321,11 @@ def _generate_audio_data(self) -> bytes: self._audio_rng, num_samples, self.config.audio.channels ) - # Scale to the appropriate bit depth range - # Note: For 8-bit, we use int16 input and let soundfile convert to PCM_U8 + # soundfile scales integer samples by their dtype, including int32 + # input for PCM_24 and int16 input for PCM_U8. bit_depth = self.config.audio.depth numpy_type, subtype = SUPPORTED_BIT_DEPTHS[bit_depth] - scale_depth = 16 if bit_depth == 8 else bit_depth - max_val = 2 ** (scale_depth - 1) - 1 + max_val = np.iinfo(numpy_type).max audio_data = (signal * max_val).astype(numpy_type) output_buffer = io.BytesIO() diff --git a/tests/unit/dataset/generator/test_audio_generator.py b/tests/unit/dataset/generator/test_audio_generator.py index c43bdab7b1..12e08fd3a7 100644 --- a/tests/unit/dataset/generator/test_audio_generator.py +++ b/tests/unit/dataset/generator/test_audio_generator.py @@ -8,6 +8,7 @@ import numpy as np import pytest import soundfile as sf +from pytest import param from aiperf.common import random_generator as rng from aiperf.common.enums import AudioFormat @@ -262,6 +263,35 @@ def test_wav_bit_depth_produces_valid_audio(self, bit_depth): assert len(audio_data) > 0 assert sample_rate == 16000 + @pytest.mark.parametrize( + "audio_format", + [param(AudioFormat.WAV, id="wav"), param(AudioFormat.MP3, id="mp3")], + ) # fmt: skip + @pytest.mark.parametrize("bit_depth", [8, 16, 24, 32]) + def test_bit_depth_preserves_signal_amplitude( + self, audio_format: AudioFormat, bit_depth: int + ) -> None: + """Each bit depth preserves signal level for WAV and MP3 output.""" + reference, _ = decode_audio( + AudioGenerator( + make_config(mean=0.1, stddev=0, depths=[32], audio_format=audio_format) + ).generate() + ) + actual, _ = decode_audio( + AudioGenerator( + make_config( + mean=0.1, stddev=0, depths=[bit_depth], audio_format=audio_format + ) + ).generate() + ) + + # Each generator derives the same seeded waveform. Changing precision + # may quantize it, but must not change its overall signal level. + actual_rms = np.sqrt(np.mean(actual**2)) + reference_rms = np.sqrt(np.mean(reference**2)) + assert actual_rms == pytest.approx(reference_rms, rel=1e-3) + assert actual_rms == pytest.approx(0.3, rel=0.1) + @pytest.mark.parametrize("bit_depth", [8, 16, 24, 32]) def test_mp3_ignores_bit_depth_uses_lossy_encoding(self, bit_depth): """MP3 format works with all bit depths (lossy encoding ignores PCM subtype).""" diff --git a/tests/unit/dataset/generator/test_video_generator.py b/tests/unit/dataset/generator/test_video_generator.py index 862abe1ddd..0e108267e0 100644 --- a/tests/unit/dataset/generator/test_video_generator.py +++ b/tests/unit/dataset/generator/test_video_generator.py @@ -7,6 +7,7 @@ from pathlib import Path from unittest.mock import Mock, patch +import numpy as np import pytest import soundfile as sf from PIL import Image @@ -621,6 +622,25 @@ def test_video_audio_bit_depth_produces_valid_audio(self, bit_depth): assert len(data) > 0 assert sr == 44100 # default sample rate + @pytest.mark.parametrize("bit_depth", [8, 16, 24, 32]) + def test_video_audio_bit_depth_preserves_signal_amplitude( + self, base_config: VideoConfig, bit_depth: int + ) -> None: + """Each video audio depth preserves amplitude relative to int32.""" + base_config.audio = VideoAudioConfig(channels=1, depth=32) + reference, _ = sf.read( + io.BytesIO(VideoGenerator(base_config)._generate_audio_data()) + ) + base_config.audio.depth = bit_depth + actual, _ = sf.read( + io.BytesIO(VideoGenerator(base_config)._generate_audio_data()) + ) + + actual_rms = np.sqrt(np.mean(actual**2)) + reference_rms = np.sqrt(np.mean(reference**2)) + assert actual_rms == pytest.approx(reference_rms, rel=1e-3) + assert actual_rms == pytest.approx(0.3, rel=0.1) + def test_video_audio_8bit_is_unsigned(self): """8-bit video audio values are in unsigned range (0-255 centered at 128). From 74876e1c7910c783d980c85fc57016aa6728a840 Mon Sep 17 00:00:00 2001 From: danisereb Date: Tue, 6 Oct 2026 01:06:54 +0300 Subject: [PATCH 073/119] fix(dataset): honor a weka trace's declared block_size over the plugin default (#1508) Signed-off-by: Daniel Serebrenik (cherry picked from commit f3a76e5a0bd1dd3888052ddcc70be61e53e5cb56) --- src/aiperf/dataset/loader/weka_trace.py | 5 +- tests/unit/dataset/loader/conftest.py | 6 +- .../test_semianalysis_cc_traces_weka.py | 9 +- .../loader/test_weka_trace_block_size.py | 107 ++++++++++++++---- 4 files changed, 97 insertions(+), 30 deletions(-) diff --git a/src/aiperf/dataset/loader/weka_trace.py b/src/aiperf/dataset/loader/weka_trace.py index e943a8fce5..6b8f7718c7 100644 --- a/src/aiperf/dataset/loader/weka_trace.py +++ b/src/aiperf/dataset/loader/weka_trace.py @@ -803,13 +803,12 @@ def __init__( ) # Only synthetic datasets carry prompts.block_size; FileDataset (the # weka home) has none, so the user override is None and the precedence - # collapses to trace-declared > 64. + # collapses to trace-declared > 64. ``default_block_size`` (plugin + # metadata) is not an override: every weka trace declares its own. prompts = getattr(dataset, "prompts", None) user_block_size = getattr(prompts, "block_size", None) if prompts else None if user_block_size is not None: self._user_block_size_override: int | None = user_block_size - elif default_block_size is not None: - self._user_block_size_override = default_block_size else: self._user_block_size_override = None # ``self._block_size`` is preserved for callbacks (``_decode_block_tokens`` diff --git a/tests/unit/dataset/loader/conftest.py b/tests/unit/dataset/loader/conftest.py index 993709bd7f..c7b9fc5408 100644 --- a/tests/unit/dataset/loader/conftest.py +++ b/tests/unit/dataset/loader/conftest.py @@ -88,9 +88,9 @@ def make_weka_run( required -- a MagicMock no longer satisfies the loader's reads. ``block_size`` has no FileDataset home in v2 (only synthetic prompts carry - it), so it is forwarded to the loader via ``default_block_size`` by callers, - not through config. It is accepted here for signature compatibility but - ignored at the config layer. + it); weka traces declare their own ``block_size``, which the loader honors + over the plugin ``default_block_size``. It is accepted here for signature + compatibility but ignored at the config layer. """ from aiperf.config import BenchmarkConfig, BenchmarkRun diff --git a/tests/unit/dataset/loader/test_semianalysis_cc_traces_weka.py b/tests/unit/dataset/loader/test_semianalysis_cc_traces_weka.py index 7de9fa12df..0fbdadc4da 100644 --- a/tests/unit/dataset/loader/test_semianalysis_cc_traces_weka.py +++ b/tests/unit/dataset/loader/test_semianalysis_cc_traces_weka.py @@ -89,14 +89,17 @@ async def test_propagates_prompt_generator_to_inner_loader( ) assert loader._weka.prompt_generator is pg - async def test_propagates_default_block_size(self, user_config) -> None: + async def test_default_block_size_is_not_a_block_size_override( + self, user_config + ) -> None: + """Each row declares its own block_size; the plugin default must not override it.""" loader = SemiAnalysisCCTracesWekaLoader( run=user_config, hf_dataset_name=_NO_SUBAGENTS_HF_DATASET_NAME, prompt_generator=MagicMock(), - default_block_size=64, + default_block_size=128, ) - assert loader._weka._block_size == 64 + assert loader._weka._user_block_size_override is None async def test_streaming_forced_off_even_when_caller_passes_true( self, user_config diff --git a/tests/unit/dataset/loader/test_weka_trace_block_size.py b/tests/unit/dataset/loader/test_weka_trace_block_size.py index 31e656274a..698781466e 100644 --- a/tests/unit/dataset/loader/test_weka_trace_block_size.py +++ b/tests/unit/dataset/loader/test_weka_trace_block_size.py @@ -2,11 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 """Per-trace block_size resolution in WekaTraceLoader.""" +from types import SimpleNamespace from unittest.mock import MagicMock import pytest +from pytest import param from aiperf.dataset.loader.weka_trace import WekaTraceLoader +from aiperf.plugin import plugins +from aiperf.plugin.enums import CustomDatasetType from tests.unit.dataset.loader._shared_helpers import _write_trace @@ -17,8 +21,8 @@ def _mk_user_config(): def _make_loader(filename, uc, monkeypatch, *, block_size=None): - # v2 FileDataset has no block_size field, so the user block-size override is - # injected via the loader's ``default_block_size`` ctor kwarg, not config. + # ``block_size`` is passed as ``default_block_size``, as CustomDatasetComposer + # does with the plugin metadata value; it must not override the trace's own. loader = WekaTraceLoader( filename=str(filename), run=uc, default_block_size=block_size ) @@ -84,35 +88,96 @@ def test_trace_block_size_honored_when_user_unset(tmp_path, monkeypatch): assert any(c.session_id == "t_bs128" for c in convs) -def test_user_block_size_overrides_trace_block_size(tmp_path, monkeypatch): - """User-config block_size overrides trace.block_size: trace declares 64, user wants 32, loader must use 32.""" - # in_tokens=128, bs=32 -> 4 hash_ids needed. The trace declares bs=64 but - # provides only 4 hash_ids; bs=64 would need 2. Either resolution works at - # turn-0 (since 4 >= 2 and 4 >= 4). What we're really checking is which - # one the loader picks. We'll check via a side-channel: the ConversationReconstructor - # constructor's recorded block_size. - trace = _trace_with_bs("t_bs_override", bs=64, in_tokens=128, hash_ids=[1, 2, 3, 4]) - path = _write_trace(tmp_path, trace) - loader = _make_loader(path, _mk_user_config(), monkeypatch, block_size=32) - # Capture every ConversationReconstructor block_size argument the loader uses - # during this convert call. +def _capture_reconstructor_block_sizes(monkeypatch) -> list[int]: + """Record the block_size of every ConversationReconstructor the loader builds.""" from aiperf.dataset.loader import weka_synth_buf as wsb - captured_block_sizes: list[int] = [] + captured: list[int] = [] orig = wsb.ConversationReconstructor.__init__ def spy(self, *args, **kw): - captured_block_sizes.append(kw.get("block_size", args[0] if args else None)) + captured.append(kw.get("block_size", args[0] if args else None)) return orig(self, *args, **kw) monkeypatch.setattr(wsb.ConversationReconstructor, "__init__", spy) + return captured + + +@pytest.mark.parametrize( + "trace_bs", + [ + param(16, id="smaller-than-plugin-default"), + param(128, id="larger-than-plugin-default"), + ], +) # fmt: skip +def test_trace_block_size_wins_over_plugin_default_block_size( + tmp_path, monkeypatch, trace_bs +): + """A trace's declared block_size is replayed as-is although the composer always passes the plugin's default_block_size (64). + + Regression: the plugin default was treated as a user override, so a 128-token + trace was rebuilt from 64-token blocks -- its hash_ids covered only half of each + prompt and the rest was synthesized fresh every turn, halving prefix-cache reuse. + """ + plugin_default = plugins.get_dataset_loader_metadata( + CustomDatasetType.WEKA_TRACE + ).default_block_size + assert plugin_default == 64, "precondition: plugin default differs from the trace" + trace = _trace_with_bs( + f"t_bs{trace_bs}_plugin_default", + bs=trace_bs, + in_tokens=4 * trace_bs, + hash_ids=[100, 200, 300, 400], + ) + path = _write_trace(tmp_path, trace) + # Build the loader the way CustomDatasetComposer does for weka_trace. + loader = _make_loader( + path, _mk_user_config(), monkeypatch, block_size=plugin_default + ) + captured = _capture_reconstructor_block_sizes(monkeypatch) loader.convert_to_conversations(loader.load_dataset()) - assert captured_block_sizes, ( - "no ConversationReconstructor built - test setup broken" + assert captured, "no ConversationReconstructor built - test setup broken" + assert all(bs == trace_bs for bs in captured), ( + f"trace-declared block_size={trace_bs} must win over the plugin default " + f"{plugin_default}. Got: {captured}" ) - assert all(bs == 32 for bs in captured_block_sizes), ( - f"user-config block_size=32 should win over trace.block_size=64. " - f"Got: {captured_block_sizes}" + + +class _DatasetWithPromptsBlockSize: + """The run's real default dataset plus ``prompts.block_size`` -- the loader's user override.""" + + def __init__(self, dataset, block_size): + self._dataset = dataset + self.prompts = SimpleNamespace(block_size=block_size) + + def __getattr__(self, name): + return getattr(self._dataset, name) + + +def test_user_block_size_overrides_trace_block_size(tmp_path, monkeypatch): + """A configured prompts.block_size overrides trace.block_size: trace declares 64, user sets 32, loader must use 32.""" + # in_tokens=128, bs=32 -> 4 hash_ids needed. The trace declares bs=64 but + # provides only 4 hash_ids; bs=64 would need 2. Either resolution works at + # turn-0 (since 4 >= 2 and 4 >= 4). What we're really checking is which + # one the loader picks, via the ConversationReconstructor's block_size. + trace = _trace_with_bs("t_bs_override", bs=64, in_tokens=128, hash_ids=[1, 2, 3, 4]) + path = _write_trace(tmp_path, trace) + uc = _mk_user_config() + dataset = uc.cfg.get_default_dataset() + monkeypatch.setattr( + type(uc.cfg), + "get_default_dataset", + lambda self: _DatasetWithPromptsBlockSize(dataset, 32), + ) + plugin_default = plugins.get_dataset_loader_metadata( + CustomDatasetType.WEKA_TRACE + ).default_block_size + loader = _make_loader(path, uc, monkeypatch, block_size=plugin_default) + captured = _capture_reconstructor_block_sizes(monkeypatch) + loader.convert_to_conversations(loader.load_dataset()) + assert captured, "no ConversationReconstructor built - test setup broken" + assert all(bs == 32 for bs in captured), ( + f"user-config block_size=32 should win over trace.block_size=64. Got: {captured}" ) From e92d63ca15c766b8a826cfb209b2063560115495 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:18:10 -0700 Subject: [PATCH 074/119] ci: add a single Unit Tests Gate check for the merge ruleset (#1515) Signed-off-by: Elias Bermudez (cherry picked from commit 4000c744f7b5cab3704919c20419756d54934720) --- .github/workflows/run-unit-tests.yml | 20 ++++++++++++++++++++ CONTRIBUTING.md | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 8fd35f8c76..ffd2fa7f32 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -183,3 +183,23 @@ jobs: run: | uv run pytest -n auto --dist=worksteal -m slow \ tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py + + # The merge ruleset requires this single check rather than each matrix leg, + # whose names change whenever a leg is added or renamed. always() keeps it + # from being skipped, because GitHub treats a skipped required check as passed. + unit-tests-gate: + name: Unit Tests Gate + needs: [build, botorch-regression] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Require every unit test job to pass + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + echo "Job results: ${RESULTS}" + for result in ${RESULTS}; do + [ "${result}" = "success" ] || exit 1 + done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8986d9e0c0..e86f71dd1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -187,6 +187,14 @@ Contributions intended to add significant new functionality must follow a more c Feature branches use `/feature-name` format, forked from `main`. +## Required Checks + +A PR can merge into `main` only after the `pre-commit` and `Unit Tests Gate` checks pass. `Unit Tests Gate` is the last job in `.github/workflows/run-unit-tests.yml` and fails unless every unit test job passed. + +Unit tests run on NVIDIA runners, so copy-pr-bot copies each PR to a `pull-request/` branch to run them. A PR is copied automatically only when its author is trusted and every commit is cryptographically signed (GPG or SSH; this is separate from the DCO sign-off below). Fork PRs, PRs with unsigned commits, and bot PRs such as Dependabot's wait for a maintainer to review them and comment `/ok to test `. Until then the PR shows `Unit Tests Gate` as expected and cannot merge. Docs-only PRs are not exempt, because several unit tests check the docs. Repository admins can bypass the requirement when needed. + +A PR must also be up to date with `main` before it merges, and every branch update creates a new head commit that needs a full unit test run. For a PR that is not copied automatically, a maintainer must comment `/ok to test ` again after each update. If a job fails from a known flake, rerun the failed jobs with `gh run rerun --failed`, which also reruns `Unit Tests Gate`. + ## Running GitHub Actions Locally You can use the `act` tool to run GitHub Actions locally. See [act usage](https://nektosact.com/introduction.html). From 070534bde08defafbdbaf8ec62778f995f6c1c33 Mon Sep 17 00:00:00 2001 From: Manu Nicholas Jacob <118115267+manunicholasjacob@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:29:34 -0500 Subject: [PATCH 075/119] fix(logging): keep log lines a Windows console cannot encode (#1470) Signed-off-by: manunicholasjacob Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 59087bab509fc88336a932851b3ed33bf13405c2) --- src/aiperf/cli_runner/_preflight.py | 17 +-- src/aiperf/cli_runner/_sweep_table.py | 19 ++- src/aiperf/common/logging.py | 113 +++++++++++++-- .../cli_runner/test_sweep_table_encoding.py | 34 +++++ tests/unit/common/test_logging_encoding.py | 132 ++++++++++++++++++ tests/unit/common/test_logging_tty.py | 60 ++++++++ 6 files changed, 350 insertions(+), 25 deletions(-) create mode 100644 tests/unit/cli_runner/test_sweep_table_encoding.py create mode 100644 tests/unit/common/test_logging_encoding.py diff --git a/src/aiperf/cli_runner/_preflight.py b/src/aiperf/cli_runner/_preflight.py index 689f24953e..f2a911d049 100644 --- a/src/aiperf/cli_runner/_preflight.py +++ b/src/aiperf/cli_runner/_preflight.py @@ -183,19 +183,16 @@ def _preflight_endpoint_ready(plan: BenchmarkPlan) -> None: assumed to share an endpoint. """ import asyncio - import logging cfg = plan.configs[0].endpoint if cfg.wait_for_model_timeout <= 0: return + from aiperf.common.logging import install_preflight_handler + # Preflight runs before rich logging is installed; install a minimal # stderr handler so probe lines are visible. - if not logging.getLogger().handlers: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) + install_preflight_handler() from aiperf.common.readiness_probe import wait_for_endpoint @@ -297,11 +294,9 @@ def _preflight_dataset_materialize(plan: BenchmarkPlan) -> None: # yet, and a multi-GB fetch with no output looks like a hang. That check # returns early when endpoint waiting is disabled, so don't rely on it # having installed the handler. - if not logging.getLogger().handlers: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) + from aiperf.common.logging import install_preflight_handler + + if install_preflight_handler(): # Root INFO also switches on httpx's per-request logging, and resolution # makes hundreds of requests -- including the 404s `datasets` expects # while probing for a legacy loading script before falling back to diff --git a/src/aiperf/cli_runner/_sweep_table.py b/src/aiperf/cli_runner/_sweep_table.py index f9b3c9d624..f942d97248 100644 --- a/src/aiperf/cli_runner/_sweep_table.py +++ b/src/aiperf/cli_runner/_sweep_table.py @@ -14,10 +14,12 @@ from io import StringIO from typing import TYPE_CHECKING, Any -from rich.box import SIMPLE_HEAVY +from rich.box import ASCII2, SIMPLE_HEAVY, Box from rich.console import Console from rich.table import Table +from aiperf.common.logging import is_utf8_encoding + if TYPE_CHECKING: from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.config.resolution.plan import BenchmarkPlan @@ -223,9 +225,22 @@ def __call__(self, variation_key: tuple, cell: dict[str, Any]) -> None: rendered = self._render() self._logger.info(f"\n{rendered}\n") + @staticmethod + def _box_style() -> Box: + """SIMPLE_HEAVY where the log sink can carry it, ASCII where it cannot. + + This table is rendered to a string and handed to the logger, so Rich + cannot see the eventual console and its own legacy-Windows substitution + never runs. A cp1252 console, the Windows default, has no box-drawing + characters, so ask the stream the handler writes to. + """ + if is_utf8_encoding(getattr(sys.stdout, "encoding", None)): + return SIMPLE_HEAVY + return ASCII2 + def _render(self) -> str: """Render the current accumulated table to a string.""" - table = Table(box=SIMPLE_HEAVY, show_header=True) + table = Table(box=self._box_style(), show_header=True) for name in self._param_names: table.add_column(name, justify="right") for metric, _stat in HEADLINE_METRICS: diff --git a/src/aiperf/common/logging.py b/src/aiperf/common/logging.py index 5954a4667a..0286f4d42a 100644 --- a/src/aiperf/common/logging.py +++ b/src/aiperf/common/logging.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import asyncio +import codecs import logging import multiprocessing import queue @@ -48,6 +49,100 @@ _BASIC_DATE_FORMAT = "%H:%M:%S" +class _EncodableFormatter(logging.Formatter): + """Formatter that degrades characters the stream cannot encode. + + A single-byte console encoding, and cp1252 is the Windows default, cannot + represent the box-drawing characters Rich puts in a rendered table. Writing + one raises inside ``StreamHandler.emit``, and logging's own handler then + discards the whole record: the operator loses the line entirely rather than + the character. Substituting keeps the line. + """ + + def __init__(self, *args, encoding: str, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._encoding = encoding + + def format(self, record: logging.LogRecord) -> str: + formatted = super().format(record) + return formatted.encode(self._encoding, errors="replace").decode(self._encoding) + + +def _stream_encoding(stream: object) -> str | None: + return getattr(stream, "encoding", None) or None + + +def is_utf8_encoding(encoding: str | None) -> bool: + """Whether a stream reporting ``encoding`` can take any message as is. + + The one rule for "is this sink narrower than the message", shared by the + basic formatter and the sweep table's box style so the two cannot drift. + """ + if encoding is None: + return False + try: + return codecs.lookup(encoding).name == "utf-8" + except LookupError: + return False + + +def _encodable_formatter( + fmt: str, datefmt: str | None, encoding: str | None +) -> logging.Formatter: + """A plain formatter on a UTF-8 sink, a substituting one on anything else. + + Anything not known to be UTF-8 is treated as possibly narrower than the + message, including a stream that reports no encoding and a codec name + Python does not recognise. Guessing UTF-8 there costs the whole record when + the guess is wrong, while substituting costs one character when it was + right, so the unknown cases fall back to ASCII. + """ + if is_utf8_encoding(encoding): + return logging.Formatter(fmt, datefmt=datefmt) + if encoding is not None: + try: + codecs.lookup(encoding) + return _EncodableFormatter(fmt, datefmt=datefmt, encoding=encoding) + except LookupError: + pass + return _EncodableFormatter(fmt, datefmt=datefmt, encoding="ascii") + + +def _basic_formatter(encoding: str | None) -> logging.Formatter: + return _encodable_formatter(_BASIC_LOG_FORMAT, _BASIC_DATE_FORMAT, encoding) + + +_PREFLIGHT_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + + +def install_preflight_handler() -> bool: + """Install a temporary stderr handler for output before rich logging starts. + + Returns False and does nothing when root already has a handler, so a + caller's own logging config wins. ``setup_rich_logging`` removes the + handler this installs; left in place it printed every later line a second + time, through a stream nobody had checked could encode it. + """ + root = logging.getLogger() + if root.handlers: + return False + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter( + _encodable_formatter(_PREFLIGHT_LOG_FORMAT, None, _stream_encoding(sys.stderr)) + ) + handler._aiperf_preflight = True # type: ignore[attr-defined] + root.addHandler(handler) + root.setLevel(logging.INFO) + return True + + +def _remove_preflight_handlers() -> None: + root = logging.getLogger() + for handler in root.handlers[:]: + if getattr(handler, "_aiperf_preflight", False): + root.removeHandler(handler) + + def _create_basic_handler(level: str | int) -> logging.StreamHandler: """Create a basic non-rich StreamHandler for non-TTY environments. @@ -55,9 +150,7 @@ def _create_basic_handler(level: str | int) -> logging.StreamHandler: """ handler = logging.StreamHandler(sys.stdout) handler.setLevel(level) - handler.setFormatter( - logging.Formatter(_BASIC_LOG_FORMAT, datefmt=_BASIC_DATE_FORMAT) - ) + handler.setFormatter(_basic_formatter(_stream_encoding(sys.stdout))) return handler @@ -196,6 +289,7 @@ def setup_rich_logging(run: "BenchmarkRun") -> None: # Set logging level for the root logger (affects all loggers) level = run.cfg.logging.level.upper() logging.root.setLevel(level) + _remove_preflight_handlers() if is_tty(): console_handler = CustomRichHandler( @@ -210,16 +304,11 @@ def setup_rich_logging(run: "BenchmarkRun") -> None: console_handler = _create_basic_handler(level) logging.root.addHandler(console_handler) - # Enable file logging for services + # Enable file logging for services. UTF-8 explicitly: without an encoding + # the file opens in the locale's code page, cp1252 on a default Windows + # box, and on a TTY this file is the one sink narrower than the messages. log_folder = run.cfg.artifacts.dir / OutputDefaults.LOG_FOLDER - log_folder.mkdir(parents=True, exist_ok=True) - file_handler = logging.FileHandler(log_folder / OutputDefaults.LOG_FILE) - file_handler.setLevel(level) - file_handler.formatter = logging.Formatter( - "%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - logging.root.addHandler(file_handler) + logging.root.addHandler(create_file_handler(log_folder, level)) _logger.debug(lambda: f"Logging initialized with level: {level}") diff --git a/tests/unit/cli_runner/test_sweep_table_encoding.py b/tests/unit/cli_runner/test_sweep_table_encoding.py new file mode 100644 index 0000000000..940ff82d6d --- /dev/null +++ b/tests/unit/cli_runner/test_sweep_table_encoding.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys +from types import SimpleNamespace + +import pytest + +from aiperf.cli_runner._sweep_table import SweepTableLogger + +_BOX_DRAWING = "\u2500\u2501" + + +@pytest.mark.parametrize( + ("encoding", "expect_box_drawing"), + [ + ("utf-8", True), + ("UTF-8", True), + (None, False), + ("cp1252", False), + ("not-a-codec", False), + ], +) +def test_box_style_follows_the_stdout_encoding( + monkeypatch: pytest.MonkeyPatch, encoding: str | None, expect_box_drawing: bool +) -> None: + monkeypatch.setattr(sys, "stdout", SimpleNamespace(encoding=encoding)) + + style = str(SweepTableLogger._box_style()) + + if expect_box_drawing: + assert any(ch in style for ch in _BOX_DRAWING) + else: + assert style.isascii(), "a narrow sink needs a style it can encode" diff --git a/tests/unit/common/test_logging_encoding.py b/tests/unit/common/test_logging_encoding.py new file mode 100644 index 0000000000..4273dab1e9 --- /dev/null +++ b/tests/unit/common/test_logging_encoding.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import io +import logging + +import pytest + +from aiperf.common.logging import ( + _basic_formatter, + _create_basic_handler, + _stream_encoding, +) + +_HEAVY_RULE = "\u2501" * 8 +_TABLE_LIKE = f"\n concurrency \n {_HEAVY_RULE} \n 1 \n" + + +def _record(msg: str) -> logging.LogRecord: + return logging.LogRecord( + name="t", + level=logging.INFO, + pathname="p", + lineno=1, + msg=msg, + args=(), + exc_info=None, + ) + + +def _cp1252_stream() -> tuple[io.BytesIO, io.TextIOWrapper]: + raw = io.BytesIO() + return raw, io.TextIOWrapper(raw, encoding="cp1252", errors="strict", newline="") + + +def test_the_line_survives_on_a_cp1252_console() -> None: + raw, stream = _cp1252_stream() + handler = logging.StreamHandler(stream) + handler.setFormatter(_basic_formatter("cp1252")) + + handler.emit(_record(_TABLE_LIKE)) + stream.flush() + + written = raw.getvalue().decode("cp1252") + assert "concurrency" in written + assert "\u2501" not in written, "the rule cannot be encoded, so it is substituted" + assert "?" in written + + +@pytest.mark.parametrize( + ("encoding", "substitutes"), + [ + ("utf-8", False), + ("UTF8", False), + ("cp1252", True), + ("not-a-codec", True), + ], +) +def test_formatter_choice_follows_the_encoding( + encoding: str | None, substitutes: bool +) -> None: + formatted = _basic_formatter(encoding).format(_record(_TABLE_LIKE)) + assert ("\u2501" in formatted) is not substitutes + + +@pytest.mark.parametrize( + ("attr", "expected"), + [("cp1252", "cp1252"), (None, None)], + ids=["reports-encoding", "reports-none"], +) +def test_stream_encoding_reports_none_when_the_stream_is_silent( + attr: str | None, expected: str | None +) -> None: + class _Stream: + encoding = attr + + assert _stream_encoding(_Stream()) == expected + + +def test_the_basic_handler_wires_the_substituting_formatter_in( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Guards the wiring, not just the helper. + + ``_basic_formatter`` being correct is no use if ``_create_basic_handler`` + does not reach for it, and that is the seam a refactor would break. + """ + raw, stream = _cp1252_stream() + monkeypatch.setattr("aiperf.common.logging.sys.stdout", stream) + + handler = _create_basic_handler(logging.INFO) + handler.emit(_record(_TABLE_LIKE)) + stream.flush() + + written = raw.getvalue().decode("cp1252") + assert "concurrency" in written + assert "\u2501" not in written + + +class _SilentAboutEncoding: + """A wrapper that forwards writes but reports no encoding. + + Stands in for the stream layers that hide the terminal's encoding. The sink + underneath is still strict cp1252, so guessing UTF-8 here would drop the + record. + """ + + def __init__(self, wrapped) -> None: + self._wrapped = wrapped + + def write(self, text: str) -> int: + return self._wrapped.write(text) + + def flush(self) -> None: + self._wrapped.flush() + + +def test_a_stream_that_reports_no_encoding_is_not_assumed_to_be_utf8( + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw, strict_cp1252 = _cp1252_stream() + monkeypatch.setattr( + "aiperf.common.logging.sys.stdout", _SilentAboutEncoding(strict_cp1252) + ) + + handler = _create_basic_handler(logging.INFO) + handler.emit(_record(_TABLE_LIKE)) + strict_cp1252.flush() + + written = raw.getvalue().decode("cp1252") + assert "concurrency" in written + assert "━" not in written diff --git a/tests/unit/common/test_logging_tty.py b/tests/unit/common/test_logging_tty.py index df7897b60f..5dc4f3ceb1 100644 --- a/tests/unit/common/test_logging_tty.py +++ b/tests/unit/common/test_logging_tty.py @@ -1,6 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import io import logging +import sys +from pathlib import Path from unittest.mock import MagicMock import pytest @@ -9,6 +12,7 @@ CustomRichHandler, MultiProcessLogHandler, _create_basic_handler, + install_preflight_handler, setup_child_process_logging, setup_rich_logging, ) @@ -121,6 +125,62 @@ def test_file_handler_always_added(self, monkeypatch, mock_run, tty): file_handlers = [h for h in root.handlers if isinstance(h, logging.FileHandler)] assert len(file_handlers) == 1 + def test_log_file_is_utf8_whatever_the_locale(self, monkeypatch, mock_run): + """The sweep table only renders on a TTY, where this file is the sink that + can be narrower than the message: a FileHandler with no encoding opens in + the locale's code page, cp1252 on a default Windows box.""" + monkeypatch.setattr("aiperf.common.logging.is_tty", lambda: True) + monkeypatch.setattr( + "io.text_encoding", lambda encoding, stacklevel=2: encoding or "cp1252" + ) + + setup_rich_logging(mock_run) + + file_handler = next( + h + for h in logging.getLogger().handlers + if isinstance(h, logging.FileHandler) + ) + rule = "━" * 8 + file_handler.emit( + logging.LogRecord("t", logging.INFO, __file__, 1, rule, None, None) + ) + file_handler.flush() + assert rule in Path(file_handler.baseFilename).read_text(encoding="utf-8") + + def test_rich_logging_replaces_the_preflight_handler(self, monkeypatch, mock_run): + """Left in place, the preflight stderr handler printed every later line a + second time next to the rich console.""" + monkeypatch.setattr("aiperf.common.logging.is_tty", lambda: True) + # pytest attaches its own capture handlers to root during a test. + monkeypatch.setattr(logging.getLogger(), "handlers", []) + assert install_preflight_handler() + + setup_rich_logging(mock_run) + + kinds = sorted(type(h).__name__ for h in logging.getLogger().handlers) + assert kinds == ["CustomRichHandler", "FileHandler"] + + def test_preflight_handler_defers_to_existing_logging(self, monkeypatch): + existing = logging.NullHandler() + monkeypatch.setattr(logging.getLogger(), "handlers", [existing]) + + assert not install_preflight_handler() + assert logging.getLogger().handlers == [existing] + + def test_preflight_handler_survives_a_narrow_stderr(self, monkeypatch): + raw = io.BytesIO() + stderr = io.TextIOWrapper(raw, encoding="cp1252", errors="strict", newline="") + monkeypatch.setattr(sys, "stderr", stderr) + monkeypatch.setattr(logging.getLogger(), "handlers", []) + assert install_preflight_handler() + + logging.getLogger("t").info("rule ━━ end") + stderr.flush() + + written = raw.getvalue().decode("cp1252") + assert "rule ?? end" in written + # --------------------------------------------------------------------------- # setup_child_process_logging From fbe9d65902febd05a0c992cd4d7f9da66f0fcb62 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:39:55 -0700 Subject: [PATCH 076/119] build(docs): update Fern CLI to 5.149.0 (#1542) Signed-off-by: Elias Bermudez (cherry picked from commit 33eaddaf1200c3c38089997b57082c32cef53570) --- .github/workflows/run-fern-checks.yml | 6 +++--- fern/fern.config.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-fern-checks.yml b/.github/workflows/run-fern-checks.yml index 16bb8d49b1..3abafc9139 100644 --- a/.github/workflows/run-fern-checks.yml +++ b/.github/workflows/run-fern-checks.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 name: "Fern Docs Checks" @@ -19,10 +19,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - name: Set up Node.js 20 + - name: Set up Node.js 22 uses: actions/setup-node@v5 with: - node-version: "20" + node-version: "22" - name: Install Fern CLI run: npm install -g fern-api@$(jq -r '.version' fern/fern.config.json) - name: Set up Python 3.12 diff --git a/fern/fern.config.json b/fern/fern.config.json index d67d985d26..a4166e8063 100644 --- a/fern/fern.config.json +++ b/fern/fern.config.json @@ -1,4 +1,4 @@ { "organization": "nvidia", - "version": "5.75.8" + "version": "5.149.0" } From dd0368d34a98a071de7aba3ee44820c420726053 Mon Sep 17 00:00:00 2001 From: Matthias Brunel Date: Fri, 9 Oct 2026 03:24:33 +0200 Subject: [PATCH 077/119] feat(h-cua-perf): add the uuid_cache and disable_structured_output knobs (#1482) Signed-off-by: Matthias Brunel Co-authored-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> (cherry picked from commit 9747baed284009cb80b879d02a2755e91c5c642e) --- docs/tutorials/h-cua-perf.md | 4 +- src/aiperf/dataset/loader/h_cua_perf.py | 4 +- .../dataset/loader/h_cua_perf_processing.py | 64 +++++++++++++++++-- src/aiperf/dataset/mmap_cache.py | 5 ++ src/aiperf/plugin/plugins.yaml | 3 +- .../loader/test_h_cua_perf_processing.py | 52 ++++++++++++++- 6 files changed, 121 insertions(+), 11 deletions(-) diff --git a/docs/tutorials/h-cua-perf.md b/docs/tutorials/h-cua-perf.md index 54bb693f76..3d92e752a3 100644 --- a/docs/tutorials/h-cua-perf.md +++ b/docs/tutorials/h-cua-perf.md @@ -65,7 +65,7 @@ Some requests are larger than the 128k-token context of the example server; `max Each session is one multi-turn conversation: a request is sent, the response awaited, the recorded wait slept, then the next recorded request is sent. There are no timestamps, so `--fixed-schedule` does not apply. -The screenshot window and the session selection are `--dataset-filter` options: +The screenshot window, the session selection and the request shaping are `--dataset-filter` options: | Filter | Effect | | --- | --- | @@ -73,6 +73,8 @@ The screenshot window and the session selection are `--dataset-filter` options: | `min_trace_length` | Drop shorter sessions; truncations never go below it (default 1). | | `max_trace_length` | Keep each session's first N turns. | | `avg_trace_length` | Scale every session's length by the same factor until the mean reaches the target. | +| `uuid_cache` | Send each screenshot in full only in the first request that carries it; later requests of the session reference it by `uuid` with `image_url` set to null. Needs a server that caches multimodal inputs by uuid, as vLLM does, and every request of a session routed to the same replica. | +| `disable_structured_output` | Drop the recorded response format from the extra body, so the server applies no guided decoding; `tool_choice` is kept. The published records carry no response format, so this only changes builds whose agent used structured outputs. | ## Related Tutorials diff --git a/src/aiperf/dataset/loader/h_cua_perf.py b/src/aiperf/dataset/loader/h_cua_perf.py index 490306526b..e9841b1e31 100644 --- a/src/aiperf/dataset/loader/h_cua_perf.py +++ b/src/aiperf/dataset/loader/h_cua_perf.py @@ -119,9 +119,7 @@ def _explicit_entries(self) -> int | None: def _read_records(self, trace: Path, plan: dict[str, int]) -> list[dict[str, Any]]: with open_dataset(trace) as lines: records = (orjson.loads(line) for line in lines if line.strip()) - return list( - iter_selected_records(records, plan, self.filters.n_screenshots) - ) + return list(iter_selected_records(records, plan, self.filters)) async def load_dataset(self) -> dict[str, list[MooncakeTrace]]: """Download, plan the trajectory selection, read that far, delegate the rest.""" diff --git a/src/aiperf/dataset/loader/h_cua_perf_processing.py b/src/aiperf/dataset/loader/h_cua_perf_processing.py index 4c91f71635..af79e4617d 100644 --- a/src/aiperf/dataset/loader/h_cua_perf_processing.py +++ b/src/aiperf/dataset/loader/h_cua_perf_processing.py @@ -14,6 +14,7 @@ from pydantic import ConfigDict, Field, model_validator from aiperf.common.aiperf_logger import AIPerfLogger +from aiperf.common.enums import AssistantResponseMode from aiperf.common.models import AIPerfBaseModel _logger = AIPerfLogger(__name__) @@ -28,6 +29,9 @@ SCREENSHOT_ROLES = frozenset({"user", "tool"}) """Observations carry the screenshots: user messages for GUI agents, tool results for tool-calling ones.""" +STRUCTURED_OUTPUT_KEYS = frozenset({"response_format", "structured_outputs"}) +"""Extra-body keys that request guided decoding: the OpenAI API's and vLLM's.""" + _BISECTION_STEPS = 32 @@ -65,6 +69,16 @@ class HCuaPerfFilters(AIPerfBaseModel): description="Target mean requests per trajectory, reached by truncating " "every trajectory by the same factor.", ) + uuid_cache: bool = Field( + default=False, + description="Send each screenshot in full once per trajectory; later requests " + "reference it by uuid only, for servers that cache media by uuid.", + ) + disable_structured_output: bool = Field( + default=False, + description="Drop the recorded response format from the extra body; " + "tool_choice is kept.", + ) @model_validator(mode="after") def _max_not_below_min(self) -> HCuaPerfFilters: @@ -149,9 +163,9 @@ def open_dataset(path: Path) -> TextIO: def iter_selected_records( records: Iterator[dict[str, Any]], plan: dict[str, int], - n_screenshots: int | None, + filters: HCuaPerfFilters, ) -> Iterator[dict[str, Any]]: - """Yield the planned prefix of every selected trajectory, re-windowed, in file order. + """Yield the planned prefix of every selected trajectory, reshaped, in file order. The plan comes from the manifest and the file is verified against the manifest's sha256 before this runs, so the plan is trusted. Trajectories @@ -164,13 +178,55 @@ def iter_selected_records( if kept_n is None: continue kept = list(islice(session, kept_n)) - if n_screenshots is not None: - apply_screenshot_window(kept, n_screenshots) + reshape_trajectory(kept, filters) + for record in kept: + # Mooncake takes a row without it for a possibly incremental one and warns. + record.setdefault("assistant_responses", AssistantResponseMode.RECORDED) yield from kept if not remaining: return +def reshape_trajectory(records: list[dict[str, Any]], filters: HCuaPerfFilters) -> None: + """Apply the request-shaping filters to one trajectory's records, in place.""" + if filters.n_screenshots is not None: + apply_screenshot_window(records, filters.n_screenshots) + if filters.uuid_cache: + reference_repeated_screenshots(records) + if filters.disable_structured_output: + for record in records: + drop_structured_output(record) + + +def reference_repeated_screenshots(records: list[dict[str, Any]]) -> None: + """Keep a screenshot's data in the first request carrying it; later ones send its uuid only.""" + seen: set[str] = set() + for record in records: + for message in record["messages"]: + content = message.get("content") + if not isinstance(content, list): + continue + for idx, part in enumerate(content): + image_uuid = part.get("uuid") if _is_image(part) else None + if image_uuid is None: + continue + if image_uuid in seen: + # Replaced, not edited: the window shares one part object across records. + content[idx] = {**part, "image_url": None} + seen.add(image_uuid) + + +def drop_structured_output(record: dict[str, Any]) -> None: + """Remove the guided-decoding keys from a record's extra body, and the body once empty.""" + extra = record.get("extra") + if not isinstance(extra, dict): + return + for key in STRUCTURED_OUTPUT_KEYS: + extra.pop(key, None) + if not extra: + del record["extra"] + + def apply_screenshot_window(records: list[dict[str, Any]], n_screenshots: int) -> None: """Resize the screenshot window of every record of one trajectory, in place, to its last n_screenshots slots.""" slots_per_record = [screenshot_slots(record["messages"]) for record in records] diff --git a/src/aiperf/dataset/mmap_cache.py b/src/aiperf/dataset/mmap_cache.py index a3032c0a8c..6004c9f2f1 100644 --- a/src/aiperf/dataset/mmap_cache.py +++ b/src/aiperf/dataset/mmap_cache.py @@ -881,6 +881,11 @@ def _settings_payload_from_run(run: BenchmarkRun) -> dict[str, object]: system_prompt.encode("utf-8") ).hexdigest() + # --dataset-filter values change the loaded records, e.g. h_cua_perf's screenshot window. + dataset_filters = getattr(dataset, "filters", None) + if dataset_filters: + payload["dataset_filters"] = dict(dataset_filters) + return payload diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index c97b71307e..450ad48f6a 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -2686,7 +2686,8 @@ public_dataset_loader: on CUA-Gym desktop tasks: full text history, 10 tool definitions and one screenshot per request. Delegates to the mooncake_trace loader, replaying the recorded inter-request delays. --dataset-filter reshapes the corpus in - place: n_screenshots, min_trace_length, max_trace_length, avg_trace_length. + place: n_screenshots, min_trace_length, max_trace_length, avg_trace_length, + uuid_cache, disable_structured_output. Usage: --public-dataset h_cua_perf --dataset-filter n_screenshots=3 metadata: hf_dataset_name: Hcompany/h_cua_perf diff --git a/tests/unit/dataset/loader/test_h_cua_perf_processing.py b/tests/unit/dataset/loader/test_h_cua_perf_processing.py index 2ba4f4b9d0..4d0234a356 100644 --- a/tests/unit/dataset/loader/test_h_cua_perf_processing.py +++ b/tests/unit/dataset/loader/test_h_cua_perf_processing.py @@ -14,6 +14,7 @@ HCuaPerfFilters, apply_screenshot_window, drop_screenshot, + drop_structured_output, fit_lengths, iter_selected_records, screenshot_slots, @@ -42,6 +43,15 @@ def _image_slots(rec: dict[str, Any]) -> list[int]: return image_slots(rec["messages"]) +def _references(rec: dict[str, Any]) -> list[tuple[int, bool]]: + """(slot, carries its data) of every image part still in the record, in order.""" + return [ + (slot, parts[idx]["image_url"] is not None) + for slot, (parts, idx) in enumerate(screenshot_slots(rec["messages"])) + if parts[idx].get("type") == "image_url" + ] + + class TestSelectTraceLengths: def test_defaults_keep_everything_in_file_order(self) -> None: assert select_trace_lengths(HCuaPerfFilters(), TURNS) == TURNS @@ -83,7 +93,7 @@ def test_floor_is_never_crossed(self) -> None: class TestIterSelectedRecords: def test_yields_planned_prefixes_in_file_order(self) -> None: plan = {"traj-a": 2, "traj-c": 2} - out = list(iter_selected_records(iter(RECORDS), plan, None)) + out = list(iter_selected_records(iter(RECORDS), plan, HCuaPerfFilters())) assert [(r["session_id"], r["output_length"]) for r in out] == [ ("traj-a", 10), ("traj-a", 11), ("traj-c", 10), ("traj-c", 11), ] # fmt: skip @@ -96,9 +106,16 @@ def records(): consumed.append(r["session_id"]) yield r - list(iter_selected_records(records(), {"traj-a": 3, "traj-b": 1}, None)) + plan = {"traj-a": 3, "traj-b": 1} + list(iter_selected_records(records(), plan, HCuaPerfFilters())) assert consumed == ["traj-a"] * 3 + ["traj-b"] + def test_marks_every_row_recorded_for_mooncake(self) -> None: + out = list( + iter_selected_records(iter(RECORDS), {"traj-a": 2}, HCuaPerfFilters()) + ) + assert [r["assistant_responses"] for r in out] == ["recorded", "recorded"] + class TestScreenshotWindow: def test_slots_span_user_and_tool_observations(self) -> None: @@ -183,6 +200,37 @@ def test_drop_screenshot_merges_neighbouring_text( assert IMAGE_PLACEHOLDER in parts[0]["text"] +class TestRequestShaping: + def test_uuid_cache_sends_each_screenshot_in_full_once_per_trajectory(self) -> None: + filters = HCuaPerfFilters(n_screenshots=2, uuid_cache=True) + plan = {"traj-a": 3, "traj-b": 1} + out = list(iter_selected_records(iter(RECORDS), plan, filters)) + + assert [_references(r) for r in out] == [ + [(0, True)], + [(0, False), (1, True)], + [(1, False), (2, True)], + [(0, True)], + ] + parts, idx = screenshot_slots(out[1]["messages"])[0] + assert parts[idx] == { + "type": "image_url", + "image_url": None, + "uuid": image("traj-a", 0)["uuid"], + } + + def test_disable_structured_output_keeps_tool_choice(self) -> None: + record = { + "extra": { + "tool_choice": "auto", + "response_format": {"type": "json_object"}, + "structured_outputs": {"json": {"type": "object"}}, + } + } + drop_structured_output(record) + assert record == {"extra": {"tool_choice": "auto"}} + + class TestFilters: def test_string_values_from_the_cli_are_coerced(self) -> None: filters = HCuaPerfFilters.model_validate( From 4bb69157c3eb368b82c0283dc16a2f0963ed77e6 Mon Sep 17 00:00:00 2001 From: Matthias Brunel Date: Fri, 9 Oct 2026 20:22:17 +0200 Subject: [PATCH 078/119] feat(h-cua-perf): replay a local build with --input-file (#1493) Signed-off-by: Matthias Brunel (cherry picked from commit b05ecc0b3dc4e691282d7b0695af97e057d29bc8) --- docs/tutorials/h-cua-perf.md | 12 ++ src/aiperf/dataset/loader/h_cua_perf.py | 52 ++------ src/aiperf/dataset/loader/h_cua_perf_file.py | 110 +++++++++++++++ .../dataset/loader/h_cua_perf_processing.py | 62 ++++++++- src/aiperf/plugin/plugins.yaml | 13 ++ .../dataset/loader/test_h_cua_perf_file.py | 125 ++++++++++++++++++ 6 files changed, 334 insertions(+), 40 deletions(-) create mode 100644 src/aiperf/dataset/loader/h_cua_perf_file.py create mode 100644 tests/unit/dataset/loader/test_h_cua_perf_file.py diff --git a/docs/tutorials/h-cua-perf.md b/docs/tutorials/h-cua-perf.md index 3d92e752a3..81af629f5a 100644 --- a/docs/tutorials/h-cua-perf.md +++ b/docs/tutorials/h-cua-perf.md @@ -76,6 +76,18 @@ The screenshot window, the session selection and the request shaping are `--data | `uuid_cache` | Send each screenshot in full only in the first request that carries it; later requests of the session reference it by `uuid` with `image_url` set to null. Needs a server that caches multimodal inputs by uuid, as vLLM does, and every request of a session routed to the same replica. | | `disable_structured_output` | Drop the recorded response format from the extra body, so the server applies no guided decoding; `tool_choice` is kept. The published records carry no response format, so this only changes builds whose agent used structured outputs. | +### Replay a local build + +A build derived with the dataset's `trace_processor.py` is replayed from disk. Point `--input-file` at the trace and name the format: + +```bash +aiperf profile ... \ + --input-file ./build/h_cua.jsonl.zst \ + --custom-dataset-type h_cua_perf +``` + +`trace_processor.py` writes the trace, zstd-compressed when the output path ends in `.zst`, and its `h_cua.meta.json` manifest beside it with the trace's sha256; any other file is refused. A manifest without a sha256 comes from an older copy of the script: download it again and derive the build again. Every session of the build is loaded. `--dataset-filter` applies to `--public-dataset` only: a derived build already carries its screenshot window and session selection, so shape it when deriving it. + ## Related Tutorials - [Trace Replay with Mooncake Traces](../benchmark-modes/trace-replay.md) diff --git a/src/aiperf/dataset/loader/h_cua_perf.py b/src/aiperf/dataset/loader/h_cua_perf.py index e9841b1e31..9e3fbbf143 100644 --- a/src/aiperf/dataset/loader/h_cua_perf.py +++ b/src/aiperf/dataset/loader/h_cua_perf.py @@ -4,12 +4,10 @@ from __future__ import annotations import asyncio -import hashlib from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar import orjson -import psutil import zstandard from pydantic import ValidationError @@ -21,9 +19,12 @@ from aiperf.dataset.loader.h_cua_perf_processing import ( HCuaPerfFilters, iter_selected_records, + memory_shortfall, open_dataset, + reject_ignore_trace_delays, select_trace_lengths, supported_filter_keys, + verify_trace, ) from aiperf.dataset.loader.models import MooncakeTrace from aiperf.dataset.loader.mooncake_trace import MooncakeTraceDatasetLoader @@ -32,9 +33,6 @@ if TYPE_CHECKING: from aiperf.config.resolution.plan import BenchmarkRun -LIVE_OBJECT_FACTOR = 2.0 -"""Live bytes per on-disk byte once a record is parsed; measured at 1.4 to 1.7, rounded up for text-heavy sessions.""" - class HCuaPerfDatasetLoader(BaseHFDatasetLoader): """Download ``Hcompany/h_cua_perf`` and replay it as a Mooncake trace.""" @@ -69,13 +67,7 @@ def __init__( raise DatasetLoaderError( f"{self.tag}: {hf_dataset_name} has no subsets; drop --hf-subset" ) - dataset = run.cfg.get_default_dataset() if run is not None else None - if getattr(dataset, "ignore_trace_delays", False): - raise DatasetLoaderError( - f"{self.tag}: --ignore-trace-delays applies to the Weka loaders only; " - "use --inter-turn-delay-cap-seconds 0 to send each session's turns " - "back to back" - ) + reject_ignore_trace_delays(self.tag, run) try: self.filters = HCuaPerfFilters.model_validate(filters or {}) except ValidationError as e: @@ -148,38 +140,20 @@ async def load_dataset(self) -> dict[str, list[MooncakeTrace]]: return data def _verify_trace(self, meta: dict[str, Any], trace: Path) -> None: - """The manifest names the sha256 of the trace it describes; any other file is refused.""" - digests = meta.get("sha256") - expected = digests.get(self.hf_filename) if isinstance(digests, dict) else None - if not expected: - raise DatasetLoaderError( - f"{self.tag}: the manifest carries no sha256 for {self.hf_filename}" - ) - with open(trace, "rb") as f: - actual = hashlib.file_digest(f, "sha256").hexdigest() - if actual != expected: - raise DatasetLoaderError( - f"{self.tag}: {self.hf_filename} does not match the manifest " - f"(sha256 {actual[:12]}, manifest says {expected[:12]}); " - "delete it from the Hub cache and download again" - ) + """The Hub cache persists across runs, so a mismatch is fixed by downloading again.""" + verify_trace( + meta, + trace, + mismatch_hint="; delete it from the Hub cache and download again", + ) def _warn_if_selection_exceeds_memory( self, meta: dict[str, Any], plan: dict[str, int] ) -> None: - """The whole corpus is parsed into memory; say so before the read when it will not fit.""" - if not meta.get("size_mb") or not meta.get("num_turns"): - return - planned = sum(plan.values()) - estimate = ( - meta["size_mb"] * 1e6 * planned / meta["num_turns"] * LIVE_OBJECT_FACTOR - ) - available = psutil.virtual_memory().available - if estimate > available: + if shortfall := memory_shortfall(meta, plan): self.warning( - f"{self.tag}: the {planned:,} selected requests need about " - f"{estimate / 1e9:.1f} GB of RAM once parsed and {available / 1e9:.1f} GB " - "is available; select fewer with --num-dataset-entries or --dataset-filter" + f"{self.tag}: {shortfall}; select fewer with --num-dataset-entries " + "or --dataset-filter" ) async def convert_to_conversations( diff --git a/src/aiperf/dataset/loader/h_cua_perf_file.py b/src/aiperf/dataset/loader/h_cua_perf_file.py new file mode 100644 index 0000000000..bd0ccfddea --- /dev/null +++ b/src/aiperf/dataset/loader/h_cua_perf_file.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +import orjson +import zstandard + +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.dataset.generator.prompt import PromptGenerator +from aiperf.dataset.loader.base_loader import LoaderProbeData +from aiperf.dataset.loader.h_cua_perf_processing import ( + HCuaPerfFilters, + iter_selected_records, + manifest_path, + memory_shortfall, + open_dataset, + reject_ignore_trace_delays, + verify_trace, +) +from aiperf.dataset.loader.mooncake_trace import MooncakeTraceDatasetLoader + +if TYPE_CHECKING: + from aiperf.config.resolution.plan import BenchmarkRun + + +class HCuaPerfFileLoader(MooncakeTraceDatasetLoader): + """Replay an H CUA Perf build from disk. + + ``--input-file .jsonl.zst --custom-dataset-type h_cua_perf`` reads the + trace beside its ``.meta.json`` manifest, which must carry the trace's + sha256. The records are the Mooncake rows the ``h_cua_perf`` public dataset + downloads from the Hub, so the replay is the same; a build derived with the + dataset's ``trace_processor.py`` already carries its shape, which is why no + ``--dataset-filter`` applies here. + """ + + tag: ClassVar[str] = "HCuaPerf" + + def __init__( + self, + *, + filename: str | Path | None = None, + prompt_generator: PromptGenerator, + run: BenchmarkRun | None = None, + default_block_size: int | None = None, + **kwargs: Any, + ) -> None: + if filename is None: + raise DatasetLoaderError( + f"{self.tag}: inline records are not supported; " + "pass --input-file .jsonl.zst" + ) + super().__init__( + filename=filename, + prompt_generator=prompt_generator, + run=run, + default_block_size=default_block_size, + **kwargs, + ) + reject_ignore_trace_delays(self.tag, run) + self.manifest = manifest_path(self.filename) + if not self.manifest.is_file(): + raise DatasetLoaderError( + f"{self.tag}: {self.manifest} not found beside {self.filename.name}" + ) + self._plan: dict[str, int] = {} + + @classmethod + def can_load( + cls, data: LoaderProbeData | None = None, filename: str | Path | None = None + ) -> bool: + """Never claim a file: selected by name only. + + The records are valid Mooncake rows, so claiming them would make every + Mooncake file ambiguous and break structural detection for + ``mooncake_trace``. + """ + return False + + def _init_trace_scope(self) -> None: + """One read of the trace serves both the manifest check and the hash_ids scope. + + The base class would hash the file again for ``trace_id``; on a 2.5 GB + build that is a second full pass before the decompressing read. + """ + try: + meta = orjson.loads(self.manifest.read_bytes()) + self._plan = meta["session_turns"] + if shortfall := memory_shortfall(meta, self._plan): + self.warning(f"{self.tag}: {shortfall}; derive a smaller build") + self._trace_id = verify_trace(meta, self.filename)[:16] + except (KeyError, ValueError) as e: + raise DatasetLoaderError(f"{self.tag}: {e}") from e + self.prompt_generator._hash_id_corpus_rng.set_trace_id(self._trace_id) + self.prompt_generator._cache.clear() + + def _iter_record_dicts( + self, source: str | Path | None = None + ) -> Iterator[dict[str, Any]]: + try: + with open_dataset(self.filename) as lines: + records = (orjson.loads(line) for line in lines if line.strip()) + yield from iter_selected_records(records, self._plan, HCuaPerfFilters()) + except (KeyError, ValueError, zstandard.ZstdError) as e: + raise DatasetLoaderError(f"{self.tag}: {e}") from e diff --git a/src/aiperf/dataset/loader/h_cua_perf_processing.py b/src/aiperf/dataset/loader/h_cua_perf_processing.py index af79e4617d..ce8e983288 100644 --- a/src/aiperf/dataset/loader/h_cua_perf_processing.py +++ b/src/aiperf/dataset/loader/h_cua_perf_processing.py @@ -3,20 +3,26 @@ from __future__ import annotations +import hashlib import statistics from collections.abc import Iterator from itertools import groupby, islice from operator import itemgetter from pathlib import Path -from typing import Any, TextIO +from typing import TYPE_CHECKING, Any, TextIO +import psutil import zstandard from pydantic import ConfigDict, Field, model_validator from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.enums import AssistantResponseMode +from aiperf.common.exceptions import DatasetLoaderError from aiperf.common.models import AIPerfBaseModel +if TYPE_CHECKING: + from aiperf.config.resolution.plan import BenchmarkRun + _logger = AIPerfLogger(__name__) # Mirrors the dataset's trace_processor.py (published beside the data); keep the two in sync. @@ -29,6 +35,9 @@ SCREENSHOT_ROLES = frozenset({"user", "tool"}) """Observations carry the screenshots: user messages for GUI agents, tool results for tool-calling ones.""" +LIVE_OBJECT_FACTOR = 2.0 +"""Live bytes per on-disk byte once a record is parsed; measured at 1.4 to 1.7, rounded up for text-heavy sessions.""" + STRUCTURED_OUTPUT_KEYS = frozenset({"response_format", "structured_outputs"}) """Extra-body keys that request guided decoding: the OpenAI API's and vLLM's.""" @@ -160,6 +169,57 @@ def open_dataset(path: Path) -> TextIO: return zstandard.open(path, "rt", encoding="utf-8") +def manifest_path(trace: Path) -> Path: + """``.meta.json`` beside the trace, ```` being its name less ``.jsonl``/``.zst``: trace_processor.py's layout.""" + name = trace.name.removesuffix(".zst").removesuffix(".jsonl") + return trace.with_name(f"{name}.meta.json") + + +def verify_trace(meta: dict[str, Any], trace: Path, *, mismatch_hint: str = "") -> str: + """Refuse any trace but the one the manifest names; return its sha256 so it is hashed once.""" + digests = meta.get("sha256") + expected = digests.get(trace.name) if isinstance(digests, dict) else None + if not expected: + raise ValueError(f"the manifest carries no sha256 for {trace.name}") + with open(trace, "rb") as f: + actual = hashlib.file_digest(f, "sha256").hexdigest() + if actual != expected: + raise ValueError( + f"{trace.name} does not match the manifest " + f"(sha256 {actual[:12]}, manifest says {expected[:12]}){mismatch_hint}" + ) + return actual + + +def reject_ignore_trace_delays(tag: str, run: BenchmarkRun | None) -> None: + """Fail on ``--ignore-trace-delays``: only the Weka loaders honor it, so the replay would keep the recorded waits.""" + dataset = run.cfg.get_default_dataset() if run is not None else None + if getattr(dataset, "ignore_trace_delays", False): + raise DatasetLoaderError( + f"{tag}: --ignore-trace-delays applies to the Weka loaders only; " + "use --inter-turn-delay-cap-seconds 0 to send each session's turns " + "back to back" + ) + + +def memory_shortfall(meta: dict[str, Any], plan: dict[str, int]) -> str | None: + """How far the selection overshoots the RAM available once parsed, or None when it fits. + + The whole selection is parsed into memory, so the caller warns before the read. + """ + if not meta.get("size_mb") or not meta.get("num_turns"): + return None + planned = sum(plan.values()) + estimate = meta["size_mb"] * 1e6 * planned / meta["num_turns"] * LIVE_OBJECT_FACTOR + available = psutil.virtual_memory().available + if estimate <= available: + return None + return ( + f"the {planned:,} selected requests need about {estimate / 1e9:.1f} GB of RAM " + f"once parsed and {available / 1e9:.1f} GB is available" + ) + + def iter_selected_records( records: Iterator[dict[str, Any]], plan: dict[str, int], diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 450ad48f6a..11ab885cfc 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -560,6 +560,19 @@ custom_dataset_loader: is_trace: true default_block_size: 512 + h_cua_perf: + class: aiperf.dataset.loader.h_cua_perf_file:HCuaPerfFileLoader + description: | + H CUA Perf computer-use agent traces read from a local build: .jsonl.zst + beside its .meta.json manifest, e.g. a variant derived from + Hcompany/h_cua_perf with the dataset's trace_processor.py. Same records + and replay as the h_cua_perf public dataset, minus its --dataset-filter + knobs: a derived build already carries its shape. Never auto-detected; + select it by name. + Usage: --input-file ./build/h_cua.jsonl.zst --custom-dataset-type h_cua_perf + metadata: + is_trace: true + raw_payload: class: aiperf.dataset.loader.raw_payload:RawPayloadDatasetLoader description: | diff --git a/tests/unit/dataset/loader/test_h_cua_perf_file.py b/tests/unit/dataset/loader/test_h_cua_perf_file.py new file mode 100644 index 0000000000..e68cc22c21 --- /dev/null +++ b/tests/unit/dataset/loader/test_h_cua_perf_file.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import orjson +import pytest +import zstandard +from pytest import param + +from aiperf.common.exceptions import DatasetLoaderError +from aiperf.common.tokenizer import Tokenizer +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.resolution.plan import BenchmarkRun +from aiperf.dataset.composer.custom import CustomDatasetComposer +from aiperf.dataset.loader.h_cua_perf_file import HCuaPerfFileLoader +from aiperf.plugin.enums import CustomDatasetType +from tests.unit.conftest import make_run_from_cli +from tests.unit.dataset.loader._h_cua_perf_records import trajectory + +SESSION_TURNS = {"traj-a": 3, "traj-b": 1, "traj-c": 2} +RECORDS = [r for sid, n in SESSION_TURNS.items() for r in trajectory(sid, n)] + + +def build(directory: Path, name: str) -> Path: + trace = directory / name + lines = b"".join(orjson.dumps(r) + b"\n" for r in RECORDS) + trace.write_bytes(zstandard.compress(lines) if name.endswith(".zst") else lines) + digest = hashlib.sha256(trace.read_bytes()).hexdigest() + (directory / "h_cua.meta.json").write_bytes( + orjson.dumps({"session_turns": SESSION_TURNS, "sha256": {name: digest}}) + ) + return trace + + +def _run(trace: Path, **cli: Any) -> BenchmarkRun: + return make_run_from_cli( + CLIConfig( + model_names=["test-model"], + endpoint_type="chat", + input_file=str(trace), + custom_dataset_type=CustomDatasetType.H_CUA_PERF, + **cli, + ) + ) + + +class TestLoader: + """The Hub loader's tests cover the replay itself; these cover the file path into it.""" + + def test_never_claims_a_file(self, tmp_path: Path) -> None: + """Its rows are Mooncake rows; claiming them would break mooncake_trace detection.""" + assert ( + HCuaPerfFileLoader.can_load(RECORDS[0], tmp_path / "h_cua.jsonl") is False + ) + + @pytest.mark.parametrize( + "name", + [ + param("h_cua.jsonl.zst", id="zstd"), + param("h_cua.jsonl", id="plain"), + ], + ) # fmt: skip + def test_composer_replays_the_build_named_by_input_file( + self, tmp_path: Path, name: str, mock_tokenizer_cls: type[Tokenizer] + ) -> None: + trace = build(tmp_path, name) + composer = CustomDatasetComposer( + run=_run(trace), tokenizer=mock_tokenizer_cls.from_pretrained("test-model") + ) + + # The manifest check's digest scopes the hash_ids; the base class must not hash again. + with patch( + "aiperf.dataset.loader.base_trace_loader._compute_file_hash" + ) as second_hash: + conversations = composer.create_dataset() + + assert isinstance(composer.loader, HCuaPerfFileLoader) + assert {c.session_id: len(c.turns) for c in conversations} == SESSION_TURNS + second_hash.assert_not_called() + + def test_missing_manifest_is_rejected_at_construction(self, tmp_path: Path) -> None: + trace = build(tmp_path, "h_cua.jsonl.zst") + (tmp_path / "h_cua.meta.json").unlink() + + with pytest.raises(DatasetLoaderError, match="h_cua.meta.json"): + HCuaPerfFileLoader( + filename=trace, prompt_generator=MagicMock(), run=_run(trace) + ) + + def test_ignore_trace_delays_is_rejected_like_the_hub_loader( + self, tmp_path: Path + ) -> None: + trace = build(tmp_path, "h_cua.jsonl.zst") + + with pytest.raises(DatasetLoaderError, match="--ignore-trace-delays"): + HCuaPerfFileLoader( + filename=trace, + prompt_generator=MagicMock(), + run=_run(trace, ignore_trace_delays=True), + ) + + @pytest.mark.parametrize( + "sha256, match", + [ + param({}, "no sha256", id="trace_not_named"), + param({"h_cua.jsonl.zst": "0" * 64}, "does not match", id="stale_digest"), + ], + ) # fmt: skip + def test_a_manifest_not_naming_the_trace_digest_is_refused( + self, tmp_path: Path, sha256: dict[str, str], match: str + ) -> None: + trace = build(tmp_path, "h_cua.jsonl.zst") + manifest = tmp_path / "h_cua.meta.json" + meta = orjson.loads(manifest.read_bytes()) | {"sha256": sha256} + manifest.write_bytes(orjson.dumps(meta)) + loader = HCuaPerfFileLoader( + filename=trace, prompt_generator=MagicMock(), run=_run(trace) + ) + + with pytest.raises(DatasetLoaderError, match=match): + loader.load_dataset() From 54025049fa175e818f507da95833bf6b1f300781 Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:33:07 -0700 Subject: [PATCH 079/119] feat(spec-decode): TensorRT-LLM per-request acceptance adapter (#1433) Signed-off-by: Elias Bermudez (cherry picked from commit 465bc7c1a0d85384d003688e1e395f6e34f67601) --- docs/reference/spec-decode-acceptance.md | 129 ++++++- docs/tutorials/spec-decode-metrics.md | 79 ++++- src/aiperf/common/models/extracted_payload.py | 7 + src/aiperf/endpoints/base_endpoint.py | 51 ++- src/aiperf/endpoints/payload_extraction.py | 3 + src/aiperf/plugin/enums.py | 2 +- src/aiperf/plugin/plugins.yaml | 15 + src/aiperf/records/inference_result_parser.py | 110 +++++- src/aiperf/spec_decode/__init__.py | 2 + src/aiperf/spec_decode/_shared.py | 69 ++++ src/aiperf/spec_decode/trtllm_adapter.py | 128 +++++++ src/aiperf/spec_decode/vllm_adapter.py | 83 ++--- tests/aiperf_mock_server/README.md | 19 + tests/aiperf_mock_server/app.py | 8 +- tests/aiperf_mock_server/config.py | 54 +++ tests/aiperf_mock_server/utils.py | 176 ++++++++-- .../endpoints/test_spec_decode_capture.py | 96 +++++ .../records/test_inference_result_parser.py | 130 +++++++ .../spec_decode/test_adapter_autodetect.py | 259 ++++++++++---- .../spec_decode/test_mock_server_payloads.py | 331 ++++++++++++++++++ tests/unit/spec_decode/test_trtllm_adapter.py | 228 ++++++++++++ tests/unit/spec_decode/test_vllm_adapter.py | 6 +- 22 files changed, 1781 insertions(+), 204 deletions(-) create mode 100644 src/aiperf/spec_decode/_shared.py create mode 100644 src/aiperf/spec_decode/trtllm_adapter.py create mode 100644 tests/unit/spec_decode/test_mock_server_payloads.py create mode 100644 tests/unit/spec_decode/test_trtllm_adapter.py diff --git a/docs/reference/spec-decode-acceptance.md b/docs/reference/spec-decode-acceptance.md index 068900d845..d2d3ee7c52 100644 --- a/docs/reference/spec-decode-acceptance.md +++ b/docs/reference/spec-decode-acceptance.md @@ -12,8 +12,10 @@ AIPerf captures this as an **engine-neutral per-request record** so the metrics layer can reason about acceptance without knowing which engine produced it. This page documents the record, the adapter interface that fills it, and the -vLLM adapter (the first supported engine). SGLang and TensorRT-LLM adapters are -future work and reuse the same record. +two supported engines: vLLM and TensorRT-LLM. They emit genuinely different +payloads -- different field names and different placement in the response -- and +the adapter layer is what keeps that difference from reaching the metrics layer. +An SGLang adapter is future work and reuses the same record. ## The engine-neutral record @@ -26,7 +28,7 @@ drafting such as DSpark-style adaptive verification. | Field | Description | | --- | --- | -| `engine` | Serving engine that produced the stats (e.g. `vllm`). | +| `engine` | Serving engine that produced the stats: `vllm` or `tensorrt_llm`. Set by the adapter for provenance; no metric branches on it. | | `mean_acceptance_length` | Mean tokens per verify step including the bonus token: `1 + num_accepted_draft_tokens / num_spec_steps`. Ranges `1.0` … `num_spec_tokens + 1`. | | `draft_acceptance_rate` | `num_accepted_draft_tokens / num_draft_tokens`. Draft-only. | | `acceptance_histogram` | Sparse `{accepted_draft_count: num_steps}` map with **integer** keys. Zero-count buckets omitted. Excludes the bonus token. | @@ -49,11 +51,31 @@ spec-decode shape. It reads the raw payload captured on the parsed responses so nothing engine-specific leaks into the metrics layer. Adapters are a plugin category (`spec_decode_adapter`) and are resolved by -**auto-detection**, mirroring custom-dataset-loader detection: the parser walks -registered adapters in priority order and uses the first whose `can_adapt` -recognizes the payload by its engine-specific signature — so an adapter claims -only its own payloads and defers on a foreign one. Both methods are classmethods -(adapters are stateless). +**auto-detection**: AIPerf never learns which engine it is pointed at, so every +payload is offered to every registered adapter, and each recognizes its own by +an engine-specific signature. Both methods are classmethods (adapters are +stateless). + +Detection therefore depends on those signatures being **disjoint**, and each +adapter's signature must contain at least one key the other engines do not emit. +The two current signatures are: + +| Adapter | Signature keys | +| --- | --- | +| vLLM | `acceptance_histogram`, `num_spec_steps`, `mean_acceptance_length` | +| TensorRT-LLM | `total_accepted_draft_tokens`, `total_draft_tokens` | + +`mean_acceptance_length` is load-bearing: the other two vLLM keys are also +emitted by TensorRT-LLM, so a signature without it lets the vLLM adapter claim +a TRT-LLM payload. Its `adapt()` then raises `KeyError` on the missing key, so +every TRT-LLM record is dropped with only a warning in the log. + +Plugin **priority does not order detection**: `priority` resolves conflicts only +between plugins registering the same *name*, so `iter_all` yields declaration +order from `plugins.yaml`. Rather than let YAML line order decide, the parser +collects every match and, if more than one adapter claims a payload, logs a +warning and produces **no record** — an ambiguous payload is a bug in AIPerf's +signatures, and dropping it is better than attributing it arbitrarily. ```python @runtime_checkable @@ -66,7 +88,7 @@ class SpecDecodeAdapterProtocol(Protocol): ```mermaid flowchart LR - R["Raw response
metrics.speculative_decoding"] + R["Raw response
root metrics.speculative_decoding (vLLM)
or choices[].speculative_decoding (TRT-LLM)"] P["ParsedResponse
.spec_decode_stats (raw dict)"] A["Engine adapter
(auto-detected)"] N["SpecDecodeAcceptanceRecord
(engine-neutral)"] @@ -132,3 +154,92 @@ The wire object maps to the record one-to-one, except: only `n > 1` triggers this. - **Behind Dynamo** the custom field is currently stripped, so this path is direct-to-vLLM only. + +## The TensorRT-LLM adapter + +`TRTLLMSpecDecodeAdapter` reads TensorRT-LLM's **per-choice** +`speculative_decoding` object. Unlike vLLM's, it rides the choice rather than the +response root — the placement TRT-LLM already uses for its +`avg_decoded_tokens_per_iter` field. In streaming it arrives on the **terminal +chunk's choice** (the one carrying `finish_reason`), not on a trailing usage +chunk. + +```json +{ + "index": 0, + "message": {"role": "assistant", "content": "..."}, + "finish_reason": "stop", + "avg_decoded_tokens_per_iter": 2.1, + "speculative_decoding": { + "acceptance_rate": 0.5, + "total_accepted_draft_tokens": 30, + "total_draft_tokens": 60, + "num_spec_steps": 20, + "acceptance_histogram": [8, 0, 6, 6], + "num_spec_tokens": 3 + } +} +``` + +Read that as: 20 verify steps; 8 accepted nothing, 6 accepted 2 drafts, 6 +accepted all 3. So 30 accepted of 60 proposed, and 2.5 tokens emitted per step +instead of 1. + +The mapping to the record differs from vLLM's in three ways: + +- **Different field names.** `acceptance_rate` → `draft_acceptance_rate`, + `total_accepted_draft_tokens` → `num_accepted_draft_tokens`, + `total_draft_tokens` → `num_draft_tokens`. These are TensorRT-LLM's own names + for the quantities, already used internally for the same counters. +- **`mean_acceptance_length` is derived, not read.** TRT-LLM does not report + it. Its per-choice `avg_decoded_tokens_per_iter` is a different quantity: it + averages decoded tokens over every decoding iteration, including iterations + that drafted nothing, so it reads lower whenever some iterations did not + draft. The adapter computes `1 + num_accepted_draft_tokens / num_spec_steps` — the + record's own definition — so the reported length can never contradict the + histogram it sits beside. +- **`per_step_accepted` / `per_step_drafted` are never populated.** TRT-LLM keeps + per-*position* vectors (a survival curve), not per-step sequences, so it has no + per-step data to report. This is a real fidelity gap versus vLLM's `detailed` + level. + +`acceptance_histogram` uses the same dense `list[int]` encoding as vLLM's and is +inflated by the same shared helper. + +### Enabling it + +TensorRT-LLM has no CLI flag for metrics; these are `TorchLlmArgs` fields set +through the YAML passed to `--extra_llm_api_options`. Enablement is two-level — +the server declares the capability once, and the client asks per request, so +requests that do not ask pay nothing: + +```yaml +# extra-llm-api-config.yaml +per_request_spec_decode_stats: true +``` + +```bash +trtllm-serve --extra_llm_api_options extra-llm-api-config.yaml +``` + +That server-side field is the entire opt-in: a client sends nothing extra, so +AIPerf discovers the payload by shape exactly as it does for vLLM. It is +deliberately independent of `return_perf_metrics`, which also mounts the +Prometheus endpoint. + +### Missing-field and edge cases + +- **`num_spec_tokens` is `null` under `draft_len_schedule`**, where the per-step + draft bound varies by batch size. This is legal and expected: the record's + field is optional, and the histogram-length cross-check is skipped. The + record's identity validators still apply in full. +- **PyTorch backend only.** The C++/TRT path populates spec metrics via + `updateNumTokensPerIteration` and has no per-position vectors, so the field is + simply absent there. +- **Absent when the request never drafted**, rather than reported as zeros. +- **`n > 1`**: because the payload is per-choice, a response carrying more than + one choice is suppressed client-side — a single per-request record cannot + attribute request-level `completion_tokens` to one sequence. +- **Tree drafting** (EAGLE3 dynamic-tree, Medusa) works unchanged: the histogram + counts how many steps produced each output length and encodes no + parent/child structure. diff --git a/docs/tutorials/spec-decode-metrics.md b/docs/tutorials/spec-decode-metrics.md index 1414e8aa68..706bdbee96 100644 --- a/docs/tutorials/spec-decode-metrics.md +++ b/docs/tutorials/spec-decode-metrics.md @@ -17,12 +17,13 @@ cleanly when they are not. > [!NOTE] > There are two ways to get acceptance metrics out of AIPerf; pick by what your server exposes: > -> - **Per-request (this guide):** the stats ride the response body, per choice, so AIPerf -> reads them inline during a normal `aiperf profile` run. Highest fidelity -- you get -> per-request distributions (avg/min/percentiles) and a pooled histogram. Direct-to-vLLM only. +> - **Per-request (this guide):** the stats ride the response body, so AIPerf reads them +> inline during a normal `aiperf profile` run. Highest fidelity -- you get per-request +> distributions (avg/min/percentiles) and a pooled histogram. Supported for vLLM and +> TensorRT-LLM; direct to the engine, not behind Dynamo. > - **Server scrape:** aggregate acceptance is read from the server's Prometheus `/metrics` > endpoint via `--server-metrics`. Works on engines with no per-request reporting -> (SGLang, TensorRT-LLM) and behind Dynamo, but the numbers cover the whole server. +> (SGLang) and behind Dynamo, but the numbers cover the whole server. > > For a per-category acceptance matrix over a ready-made speculative-decoding dataset, see the > [SPEED-Bench tutorial](speed-bench.md), which covers both paths: on the per-request path a @@ -38,15 +39,21 @@ of the metrics reference. ## Prerequisites -- A vLLM server running **speculative decoding** with **per-request metrics enabled** via - `--per-request-spec-decode-metrics summary` (or `detailed`). The field shape tracks vLLM - [PR #48915](https://github.com/vllm-project/vllm/pull/48915) -- confirm your vLLM build - includes it. -- **Direct-to-vLLM only.** Behind Dynamo the custom stats field is currently stripped, so +- A server running **speculative decoding** with **per-request metrics enabled**. Either: + - **vLLM** with `--per-request-spec-decode-metrics summary` (or `detailed`). The field + shape tracks vLLM [PR #48915](https://github.com/vllm-project/vllm/pull/48915) -- + confirm your vLLM build includes it. + - **TensorRT-LLM** with `per_request_spec_decode_stats: true` in the YAML passed to + `--extra_llm_api_options`. PyTorch backend only. + + You do not tell AIPerf which engine you are pointed at -- it recognizes each engine's + payload by shape. +- **Direct to the engine.** Behind Dynamo the custom stats field is currently stripped, so the per-request path is unavailable there (use the server-scrape path instead). -- Streaming works out of the box. In streaming, vLLM sends the metrics on the trailing - usage chunk, so AIPerf requests `stream_options.include_usage` on every streaming run; - no extra flag is needed. +- Streaming works out of the box, though the two engines place the payload differently: + vLLM sends it on the trailing usage chunk -- so AIPerf requests + `stream_options.include_usage` on every streaming run -- while TensorRT-LLM attaches it + to the terminal chunk's choice. No extra flag is needed for either. --- @@ -75,6 +82,42 @@ curl -s localhost:8000/v1/chat/completions \ --- +## Or start a TensorRT-LLM server + +TensorRT-LLM has no CLI flag for metrics — they are `TorchLlmArgs` fields set through the +YAML passed to `--extra_llm_api_options`: + +```yaml +# extra-llm-api-config.yaml +per_request_spec_decode_stats: true +speculative_config: + decoding_type: Eagle + speculative_model_dir: + max_draft_len: 3 +``` + +```bash +trtllm-serve meta-llama/Llama-3.1-8B-Instruct \ + --extra_llm_api_options extra-llm-api-config.yaml +``` + +That YAML field is the whole opt-in -- clients need send nothing, so AIPerf detects the +payload the same way it detects vLLM's. Verify the server is emitting the field — note it +sits on the **choice**, not at the response root: + +```bash +curl -s localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"meta-llama/Llama-3.1-8B-Instruct","messages":[{"role":"user","content":"test"}],"max_tokens":32}' +``` + +Look for `choices[0].speculative_decoding`. A few things differ from vLLM and are expected: +`num_spec_tokens` is `null` when the server runs with `draft_len_schedule`; the per-step +arrays are never populated; and the field is absent entirely for a request that drafted +nothing. Support is PyTorch-backend only. + +--- + ## Run AIPerf No spec-decode-specific flag is required -- run a normal profile. Add `--export-level @@ -193,10 +236,18 @@ If the Spec Decode section, histogram, and `spec_decode_*` fields are all absent the expected clean-degradation behavior -- not an error. Common causes: - speculative decoding is off, or the requests had no verify steps; -- the server was not started with `--per-request-spec-decode-metrics`; +- the server was not started with per-request reporting enabled -- vLLM's + `--per-request-spec-decode-metrics`, or TensorRT-LLM's + `per_request_spec_decode_stats: true`; - the server is behind Dynamo, which strips the custom field (use the [server-scrape path](speed-bench.md#portable-path-server-scrape) instead); -- the vLLM build predates [PR #48915](https://github.com/vllm-project/vllm/pull/48915). +- the vLLM build predates [PR #48915](https://github.com/vllm-project/vllm/pull/48915); +- on TensorRT-LLM, the run used the C++/TRT backend, which has no per-position + vectors -- per-request reporting is PyTorch-backend only; +- on TensorRT-LLM, drafting was switched off at the batch sizes the run reached: + speculation stops above `max_concurrency`, or above the largest batch size in + `draft_len_schedule`, so a run held above that concurrency drafts nothing and + reports no per-request stats. --- diff --git a/src/aiperf/common/models/extracted_payload.py b/src/aiperf/common/models/extracted_payload.py index 9759ec1771..eef9f6c94b 100644 --- a/src/aiperf/common/models/extracted_payload.py +++ b/src/aiperf/common/models/extracted_payload.py @@ -33,6 +33,13 @@ class ExtractedPayload(AIPerfBaseModel): "in ``messages`` are deliberately excluded here so the chat-template " "path does not count them twice.", ) + num_choices: int = Field( + default=1, + ge=1, + description="Sequences requested per prompt (the OpenAI ``n`` " + "parameter); 1 when the payload does not set it. Usage is reported per " + "request, so it spans all of them.", + ) image_count: int = Field( default=0, description="Count of image content parts in the payload.", diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index 76fe36d105..9dfb2a9b81 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -115,22 +115,47 @@ def process_responses( @staticmethod def extract_spec_decode_stats(json_obj: dict[str, Any]) -> dict[str, Any] | None: - """Capture the raw speculative-decoding payload from the response root. - - vLLM nests it at ``metrics.speculative_decoding`` -- on the response body - non-streaming, or on the trailing usage chunk (empty ``choices``) - streaming -- when the server runs with ``--per-request-spec-decode-metrics``. - Captured verbatim and uninterpreted so a ``SpecDecodeAdapterProtocol`` - owns the engine-specific interpretation downstream; None when absent. - - vLLM populates it only for single-sequence requests (``n == 1``), leaving - it ``null`` otherwise, so no client-side ``n > 1`` suppression is needed: - a mixed per-request record can never arise here. + """Capture the raw speculative-decoding payload, wherever the engine put it. + + Two placements are recognized, because the engines genuinely differ: + + - **Response root** ``metrics.speculative_decoding`` (vLLM), on the body + non-streaming or on the trailing usage chunk (empty ``choices``) + streaming, when the server runs with + ``--per-request-spec-decode-metrics``. + - **Per choice** ``choices[0].speculative_decoding`` (TensorRT-LLM), on + the body non-streaming or on the terminal chunk's choice (the one + carrying ``finish_reason``) streaming, when the server sets + ``per_request_spec_decode_stats``. + + Root wins when both somehow appear. Captured verbatim and uninterpreted + so a ``SpecDecodeAdapterProtocol`` owns the engine-specific + interpretation downstream; None when absent. + + A response with more than one choice is an ``n > 1`` non-streaming + request; its per-choice payload is suppressed because a single + per-request record cannot attribute request-level usage to one sequence + -- reporting one choice's acceptance alongside all choices' token count + would be a mixed, misleading record. (``n > 1`` streaming is suppressed + in the parser from the request's own ``n``, since each sequence's stats + arrive on separate chunks and a sequence that never drafted sends none.) + The root placement needs no such guard: vLLM populates it only for + single-sequence requests, leaving it ``null`` otherwise. """ metrics = json_obj.get("metrics") - if not isinstance(metrics, dict): + if isinstance(metrics, dict): + stats = metrics.get("speculative_decoding") + if isinstance(stats, dict): + return stats + + choices = json_obj.get("choices") + if ( + not isinstance(choices, list) + or len(choices) != 1 + or not isinstance(choices[0], dict) + ): return None - stats = metrics.get("speculative_decoding") + stats = choices[0].get("speculative_decoding") return stats if isinstance(stats, dict) else None def build_assistant_turn(self, record: RequestRecord) -> Turn | None: diff --git a/src/aiperf/endpoints/payload_extraction.py b/src/aiperf/endpoints/payload_extraction.py index 92650c8a4e..1a1c978b67 100644 --- a/src/aiperf/endpoints/payload_extraction.py +++ b/src/aiperf/endpoints/payload_extraction.py @@ -32,6 +32,9 @@ def extract_inputs( ``BaseEndpoint.extract_payload_inputs`` directly. """ result = ExtractedPayload() + n = payload.get("n") + if isinstance(n, int) and n > 1: + result.num_choices = n # Reverse index the part-type set: ``{"text": MediaType.TEXT, # "image_url": MediaType.IMAGE, ...}``. Built per-call - the map is # small and per-part lookup is O(1). diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index fb38d66b03..4831126fb2 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -95,7 +95,7 @@ SpecDecodeAdapterTypeStr: TypeAlias = str SpecDecodeAdapterType = plugins.create_enum(PluginType.SPEC_DECODE_ADAPTER, "SpecDecodeAdapterType", module=__name__) -"""Dynamic enum for spec decode adapter. Example: SpecDecodeAdapterType.VLLM""" +"""Dynamic enum for spec decode adapter. Example: SpecDecodeAdapterType.TENSORRT_LLM, SpecDecodeAdapterType.VLLM""" AccuracyGraderTypeStr: TypeAlias = str AccuracyGraderType = plugins.create_enum(PluginType.ACCURACY_GRADER, "AccuracyGraderType", module=__name__) diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 11ab885cfc..89a5c038e3 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -1395,6 +1395,21 @@ spec_decode_adapter: list into the record's sparse map. metadata: {} + tensorrt_llm: + class: aiperf.spec_decode.trtllm_adapter:TRTLLMSpecDecodeAdapter + description: | + Reads TensorRT-LLM's per-choice ``speculative_decoding`` object (emitted + when the server sets ``per_request_spec_decode_stats``) and fills the + engine-neutral + SpecDecodeAcceptanceRecord. Handles chat and completions, streaming and + non-streaming. Derives mean acceptance length from the counts, since + TRT-LLM does not report it (its ``avg_decoded_tokens_per_iter`` averages + over all decoding iterations, not just drafting steps). + Declaration order here is not a detection mechanism: adapters are + distinguished by disjoint payload signatures, and the parser rejects a + payload that more than one adapter claims. + metadata: {} + accuracy_grader: exact_match: class: aiperf.accuracy.graders.exact_match:ExactMatchGrader diff --git a/src/aiperf/records/inference_result_parser.py b/src/aiperf/records/inference_result_parser.py index 0a5683cd69..806eb74945 100644 --- a/src/aiperf/records/inference_result_parser.py +++ b/src/aiperf/records/inference_result_parser.py @@ -2,11 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio import time +from collections.abc import Callable from contextlib import suppress from typing import TYPE_CHECKING, Any import orjson +from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.enums import ExportLevel from aiperf.common.hooks import on_init from aiperf.common.mixins import CommunicationMixin @@ -37,6 +39,30 @@ from aiperf.config.resolution.plan import BenchmarkRun +# ``_extract_spec_decode_acceptance`` is a staticmethod, so it cannot use the +# instance logger CommunicationMixin provides. +_logger = AIPerfLogger(__name__) + + +def _warn_once(warned: set[str] | None, key: str, message: Callable[[], str]) -> None: + """Warn on a detection problem's first occurrence in a run, debug after. + + Ambiguity and a raising adapter come from the installed adapters, not from + any one response, so they recur on every record. ``warned`` is the parser's + per-run memory of what it has already reported; ``None`` warns every time. + """ + if warned is None: + _logger.warning(message) + return + if key in warned: + _logger.debug(message) + return + warned.add(key) + _logger.warning( + lambda: f"{message()} (further occurrences this run are logged at debug)" + ) + + # TODO: Should we create non-tokenizer based parsers? class InferenceResultParser(CommunicationMixin): """InferenceResultParser is responsible for parsing the inference results.""" @@ -49,6 +75,9 @@ def __init__( run=run, ) self.tokenizers: dict[str, Tokenizer] = {} + # Spec-decode detection problems already warned about this run; see + # _warn_once. + self._spec_decode_warned: set[str] = set() self.tokenizer_lock: asyncio.Lock = asyncio.Lock() self.model_endpoint: ModelEndpointInfo = ModelEndpointInfo.from_run(run) EndpointClass = plugins.get_class( @@ -321,12 +350,19 @@ async def process_valid_record( responses=resp, token_counts=token_counts, media_counts=media_counts or MediaCounts(), - spec_decode_acceptance=self._extract_spec_decode_acceptance(resp), + spec_decode_acceptance=self._extract_spec_decode_acceptance( + resp, + self._spec_decode_warned, + num_choices=inputs.num_choices if inputs is not None else 1, + ), ) @staticmethod def _extract_spec_decode_acceptance( responses: list[ParsedResponse], + warned: set[str] | None = None, + *, + num_choices: int = 1, ) -> SpecDecodeAcceptanceRecord | None: """Build the engine-neutral acceptance record via adapter auto-detection. @@ -336,25 +372,73 @@ def _extract_spec_decode_acceptance( priority order and uses the first whose ``can_adapt`` recognizes the payload -- mirroring custom-dataset-loader auto-detection. - Suppresses the record when more than one response carried stats (an - ``n > 1`` streaming request, where each sequence's stats ride its own - finish chunk): the per-request record can't attribute request-level - ``completion_tokens`` to a single sequence, so a mixed record is worse - than none. ``n > 1`` non-streaming needs no client-side guard: vLLM - populates ``metrics.speculative_decoding`` only for single-sequence - requests and leaves it null otherwise. + Suppresses the record for any ``n > 1`` request: the per-request record + can't attribute request-level ``completion_tokens`` to a single + sequence, so a mixed record is worse than none. ``num_choices`` is the + request's own ``n`` and is the primary guard -- a sibling that never + drafted carries no payload, so counting payloads alone would let an + ``n = 2`` stream through with one. More than one response carrying + stats is still refused as a backstop for callers without the request. Counts payloads by truthiness (not ``is not None``) to match the - adapter's ``_find_spec_decode_payload``: an empty ``{}`` is treated as + adapter's ``find_spec_decode_payload``: an empty ``{}`` is treated as absent at both sites, so it never spuriously trips the n > 1 guard. + + Collects every match rather than returning on the first, so a payload + two adapters both claim is a reported error and a dropped record instead + of whichever adapter ``iter_all`` happened to reach first. That order is + ``plugins.yaml`` declaration order -- ``priority`` only resolves + conflicts between plugins registering the same name -- so first-match + would silently make YAML line order decide which engine a record is + attributed to. """ + if num_choices > 1: + return None with_stats = [r for r in responses if r.spec_decode_stats] if len(with_stats) != 1: return None - for _entry, AdapterClass in plugins.iter_all(PluginType.SPEC_DECODE_ADAPTER): - if AdapterClass.can_adapt(responses): - return AdapterClass.adapt(responses) - return None + matches = [] + for entry, AdapterClass in plugins.iter_all(PluginType.SPEC_DECODE_ADAPTER): + try: + if AdapterClass.can_adapt(responses): + matches.append(AdapterClass) + except Exception as e: # noqa: BLE001 - one bad plugin must not fail the record + # ``can_adapt`` is a third-party plugin callback: the protocol + # asks for it to be cheap and side-effect free but cannot stop + # an implementation from raising. Isolating each call keeps one + # bad adapter from failing the whole record and from hiding + # every adapter registered after it. + error = e + _warn_once( + warned, + f"raised:{entry.name}", + lambda name=entry.name, + error=error: f"Spec-decode adapter {name!r} raised during " + f"detection; skipping it: {error!r}", + ) + if len(matches) > 1: + # Always a bug in AIPerf's own signatures, not in the payload: two + # adapters cannot both be right about which engine produced it. + claimants = sorted(a.__name__ for a in matches) + _warn_once( + warned, + f"ambiguous:{','.join(claimants)}", + lambda: f"Ambiguous spec-decode payload claimed by {claimants}; " + "dropping record", + ) + return None + if not matches: + # A payload is present -- the guard above returned otherwise -- yet + # no adapter recognized it. Either an engine changed its wire format + # or this run targets an engine AIPerf has no adapter for. Debug + # rather than warning: on an unsupported engine it would fire for + # every record in the run. + _logger.debug( + lambda: "Spec-decode payload present but claimed by no adapter: " + f"{with_stats[0].spec_decode_stats!r}" + ) + return None + return matches[0].adapt(responses) async def compute_input_token_count( self, diff --git a/src/aiperf/spec_decode/__init__.py b/src/aiperf/spec_decode/__init__.py index 4d780771bf..3f5bfa657e 100644 --- a/src/aiperf/spec_decode/__init__.py +++ b/src/aiperf/spec_decode/__init__.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 from aiperf.spec_decode.protocols import SpecDecodeAdapterProtocol +from aiperf.spec_decode.trtllm_adapter import TRTLLMSpecDecodeAdapter from aiperf.spec_decode.vllm_adapter import VLLMSpecDecodeAdapter __all__ = [ "SpecDecodeAdapterProtocol", + "TRTLLMSpecDecodeAdapter", "VLLMSpecDecodeAdapter", ] diff --git a/src/aiperf/spec_decode/_shared.py b/src/aiperf/spec_decode/_shared.py new file mode 100644 index 0000000000..5841a35626 --- /dev/null +++ b/src/aiperf/spec_decode/_shared.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The engine-neutral parts of adapting a spec-decode payload. + +An adapter exists to absorb one engine's on-the-wire shape, but two pieces of +that work are identical for every engine: locating the captured payload among a +request's responses, and converting the dense acceptance histogram every engine +sends into the sparse map the neutral record stores. Both are subtle enough that +duplicating them across adapters would let the copies drift, so they live here. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from aiperf.common.models import ParsedResponse + + +def find_spec_decode_payload( + responses: list[ParsedResponse], +) -> dict[str, Any] | None: + """Return the last non-empty ``spec_decode_stats`` payload across responses. + + An engine attaches the payload once per request: on the body non-streaming, + or on a trailing chunk streaming. Walking from the end mirrors + ``find_last_non_empty_usage`` and tolerates either layout. + """ + for response in reversed(responses): + stats = response.spec_decode_stats + if stats: + return stats + return None + + +def inflate_dense_histogram( + raw_histogram: Any, num_spec_tokens: int | None +) -> dict[int, int]: + """Validate a dense on-the-wire histogram and inflate it to the sparse map. + + Raises ``TypeError`` or ``ValueError`` on a malformed histogram; callers + catch and degrade to ``None`` so one bad payload cannot abort a run. + """ + # Validate the shape AND every element before filtering: a str/dict is + # iterable (enumerate would build a bucket per character/key), and filtering + # on truthiness first would silently drop a falsey malformed entry + # (None/False/0.0) while keeping a truthy one that coerces to zero ("0"), + # which would violate the record's zero-buckets-omitted invariant. + # ``type(...) is int`` rather than isinstance so bools -- an int subclass -- + # are rejected too. + if not isinstance(raw_histogram, list) or not all( + type(count) is int for count in raw_histogram + ): + raise TypeError( + f"acceptance_histogram must be a list of ints, got {raw_histogram!r}" + ) + # Length is num_spec_tokens + 1 (one bucket per accepted count from 0..k). + # Enforcing it rejects a payload whose bucket indices exceed the draft + # budget -- j > k is physically impossible, yet satisfies the record's + # arithmetic validators, so it would otherwise be reported as a real + # acceptance length. Optional on the record, so only enforce it when the + # server sent a fixed bound; engines that draft variable-length report None. + if num_spec_tokens is not None and len(raw_histogram) != num_spec_tokens + 1: + raise ValueError( + f"acceptance_histogram has {len(raw_histogram)} buckets, " + f"expected num_spec_tokens + 1 = {num_spec_tokens + 1}" + ) + return {j: count for j, count in enumerate(raw_histogram) if count != 0} diff --git a/src/aiperf/spec_decode/trtllm_adapter.py b/src/aiperf/spec_decode/trtllm_adapter.py new file mode 100644 index 0000000000..492ce8fcb8 --- /dev/null +++ b/src/aiperf/spec_decode/trtllm_adapter.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from pydantic import ValidationError + +from aiperf.common.aiperf_logger import AIPerfLogger +from aiperf.common.models import SpecDecodeAcceptanceRecord +from aiperf.common.models.record_models import find_last_non_empty_usage +from aiperf.spec_decode._shared import ( + find_spec_decode_payload, + inflate_dense_histogram, +) + +if TYPE_CHECKING: + from aiperf.common.models import ParsedResponse + +_logger = AIPerfLogger(__name__) + +ENGINE = "tensorrt_llm" + +# Keys that identify a payload as TensorRT-LLM's ``speculative_decoding`` shape. +# Both are TRT-LLM's own names for these quantities and neither appears in +# vLLM's payload, which uses ``num_accepted_draft_tokens`` / ``num_draft_tokens`` +# -- so the two signatures are disjoint in both directions and each adapter +# claims only its own payloads. The keys the two engines *do* share +# (``acceptance_histogram``, ``num_spec_steps``, ``num_spec_tokens``) are +# deliberately not used to discriminate. +_TRTLLM_SIGNATURE_KEYS = ("total_accepted_draft_tokens", "total_draft_tokens") + + +def _is_trtllm_payload(payload: Any) -> bool: + return isinstance(payload, dict) and all( + key in payload for key in _TRTLLM_SIGNATURE_KEYS + ) + + +class TRTLLMSpecDecodeAdapter: + """Fills the acceptance record from TensorRT-LLM's ``speculative_decoding``. + + Reads the per-choice ``speculative_decoding`` object emitted when the server + sets ``per_request_spec_decode_stats`` in the YAML passed to + ``--extra_llm_api_options``. That is the whole opt-in -- the client sends + nothing extra. Present on chat and completions; in streaming + it rides the terminal chunk's choice (the one carrying ``finish_reason``), + alongside TRT-LLM's existing ``avg_decoded_tokens_per_iter``. + + Two differences from vLLM's payload, both deliberate upstream: + + - **No mean acceptance length.** TRT-LLM does not report it. Its per-choice + ``avg_decoded_tokens_per_iter`` is a different quantity: it averages over + every decoding iteration, including ones that drafted nothing. This + adapter derives acceptance length from the counts instead, so the value + can never disagree with the histogram it is reported beside. + - **Its own field names** -- ``acceptance_rate`` and ``total_*`` rather than + ``draft_acceptance_rate`` and ``num_*`` -- matching the vocabulary TRT-LLM + already uses internally for these counters. + + ``per_step_accepted`` / ``per_step_drafted`` are never populated: TRT-LLM + keeps per-*position* vectors, not per-step sequences, so it has no per-step + data to report. Support is PyTorch-backend only; the C++/TRT path has no + per-position vectors and emits nothing. + """ + + @classmethod + def can_adapt(cls, responses: list[ParsedResponse]) -> bool: + return _is_trtllm_payload(find_spec_decode_payload(responses)) + + @classmethod + def adapt( + cls, responses: list[ParsedResponse] + ) -> SpecDecodeAcceptanceRecord | None: + payload = find_spec_decode_payload(responses) + if payload is None: + return None + + try: + num_spec_tokens = payload.get("num_spec_tokens") + histogram = inflate_dense_histogram( + payload["acceptance_histogram"], num_spec_tokens + ) + num_spec_steps = payload["num_spec_steps"] + num_accepted = payload["total_accepted_draft_tokens"] + if num_spec_steps <= 0: + # The record's own definition of mean acceptance length divides + # by this. TRT-LLM omits the block entirely for a request that + # never drafted, so reaching here means a malformed payload + # rather than a legitimately step-free request. + raise ValueError( + f"num_spec_steps must be positive, got {num_spec_steps}" + ) + usage = find_last_non_empty_usage(responses) + return SpecDecodeAcceptanceRecord( + engine=ENGINE, + # Derived rather than read: both operands come from this same + # payload, so the reported length cannot contradict the counts. + mean_acceptance_length=1 + num_accepted / num_spec_steps, + draft_acceptance_rate=payload["acceptance_rate"], + acceptance_histogram=histogram, + num_accepted_draft_tokens=num_accepted, + num_draft_tokens=payload["total_draft_tokens"], + num_spec_steps=num_spec_steps, + num_spec_tokens=num_spec_tokens, + completion_tokens=usage.completion_tokens if usage else None, + ) + except ( + KeyError, + TypeError, + ValueError, + ZeroDivisionError, + AttributeError, + ValidationError, + ) as e: + # Degrade to None on an unexpected shape: a single malformed payload + # must not abort a run. can_adapt already matched the TRT-LLM + # signature, so this only fires when a signature-matching body still + # has a broken value -- including one whose counts violate the + # record's aggregate identities, which is the case worth dropping + # rather than trusting. + error = e + _logger.warning( + lambda: f"Ignoring malformed TensorRT-LLM spec-decode payload " + f"{payload!r}: {error!r}" + ) + return None diff --git a/src/aiperf/spec_decode/vllm_adapter.py b/src/aiperf/spec_decode/vllm_adapter.py index 32d60637bf..8936d0409c 100644 --- a/src/aiperf/spec_decode/vllm_adapter.py +++ b/src/aiperf/spec_decode/vllm_adapter.py @@ -10,6 +10,10 @@ from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.models import SpecDecodeAcceptanceRecord from aiperf.common.models.record_models import find_last_non_empty_usage +from aiperf.spec_decode._shared import ( + find_spec_decode_payload, + inflate_dense_histogram, +) if TYPE_CHECKING: from aiperf.common.models import ParsedResponse @@ -21,9 +25,24 @@ # Keys that identify a payload as vLLM's ``speculative_decoding`` shape. # ``can_adapt`` checks for these so that, once other engines populate the same # raw slot, this adapter claims only its own payloads (auto-detection) rather -# than greedily matching on mere presence. Both are always emitted by vLLM, +# than greedily matching on mere presence. All three are always emitted by vLLM, # including the zero-step case (``acceptance_histogram`` is all-zero but present). -_VLLM_SIGNATURE_KEYS = ("acceptance_histogram", "num_spec_steps") +# +# ``mean_acceptance_length`` is the discriminating key and must stay here. +# TensorRT-LLM emits ``acceptance_histogram`` and ``num_spec_steps`` too, so a +# signature without a vLLM-unique key matches a TRT-LLM payload. That happened: +# this adapter claimed every TRT-LLM payload, ``adapt`` raised ``KeyError`` on +# the missing key, and the records were lost -- under first-match detection the +# TRT-LLM adapter was never even tried. TRT-LLM does not report mean acceptance +# length at all; its per-choice ``avg_decoded_tokens_per_iter`` averages over +# every decoding iteration, not just drafting steps. Requiring the key costs +# nothing: ``adapt`` already reads it unguarded, so a payload lacking it could +# never produce a record. +_VLLM_SIGNATURE_KEYS = ( + "acceptance_histogram", + "num_spec_steps", + "mean_acceptance_length", +) def _is_vllm_payload(payload: Any) -> bool: @@ -32,22 +51,6 @@ def _is_vllm_payload(payload: Any) -> bool: ) -def _find_spec_decode_payload( - responses: list[ParsedResponse], -) -> dict[str, Any] | None: - """Return the last non-empty ``spec_decode_stats`` payload across responses. - - vLLM attaches the payload once per request at the response root: on the body - non-streaming, or on the trailing usage chunk streaming. Walking from the end - mirrors ``find_last_non_empty_usage`` and tolerates either layout. - """ - for response in reversed(responses): - stats = response.spec_decode_stats - if stats: - return stats - return None - - class VLLMSpecDecodeAdapter: """Fills the acceptance record from vLLM's ``metrics.speculative_decoding``. @@ -66,53 +69,21 @@ class VLLMSpecDecodeAdapter: @classmethod def can_adapt(cls, responses: list[ParsedResponse]) -> bool: - return _is_vllm_payload(_find_spec_decode_payload(responses)) + return _is_vllm_payload(find_spec_decode_payload(responses)) @classmethod def adapt( cls, responses: list[ParsedResponse] ) -> SpecDecodeAcceptanceRecord | None: - payload = _find_spec_decode_payload(responses) + payload = find_spec_decode_payload(responses) if payload is None: return None try: - # vLLM sends a dense ``list[int]`` (index j -> step count); inflate - # into the neutral record's sparse map, dropping zero-count buckets. - # Validate the shape AND every element before filtering: a str/dict - # is iterable (enumerate would build a bucket per character/key), - # and filtering on truthiness first would silently drop a falsey - # malformed entry (None/False/0.0) while keeping a truthy one that - # coerces to zero ("0"), which would violate the record's - # zero-buckets-omitted invariant. ``type(...) is int`` rather than - # isinstance so bools -- an int subclass -- are rejected too. - raw_histogram = payload["acceptance_histogram"] - if not isinstance(raw_histogram, list) or not all( - type(count) is int for count in raw_histogram - ): - raise TypeError( - "acceptance_histogram must be a list of ints, got " - f"{raw_histogram!r}" - ) - # Length is num_spec_tokens + 1 (one bucket per accepted count from - # 0..k). Enforcing it rejects a payload whose bucket indices exceed - # the draft budget -- j > k is physically impossible, yet satisfies - # the record's arithmetic validators, so it would otherwise be - # reported as a real acceptance length. Doubles as a tripwire if the - # still-unmerged upstream PR changes the histogram shape. - # Optional on the record, so only enforce when the server sent it. num_spec_tokens = payload.get("num_spec_tokens") - if ( - num_spec_tokens is not None - and len(raw_histogram) != num_spec_tokens + 1 - ): - raise ValueError( - f"acceptance_histogram has {len(raw_histogram)} buckets, " - f"expected num_spec_tokens + 1 = {num_spec_tokens + 1}" - ) - histogram = { - j: count for j, count in enumerate(raw_histogram) if count != 0 - } + histogram = inflate_dense_histogram( + payload["acceptance_histogram"], num_spec_tokens + ) usage = find_last_non_empty_usage(responses) return SpecDecodeAcceptanceRecord( engine=ENGINE, @@ -122,7 +93,7 @@ def adapt( num_accepted_draft_tokens=payload["num_accepted_draft_tokens"], num_draft_tokens=payload["num_draft_tokens"], num_spec_steps=payload["num_spec_steps"], - num_spec_tokens=payload.get("num_spec_tokens"), + num_spec_tokens=num_spec_tokens, completion_tokens=usage.completion_tokens if usage else None, per_step_accepted=payload.get("per_step_accepted"), per_step_drafted=payload.get("per_step_drafted"), diff --git a/tests/aiperf_mock_server/README.md b/tests/aiperf_mock_server/README.md index e5988994f0..9ee848ddc1 100644 --- a/tests/aiperf_mock_server/README.md +++ b/tests/aiperf_mock_server/README.md @@ -140,6 +140,25 @@ Configuration via CLI arguments or environment variables (`MOCK_SERVER_` prefix) | `--error-rate` | `0.0` | Error injection rate (0-100%) | | `--random-seed` | `None` | Seed for reproducible errors | +### Speculative Decoding + +Emits a per-request speculative-decoding acceptance payload on chat and +completions responses (streaming and non-streaming), so AIPerf's spec-decode +adapters and metrics can be exercised without a GPU. Off by default, so +responses are unchanged unless enabled. + +| Option | Default | Description | +|--------|---------|-------------| +| `--spec-decode-enabled` | `false` | Emit the acceptance payload | +| `--spec-decode-flavor` | `vllm` | Wire format: `vllm` (response-root `metrics.speculative_decoding`) or `trtllm` (per-choice `speculative_decoding`, TensorRT-LLM field names) | +| `--spec-decode-num-spec-tokens` | `3` | Draft budget per step, emitted as `num_spec_tokens` | +| `--spec-decode-acceptance-rate` | `0.5` | Target fraction of drafted tokens accepted (0.0-1.0) | + +The emitted counts are derived from each response's own `completion_tokens`, so +acceptance and usage stay consistent. As with every option, these can also be +set through `MOCK_SERVER_`-prefixed environment variables, e.g. +`MOCK_SERVER_SPEC_DECODE_ENABLED=true`. + ### GPU Telemetry | Option | Default | Description | diff --git a/tests/aiperf_mock_server/app.py b/tests/aiperf_mock_server/app.py index af6de7c309..eb1fcedfa5 100644 --- a/tests/aiperf_mock_server/app.py +++ b/tests/aiperf_mock_server/app.py @@ -65,6 +65,8 @@ from aiperf_mock_server.utils import ( RequestCtx, anthropic_stop_reason, + attach_spec_decode, + build_spec_decode_payload, make_ctx, simulate_anthropic_cache, stream_anthropic_messages, @@ -339,7 +341,7 @@ def _build_chat_response_data(ctx: RequestCtx) -> dict[str, Any]: message: dict[str, Any] = {"role": "assistant", "content": ctx.content} if ctx.reasoning_content: message["reasoning_content"] = ctx.reasoning_content - return { + response = { "id": ctx.request_id, "object": "chat.completion", "created": int(time.time()), @@ -349,6 +351,7 @@ def _build_chat_response_data(ctx: RequestCtx) -> dict[str, Any]: ], "usage": ctx.usage, } + return attach_spec_decode(response, build_spec_decode_payload(ctx)) @app.post("/v1/chat/completions", response_model=None) @@ -573,7 +576,7 @@ async def _anthropic_stream_wrapper( def _build_completion_response_data(ctx: RequestCtx) -> dict[str, Any]: """Build non-streaming text completion response data.""" - return { + response = { "id": ctx.request_id, "object": "text_completion", "created": int(time.time()), @@ -587,6 +590,7 @@ def _build_completion_response_data(ctx: RequestCtx) -> dict[str, Any]: ], "usage": ctx.usage, } + return attach_spec_decode(response, build_spec_decode_payload(ctx)) @app.post("/v1/completions", response_model=None) diff --git a/tests/aiperf_mock_server/config.py b/tests/aiperf_mock_server/config.py index 1d4c749a0d..ca8cc1e4df 100644 --- a/tests/aiperf_mock_server/config.py +++ b/tests/aiperf_mock_server/config.py @@ -317,6 +317,60 @@ def apply_flags(self) -> Self: Parameter(name="--itl-jitter-cv"), ] = 0.0 + # Speculative decoding: synthesises a self-consistent per-request + # acceptance payload so the spec-decode path can be exercised end to end + # without a GPU. Off by default so every other test's responses are + # unchanged. + spec_decode_enabled: Annotated[ + bool, + Field( + description=( + "Emit a per-request speculative-decoding acceptance payload on " + "chat and completions responses." + ) + ), + Parameter(name="--spec-decode-enabled"), + ] = False + + spec_decode_flavor: Annotated[ + Literal["vllm", "trtllm"], + Field( + description=( + "Which engine's wire format to emit. 'vllm' nests the payload " + "at the response root under metrics.speculative_decoding; " + "'trtllm' attaches it per choice as speculative_decoding with " + "TensorRT-LLM's field names." + ) + ), + Parameter(name="--spec-decode-flavor"), + ] = "vllm" + + # None (num_spec_tokens: null, as under TensorRT-LLM's draft_len_schedule) + # is reachable only by setting the config directly, as the unit tests do; + # the CLI and environment parse an int. + spec_decode_num_spec_tokens: Annotated[ + int | None, + Field( + description="Draft budget k per step, emitted as num_spec_tokens.", + ge=1, + ), + Parameter(name="--spec-decode-num-spec-tokens"), + ] = 3 + + spec_decode_acceptance_rate: Annotated[ + float, + Field( + description=( + "Target fraction of proposed draft tokens accepted. The emitted " + "counts are derived from this and the response's own token " + "count, so acceptance and usage stay mutually consistent." + ), + ge=0.0, + le=1.0, + ), + Parameter(name="--spec-decode-acceptance-rate"), + ] = 0.5 + # Embedding latency: base + per_input * num_inputs embedding_base_latency: Annotated[ float, diff --git a/tests/aiperf_mock_server/utils.py b/tests/aiperf_mock_server/utils.py index 71c8a113ec..d1c5dc8563 100644 --- a/tests/aiperf_mock_server/utils.py +++ b/tests/aiperf_mock_server/utils.py @@ -521,16 +521,114 @@ def _partial_usage(ctx: "RequestCtx", completion_tokens: int) -> dict[str, Any]: } +def build_spec_decode_payload(ctx: "RequestCtx") -> dict[str, Any] | None: + """Synthesise a self-consistent per-request spec-decode acceptance payload. + + Derived from the response's own ``completion_tokens`` so the acceptance + numbers reconcile with the usage the mock reports beside them -- the same + reconciliation a live run performs against a real engine. + + The three identities AIPerf enforces hold **by construction**, which is the + point: an end-to-end test should prove the plumbing carries a valid record, + not that the mock happened to guess consistent numbers. + + ``num_accepted`` is spread as evenly as possible across the steps with no + RNG, so a run is reproducible. The resulting distribution is deliberately + degenerate (two adjacent buckets) -- modelling realistic acceptance + statistics is out of scope; the zero buckets below ``base`` still exercise + the adapters' sparse-map inflation. + """ + if not server_config.spec_decode_enabled: + return None + + # None means "no fixed per-step bound" on the wire, but generation still + # needs a concrete draft length to work from. + num_spec_tokens = server_config.spec_decode_num_spec_tokens + k = num_spec_tokens if num_spec_tokens is not None else 3 + rate = server_config.spec_decode_acceptance_rate + + completion_tokens = int(ctx.usage.get("completion_tokens") or 0) + if completion_tokens <= 0: + return None + + # The mock attributes every completion token to a verify step: one + # always-accepted bonus token per step plus whatever drafts it accepted, so + # ``num_spec_steps + num_accepted == completion_tokens``. That is a modelling + # choice, not a physical identity -- on a real engine prefill and iterations + # that drafted nothing also emit tokens. Derive the accepted count from the + # token count instead of rounding it independently: independent rounding + # disagreed with the usage reported beside it for ``completion_tokens <= 2``, + # emitting acceptance that implied more tokens than the response contained. + # + # The floor on steps is what keeps accepted <= drafted. A step contributes at + # most k + 1 tokens, so fewer than ceil(completion_tokens / (k + 1)) steps + # could not have produced the tokens being claimed. + minimum_steps = math.ceil(completion_tokens / (k + 1)) + num_spec_steps = max(1, minimum_steps, round(completion_tokens / (1 + rate * k))) + num_draft_tokens = k * num_spec_steps + num_accepted = completion_tokens - num_spec_steps + + # base <= k always, since num_accepted <= num_draft_tokens = k * steps; and + # base == k forces rem == 0, so base + 1 never runs off the histogram. + base, rem = divmod(num_accepted, num_spec_steps) + histogram = [0] * (k + 1) + histogram[base] += num_spec_steps - rem + if rem: + histogram[base + 1] += rem + + if server_config.spec_decode_flavor == "trtllm": + return { + "acceptance_rate": num_accepted / num_draft_tokens, + "total_accepted_draft_tokens": num_accepted, + "total_draft_tokens": num_draft_tokens, + "num_spec_steps": num_spec_steps, + "acceptance_histogram": histogram, + "num_spec_tokens": num_spec_tokens, + } + return { + # Recomputed from the final integers rather than the targets, so + # rounding cannot make the emitted floats disagree with the counts. + "mean_acceptance_length": 1 + num_accepted / num_spec_steps, + "draft_acceptance_rate": num_accepted / num_draft_tokens, + "acceptance_histogram": histogram, + "num_spec_steps": num_spec_steps, + "num_accepted_draft_tokens": num_accepted, + "num_draft_tokens": num_draft_tokens, + "num_spec_tokens": num_spec_tokens, + } + + +def attach_spec_decode( + response: dict[str, Any], payload: dict[str, Any] | None +) -> dict[str, Any]: + """Attach the payload where the configured engine flavour puts it. + + vLLM nests it at the response root under ``metrics``; TensorRT-LLM attaches + it to the choice. No key is added at all when disabled, so responses stay + byte-identical for every test that does not ask for this. + """ + if payload is None: + return response + if server_config.spec_decode_flavor == "trtllm": + choices = response.get("choices") + if isinstance(choices, list) and len(choices) == 1: + choices[0]["speculative_decoding"] = payload + else: + response["metrics"] = {"speculative_decoding": payload} + return response + + async def stream_chat_completion( ctx: RequestCtx, endpoint: str, include_usage: bool ) -> AsyncGenerator[bytes, None]: """Stream chat completion tokens as SSE chunks.""" has_reasoning = bool(ctx.reasoning_content_tokens) + num_reasoning = len(ctx.reasoning_content_tokens) try: # Stream reasoning tokens first (if any) completion_so_far = 0 - for token in ctx.reasoning_content_tokens: + for ri, token in enumerate(ctx.reasoning_content_tokens): await ctx.latency_sim.wait_for_next_token() record_streamed_token(endpoint, ctx.model) completion_so_far += 1 @@ -548,6 +646,15 @@ async def stream_chat_completion( } if ctx.continuous_usage: chunk["usage"] = _partial_usage(ctx, completion_so_far) + if ( + not ctx.tokens + and ri == num_reasoning - 1 + and server_config.spec_decode_flavor == "trtllm" + ): + # Reasoning used the whole budget, so no content chunk follows: + # this is the last chunk with a choice, and the payload must + # ride it or it is never sent. + attach_spec_decode(chunk, build_spec_decode_payload(ctx)) yield _sse(chunk) # Stream output tokens, bundling the first chunk when requested. @@ -576,20 +683,25 @@ async def stream_chat_completion( } if ctx.continuous_usage: chunk["usage"] = _partial_usage(ctx, completion_so_far) + if gi == num_groups - 1 and server_config.spec_decode_flavor == "trtllm": + # TRT-LLM attaches acceptance to the choice bearing + # finish_reason, not to a trailing usage chunk. + attach_spec_decode(chunk, build_spec_decode_payload(ctx)) yield _sse(chunk) # Final usage chunk (if requested) if include_usage: - yield _sse( - { - "id": ctx.request_id, - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": ctx.model, - "choices": [], - "usage": ctx.usage, - } - ) + final_chunk = { + "id": ctx.request_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": ctx.model, + "choices": [], + "usage": ctx.usage, + } + if server_config.spec_decode_flavor == "vllm": + attach_spec_decode(final_chunk, build_spec_decode_payload(ctx)) + yield _sse(final_chunk) ctx.latency_sim.mark_finished() yield _SSE_DONE @@ -612,27 +724,31 @@ async def stream_text_completion( if i == num_tokens - 1: choice["finish_reason"] = ctx.finish_reason - yield _sse( - { - "id": ctx.request_id, - "object": "text_completion", - "created": int(time.time()), - "model": ctx.model, - "choices": [choice], - } - ) + chunk = { + "id": ctx.request_id, + "object": "text_completion", + "created": int(time.time()), + "model": ctx.model, + "choices": [choice], + } + if i == num_tokens - 1 and server_config.spec_decode_flavor == "trtllm": + # TRT-LLM attaches acceptance to the choice bearing + # finish_reason, not to a trailing usage chunk. + attach_spec_decode(chunk, build_spec_decode_payload(ctx)) + yield _sse(chunk) if include_usage: - yield _sse( - { - "id": ctx.request_id, - "object": "text_completion", - "created": int(time.time()), - "model": ctx.model, - "choices": [], - "usage": ctx.usage, - } - ) + final_chunk = { + "id": ctx.request_id, + "object": "text_completion", + "created": int(time.time()), + "model": ctx.model, + "choices": [], + "usage": ctx.usage, + } + if server_config.spec_decode_flavor == "vllm": + attach_spec_decode(final_chunk, build_spec_decode_payload(ctx)) + yield _sse(final_chunk) ctx.latency_sim.mark_finished() yield _SSE_DONE diff --git a/tests/unit/endpoints/test_spec_decode_capture.py b/tests/unit/endpoints/test_spec_decode_capture.py index bf6d07b81e..4028541b73 100644 --- a/tests/unit/endpoints/test_spec_decode_capture.py +++ b/tests/unit/endpoints/test_spec_decode_capture.py @@ -172,3 +172,99 @@ def test_parse_response_absent_stats_returns_none( parsed = completions_endpoint.parse_response(_mock_response(json_obj)) assert parsed is not None assert parsed.spec_decode_stats is None + + +# TensorRT-LLM attaches its payload per choice rather than at the response root, +# using its own field names. Shape mirrors the TRT-LLM adapter's tests. +TRTLLM_STATS = { + "acceptance_rate": 0.5, + "total_accepted_draft_tokens": 30, + "total_draft_tokens": 60, + "num_spec_steps": 20, + "acceptance_histogram": [8, 0, 6, 6], + "num_spec_tokens": 3, +} + + +class TestPerChoiceCapture: + """The per-choice placement TensorRT-LLM uses, on both endpoints. + + One case per endpoint rather than per endpoint-and-stream-shape: the capture + helper is a pure function of the response body and never inspects the + endpoint type, ``object``, or whether the payload arrived on a stream chunk. + Both endpoints are kept because they have separate ``_parse_json_response`` + implementations, each calling the helper independently, so a single case + would leave the other call site untested. + """ + + def test_chat_non_streaming(self, chat_endpoint: ChatEndpoint) -> None: + json_obj = { + "object": "chat.completion", + "choices": [ + { + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + "speculative_decoding": TRTLLM_STATS, + } + ], + } + parsed = chat_endpoint.parse_response(_mock_response(json_obj)) + assert parsed is not None + assert parsed.spec_decode_stats == TRTLLM_STATS + + def test_completions_non_streaming( + self, completions_endpoint: CompletionsEndpoint + ) -> None: + json_obj = { + "object": "text_completion", + "choices": [ + { + "text": "hi", + "finish_reason": "stop", + "speculative_decoding": TRTLLM_STATS, + } + ], + } + parsed = completions_endpoint.parse_response(_mock_response(json_obj)) + assert parsed is not None + assert parsed.spec_decode_stats == TRTLLM_STATS + + def test_multi_choice_is_suppressed(self, chat_endpoint: ChatEndpoint) -> None: + """``n > 1`` non-streaming: a single per-request record cannot attribute + request-level usage to one sequence, so no payload is captured.""" + json_obj = { + "object": "chat.completion", + "choices": [ + {"message": {"content": "a"}, "speculative_decoding": TRTLLM_STATS}, + {"message": {"content": "b"}, "speculative_decoding": TRTLLM_STATS}, + ], + } + parsed = chat_endpoint.parse_response(_mock_response(json_obj)) + assert parsed is not None + assert parsed.spec_decode_stats is None + + def test_root_placement_wins_when_both_present( + self, chat_endpoint: ChatEndpoint + ) -> None: + """Not expected on the wire, but the precedence must be deterministic.""" + json_obj = { + "object": "chat.completion", + "choices": [ + {"message": {"content": "hi"}, "speculative_decoding": TRTLLM_STATS} + ], + "metrics": {"speculative_decoding": STATS}, + } + parsed = chat_endpoint.parse_response(_mock_response(json_obj)) + assert parsed is not None + assert parsed.spec_decode_stats == STATS + + def test_non_dict_per_choice_payload_ignored( + self, chat_endpoint: ChatEndpoint + ) -> None: + json_obj = { + "object": "chat.completion", + "choices": [{"message": {"content": "hi"}, "speculative_decoding": "x"}], + } + parsed = chat_endpoint.parse_response(_mock_response(json_obj)) + assert parsed is not None + assert parsed.spec_decode_stats is None diff --git a/tests/unit/records/test_inference_result_parser.py b/tests/unit/records/test_inference_result_parser.py index 395b0dc197..620ef30d44 100644 --- a/tests/unit/records/test_inference_result_parser.py +++ b/tests/unit/records/test_inference_result_parser.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import logging from unittest.mock import AsyncMock, MagicMock, patch import orjson @@ -1003,3 +1004,132 @@ async def test_apply_chat_template_off_falls_back_to_bare_encode( assert result == 8 tokenizer._tokenizer.apply_chat_template.assert_not_called() tokenizer.encode.assert_called_once() + + +@pytest.mark.asyncio +class TestSpecDecodeDetectionWarnings: + """A detection problem is reported once per run, not once per record. + + Ambiguity and a raising adapter are properties of the installed adapters, + not of any one response, so they recur on every record of a run. One + warning carries the signal; repeats are kept at debug for diagnosis. + """ + + LOGGER = "aiperf.records.inference_result_parser" + + class _GreedyAdapter: + """Claims any payload, so it collides with the vLLM adapter.""" + + @classmethod + def can_adapt(cls, responses: list[ParsedResponse]) -> bool: + return any(r.spec_decode_stats for r in responses) + + @classmethod + def adapt(cls, responses: list[ParsedResponse]) -> None: + raise AssertionError("an ambiguous payload must never be adapted") + + class _ExplodingAdapter: + @classmethod + def can_adapt(cls, responses: list[ParsedResponse]) -> bool: + raise RuntimeError("third-party adapter blew up") + + @classmethod + def adapt(cls, responses: list[ParsedResponse]) -> None: + raise AssertionError("must never be reached") + + @staticmethod + def _responses() -> list[ParsedResponse]: + from tests.unit.spec_decode.test_vllm_adapter import SUMMARY_PAYLOAD + + return [ + ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), + make_parsed_response(prompt_tokens=10, completion_tokens=7), + ] + + async def _parse_twice(self, parser, request_record, caplog, adapter, name): + from aiperf.plugin.enums import PluginType + from tests.harness import mock_plugin + + setup_parser_responses(parser, self._responses()) + with ( + mock_plugin(PluginType.SPEC_DECODE_ADAPTER, name, adapter), + caplog.at_level(logging.DEBUG, logger=self.LOGGER), + ): + await parser.process_valid_record(request_record) + await parser.process_valid_record(request_record) + return [r for r in caplog.records if r.name == self.LOGGER] + + async def test_ambiguity_warns_once_per_run( + self, server_token_parser, request_record, caplog + ): + records = await self._parse_twice( + server_token_parser, request_record, caplog, self._GreedyAdapter, "greedy" + ) + ambiguous = [r for r in records if "Ambiguous" in r.getMessage()] + assert [r.levelno for r in ambiguous] == [logging.WARNING, logging.DEBUG] + + async def test_raising_adapter_warns_once_per_run( + self, server_token_parser, request_record, caplog + ): + records = await self._parse_twice( + server_token_parser, + request_record, + caplog, + self._ExplodingAdapter, + "exploding", + ) + raised = [r for r in records if "raised during detection" in r.getMessage()] + assert [r.levelno for r in raised] == [logging.WARNING, logging.DEBUG] + + +@pytest.mark.asyncio +class TestSpecDecodeRequestCardinality: + """An ``n > 1`` request never yields a single-sequence acceptance record. + + Counting payloads alone is not enough: a sibling sequence that never + drafted carries no payload, so an ``n = 2`` stream can arrive with exactly + one, beside request-level usage covering both sequences. The request's own + ``n`` is the ground truth for how many sequences the usage spans. + """ + + @staticmethod + def _with_n(request_record, n: int | None): + payload = orjson.loads(request_record.request_info.payload_bytes) + if n is None: + payload.pop("n", None) + else: + payload["n"] = n + request_record.request_info.payload_bytes = orjson.dumps(payload) + return request_record + + async def _parse(self, parser, request_record, spy_tokenizer): + from tests.unit.spec_decode.test_vllm_adapter import SUMMARY_PAYLOAD + + parser.get_tokenizer = AsyncMock(return_value=spy_tokenizer) + setup_parser_responses( + parser, + [ + ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), + make_parsed_response(prompt_tokens=10, completion_tokens=7), + ], + ) + # The same single decode parse_request_record hands to + # process_valid_record. + inputs = parser._extract_payload_inputs_for_record(request_record) + return await parser.process_valid_record(request_record, inputs=inputs) + + async def test_single_sequence_request_yields_a_record( + self, server_token_parser, request_record, spy_tokenizer + ): + result = await self._parse( + server_token_parser, self._with_n(request_record, None), spy_tokenizer + ) + assert result.spec_decode_acceptance is not None + + async def test_multi_sequence_request_with_one_payload_yields_none( + self, server_token_parser, request_record, spy_tokenizer + ): + result = await self._parse( + server_token_parser, self._with_n(request_record, 2), spy_tokenizer + ) + assert result.spec_decode_acceptance is None diff --git a/tests/unit/spec_decode/test_adapter_autodetect.py b/tests/unit/spec_decode/test_adapter_autodetect.py index 7973772f46..fbf71c0a5a 100644 --- a/tests/unit/spec_decode/test_adapter_autodetect.py +++ b/tests/unit/spec_decode/test_adapter_autodetect.py @@ -5,86 +5,215 @@ Exercises ``InferenceResultParser._extract_spec_decode_acceptance`` -- the auto-detection seam that walks registered ``spec_decode_adapter`` plugins -- so -the vLLM adapter is reachable end-to-end through the registry, not just when +each adapter is reachable end-to-end through the registry, not just when imported directly. + +AIPerf never learns which engine it is pointed at, by design: the user should +not have to declare it. Every payload is therefore offered to every registered +adapter, and detection soundness rests entirely on ``can_adapt`` signatures +being disjoint. These tests pin that contract from both directions -- each +adapter must claim its own payload and defer on the other's -- and pin the +parser's behaviour when a signature collision happens anyway. """ +from typing import Any + from aiperf.common.models import ParsedResponse, SpecDecodeAcceptanceRecord from aiperf.plugin import plugins from aiperf.plugin.enums import PluginType from aiperf.records.inference_result_parser import InferenceResultParser +from aiperf.spec_decode.trtllm_adapter import TRTLLMSpecDecodeAdapter +from aiperf.spec_decode.vllm_adapter import VLLMSpecDecodeAdapter +from tests.harness import mock_plugin +from tests.unit.spec_decode.test_trtllm_adapter import ( + SUMMARY_PAYLOAD as TRTLLM_PAYLOAD, +) from tests.unit.spec_decode.test_vllm_adapter import SUMMARY_PAYLOAD -def test_vllm_adapter_is_registered() -> None: - """The vllm adapter resolves through the spec_decode_adapter category.""" - adapter = plugins.get_class(PluginType.SPEC_DECODE_ADAPTER, "vllm") - assert adapter.__name__ == "VLLMSpecDecodeAdapter" - +def test_adapters_are_registered() -> None: + assert ( + plugins.get_class(PluginType.SPEC_DECODE_ADAPTER, "vllm").__name__ + == "VLLMSpecDecodeAdapter" + ) + assert ( + plugins.get_class(PluginType.SPEC_DECODE_ADAPTER, "tensorrt_llm").__name__ + == "TRTLLMSpecDecodeAdapter" + ) -def test_extract_returns_record_when_stats_present() -> None: - responses = [ - ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), - ParsedResponse(perf_ns=2, usage={"completion_tokens": 7}), - ] - record = InferenceResultParser._extract_spec_decode_acceptance(responses) - assert isinstance(record, SpecDecodeAcceptanceRecord) - assert record.engine == "vllm" - assert record.completion_tokens == 7 +class TestCrossClaim: + """Neither adapter may claim the other engine's payload. + The two payloads share ``acceptance_histogram``, ``num_spec_steps``, and + ``num_spec_tokens``, so a signature built only from shared keys would match + both. What follows depends on whether the foreign payload parses. The vLLM + adapter's old signature claimed TensorRT-LLM payloads and then failed on + the missing ``mean_acceptance_length``, silently dropping every TensorRT-LLM + record; an adapter whose fields did parse would instead attribute the run + to the wrong engine. Both failures are silent, so neither may claim. + """ -def test_extract_returns_none_when_no_stats() -> None: - """Records whose responses carry no spec-decode payload yield None.""" - responses = [ - ParsedResponse(perf_ns=1), - ParsedResponse(perf_ns=2, usage={"completion_tokens": 7}), - ] - assert InferenceResultParser._extract_spec_decode_acceptance(responses) is None - + def test_vllm_defers_on_trtllm_payload(self) -> None: + responses = [ParsedResponse(perf_ns=1, spec_decode_stats=TRTLLM_PAYLOAD)] + assert VLLMSpecDecodeAdapter.can_adapt(responses) is False + + def test_trtllm_defers_on_vllm_payload(self) -> None: + responses = [ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD)] + assert TRTLLMSpecDecodeAdapter.can_adapt(responses) is False + + def test_each_payload_resolves_to_its_own_engine(self) -> None: + for payload, engine in ( + (SUMMARY_PAYLOAD, "vllm"), + (TRTLLM_PAYLOAD, "tensorrt_llm"), + ): + responses = [ + ParsedResponse(perf_ns=1, spec_decode_stats=payload), + ParsedResponse(perf_ns=2, usage={"completion_tokens": 7}), + ] + record = InferenceResultParser._extract_spec_decode_acceptance(responses) + assert isinstance(record, SpecDecodeAcceptanceRecord) + assert record.engine == engine + assert record.completion_tokens == 7 + + +class TestAmbiguityIsRejected: + """Two adapters claiming one payload is a bug, and must not be resolved. + + Iteration order is ``plugins.yaml`` declaration order -- ``priority`` only + resolves conflicts between plugins registering the same *name*, and never + orders iteration. So a first-match walk would silently let YAML line order + decide which engine a record is attributed to. + """ -def test_extract_returns_none_for_empty_responses() -> None: - """No responses at all yields None.""" - assert InferenceResultParser._extract_spec_decode_acceptance([]) is None + class _GreedyAdapter: + """Claims anything, standing in for a future adapter with a loose signature.""" + + @classmethod + def can_adapt(cls, responses: list[ParsedResponse]) -> bool: + return any(r.spec_decode_stats for r in responses) + + @classmethod + def adapt(cls, responses: list[ParsedResponse]) -> None: + raise AssertionError("ambiguous payload must never be adapted") + + def test_contested_payload_yields_no_record(self) -> None: + responses = [ + ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), + ParsedResponse(perf_ns=2, usage={"completion_tokens": 7}), + ] + with mock_plugin( + PluginType.SPEC_DECODE_ADAPTER, "greedy-stub", self._GreedyAdapter + ): + # Two adapters now match; ``adapt`` raises if either is chosen, so a + # passing test also proves no adapter was picked arbitrarily. + assert ( + InferenceResultParser._extract_spec_decode_acceptance(responses) is None + ) + + +class TestAdapterFailureIsIsolated: + """One adapter raising must not fail the record or hide the others. + + ``can_adapt`` is a third-party plugin callback: the protocol asks for it to + be cheap and side-effect free, but nothing stops an implementation raising. + Without isolation the exception propagates out of the parser and the whole + inference record fails, even though the response was perfectly usable and a + later adapter could have read it. + """ + class _ExplodingAdapter: + @classmethod + def can_adapt(cls, responses: list[ParsedResponse]) -> bool: + raise RuntimeError("third-party adapter blew up") + + @classmethod + def adapt(cls, responses: list[ParsedResponse]) -> None: + raise AssertionError("must never be reached") + + def test_raising_adapter_does_not_fail_the_record(self) -> None: + responses = [ + ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), + ParsedResponse(perf_ns=2, usage={"completion_tokens": 7}), + ] + with mock_plugin( + PluginType.SPEC_DECODE_ADAPTER, "exploding-stub", self._ExplodingAdapter + ): + record = InferenceResultParser._extract_spec_decode_acceptance(responses) + + assert record is not None + assert record.engine == "vllm" + + +class TestNoAdapterMatches: + def test_unrecognized_engine_payload_yields_none(self) -> None: + """A payload no adapter recognizes yields no record rather than a guess. + + Guards the auto-detection contract: an adapter must not greedily claim a + foreign payload just because the raw slot is populated. + """ + responses = [ + ParsedResponse( + perf_ns=1, + spec_decode_stats={ + "spec_correct_drafts_histogram": {"0": 5}, + "steps": 5, + }, + ) + ] + assert InferenceResultParser._extract_spec_decode_acceptance(responses) is None + + +class TestPayloadPresence: + def test_returns_none_when_no_stats(self) -> None: + responses = [ + ParsedResponse(perf_ns=1), + ParsedResponse(perf_ns=2, usage={"completion_tokens": 7}), + ] + assert InferenceResultParser._extract_spec_decode_acceptance(responses) is None + + def test_returns_none_for_empty_responses(self) -> None: + assert InferenceResultParser._extract_spec_decode_acceptance([]) is None + + def test_suppresses_multi_sequence_stats(self) -> None: + """n > 1 streaming: more than one response carries stats, so the + per-request record is suppressed rather than mixing one sequence's + acceptance with the request-level completion_tokens.""" + responses = [ + ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), + ParsedResponse(perf_ns=2, spec_decode_stats=SUMMARY_PAYLOAD), + ParsedResponse(perf_ns=3, usage={"completion_tokens": 10}), + ] + assert InferenceResultParser._extract_spec_decode_acceptance(responses) is None + + def test_treats_empty_dict_payload_as_absent(self) -> None: + """An empty ``{}`` payload is counted by truthiness (like the adapter), so + a real payload beside it still yields a record instead of tripping the + n > 1 guard.""" + responses: list[ParsedResponse] = [ + ParsedResponse(perf_ns=1, spec_decode_stats={}), + ParsedResponse(perf_ns=2, spec_decode_stats=SUMMARY_PAYLOAD), + ParsedResponse(perf_ns=3, usage={"completion_tokens": 5}), + ] + record = InferenceResultParser._extract_spec_decode_acceptance(responses) + assert record is not None + assert record.engine == "vllm" + + +def test_signature_keys_are_disjoint() -> None: + """Mechanical guard on the property the whole design rests on. + + Fails if either signature stops requiring a key unique to its own engine -- + removing ``mean_acceptance_length`` from the vLLM signature, say -- because + the other engine's payload would then satisfy it. Adding a key, even a + shared one, only narrows a signature, so it cannot cause a cross-claim and + is not what this catches. + """ + from aiperf.spec_decode.trtllm_adapter import _TRTLLM_SIGNATURE_KEYS + from aiperf.spec_decode.vllm_adapter import _VLLM_SIGNATURE_KEYS -def test_extract_returns_none_for_unrecognized_engine_payload() -> None: - """A payload present but matching no registered adapter yields no record. + vllm_payload: dict[str, Any] = SUMMARY_PAYLOAD + trtllm_payload: dict[str, Any] = TRTLLM_PAYLOAD - Guards the auto-detection contract: the vLLM adapter must not greedily claim - a foreign (e.g. future SGLang/TRT-LLM) payload just because the raw slot is - populated. When another adapter is added, it -- not vLLM -- should win here. - """ - responses = [ - ParsedResponse( - perf_ns=1, - spec_decode_stats={"spec_correct_drafts_histogram": {"0": 5}, "steps": 5}, - ) - ] - assert InferenceResultParser._extract_spec_decode_acceptance(responses) is None - - -def test_extract_suppresses_multi_sequence_stats() -> None: - """n > 1 streaming: more than one response carries stats, so the per-request - record is suppressed rather than mixing one sequence's acceptance with the - request-level completion_tokens.""" - responses = [ - ParsedResponse(perf_ns=1, spec_decode_stats=SUMMARY_PAYLOAD), - ParsedResponse(perf_ns=2, spec_decode_stats=SUMMARY_PAYLOAD), - ParsedResponse(perf_ns=3, usage={"completion_tokens": 10}), - ] - assert InferenceResultParser._extract_spec_decode_acceptance(responses) is None - - -def test_extract_treats_empty_dict_payload_as_absent() -> None: - """An empty ``{}`` payload is counted by truthiness (like the adapter), so a - real payload beside it still yields a record instead of tripping the n > 1 - guard.""" - responses = [ - ParsedResponse(perf_ns=1, spec_decode_stats={}), - ParsedResponse(perf_ns=2, spec_decode_stats=SUMMARY_PAYLOAD), - ParsedResponse(perf_ns=3, usage={"completion_tokens": 5}), - ] - record = InferenceResultParser._extract_spec_decode_acceptance(responses) - assert record is not None - assert record.engine == "vllm" + assert not all(key in trtllm_payload for key in _VLLM_SIGNATURE_KEYS) + assert not all(key in vllm_payload for key in _TRTLLM_SIGNATURE_KEYS) diff --git a/tests/unit/spec_decode/test_mock_server_payloads.py b/tests/unit/spec_decode/test_mock_server_payloads.py new file mode 100644 index 0000000000..096cefd10f --- /dev/null +++ b/tests/unit/spec_decode/test_mock_server_payloads.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The mock server's spec-decode payloads must be consumable by the adapters. + +Every other test in this directory feeds the adapters hand-written payloads, +which only ever proves the adapters agree with our beliefs about the wire +format. This one closes the loop the other direction: it takes what the mock +server actually generates and runs it through the real adapter and the real +record validators, so a payload that could not produce a record fails here +rather than during a benchmark. + +It also pins the mock's own contract -- that its acceptance numbers reconcile +with the ``completion_tokens`` it reports beside them, and that the record's +three identities hold by construction rather than by luck. +""" + +from types import SimpleNamespace +from typing import Any + +import pytest +from aiperf_mock_server.config import server_config +from aiperf_mock_server.utils import attach_spec_decode, build_spec_decode_payload +from pytest import param + +from aiperf.common.models import ParsedResponse, SpecDecodeAcceptanceRecord +from aiperf.endpoints.base_endpoint import BaseEndpoint +from aiperf.spec_decode.trtllm_adapter import TRTLLMSpecDecodeAdapter +from aiperf.spec_decode.vllm_adapter import VLLMSpecDecodeAdapter + + +@pytest.fixture +def spec_decode_config(): + """Enable mock spec-decode emission, restoring prior settings afterwards.""" + saved = { + key: getattr(server_config, key) + for key in ( + "spec_decode_enabled", + "spec_decode_flavor", + "spec_decode_num_spec_tokens", + "spec_decode_acceptance_rate", + ) + } + + def configure( + *, flavor: str, num_spec_tokens: int | None = 3, acceptance_rate: float = 0.5 + ) -> None: + server_config.spec_decode_enabled = True + server_config.spec_decode_flavor = flavor + server_config.spec_decode_num_spec_tokens = num_spec_tokens + server_config.spec_decode_acceptance_rate = acceptance_rate + + yield configure + + for key, value in saved.items(): + setattr(server_config, key, value) + + +def _ctx(completion_tokens: int) -> Any: + return SimpleNamespace(usage={"completion_tokens": completion_tokens}) + + +def _assert_identities(record: SpecDecodeAcceptanceRecord) -> None: + """The invariants the record's validators enforce, restated explicitly. + + Constructing the record already checks these, so this documents what the + mock is required to satisfy rather than adding coverage. + """ + assert sum(record.acceptance_histogram.values()) == record.num_spec_steps + assert ( + sum(j * c for j, c in record.acceptance_histogram.items()) + == record.num_accepted_draft_tokens + ) + assert record.num_accepted_draft_tokens <= record.num_draft_tokens + + +class TestPayloadsProduceRecords: + @pytest.mark.parametrize( + "flavor, adapter, engine", + [ + param("vllm", VLLMSpecDecodeAdapter, "vllm", id="vllm"), + param("trtllm", TRTLLMSpecDecodeAdapter, "tensorrt_llm", id="trtllm"), + ], + ) # fmt: skip + @pytest.mark.parametrize( + "completion_tokens", [param(1, id="single_token"), param(100, id="typical")] + ) # fmt: skip + def test_adapter_builds_a_valid_record( + self, spec_decode_config, flavor, adapter, engine, completion_tokens + ) -> None: + spec_decode_config(flavor=flavor) + payload = build_spec_decode_payload(_ctx(completion_tokens)) + assert payload is not None + + responses = [ParsedResponse(perf_ns=1, spec_decode_stats=payload)] + assert adapter.can_adapt(responses) is True + record = adapter.adapt(responses) + + assert isinstance(record, SpecDecodeAcceptanceRecord) + assert record.engine == engine + _assert_identities(record) + + @pytest.mark.parametrize( + "rate", [param(0.0, id="all_rejected"), param(1.0, id="all_accepted")] + ) # fmt: skip + def test_boundary_acceptance_rates(self, spec_decode_config, rate) -> None: + """0.0 and 1.0 are the cases most likely to produce a degenerate payload.""" + spec_decode_config(flavor="vllm", acceptance_rate=rate) + payload = build_spec_decode_payload(_ctx(100)) + record = VLLMSpecDecodeAdapter.adapt( + [ParsedResponse(perf_ns=1, spec_decode_stats=payload)] + ) + assert record is not None + _assert_identities(record) + assert record.draft_acceptance_rate == rate + + @pytest.mark.parametrize( + "completion_tokens", + [param(n, id=f"ct_{n}") for n in (1, 2, 3, 5, 17, 100)], + ) # fmt: skip + def test_acceptance_reconciles_with_usage( + self, spec_decode_config, completion_tokens + ) -> None: + """Emitted acceptance must account for exactly the tokens usage reports. + + Each verify step emits one always-accepted bonus token plus its accepted + drafts, so ``num_spec_steps + num_accepted == completion_tokens`` is a + physical identity. The record's own validators do not cross-check against + usage, so without this a payload can satisfy every identity while still + claiming more emitted tokens than the response contained -- which it did + for ``completion_tokens <= 2``. + """ + spec_decode_config(flavor="vllm") + payload = build_spec_decode_payload(_ctx(completion_tokens)) + assert ( + payload["num_spec_steps"] + payload["num_accepted_draft_tokens"] + == completion_tokens + ) + assert payload["num_accepted_draft_tokens"] <= payload["num_draft_tokens"] + + def test_adaptive_bound_round_trips(self, spec_decode_config) -> None: + spec_decode_config(flavor="trtllm", num_spec_tokens=None) + payload = build_spec_decode_payload(_ctx(100)) + assert payload["num_spec_tokens"] is None + + record = TRTLLMSpecDecodeAdapter.adapt( + [ParsedResponse(perf_ns=1, spec_decode_stats=payload)] + ) + assert record is not None + assert record.num_spec_tokens is None + _assert_identities(record) + + +class TestPlacementMatchesTheEngine: + """``attach_spec_decode`` must put the payload where the capture layer looks.""" + + @pytest.mark.parametrize( + "flavor", [param("vllm", id="vllm"), param("trtllm", id="trtllm")] + ) # fmt: skip + def test_capture_layer_finds_the_payload(self, spec_decode_config, flavor) -> None: + spec_decode_config(flavor=flavor) + payload = build_spec_decode_payload(_ctx(100)) + response = attach_spec_decode( + { + "object": "chat.completion", + "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], + "usage": {"completion_tokens": 100}, + }, + payload, + ) + + assert BaseEndpoint.extract_spec_decode_stats(response) == payload + + def test_disabled_adds_no_key(self, spec_decode_config) -> None: + """Off by default, responses must be untouched -- no null-filled key.""" + server_config.spec_decode_enabled = False + response = {"object": "chat.completion", "choices": [{"text": "hi"}]} + before = dict(response) + assert ( + attach_spec_decode(response, build_spec_decode_payload(_ctx(100))) == before + ) + assert "metrics" not in response + assert "speculative_decoding" not in response["choices"][0] + + +class TestStreamingEndToEnd: + """Each mock streaming path, through the real parser and adapter. + + The tests above call the payload helpers directly, so they cannot see which + chunk a generator attaches the payload to. These drive the mock's streaming + routes over HTTP, parse every chunk with AIPerf's own endpoint, and run the + result through adapter auto-detection -- so a payload attached to the wrong + chunk, attached twice, or dropped fails here. + """ + + @staticmethod + def _stream(route: str, body: dict[str, Any]) -> list[dict[str, Any]]: + import orjson + from aiperf_mock_server.app import asgi_app + from fastapi.testclient import TestClient + + with TestClient(asgi_app) as client: + resp = client.post(route, json=body) + assert resp.status_code == 200 + return [ + orjson.loads(line[len("data: ") :]) + for line in resp.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ] + + @pytest.mark.parametrize( + "flavor, engine", [("vllm", "vllm"), ("trtllm", "tensorrt_llm")] + ) + @pytest.mark.parametrize( + "route, body_key, endpoint_name", + [ + param("/v1/chat/completions", "messages", "chat", id="chat"), + param("/v1/completions", "prompt", "completions", id="completions"), + ], + ) + def test_streamed_payload_yields_a_record( + self, + spec_decode_config, + monkeypatch, + flavor, + engine, + route, + body_key, + endpoint_name, + ) -> None: + from aiperf_mock_server import utils as mock_utils + + from aiperf.endpoints.openai_chat import ChatEndpoint + from aiperf.endpoints.openai_completions import CompletionsEndpoint + from aiperf.plugin.enums import EndpointType + from aiperf.records.inference_result_parser import InferenceResultParser + from tests.unit.endpoints.test_spec_decode_capture import ( + _make_endpoint, + _mock_response, + ) + + monkeypatch.setattr(mock_utils.server_config, "ttft", 0.0) + monkeypatch.setattr(mock_utils.server_config, "itl", 0.0) + spec_decode_config(flavor=flavor) + prompt = ( + [{"role": "user", "content": "Write a few sentences."}] + if body_key == "messages" + else "Write a few sentences." + ) + chunks = self._stream( + route, + { + "model": "m", + body_key: prompt, + "max_tokens": 24, + "stream": True, + "stream_options": {"include_usage": True}, + }, + ) + endpoint = ( + _make_endpoint(EndpointType.CHAT, ChatEndpoint) + if endpoint_name == "chat" + else _make_endpoint(EndpointType.COMPLETIONS, CompletionsEndpoint) + ) + parsed = [ + p + for p in (endpoint.parse_response(_mock_response(c)) for c in chunks) + if p is not None + ] + + record = InferenceResultParser._extract_spec_decode_acceptance(parsed) + + assert record is not None + assert record.engine == engine + streamed_usage = [c["usage"] for c in chunks if c.get("usage")] + assert record.completion_tokens == streamed_usage[-1]["completion_tokens"] + _assert_identities(record) + + @pytest.mark.parametrize( + "flavor, engine", [("vllm", "vllm"), ("trtllm", "tensorrt_llm")] + ) + def test_reasoning_only_stream_still_carries_the_payload( + self, spec_decode_config, monkeypatch, flavor, engine + ) -> None: + """A reasoning model can spend every token on reasoning. + + The content loop then never runs, so a payload attached only to the + last content chunk is never sent. It must ride the last chunk sent. + """ + from aiperf_mock_server import utils as mock_utils + + from aiperf.endpoints.openai_chat import ChatEndpoint + from aiperf.plugin.enums import EndpointType + from aiperf.records.inference_result_parser import InferenceResultParser + from tests.unit.endpoints.test_spec_decode_capture import ( + _make_endpoint, + _mock_response, + ) + + monkeypatch.setattr(mock_utils.server_config, "ttft", 0.0) + monkeypatch.setattr(mock_utils.server_config, "itl", 0.0) + spec_decode_config(flavor=flavor) + chunks = self._stream( + "/v1/chat/completions", + { + # "qwen" marks a reasoning model; 24 tokens is below the + # default reasoning budget, so no content token is produced. + "model": "Qwen/Qwen3-0.6B", + "messages": [{"role": "user", "content": "Write a few sentences."}], + "max_tokens": 24, + "stream": True, + "stream_options": {"include_usage": True}, + }, + ) + assert not any( + c["choices"] and c["choices"][0].get("delta", {}).get("content") + for c in chunks + ), "precondition: the stream must contain reasoning only" + endpoint = _make_endpoint(EndpointType.CHAT, ChatEndpoint) + parsed = [ + p + for p in (endpoint.parse_response(_mock_response(c)) for c in chunks) + if p is not None + ] + + record = InferenceResultParser._extract_spec_decode_acceptance(parsed) + + assert record is not None + assert record.engine == engine + streamed_usage = [c["usage"] for c in chunks if c.get("usage")] + assert record.completion_tokens == streamed_usage[-1]["completion_tokens"] diff --git a/tests/unit/spec_decode/test_trtllm_adapter.py b/tests/unit/spec_decode/test_trtllm_adapter.py new file mode 100644 index 0000000000..a346450ea8 --- /dev/null +++ b/tests/unit/spec_decode/test_trtllm_adapter.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the TensorRT-LLM per-request spec-decode adapter. + +Covers the engine-neutral ``SpecDecodeAcceptanceRecord`` filled from TRT-LLM's +per-choice ``speculative_decoding`` payload: present, absent, fully-rejected, +fully-accepted, streaming and non-streaming, and malformed degradation. + +Two differences from vLLM drive most of what is tested here. TRT-LLM does not +send a mean acceptance length (its per-choice ``avg_decoded_tokens_per_iter`` +averages over every decoding iteration, so it is a different quantity), so the +adapter derives it from the counts. +And it sends ``num_spec_tokens: null`` under ``draft_len_schedule``, where the +per-step draft bound varies by batch size; that must relax the histogram-length +cross-check without relaxing any of the record's identity validators. +""" + +from typing import Any + +import pytest +from pytest import param + +from aiperf.common.models import ParsedResponse, SpecDecodeAcceptanceRecord +from aiperf.spec_decode.trtllm_adapter import TRTLLMSpecDecodeAdapter + +# A representative TRT-LLM payload. The dense acceptance_histogram [8, 0, 6, 6] +# means 8 steps accepted 0 drafts, 6 accepted 2, 6 accepted all 3: +# num_spec_steps = 8 + 0 + 6 + 6 = 20 +# accepted = 0*8 + 1*0 + 2*6 + 3*6 = 30 +# mean acceptance len = 1 + 30 / 20 = 2.5 (derived, not sent) +SUMMARY_PAYLOAD: dict[str, Any] = { + "acceptance_rate": 0.5, + "total_accepted_draft_tokens": 30, + "total_draft_tokens": 60, + "num_spec_steps": 20, + "acceptance_histogram": [8, 0, 6, 6], + "num_spec_tokens": 3, +} + +# draft_len_schedule: no fixed per-step bound, so num_spec_tokens is null and +# the histogram is sized to the depth actually reached. +ADAPTIVE_PAYLOAD: dict[str, Any] = { + "acceptance_rate": 0.5, + "total_accepted_draft_tokens": 30, + "total_draft_tokens": 60, + "num_spec_steps": 20, + "acceptance_histogram": [8, 0, 6, 6], + "num_spec_tokens": None, +} + + +def _response( + *, + spec_decode_stats: dict[str, Any] | None = None, + usage: dict[str, Any] | None = None, +) -> ParsedResponse: + return ParsedResponse(perf_ns=123, usage=usage, spec_decode_stats=spec_decode_stats) + + +def _non_streaming(payload: dict[str, Any]) -> list[ParsedResponse]: + return [_response(spec_decode_stats=payload, usage={"completion_tokens": 50})] + + +def _streaming(payload: dict[str, Any]) -> list[ParsedResponse]: + """Streaming layout: content chunks, then the terminal chunk's choice. + + Unlike vLLM -- whose payload rides the trailing empty-choices usage chunk -- + TRT-LLM attaches it to the choice bearing ``finish_reason``, so the payload + and the usage arrive on different responses. + """ + return [ + _response(), + _response(spec_decode_stats=payload), + _response(usage={"completion_tokens": 50}), + ] + + +class TestCanAdapt: + def test_claims_its_own_payload(self) -> None: + assert TRTLLMSpecDecodeAdapter.can_adapt(_non_streaming(SUMMARY_PAYLOAD)) + + def test_defers_when_no_payload(self) -> None: + assert not TRTLLMSpecDecodeAdapter.can_adapt([_response()]) + + def test_defers_on_empty_payload(self) -> None: + assert not TRTLLMSpecDecodeAdapter.can_adapt([_response(spec_decode_stats={})]) + + +class TestAdapt: + @pytest.mark.parametrize( + "build", + [ + param(_non_streaming, id="non_streaming"), + param(_streaming, id="streaming"), + ], + ) # fmt: skip + def test_fills_the_record(self, build) -> None: + record = TRTLLMSpecDecodeAdapter.adapt(build(SUMMARY_PAYLOAD)) + + assert isinstance(record, SpecDecodeAcceptanceRecord) + assert record.engine == "tensorrt_llm" + assert record.num_spec_steps == 20 + assert record.num_accepted_draft_tokens == 30 + assert record.num_draft_tokens == 60 + assert record.draft_acceptance_rate == 0.5 + assert record.num_spec_tokens == 3 + assert record.completion_tokens == 50 + # Zero-count buckets are dropped by the dense -> sparse inflation. + assert record.acceptance_histogram == {0: 8, 2: 6, 3: 6} + + def test_mean_acceptance_length_is_derived_from_counts(self) -> None: + """TRT-LLM sends no mean acceptance length; it comes from the counts. + + Deriving it rather than reading a transmitted float means the reported + length can never contradict the histogram it is reported beside. + """ + record = TRTLLMSpecDecodeAdapter.adapt(_non_streaming(SUMMARY_PAYLOAD)) + assert record.mean_acceptance_length == 2.5 + + def test_per_step_arrays_are_never_populated(self) -> None: + """TRT-LLM keeps per-position vectors, not per-step sequences.""" + record = TRTLLMSpecDecodeAdapter.adapt(_non_streaming(SUMMARY_PAYLOAD)) + assert record.per_step_accepted is None + assert record.per_step_drafted is None + + def test_missing_usage_yields_no_completion_tokens(self) -> None: + record = TRTLLMSpecDecodeAdapter.adapt( + [_response(spec_decode_stats=SUMMARY_PAYLOAD)] + ) + assert record is not None + assert record.completion_tokens is None + + def test_all_rejected(self) -> None: + payload = { + "acceptance_rate": 0.0, + "total_accepted_draft_tokens": 0, + "total_draft_tokens": 30, + "num_spec_steps": 10, + "acceptance_histogram": [10, 0, 0, 0], + "num_spec_tokens": 3, + } + record = TRTLLMSpecDecodeAdapter.adapt(_non_streaming(payload)) + assert record.acceptance_histogram == {0: 10} + assert record.mean_acceptance_length == 1.0 + + def test_all_accepted(self) -> None: + payload = { + "acceptance_rate": 1.0, + "total_accepted_draft_tokens": 15, + "total_draft_tokens": 15, + "num_spec_steps": 5, + "acceptance_histogram": [0, 0, 0, 5], + "num_spec_tokens": 3, + } + record = TRTLLMSpecDecodeAdapter.adapt(_non_streaming(payload)) + assert record.acceptance_histogram == {3: 5} + assert record.mean_acceptance_length == 4.0 + + +class TestAdaptiveDrafting: + """``draft_len_schedule`` runs report no fixed per-step bound.""" + + def test_null_num_spec_tokens_is_accepted(self) -> None: + record = TRTLLMSpecDecodeAdapter.adapt(_non_streaming(ADAPTIVE_PAYLOAD)) + assert record is not None + assert record.num_spec_tokens is None + assert record.acceptance_histogram == {0: 8, 2: 6, 3: 6} + + def test_null_num_spec_tokens_relaxes_only_the_length_check(self) -> None: + """A null bound must not disable the record's identity validators. + + Without a bound there is no expected histogram length, so that one + cross-check cannot run -- but a payload whose counts contradict each + other is still corrupt and must be dropped. + """ + broken = {**ADAPTIVE_PAYLOAD, "num_spec_steps": 19} # histogram sums to 20 + assert TRTLLMSpecDecodeAdapter.adapt(_non_streaming(broken)) is None + + +class TestMalformedDegradesToNone: + """A signature-matching but broken payload must not raise -- no record.""" + + @pytest.mark.parametrize( + "bad_payload", + [ + param( + {"total_accepted_draft_tokens": 1, "total_draft_tokens": 2}, + id="signature_only_missing_rest", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": [8, 0, "x", 6]}, + id="non_integer_histogram_element", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": {"0": 8}}, + id="histogram_not_a_list", + ), + param( + {**SUMMARY_PAYLOAD, "acceptance_histogram": [8, 0, 6, 6, 0]}, + id="histogram_longer_than_num_spec_tokens", + ), + param( + {**SUMMARY_PAYLOAD, "num_spec_steps": 19}, + id="histogram_does_not_sum_to_steps", + ), + param( + {**SUMMARY_PAYLOAD, "total_accepted_draft_tokens": 29}, + id="weighted_sum_disagrees_with_accepted", + ), + param( + {**SUMMARY_PAYLOAD, "total_draft_tokens": 29}, + id="accepted_exceeds_drafted", + ), + param( + { + **SUMMARY_PAYLOAD, + "num_spec_steps": 0, + "acceptance_histogram": [0, 0, 0, 0], + "total_accepted_draft_tokens": 0, + }, + id="zero_steps_would_divide_by_zero", + ), + ], + ) # fmt: skip + def test_degrades_to_none(self, bad_payload: dict[str, Any]) -> None: + responses = [_response(spec_decode_stats=bad_payload)] + assert TRTLLMSpecDecodeAdapter.adapt(responses) is None diff --git a/tests/unit/spec_decode/test_vllm_adapter.py b/tests/unit/spec_decode/test_vllm_adapter.py index 03352c5341..56286bdf34 100644 --- a/tests/unit/spec_decode/test_vllm_adapter.py +++ b/tests/unit/spec_decode/test_vllm_adapter.py @@ -252,7 +252,11 @@ def test_adapt_no_usage_leaves_completion_tokens_none(self) -> None: # Signature keys present (so can_adapt matches) but the rest of the # required body is missing. param( - {"acceptance_histogram": [0, 1], "num_spec_steps": 1}, + { + "acceptance_histogram": [0, 1], + "num_spec_steps": 1, + "mean_acceptance_length": 2.0, + }, id="signature_only_missing_rest", ), param( From 10bc0242010a1a6efe67ff2caa763d5ece5c2ae0 Mon Sep 17 00:00:00 2001 From: Shiwei Ma <80623564+shm197@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:03:54 -0700 Subject: [PATCH 080/119] fix(search): prevent zero DSP lengthscales during GP fitting (#1431) Signed-off-by: Shiwei Ma (cherry picked from commit bfa5ab1d4bb5cd1292862d44e4f3a5441cf00981) --- .../search_planner/_botorch_kernel.py | 9 +- .../search_planner/test_botorch_kernel.py | 92 +++++++++++++++++++ .../search_planner/test_optuna_dsp_kernel.py | 25 +++++ 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/src/aiperf/orchestrator/search_planner/_botorch_kernel.py b/src/aiperf/orchestrator/search_planner/_botorch_kernel.py index a1f197fae0..bb247cf6b6 100644 --- a/src/aiperf/orchestrator/search_planner/_botorch_kernel.py +++ b/src/aiperf/orchestrator/search_planner/_botorch_kernel.py @@ -49,6 +49,7 @@ def make_dsp_kernel(d: int, batch_shape: torch.Size | None = None) -> ScaleKerne normalization itself. """ import torch + from gpytorch.constraints import GreaterThan from gpytorch.kernels import MaternKernel, ScaleKernel from gpytorch.priors import GammaPrior, LogNormalPrior @@ -56,11 +57,17 @@ def make_dsp_kernel(d: int, batch_shape: torch.Size | None = None) -> ScaleKerne raise ValueError(f"d must be >= 1; got {d}") loc = torch.tensor(math.sqrt(2.0) + 0.5 * math.log(d), dtype=torch.float64) scale = torch.tensor(math.sqrt(3.0), dtype=torch.float64) + lengthscale_prior = LogNormalPrior(loc=loc, scale=scale) return ScaleKernel( MaternKernel( nu=2.5, ard_num_dims=d, - lengthscale_prior=LogNormalPrior(loc=loc, scale=scale), + lengthscale_prior=lengthscale_prior, + # Match BoTorch's dimension-scaled prior: optimize directly in a + # bounded domain so Softplus cannot underflow to an invalid zero. + lengthscale_constraint=GreaterThan( + 2.5e-2, transform=None, initial_value=lengthscale_prior.mode + ), batch_shape=batch_shape, ), outputscale_prior=GammaPrior(2.0, 0.15), diff --git a/tests/unit/orchestrator/search_planner/test_botorch_kernel.py b/tests/unit/orchestrator/search_planner/test_botorch_kernel.py index c99c445a84..c864610843 100644 --- a/tests/unit/orchestrator/search_planner/test_botorch_kernel.py +++ b/tests/unit/orchestrator/search_planner/test_botorch_kernel.py @@ -7,6 +7,7 @@ import math import pytest +from pytest import param torch = pytest.importorskip("torch") gpytorch = pytest.importorskip("gpytorch") @@ -15,6 +16,12 @@ # exits 5 instead, which is loud. botorch = pytest.importorskip("botorch") +from botorch.fit import fit_gpytorch_mll # noqa: E402 +from botorch.models import SingleTaskGP # noqa: E402 +from botorch.models.transforms import Standardize # noqa: E402 +from botorch.optim.utils import get_parameters_and_bounds # noqa: E402 +from gpytorch.mlls import ExactMarginalLogLikelihood # noqa: E402 + from aiperf.orchestrator.search_planner._botorch_kernel import ( # noqa: E402 make_dsp_kernel, ) @@ -90,3 +97,88 @@ def test_gp_fit_succeeds_for_each_output_count(self, n_outputs: int) -> None: outcome_transform=Standardize(m=n_outputs), ) fit_gpytorch_mll(ExactMarginalLogLikelihood(model.likelihood, model)) + + +class TestDspKernelLengthscaleBounds: + @pytest.mark.parametrize( + "d, batch_shape", + [ + param(1, torch.Size([]), id="one-dimension-unbatched"), + param(4, torch.Size([2]), id="four-dimensions-batched"), + ], + ) # fmt: skip + def test_lengthscale_initializes_at_prior_mode( + self, d: int, batch_shape: torch.Size + ) -> None: + base = make_dsp_kernel(d=d, batch_shape=batch_shape).double().base_kernel + + torch.testing.assert_close( + base.lengthscale, + base.lengthscale_prior.mode.expand_as(base.lengthscale), + ) + + @pytest.mark.parametrize( + "dtype, batch_shape", + [ + param(torch.float32, torch.Size([]), id="float32-unbatched"), + param(torch.float64, torch.Size([2]), id="float64-batched"), + ], + ) # fmt: skip + def test_prior_is_finite_at_optimizer_lower_bound( + self, dtype: torch.dtype, batch_shape: torch.Size + ) -> None: + base = make_dsp_kernel(d=2, batch_shape=batch_shape).to(dtype).base_kernel + _, bounds = get_parameters_and_bounds(base) + lower_bound, _ = bounds["raw_lengthscale"] + + # Positive()/Softplus exposes -inf to SciPy, where the transformed + # lengthscale becomes zero and falls outside LogNormalPrior support. + with torch.no_grad(): + base.raw_lengthscale.fill_(lower_bound) + log_prob = base.lengthscale_prior.log_prob(base.lengthscale) + log_prob.sum().backward() + + assert math.isfinite(lower_bound) and lower_bound > 0 + assert torch.isfinite(log_prob).all() + assert torch.isfinite(base.raw_lengthscale.grad).all() + + @pytest.mark.parametrize( + "n_outputs", + [ + param(1, id="single-output"), + param(2, id="two-outputs"), + ], + ) # fmt: skip + def test_gp_fit_projects_out_of_bounds_lengthscale(self, n_outputs: int) -> None: + torch.manual_seed(0) + train_x = torch.rand(8, 2, dtype=torch.float64) + train_y = torch.cat( + [ + torch.sin((i + 1) * train_x[:, :1] * 3) + train_x[:, 1:] + for i in range(n_outputs) + ], + dim=-1, + ) + model = SingleTaskGP( + train_x, + train_y, + covar_module=make_dsp_kernel( + d=2, + batch_shape=torch.Size([n_outputs]) if n_outputs > 1 else None, + ), + outcome_transform=Standardize(m=n_outputs), + ) + base = model.covar_module.base_kernel + # Verify SciPy projects this out-of-bounds value before prior evaluation. + with torch.no_grad(): + base.raw_lengthscale.fill_(-1000.0) + + mll = ExactMarginalLogLikelihood(model.likelihood, model) + fit_gpytorch_mll(mll, max_attempts=1) + + assert not mll.training + assert (base.lengthscale >= base.raw_lengthscale_constraint.lower_bound).all() + assert torch.isfinite(base.lengthscale_prior.log_prob(base.lengthscale)).all() + posterior = model.posterior(train_x) + assert torch.isfinite(posterior.mean).all() + assert torch.isfinite(posterior.variance).all() diff --git a/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py b/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py index ff5bc38628..6ba4652779 100644 --- a/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py +++ b/tests/unit/orchestrator/search_planner/test_optuna_dsp_kernel.py @@ -108,6 +108,31 @@ def test_qlognei_candidates_func_fits_with_sla_constraints(n_sla_filters: int) - assert candidates.shape == (1, d) +@pytest.mark.parametrize( + "n_sla_filters", + [param(0, id="no-sla"), param(1, id="one-sla")], +) # fmt: skip +def test_qlognei_candidates_func_supports_repeated_fits(n_sla_filters: int) -> None: + torch.manual_seed(0) + func = build_qlognei_candidates_func() + train_x = torch.linspace(0, 1, 6, dtype=torch.float64).unsqueeze(-1) + bounds = torch.tensor([[0.0], [1.0]], dtype=torch.float64) + + for _ in range(3): + train_obj = torch.sin(2 * math.pi * train_x) + train_con = ( + torch.cat([train_x - 0.65, 0.1 - train_x], dim=-1)[:, :n_sla_filters] + if n_sla_filters + else None + ) + candidates = func(train_x, train_obj, train_con, bounds, None) + + assert candidates.shape == (1, 1) + assert torch.isfinite(candidates).all() + assert ((candidates >= bounds[0]) & (candidates <= bounds[1])).all() + train_x = torch.cat([train_x, candidates], dim=0) + + def test_qnehvi_candidates_func_fits_dsp_kernel_per_objective(monkeypatch): captured: list = [] _capture_built_models(monkeypatch, captured) From cd835489a739e3d7a12714fbfcdc82cd4cc2ec88 Mon Sep 17 00:00:00 2001 From: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:10:20 -0700 Subject: [PATCH 081/119] fix(spec-decode): drop zero-step acceptance records (#1548) (cherry picked from commit e1daadd10faa395100384358131d784d6858c5ae) --- docs/metrics-reference.md | 23 ++- docs/reference/spec-decode-acceptance.md | 23 ++- docs/tutorials/spec-decode-metrics.md | 9 +- src/aiperf/common/models/record_models.py | 13 +- .../common/models/spec_decode_models.py | 5 + .../metrics/types/spec_decode_metrics.py | 29 ++++ src/aiperf/records/inference_result_parser.py | 33 +++- .../unit/metrics/test_spec_decode_metrics.py | 48 ++++++ .../records/test_inference_result_parser.py | 158 ++++++++++++++++++ tests/unit/spec_decode/test_vllm_adapter.py | 36 ++-- 10 files changed, 350 insertions(+), 27 deletions(-) diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index 1cb355cec4..c8e5c228fc 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -55,6 +55,7 @@ This document provides a comprehensive reference of all metrics available in AIP - [Overall Draft Acceptance Rate](#overall-draft-acceptance-rate) - [Accepted per Verified](#accepted-per-verified) - [Spec Decode Steps](#spec-decode-steps) + - [Zero-Step Requests](#zero-step-requests) - [Accepted / Draft Token Counts](#accepted--draft-token-counts) - [Pooled Acceptance Histogram](#pooled-acceptance-histogram) - [Usage Field Metrics](#usage-field-metrics) @@ -828,7 +829,9 @@ total_reasoning_tokens = sum(r.reasoning_token_count for r in records if r.valid > each record. They are fully engine-agnostic: any engine whose adapter fills the > record lights them up unchanged, and none of them branch on the engine. When > spec decode is off (or a request had no verify steps) the record is absent and -> every metric here drops out cleanly -- nothing is shown or exported. +> every metric here drops out cleanly -- nothing is shown or exported. The one +> exception is [Zero-Step Requests](#zero-step-requests), which counts the +> requests that reported stats but never ran a verify step. @@ -923,6 +926,24 @@ Per-request number of speculative verification steps (`num_spec_steps`). Equals --- +### Zero-Step Requests + +**Type:** [Aggregate Metric](#aggregate-metrics) · **Unit:** requests + +The number of requests whose engine reported spec-decode stats but that ran no verify step (`num_spec_steps == 0`), e.g. `max_tokens: 1`, EOS as the first token, or an NGram drafter that never matched. These requests have no acceptance record, so none of the per-request metrics above include them; this count says how many were excluded. + +**Formula:** +```python +spec_decode_zero_step_requests = sum(1 for r in records if r.spec_decode_zero_step) +``` + +**Notes:** +- Absent when no request was zero-step. A Spec Decode section showing only this row means speculation never fired, as opposed to spec decode being off, which shows no section at all. +- Only vLLM reports zero-step requests; TensorRT-LLM omits the payload for them, so they are indistinguishable from spec decode being off. +- Aggregate only (`NO_INDIVIDUAL_RECORDS`): not written to the per-record JSONL. + +--- + ### Accepted / Draft Token Counts **Type:** [Record Metrics](#record-metrics) · **Unit:** tokens · `console_group = NONE` diff --git a/docs/reference/spec-decode-acceptance.md b/docs/reference/spec-decode-acceptance.md index d2d3ee7c52..3ed69281ed 100644 --- a/docs/reference/spec-decode-acceptance.md +++ b/docs/reference/spec-decode-acceptance.md @@ -131,15 +131,28 @@ The wire object maps to the record one-to-one, except: - The `detailed` level adds `per_step_accepted` / `per_step_drafted`; `summary` omits them (they stay `None`). - `mean_acceptance_length` / `draft_acceptance_rate` are taken verbatim - (the server already computes them safely, including the zero-step case). + (the server already computes them safely). ### Missing-field and edge cases -- **Field absent** (spec decode off, or the request had no verify steps): the +- **Field absent** (spec decode off, or per-request reporting not enabled): the record is `None` and dependent metrics simply do not show. This is the common case and is not an error. -- **Zero-step / fully-rejected**: reported verbatim (empty or `{0: N}` - histogram, `mean_acceptance_length == 1.0`). +- **Zero-step** (`num_spec_steps == 0`): the record is `None`. vLLM always + sends a payload, even for a request that never ran a verify step + (`max_tokens: 1`, EOS as the first token, an NGram drafter that never + matched), while TensorRT-LLM sends none. AIPerf drops the vLLM record, so the + per-request metrics average only over requests that speculated instead of + counting these as acceptance length 1.0 and rate 0%. The drop is logged at + debug only and counted in + [`spec_decode_zero_step_requests`](../metrics-reference.md#zero-step-requests). + The raw payload is still visible on the wire and in `--export-level raw` + output. A zero-step payload must also report zero drafts; one with + `num_draft_tokens > 0` is malformed (see below). +- **Fully-rejected** (`num_spec_steps > 0`, every step in bucket `j=0`): + reported verbatim (`{0: N}` histogram, `mean_acceptance_length == 1.0`, + `draft_acceptance_rate == 0.0`). This is real speculation that failed, so it + counts. - **Malformed payload**: the adapter degrades to `None` rather than raising, so one bad response cannot abort a run. Records whose aggregate counts contradict each other (histogram not summing to `num_spec_steps`, etc.) are rejected the @@ -237,6 +250,8 @@ Prometheus endpoint. `updateNumTokensPerIteration` and has no per-position vectors, so the field is simply absent there. - **Absent when the request never drafted**, rather than reported as zeros. + Because TensorRT-LLM never sends a zero-step payload, the adapter treats one + as malformed and logs a warning, unlike the quiet drop on vLLM. - **`n > 1`**: because the payload is per-choice, a response carrying more than one choice is suppressed client-side — a single per-request record cannot attribute request-level `completion_tokens` to one sequence. diff --git a/docs/tutorials/spec-decode-metrics.md b/docs/tutorials/spec-decode-metrics.md index 706bdbee96..89e4654861 100644 --- a/docs/tutorials/spec-decode-metrics.md +++ b/docs/tutorials/spec-decode-metrics.md @@ -77,7 +77,7 @@ in the response): ```bash curl -s localhost:8000/v1/chat/completions \ -H "Content-Type: application/json" \ - -d '{"model":"meta-llama/Llama-3.1-8B-Instruct","messages":[{"role":"user","content":"test"}],"max_tokens":1}' + -d '{"model":"meta-llama/Llama-3.1-8B-Instruct","messages":[{"role":"user","content":"test"}],"max_tokens":32}' ``` --- @@ -178,6 +178,9 @@ Quick reading (full definitions in the - **Accepted per Verified** -- `(j + 1) / (l + 1)`, a `[~0, 1]` utilization: how close each step got to accepting everything it proposed. - **Spec Decode Steps** -- verify steps per request. +- **Zero-Step Requests** -- requests that reported stats but never ran a verify step (e.g. + `max_tokens: 1`, or an NGram drafter that never matched). They are excluded from the + per-request metrics above; the row appears only when the count is nonzero. - **Accepted-draft histogram** -- share of verify steps that accepted exactly `j` draft tokens, pooled across the run. Capped to buckets `0..7` on the console (any `j >= 8` folds into a trailing `>=8` bucket); the full histogram is in the JSON export. @@ -235,7 +238,9 @@ export preserves, so no additional field is needed there. If the Spec Decode section, histogram, and `spec_decode_*` fields are all absent, that is the expected clean-degradation behavior -- not an error. Common causes: -- speculative decoding is off, or the requests had no verify steps; +- speculative decoding is off, or, on TensorRT-LLM, the requests had no verify steps. On + vLLM such requests still send a payload with `num_spec_steps: 0`, so instead of no + section you see a lone **Zero-Step Requests** row: speculation never fired; - the server was not started with per-request reporting enabled -- vLLM's `--per-request-spec-decode-metrics`, or TensorRT-LLM's `per_request_spec_decode_stats: true`; diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 1fea04c600..41d7e33362 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -1591,9 +1591,16 @@ class ParsedResponseRecord: """Engine-neutral per-request speculative-decoding acceptance record, filled by a ``SpecDecodeAdapterProtocol`` when the response carried spec-decode stats. ``None`` when: spec decode is off or the request had no verify steps - (no payload); the request produced multiple sequences (``n > 1``, which is - suppressed); no registered adapter recognized the payload; or the payload - was malformed and the adapter rejected it.""" + (no payload, or a zero-step payload the parser drops); the request produced + multiple sequences (``n > 1``, which is suppressed); no registered adapter + recognized the payload; or the payload was malformed and the adapter + rejected it.""" + + spec_decode_zero_step: bool = False + """True when the engine reported spec-decode stats for this request but it + ran no verify step (``num_spec_steps == 0``). Its record is dropped, so + ``spec_decode_acceptance`` is None; this flag lets the run count such + requests separately from runs where spec decode is off.""" @cached_property def final_usage(self) -> Usage | None: diff --git a/src/aiperf/common/models/spec_decode_models.py b/src/aiperf/common/models/spec_decode_models.py index 4b6a139e9a..f4906dff3e 100644 --- a/src/aiperf/common/models/spec_decode_models.py +++ b/src/aiperf/common/models/spec_decode_models.py @@ -132,6 +132,11 @@ def _check_aggregate_invariants(self) -> Self: f"num_accepted_draft_tokens ({self.num_accepted_draft_tokens}) " f"exceeds num_draft_tokens ({self.num_draft_tokens})" ) + if self.num_spec_steps == 0 and self.num_draft_tokens > 0: + raise ValueError( + f"num_draft_tokens is {self.num_draft_tokens}, but num_spec_steps " + "is 0: every proposed draft is verified in some step" + ) self._check_per_step_invariants() return self diff --git a/src/aiperf/metrics/types/spec_decode_metrics.py b/src/aiperf/metrics/types/spec_decode_metrics.py index 41c3fc22b8..f76d477ca3 100644 --- a/src/aiperf/metrics/types/spec_decode_metrics.py +++ b/src/aiperf/metrics/types/spec_decode_metrics.py @@ -27,6 +27,7 @@ from aiperf.common.models import ParsedResponseRecord from aiperf.common.models.spec_decode_models import SpecDecodeAcceptanceRecord from aiperf.metrics import BaseDerivedMetric, BaseRecordMetric +from aiperf.metrics.base_aggregate_counter_metric import BaseAggregateCounterMetric from aiperf.metrics.derived_sum_metric import DerivedSumMetric from aiperf.metrics.metric_dicts import MetricRecordDict, MetricResultsDict @@ -181,6 +182,34 @@ class SpecDecodeStepsMetric(BaseSpecDecodeRecordMetric[int]): spec_decode_field = "num_spec_steps" +class SpecDecodeZeroStepRequestsMetric(BaseAggregateCounterMetric[int]): + """Requests whose engine reported spec-decode stats but ran no verify step. + + These carry no acceptance record, so every per-request metric above skips + them. Counting them separately tells "speculation never fired" (e.g. an + NGram drafter that never matched, or ``max_tokens: 1``) apart from spec + decode being off, and says how many requests the per-request means exclude. + Absent when the count would be zero. + """ + + tag = "spec_decode_zero_step_requests" + header = "Zero-Step Requests" + short_header = "Zero-Step Requests" + short_header_hide_unit = True + unit = GenericMetricUnit.REQUESTS + flags = MetricFlags.NO_INDIVIDUAL_RECORDS + console_group = MetricConsoleGroup.SPEC_DECODE + display_order = 5045 + required_metrics = None + + def _parse_record( + self, record: ParsedResponseRecord, record_metrics: MetricRecordDict + ) -> int: + if not record.spec_decode_zero_step: + raise NoMetricValue("The request ran at least one verify step.") + return 1 + + class SpecDecodeAcceptedDraftTokensMetric(BaseSpecDecodeRecordMetric[int]): """Per-request accepted draft tokens (``num_accepted_draft_tokens``), excluding bonus tokens. Exported, not shown in the console. diff --git a/src/aiperf/records/inference_result_parser.py b/src/aiperf/records/inference_result_parser.py index 806eb74945..0a5718bee7 100644 --- a/src/aiperf/records/inference_result_parser.py +++ b/src/aiperf/records/inference_result_parser.py @@ -345,16 +345,36 @@ async def process_valid_record( else: token_counts = TokenCounts() + spec_decode_acceptance = self._extract_spec_decode_acceptance( + resp, + self._spec_decode_warned, + num_choices=inputs.num_choices if inputs is not None else 1, + ) + # A record exists only for a request that speculated. vLLM sends a + # payload even when no verify step ran (max_tokens: 1, EOS first, an + # NGram drafter that never matched) while TensorRT-LLM sends none; + # counting it would add AL 1.0 / rate 0% to the per-request means. The + # flag keeps the request countable, so "never speculated" stays + # distinguishable from "spec decode off". + spec_decode_zero_step = ( + spec_decode_acceptance is not None + and spec_decode_acceptance.num_spec_steps == 0 + ) + if spec_decode_zero_step: + engine = spec_decode_acceptance.engine + self.debug( + lambda: f"Dropping zero-step {engine} spec-decode record: " + "the request never ran a verify step" + ) + spec_decode_acceptance = None + return ParsedResponseRecord( request=request_record, responses=resp, token_counts=token_counts, media_counts=media_counts or MediaCounts(), - spec_decode_acceptance=self._extract_spec_decode_acceptance( - resp, - self._spec_decode_warned, - num_choices=inputs.num_choices if inputs is not None else 1, - ), + spec_decode_acceptance=spec_decode_acceptance, + spec_decode_zero_step=spec_decode_zero_step, ) @staticmethod @@ -372,6 +392,9 @@ def _extract_spec_decode_acceptance( priority order and uses the first whose ``can_adapt`` recognizes the payload -- mirroring custom-dataset-loader auto-detection. + Returns zero-step records as adapted; ``process_valid_record`` drops + them engine-neutrally, so no adapter has to repeat that rule. + Suppresses the record for any ``n > 1`` request: the per-request record can't attribute request-level ``completion_tokens`` to a single sequence, so a mixed record is worse than none. ``num_choices`` is the diff --git a/tests/unit/metrics/test_spec_decode_metrics.py b/tests/unit/metrics/test_spec_decode_metrics.py index 01cbaacb17..1ca1e812a9 100644 --- a/tests/unit/metrics/test_spec_decode_metrics.py +++ b/tests/unit/metrics/test_spec_decode_metrics.py @@ -48,6 +48,7 @@ SpecDecodeOverallDraftAcceptanceRateMetric, SpecDecodeStepsMetric, SpecDecodeTokenWeightedAcceptanceLengthMetric, + SpecDecodeZeroStepRequestsMetric, TotalAcceptedDraftTokensMetric, TotalDraftTokensMetric, TotalSpecDecodeStepsMetric, @@ -368,6 +369,53 @@ async def run(): assert summary.pooled_spec_decode_acceptance_histogram is None +class TestZeroStepRequests: + """Requests that reported stats but never ran a verify step are counted, + not averaged: they carry no acceptance record, only the parser's flag.""" + + def test_parse_record_flagged_request_counts_one(self): + record = spec_record(None) + record.spec_decode_zero_step = True + metric = SpecDecodeZeroStepRequestsMetric() + assert metric.parse_record(record, MetricRecordDict()) == 1 + + def test_parse_record_unflagged_request_raises_no_metric_value(self): + with pytest.raises(NoMetricValue): + SpecDecodeZeroStepRequestsMetric().parse_record( + spec_record(None), MetricRecordDict() + ) + + def test_metadata_is_aggregate_only_in_spec_decode_section(self): + metric_cls = SpecDecodeZeroStepRequestsMetric + assert metric_cls.unit == GenericMetricUnit.REQUESTS + assert metric_cls.console_group == MetricConsoleGroup.SPEC_DECODE + assert metric_cls.has_flags(MetricFlags.NO_INDIVIDUAL_RECORDS) + + def test_summary_counts_zero_step_requests_beside_speculated_ones(self): + records = [ + _spec_metric_records_data(0, make_spec(WORKED_EXAMPLE)), + _spec_metric_records_data(1, None), + _spec_metric_records_data(2, None), + ] + for zero_step in records[1:]: + zero_step.metrics[SpecDecodeZeroStepRequestsMetric.tag] = 1 + + async def summarize(): + acc = MetricsAccumulator(make_benchmark_run()) + for record in records: + await acc.process_record(record) + return await acc.summarize() + + results = asyncio.run(summarize()).results + assert results[SpecDecodeZeroStepRequestsMetric.tag].avg == 2 + # The per-request mean covers only the request that speculated. + assert results[TotalSpecDecodeStepsMetric.tag].avg == len(WORKED_EXAMPLE) + + def test_summary_omits_count_when_no_request_was_zero_step(self): + summary = asyncio.run(_summarize_specs(make_spec(WORKED_EXAMPLE), None)) + assert SpecDecodeZeroStepRequestsMetric.tag not in summary.results + + class TestRecordsManagerHistogramSelection: """``_pooled_spec_decode_histogram`` picks the one accumulator that pooled a histogram. Only ``metric_results`` should, so more than one populated is a diff --git a/tests/unit/records/test_inference_result_parser.py b/tests/unit/records/test_inference_result_parser.py index 620ef30d44..c9afcafa47 100644 --- a/tests/unit/records/test_inference_result_parser.py +++ b/tests/unit/records/test_inference_result_parser.py @@ -12,11 +12,14 @@ ErrorDetails, ParsedResponse, RequestRecord, + SpecDecodeAcceptanceRecord, TextResponse, TextResponseData, Usage, ) from aiperf.endpoints.openai_chat import ChatEndpoint +from aiperf.plugin.enums import PluginType +from tests.harness import mock_plugin from tests.unit.records.conftest import ( create_invalid_record, create_test_request_info, @@ -1133,3 +1136,158 @@ async def test_multi_sequence_request_with_one_payload_yields_none( server_token_parser, self._with_n(request_record, 2), spy_tokenizer ) assert result.spec_decode_acceptance is None + + +@pytest.mark.asyncio +class TestSpecDecodeZeroStepRecords: + """A record exists only for a request that speculated, on every engine. + + vLLM always serializes its per-request accumulator, so a request that never + ran a verify step (``max_tokens: 1``, EOS first, an NGram drafter that never + matched) arrives with a zero-step payload; TensorRT-LLM omits the payload in + the same situation. The parser normalizes both to an absent record so the + per-request means cover only requests that speculated, and flags the + request so the run can still count it. + """ + + ZERO_STEP_VLLM_PAYLOAD = { + "mean_acceptance_length": 1.0, + "draft_acceptance_rate": 0.0, + "acceptance_histogram": [0, 0, 0, 0], + "num_spec_steps": 0, + "num_accepted_draft_tokens": 0, + "num_draft_tokens": 0, + "num_spec_tokens": 3, + "per_step_accepted": None, + "per_step_drafted": None, + } + + FULLY_REJECTED_VLLM_PAYLOAD = { + "mean_acceptance_length": 1.0, + "draft_acceptance_rate": 0.0, + "acceptance_histogram": [5, 0, 0, 0], + "num_spec_steps": 5, + "num_accepted_draft_tokens": 0, + "num_draft_tokens": 15, + "num_spec_tokens": 3, + } + + class _ZeroStepAdapter: + """A non-vLLM engine that, like vLLM, reports requests that never drafted.""" + + @classmethod + def can_adapt(cls, responses: list[ParsedResponse]) -> bool: + return any( + (r.spec_decode_stats or {}).get("zero_step_engine") for r in responses + ) + + @classmethod + def adapt(cls, responses: list[ParsedResponse]) -> SpecDecodeAcceptanceRecord: + return SpecDecodeAcceptanceRecord( + engine="zero_step_engine", + mean_acceptance_length=1.0, + draft_acceptance_rate=0.0, + acceptance_histogram={}, + num_spec_steps=0, + num_accepted_draft_tokens=0, + num_draft_tokens=0, + num_spec_tokens=3, + ) + + @staticmethod + async def _parse(parser, request_record, payload: dict | None): + setup_parser_responses( + parser, + [ + ParsedResponse(perf_ns=1, spec_decode_stats=payload), + make_parsed_response(prompt_tokens=10, completion_tokens=1), + ], + ) + return await parser.process_valid_record(request_record) + + async def test_process_valid_record_zero_step_payload_drops_record_and_flags( + self, server_token_parser, request_record + ): + result = await self._parse( + server_token_parser, request_record, self.ZERO_STEP_VLLM_PAYLOAD + ) + + assert result.spec_decode_acceptance is None + assert result.spec_decode_zero_step is True + + async def test_process_valid_record_zero_step_from_any_adapter_drops_record( + self, server_token_parser, request_record + ): + """The drop lives in the parser, so an adapter need not repeat the rule.""" + with mock_plugin( + PluginType.SPEC_DECODE_ADAPTER, "zero_step_engine", self._ZeroStepAdapter + ): + result = await self._parse( + server_token_parser, request_record, {"zero_step_engine": True} + ) + + assert result.spec_decode_acceptance is None + assert result.spec_decode_zero_step is True + + async def test_process_valid_record_fully_rejected_payload_keeps_record( + self, server_token_parser, request_record + ): + result = await self._parse( + server_token_parser, request_record, self.FULLY_REJECTED_VLLM_PAYLOAD + ) + + record = result.spec_decode_acceptance + assert record is not None + assert record.num_spec_steps == 5 + assert record.acceptance_histogram == {0: 5} + assert record.mean_acceptance_length == 1.0 + assert record.draft_acceptance_rate == 0.0 + assert result.spec_decode_zero_step is False + + @pytest.mark.parametrize( + "payload", + [ + param(None, id="spec_decode_off"), + param({**ZERO_STEP_VLLM_PAYLOAD, "num_draft_tokens": 3}, id="drafts_without_steps"), + ], + ) # fmt: skip + async def test_process_valid_record_without_clean_zero_step_is_not_flagged( + self, server_token_parser, request_record, payload + ): + """Only a clean zero-step payload counts as "never speculated": spec + decode off has no payload, and drafts without steps is malformed.""" + result = await self._parse(server_token_parser, request_record, payload) + + assert result.spec_decode_acceptance is None + assert result.spec_decode_zero_step is False + + async def test_process_valid_record_drafts_without_steps_warns( + self, server_token_parser, request_record, caplog + ): + payload = {**self.ZERO_STEP_VLLM_PAYLOAD, "num_draft_tokens": 3} + with caplog.at_level(logging.WARNING): + await self._parse(server_token_parser, request_record, payload) + + assert any( + r.levelno == logging.WARNING and "malformed" in r.getMessage() + for r in caplog.records + ) + + async def test_process_valid_record_zero_step_drop_logs_below_warning( + self, server_token_parser, request_record, caplog + ): + # The fixture mocks the parser's own log methods; adapters and the + # detection path log through module loggers, which caplog sees. + with caplog.at_level(logging.WARNING): + await self._parse( + server_token_parser, request_record, self.ZERO_STEP_VLLM_PAYLOAD + ) + + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + server_token_parser.warning.assert_not_called() + server_token_parser.error.assert_not_called() + messages = [ + arg() if callable(arg) else arg + for arg in (c.args[0] for c in server_token_parser.debug.call_args_list) + ] + assert any("zero-step" in m for m in messages) diff --git a/tests/unit/spec_decode/test_vllm_adapter.py b/tests/unit/spec_decode/test_vllm_adapter.py index 56286bdf34..95ba420481 100644 --- a/tests/unit/spec_decode/test_vllm_adapter.py +++ b/tests/unit/spec_decode/test_vllm_adapter.py @@ -7,7 +7,8 @@ root ``metrics.speculative_decoding`` payload across the shapes the ticket calls out: present, absent, zero-step, and fully-rejected, in both streaming and non-streaming layouts, plus the detailed per-step arrays and malformed -degradation. +degradation. The adapter adapts a zero-step payload faithfully; it is +``InferenceResultParser``, not the adapter, that drops the resulting record. The sample payloads mirror the wire format from vLLM PR https://github.com/vllm-project/vllm/pull/48915: ``acceptance_histogram`` is a @@ -55,6 +56,18 @@ } +# What vLLM sends for a request that never ran a verify step, e.g. max_tokens: 1. +ZERO_STEP_PAYLOAD: dict[str, Any] = { + "mean_acceptance_length": 1.0, + "draft_acceptance_rate": 0.0, + "acceptance_histogram": [0, 0, 0, 0], + "num_spec_steps": 0, + "num_accepted_draft_tokens": 0, + "num_draft_tokens": 0, + "num_spec_tokens": 3, +} + + def _response( *, spec_decode_stats: dict[str, Any] | None = None, @@ -153,17 +166,12 @@ def test_adapt_absent_or_unrecognized_payload_yields_no_record( assert VLLMSpecDecodeAdapter.adapt(responses) is None def test_adapt_zero_step_payload_fills_record(self) -> None: - """No verify steps: empty histogram, mean 1.0, rate 0.0 (server-computed).""" - payload = { - "mean_acceptance_length": 1.0, - "draft_acceptance_rate": 0.0, - "acceptance_histogram": [0, 0, 0, 0], - "num_spec_steps": 0, - "num_accepted_draft_tokens": 0, - "num_draft_tokens": 0, - "num_spec_tokens": 3, - } - record = VLLMSpecDecodeAdapter.adapt(_non_streaming(payload)) + """No verify steps: empty histogram, mean 1.0, rate 0.0 (server-computed). + + Adapter-level only: ``InferenceResultParser`` drops a zero-step record, + so it never reaches the metrics. + """ + record = VLLMSpecDecodeAdapter.adapt(_non_streaming(ZERO_STEP_PAYLOAD)) assert record is not None assert record.mean_acceptance_length == 1.0 @@ -417,6 +425,10 @@ def test_record_rejects_negative_counts(self, override: dict[str, Any]) -> None: }, id="accepted_exceeds_drafted", ), + param( + {"acceptance_histogram": {}, "num_spec_steps": 0}, # 1 draft, 0 steps + id="drafts_without_steps", + ), ], ) # fmt: skip def test_record_rejects_inconsistent_aggregates( From 07d83359d4b8c53b7fc9cdc219296fd581e825cd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:21:35 -0500 Subject: [PATCH 082/119] fix(records): fail a stalled benchmark instead of waiting indefinitely Port of ai-dynamo/aiperf#1501 (aee1bea8b) onto the fork's ProfileCancelCommand path. The AgentX scenario disables the watchdog by default in 1.0.x, so a long request at low concurrency behaves as in 1.0.9; set AIPERF_RECORD_PROGRESS_STALL_TIMEOUT to enable it. (cherry picked from commit aee1bea8b7321c1e762e5466ff76de1a2bccc0c4) --- src/aiperf/common/enums/enums.py | 1 + src/aiperf/common/environment.py | 30 ++ .../common/scenario/inferencex_agentx_mvp.py | 3 + src/aiperf/records/records_manager.py | 186 ++++++++++- .../integration/test_stall_terminates_run.py | 64 ++++ tests/unit/records/test_records_manager.py | 35 +++ tests/unit/records/test_stall_watchdog.py | 289 ++++++++++++++++++ 7 files changed, 597 insertions(+), 11 deletions(-) create mode 100644 tests/integration/test_stall_terminates_run.py create mode 100644 tests/unit/records/test_stall_watchdog.py diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index 790b588df4..510f259689 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -108,6 +108,7 @@ class ProfileCancelReason(CaseInsensitiveStrEnum): USER = "user" WARMUP_FAILURE = "warmup_failure" FAILED_REQUEST_THRESHOLD = "failed_request_threshold" + PROGRESS_STALL = "progress_stall" @property def is_abort(self) -> bool: diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 08844b2fca..f806d119c8 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -1065,6 +1065,36 @@ class _RecordSettings(BaseSettings): "set False explicitly for that case.", ) + PROGRESS_STALL_TIMEOUT: float = Field( + ge=0.0, + le=86400.0, + default=600.0, + description="Seconds during which a profiling phase has requests in " + "flight and none of them complete, after which the run fails instead " + "of waiting indefinitely. A phase completes only once the credit " + "phase reports its final count, so a credit that is never returned " + "leaves that condition unevaluable and the run waits forever with no " + "output. Per-request timeouts do not cover this: they default to 6 " + "hours and never apply to a request that was not dispatched. Note " + "that a single legitimately long request will fail the run if it " + "exceeds this timeout while nothing else completes, so raise this " + "value for workloads whose individual requests run longer. Each " + "profiling phase is judged on its own counters, so an overlapping " + "`seamless` phase that keeps completing work does not mask a stall in " + "another. Set to 0 to disable and restore the previous unbounded " + "wait.", + ) + + PROGRESS_STALL_CHECK_INTERVAL: float = Field( + ge=1.0, + le=600.0, + default=30.0, + description="Seconds between stall checks. Each check with no " + "completions also logs how many requests are still in flight and " + "which phase they belong to, so a stalled run is diagnosable from the " + "log rather than silent.", + ) + class _SearchPlannerSettings(BaseSettings): """Adaptive-search planner tunables. diff --git a/src/aiperf/common/scenario/inferencex_agentx_mvp.py b/src/aiperf/common/scenario/inferencex_agentx_mvp.py index 86145b9de7..3f2981c1de 100644 --- a/src/aiperf/common/scenario/inferencex_agentx_mvp.py +++ b/src/aiperf/common/scenario/inferencex_agentx_mvp.py @@ -65,6 +65,9 @@ "SERVICE": {"PROFILE_CONFIGURE_TIMEOUT": 1800}, "UI": {"REALTIME_METRICS_ENABLED": True}, "HTTP": {"TCP_USER_TIMEOUT": 900000}, + # 1.0.x keeps the pre-watchdog behavior: one long request at low + # concurrency can legitimately run past the 600s default. + "RECORD": {"PROGRESS_STALL_TIMEOUT": 0}, }, } ) diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index bad2a3acb8..2d948e8e6e 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -51,7 +51,9 @@ from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.mixins import PullClientMixin from aiperf.common.models import ( + BasePhaseStats, BranchStats, + CreditPhaseStats, ErrorDetails, ErrorDetailsCount, MetricResult, @@ -600,6 +602,18 @@ def __init__( int(conc_val) if isinstance(conc_val, (int, float)) else 1, 10 ) self._failed_request_abort_triggered = False + self._profiling_started = False + # Keyed by phase_index: under `seamless` a later profiling phase starts + # while the previous one drains, and both keep publishing. + self._progress_stall_last_total: dict[int | None, int] = {} + self._progress_stall_since: dict[int | None, float] = {} + # Latest profiling CreditPhaseStats per phase. The credit side is the + # only source of sent/completed/cancelled counts; the records tracker's + # PhaseRecordsStats carries record counts and has no notion of what is + # still in flight. + self._latest_profiling_credit_stats: dict[int | None, CreditPhaseStats] = {} + # Latch so a stalled run requests cancellation once, not every tick. + self._progress_stall_triggered: bool = False def _build_routing_table(self) -> dict[str, list[Any]]: """Build record_type string -> handler mapping from plugin metadata.""" @@ -739,26 +753,171 @@ async def _maybe_trigger_failed_request_abort( f"(grace floor {self._failed_request_grace_floor}). " "Broadcasting ProfileCancelCommand to terminate the run." ) + if not await self._request_profile_cancel( + ProfileCancelReason.FAILED_REQUEST_THRESHOLD, + f"{error_records}/{total} profiling requests failed " + f"({rate:.1%}), exceeding the " + f"--failed-request-threshold limit of " + f"{self._failed_request_threshold:.1%}. " + "Check inference server logs.", + ): + self._failed_request_abort_triggered = False + + async def _request_profile_cancel( + self, reason: ProfileCancelReason, reason_detail: str + ) -> bool: + """End the run through the cancel path. True if the request went out. + + A PUB socket does not receive its own message, so the local cancel + handler runs directly after the broadcast; relying on the broadcast for + local finalization would leave the run hanging. + """ try: command = ProfileCancelCommand( service_id=self.service_id, - reason=ProfileCancelReason.FAILED_REQUEST_THRESHOLD, - reason_detail=( - f"{error_records}/{total} profiling requests failed " - f"({rate:.1%}), exceeding the " - f"--failed-request-threshold limit of " - f"{self._failed_request_threshold:.1%}. " - "Check inference server logs." - ), + reason=reason, + reason_detail=reason_detail, ) await self.publish(command) except Exception as exc: self.warning( - f"Failed to publish ProfileCancelCommand for threshold abort: {exc!r}" + f"Failed to publish ProfileCancelCommand for {reason}: {exc!r}" ) - self._failed_request_abort_triggered = False - return + return False await self._on_profile_cancel_command(command) + return True + + def _remember_profiling_credit_stats(self, stats: BasePhaseStats) -> None: + """Keep the newest profiling credit-phase snapshot for the watchdog. + + ``RecordsTracker.create_aggregate_stats_for_phase`` returns a + ``PhaseRecordsStats``, which counts *records* and has no + ``in_flight_requests``/``requests_completed``. Only the credit side + tracks what was sent versus what came back, so the watchdog reads this + snapshot instead. + + Keyed by ``phase_index`` rather than held in a single slot: under + ``seamless`` a later profiling phase starts while the previous one + drains, and both keep publishing every interval. Sharing one slot let + the two interleave, and because each carries its own + ``requests_completed`` the count moved on nearly every tick -- which + reads as progress and resets the stall clock, so a stuck request in + either phase was never caught. + """ + if isinstance(stats, CreditPhaseStats) and stats.phase == CreditPhase.PROFILING: + self._latest_profiling_credit_stats[stats.phase_index] = stats + + def _longest_stalled_phase( + self, now: float + ) -> tuple[int | None, CreditPhaseStats, float] | None: + """The profiling phase stuck longest, or ``None`` if all are moving. + + Each phase is judged on its own counters, and advances its own clock as + a side effect. A single shared clock cannot work here: under + ``seamless`` two profiling phases publish concurrently, and either one + alone failing to complete is enough to hang the run. + """ + worst: tuple[int | None, CreditPhaseStats, float] | None = None + for phase_index, stats in self._latest_profiling_credit_stats.items(): + # A missing entry reads as "never seen", so the first sight of a + # phase starts its clock rather than judging it. + progressed = ( + stats.requests_completed + != self._progress_stall_last_total.get(phase_index) + ) + if stats.in_flight_requests <= 0 or progressed: + # Nothing outstanding, or progress since the last tick. Reset + # this phase's clock so a quiet stretch does not accumulate + # toward a later, unrelated stall. + self._progress_stall_last_total[phase_index] = stats.requests_completed + self._progress_stall_since[phase_index] = now + continue + stalled_for = now - self._progress_stall_since.get(phase_index, now) + if worst is None or stalled_for > worst[2]: + worst = (phase_index, stats, stalled_for) + return worst + + @background_task( + interval=lambda self: Environment.RECORD.PROGRESS_STALL_CHECK_INTERVAL, + immediate=False, + ) + async def _watch_for_progress_stall(self) -> None: + """Fail a run whose in-flight requests stop completing. + + Distinct from the record-stall handling, which covers aggregation + falling behind *after* every credit has come back and finalizes with + partial results. Here no credit ever comes back: a request that is + dispatched and never completes leaves ``final_requests_completed`` + unset, so the completion barrier is not merely unmet but unevaluable, + and nothing downstream can re-trigger it. + ``--request-timeout-seconds`` is not a backstop -- it defaults to six + hours and never applies to a request that was never dispatched. + + The condition is "requests are in flight and none are completing", not + "no records arrived". Zero records is normal and expected whenever + nothing is pending: a low request rate, a fixed-schedule replay sitting + in a recorded idle gap, a slow dataset build. Only an outstanding + request that never lands is a stall, and that is what makes the run + unable to finish on its own. + + That also removes the need to gate on phase completion. A completed + phase has nothing in flight, and ``_complete_credit_phases`` holds the + phase *kind*, so gating on it would have disarmed the watchdog for + every profiling phase after the first in a multi-phase run. + """ + timeout = Environment.RECORD.PROGRESS_STALL_TIMEOUT + if timeout <= 0: + raise asyncio.CancelledError("progress stall watchdog disabled") + if not self._profiling_started: + return + + if not self._latest_profiling_credit_stats: + # No profiling progress has been reported yet; there is nothing to + # judge, and no clock may start before the first snapshot. + return + + worst = self._longest_stalled_phase(time.monotonic()) + if worst is None: + return + stalled_index, stats, stalled_for = worst + in_flight = stats.in_flight_requests + total = stats.requests_completed + phase_label = "" if stalled_index is None else f" in phase {stalled_index}" + + if stalled_for < timeout: + # Stay quiet until a full check interval has genuinely elapsed: + # the tick right after the clock resets would otherwise report + # "0s with no completion", which reads as alarming and is not. + if stalled_for >= Environment.RECORD.PROGRESS_STALL_CHECK_INTERVAL: + self.warning( + f"{in_flight:,} request(s) in flight{phase_label} with no " + f"completion for {stalled_for:.0f}s " + f"({total:,} completed so far). " + f"Failing at {timeout:.0f}s " + "(AIPERF_RECORD_PROGRESS_STALL_TIMEOUT=0 to wait " + "indefinitely)." + ) + return + + # Raising here would only be logged: @background_task defaults to + # stop_on_error=False, so the task would retry forever and the run + # would still hang -- the exact outcome this watchdog exists to end. + # Cancel rather than publish a terminal result directly: a terminal + # result fills only the profile domain, and the controller would still + # wait on server_metrics, which cannot arrive until the profiling phase + # completes -- which the stuck request is what prevents. + if self._progress_stall_triggered: + return + self._progress_stall_triggered = True + await self._request_profile_cancel( + ProfileCancelReason.PROGRESS_STALL, + f"Benchmark stalled: {in_flight:,} request(s) in flight" + f"{phase_label} with no completion for {stalled_for:.0f}s " + f"({total:,} completed). A " + f"request was dispatched but never returned, so the phase can " + f"never report complete. Set " + f"AIPERF_RECORD_PROGRESS_STALL_TIMEOUT=0 to disable this check.", + ) def _maybe_hint_missing_cache_reporting( self, record_data: MetricRecordsData @@ -1007,6 +1166,8 @@ async def _on_credit_phase_start( """Handle a credit phase start message in order to track the total number of expected requests.""" self._records_tracker.update_phase_info(phase_start_msg.stats) await self._dispatch_record(phase_start_msg.stats, warn_if_unrouted=False) + if phase_start_msg.config.phase == CreditPhase.PROFILING: + self._profiling_started = True self.info(f"Credit phase start: {phase_start_msg.config.phase}") @on_message(MessageType.CREDIT_PHASE_PROGRESS) @@ -1014,6 +1175,7 @@ async def _on_credit_phase_progress( self, message: CreditPhaseProgressMessage ) -> None: """Handle a credit phase progress message to track and stream live timing snapshots.""" + self._remember_profiling_credit_stats(message.stats) self._records_tracker.update_phase_info(message.stats) await self._dispatch_record(message.stats, warn_if_unrouted=False) @@ -1026,6 +1188,7 @@ async def _on_credit_phase_sending_complete( self.info( f"Sent {message.stats.final_requests_sent:,} requests. Waiting for all to complete..." ) + self._remember_profiling_credit_stats(message.stats) self._records_tracker.update_phase_info(message.stats) await self._dispatch_record(message.stats, warn_if_unrouted=False) @@ -1034,6 +1197,7 @@ async def _on_credit_phase_complete( self, message: CreditPhaseCompleteMessage ) -> None: """Handle a credit phase complete message in order to track the end time, and check if all records have been received.""" + self._remember_profiling_credit_stats(message.stats) self._records_tracker.update_phase_info(message.stats) await self._dispatch_record(message.stats, warn_if_unrouted=False) self._complete_credit_phases.add(message.stats.phase) diff --git a/tests/integration/test_stall_terminates_run.py b/tests/integration/test_stall_terminates_run.py new file mode 100644 index 0000000000..c8367ecde3 --- /dev/null +++ b/tests/integration/test_stall_terminates_run.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A stalled benchmark must end the process, not just report a stall. + +The watchdog's unit tests stub ``_request_profile_cancel``, so they prove the +stall is noticed and prove nothing about whether the run then stops. It did +not: publishing a terminal result fills only the ``profile`` result domain, +and the controller keeps waiting on ``server_metrics``, whose result cannot +arrive until the profiling phase completes -- which the stuck request is +exactly what prevents. The run hung until it was killed. + +Server metrics are enabled by default whenever the endpoint exposes +``/metrics``, which the mock server does, so this must run without +``--no-server-metrics`` to cover the default configuration at all. +""" + +import pytest + +from tests.harness.utils import AIPerfCLI +from tests.integration.conftest import IntegrationTestDefaults as defaults + +# Long enough that the request never returns on its own, so the only way the +# process can exit is the watchdog. +_NEVER_RETURNS_MS = 600_000 +_STALL_TIMEOUT_S = 5 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_a_stalled_run_exits_instead_of_waiting_on_server_metrics( + cli: AIPerfCLI, mock_server_factory, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AIPERF_RECORD_PROGRESS_STALL_TIMEOUT", str(_STALL_TIMEOUT_S)) + monkeypatch.setenv("AIPERF_RECORD_PROGRESS_STALL_CHECK_INTERVAL", "1") + + async with mock_server_factory(ttft=_NEVER_RETURNS_MS) as server: + # A timeout here raises rather than returning, so a run that hangs + # fails this test -- which is the regression being guarded. + result = await cli.run( + f""" + aiperf profile \ + --model {defaults.model} \ + --tokenizer builtin \ + --url {server.url} \ + --endpoint-type chat \ + --streaming \ + --concurrency 1 \ + --request-count 5 \ + --ui none + """, + timeout=90.0, + assert_success=False, + ) + + output = result.stdout + result.stderr + assert result.exit_code == 1, output[-2000:] + assert "Benchmark stalled" in output, output[-2000:] + # Without this the test could pass vacuously: if server metrics never + # engaged, the profile domain would be the only one to finalize and the + # original defect would be invisible here. + assert "server_metrics" in output, ( + f"server metrics never engaged, so this run did not cover the " + f"configuration the defect needs: {output[-2000:]}" + ) diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index da71f55e3b..5c7f8e3f2e 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -1032,6 +1032,12 @@ def _create_credit_phase_stats() -> CreditPhaseStats: def _create_manager_for_timing_dispatch() -> RecordsManager: manager = RecordsManager.__new__(RecordsManager) + # Built via __new__, so __init__ never ran. The stall watchdog indexes + # these by phase_index, which needs the containers to already exist. + manager._latest_profiling_credit_stats = {} + manager._progress_stall_last_total = {} + manager._progress_stall_since = {} + manager._progress_stall_triggered = False manager._dataset_configured_event = asyncio.Event() manager._dataset_configured_event.set() manager._records_tracker = MagicMock() @@ -1693,3 +1699,32 @@ async def test_no_accumulator_returns_empty_results(self) -> None: result = await manager._process_server_metrics_results() assert result.results is None + + +@pytest.mark.asyncio +async def test_a_failed_abort_request_clears_the_latch_so_a_later_record_retries() -> ( + None +): + """A cancel that never left the process must not look like one in flight. + + ``_failed_request_abort_triggered`` is set before the request goes out, so + leaving it set after a failed send would suppress every later attempt and + the run would keep going past its own failure threshold in silence. + """ + manager = _create_manager_for_timing_dispatch() + manager._failed_request_threshold = 0.2 + manager._failed_request_grace_floor = 1 + manager._failed_request_abort_triggered = False + manager.warning = MagicMock() + manager._records_tracker.total_records_for_phase.return_value = 10 + manager._records_tracker.error_records_for_phase.return_value = 10 + manager._request_profile_cancel = AsyncMock(return_value=False) + + await RecordsManager._maybe_trigger_failed_request_abort( + manager, CreditPhase.PROFILING, phase_index=0 + ) + + manager._request_profile_cancel.assert_awaited_once() + assert manager._failed_request_abort_triggered is False, ( + "a cancel that could not be sent must leave the abort retryable" + ) diff --git a/tests/unit/records/test_stall_watchdog.py b/tests/unit/records/test_stall_watchdog.py new file mode 100644 index 0000000000..f40c350828 --- /dev/null +++ b/tests/unit/records/test_stall_watchdog.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A run that stops making progress must fail, not wait forever. + +A phase is only complete once the credit phase reports its final count, so a +credit that is never returned leaves that condition unevaluable: the run waits +with no output until something external kills it. Per-request timeouts do not +cover it -- they default to six hours, and never apply to a request that was +never dispatched. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from aiperf.common.enums import CreditPhase, ProfileCancelReason +from aiperf.common.environment import Environment +from aiperf.common.models import CreditPhaseStats +from aiperf.records.records_manager import RecordsManager + + +class _Manager: + """Drives the real watchdog body against real credit-phase snapshots. + + The snapshot is a genuine ``CreditPhaseStats`` pushed through the real + ``_remember_profiling_credit_stats``. An earlier version of this file + defined a local fake carrying ``in_flight_requests``/``requests_completed``, + which let the watchdog ship reading those off ``PhaseRecordsStats`` -- a + type that has neither. Every tick raised ``AttributeError``, the background + task swallowed it, and the suite stayed green while the watchdog never ran. + """ + + # The watchdog delegates per-phase bookkeeping here; use the real + # implementation so the fake cannot drift from what ships. + _longest_stalled_phase = RecordsManager._longest_stalled_phase + + def __init__(self) -> None: + self._profiling_started = True + self._progress_stall_last_total: dict[int | None, int] = {} + self._progress_stall_since: dict[int | None, float] = {} + self._latest_profiling_credit_stats: dict[int | None, CreditPhaseStats] = {} + self._progress_stall_triggered = False + self.warnings: list[str] = [] + self.cancels: list[tuple[str, str]] = [] + + async def _request_profile_cancel(self, reason, reason_detail) -> bool: + self.cancels.append((reason, reason_detail)) + return True + + def warning(self, msg) -> None: + self.warnings.append(msg() if callable(msg) else msg) + + +def _observe( + mgr: _Manager, completed: int, in_flight: int, phase_index: int | None = None +) -> None: + """Publish a real profiling snapshot the way the message handlers do.""" + RecordsManager._remember_profiling_credit_stats( + mgr, + CreditPhaseStats( + phase=CreditPhase.PROFILING, + phase_index=phase_index, + requests_sent=completed + in_flight, + requests_completed=completed, + requests_cancelled=0, + ), + ) + + +async def _tick( + mgr: _Manager, monkeypatch, total: int, now: float, in_flight: int = 1 +) -> None: + _observe(mgr, completed=total, in_flight=in_flight) + monkeypatch.setattr("aiperf.records.records_manager.time.monotonic", lambda: now) + await RecordsManager._watch_for_progress_stall(mgr) + + +@pytest.mark.asyncio +async def test_progress_keeps_the_watchdog_quiet(monkeypatch) -> None: + mgr = _Manager() + for i, now in enumerate([0.0, 100.0, 200.0, 5000.0]): + await _tick(mgr, monkeypatch, total=i + 1, now=now) + assert mgr.warnings == [] + + +@pytest.mark.asyncio +async def test_stall_warns_then_fails(monkeypatch) -> None: + mgr = _Manager() + await _tick(mgr, monkeypatch, total=198, now=0.0) # stall clock starts here + await _tick(mgr, monkeypatch, total=198, now=100.0) # warn, under the limit + assert len(mgr.warnings) == 1 + assert "198" in mgr.warnings[0] + + await _tick(mgr, monkeypatch, total=198, now=1000.0) + assert len(mgr.cancels) == 1, ( + "a stall must end the run through the cancel path: publishing a " + "terminal result fills only the profile domain, leaving the " + "controller waiting on server_metrics, which cannot arrive until the " + "profiling phase completes" + ) + reason, detail = mgr.cancels[0] + assert reason == ProfileCancelReason.PROGRESS_STALL + assert "stalled" in detail + + # Latched: a stalled run asks once, not on every subsequent tick. + await _tick(mgr, monkeypatch, total=198, now=2000.0) + assert len(mgr.cancels) == 1 + + +@pytest.mark.asyncio +async def test_stall_is_measured_from_last_progress_not_from_start( + monkeypatch, +) -> None: + """Elapsed run time is not the measure; time since the last completion is.""" + mgr = _Manager() + await _tick(mgr, monkeypatch, total=1, now=0.0) + await _tick(mgr, monkeypatch, total=2, now=5000.0) # progress 5000s in + # Only 100s without progress, even though the run is 5100s old. + await _tick(mgr, monkeypatch, total=2, now=5100.0) + assert len(mgr.warnings) == 1 + assert "100s" in mgr.warnings[0] + + +@pytest.mark.asyncio +async def test_watchdog_disarmed_until_profiling_starts(monkeypatch) -> None: + """Dataset generation can run arbitrarily long before the first request.""" + mgr = _Manager() + mgr._profiling_started = False + await _tick(mgr, monkeypatch, total=0, now=100_000.0) + assert mgr.warnings == [] + assert mgr.cancels == [] + + +@pytest.mark.asyncio +async def test_zero_timeout_disables_the_watchdog(monkeypatch) -> None: + monkeypatch.setattr(Environment.RECORD, "PROGRESS_STALL_TIMEOUT", 0.0) + mgr = _Manager() + with pytest.raises(asyncio.CancelledError): + await _tick(mgr, monkeypatch, total=0, now=0.0) + + +@pytest.mark.asyncio +async def test_no_requests_in_flight_is_not_a_stall(monkeypatch) -> None: + """Zero records is normal whenever nothing is pending. + + A low request rate, a fixed-schedule replay sitting in a recorded idle gap, + or a slow dataset build all produce long quiet stretches on a perfectly + healthy run. Only an outstanding request that never lands leaves the phase + unable to finish. + """ + mgr = _Manager() + for now in (0.0, 10.0, 100_000.0): + await _tick(mgr, monkeypatch, total=5, now=now, in_flight=0) + assert mgr.warnings == [] + assert mgr.cancels == [] + + +@pytest.mark.asyncio +async def test_a_quiet_stretch_does_not_accumulate_toward_a_later_stall( + monkeypatch, +) -> None: + mgr = _Manager() + await _tick(mgr, monkeypatch, total=5, now=0.0, in_flight=0) + await _tick(mgr, monkeypatch, total=5, now=5_000.0, in_flight=0) + # Work starts; only 100s of genuine stall has elapsed. + await _tick(mgr, monkeypatch, total=5, now=5_000.0, in_flight=2) + await _tick(mgr, monkeypatch, total=5, now=5_100.0, in_flight=2) + + assert mgr.cancels == [] + assert len(mgr.warnings) == 1 + assert "100s" in mgr.warnings[0] + + +def test_the_watchdog_is_actually_scheduled() -> None: + """The watchdog must stay attached to ``@background_task``. + + Every other test here calls ``_watch_for_progress_stall`` directly, so none + of them notice if the decorator stops applying to it -- which is exactly + what happened when a helper was inserted between the decorator and the + function: the helper became the background task, the watchdog was never + scheduled, and the suite stayed green. + """ + hook_type = getattr( + RecordsManager._watch_for_progress_stall, "__aiperf_hook_type__", None + ) + assert hook_type == "@background_task", ( + "the progress-stall watchdog is not registered as a background task, " + "so it will never run" + ) + assert not hasattr( + RecordsManager._remember_profiling_credit_stats, "__aiperf_hook_type__" + ), "the snapshot helper is called from message handlers, not scheduled" + + +@pytest.mark.asyncio +async def test_a_stuck_phase_is_caught_while_another_phase_keeps_completing( + monkeypatch, +) -> None: + """Overlapping profiling phases must each be judged on their own counters. + + Under ``seamless`` a later profiling phase starts while the previous one + drains, and both publish every interval. Held in one slot the snapshots + interleaved, and since each phase carries its own ``requests_completed`` + the count changed on nearly every tick -- which reads as progress. The + stall clock reset perpetually and a request stuck in either phase was + never caught, so the run hung with the watchdog armed and silent. + """ + monkeypatch.setattr(Environment.RECORD, "PROGRESS_STALL_TIMEOUT", 100.0) + monkeypatch.setattr(Environment.RECORD, "PROGRESS_STALL_CHECK_INTERVAL", 10.0) + mgr = _Manager() + + # Phase 0 is wedged at 5 completed with one request outstanding, while + # phase 1 keeps completing work the whole time. + for i, now in enumerate([0.0, 50.0, 150.0, 250.0]): + _observe(mgr, completed=5, in_flight=1, phase_index=0) + _observe(mgr, completed=100 + i, in_flight=4, phase_index=1) + monkeypatch.setattr( + "aiperf.records.records_manager.time.monotonic", lambda now=now: now + ) + await RecordsManager._watch_for_progress_stall(mgr) + + assert mgr.cancels, ( + "a request stuck in phase 0 never triggered the watchdog while phase 1 " + "kept reporting completions" + ) + assert "in phase 0" in mgr.cancels[0][1], mgr.cancels[0][1] + + +@pytest.mark.asyncio +async def test_two_phases_both_progressing_stay_quiet(monkeypatch) -> None: + """The per-phase clocks must not manufacture a stall out of two healthy phases.""" + monkeypatch.setattr(Environment.RECORD, "PROGRESS_STALL_TIMEOUT", 100.0) + mgr = _Manager() + for i, now in enumerate([0.0, 50.0, 150.0, 250.0, 5_000.0]): + _observe(mgr, completed=i, in_flight=2, phase_index=0) + _observe(mgr, completed=i * 3, in_flight=2, phase_index=1) + monkeypatch.setattr( + "aiperf.records.records_manager.time.monotonic", lambda now=now: now + ) + await RecordsManager._watch_for_progress_stall(mgr) + + assert mgr.cancels == [] + assert mgr.warnings == [] + + +@pytest.mark.asyncio +async def test_no_snapshot_yet_is_not_a_stall(monkeypatch) -> None: + """Before the first credit snapshot there is nothing to judge. + + Ticking in that window must not start a clock: a run whose first phase has + not reported yet would otherwise accumulate toward a stall it was never in. + """ + monkeypatch.setattr(Environment.RECORD, "PROGRESS_STALL_TIMEOUT", 100.0) + mgr = _Manager() + + for now in (0.0, 500.0, 50_000.0): + monkeypatch.setattr( + "aiperf.records.records_manager.time.monotonic", lambda now=now: now + ) + await RecordsManager._watch_for_progress_stall(mgr) + + assert mgr.cancels == [] + assert mgr.warnings == [] + + +@pytest.mark.asyncio +async def test_a_cancel_that_cannot_be_sent_reports_failure(monkeypatch) -> None: + """If the ProfileCancelCommand cannot go out, say so and report False. + + The caller uses the return value to decide whether its own abort latch + stays set, so swallowing the error would leave the run believing a + cancellation is in flight that never left this process. + """ + mgr = SimpleNamespace( + service_id="records-manager-test", + warning=MagicMock(), + publish=AsyncMock(side_effect=RuntimeError("bus down")), + ) + + sent = await RecordsManager._request_profile_cancel( + mgr, ProfileCancelReason.PROGRESS_STALL, "stalled" + ) + + assert sent is False + assert "Failed to publish ProfileCancelCommand" in mgr.warning.call_args[0][0] From 8b54fb773cb821b9c6ea820d2d4cb6652bc85eac Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:21:35 -0500 Subject: [PATCH 083/119] test: drop BranchStats expectations for the skipped DAG spine fields Reverts the pick of upstream f8c8e3653, whose expectations cover fields from #1218, which is not in 1.0.x. --- tests/unit/records/test_dag_metadata_tagging.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/unit/records/test_dag_metadata_tagging.py b/tests/unit/records/test_dag_metadata_tagging.py index 699ee04236..6b875f2d82 100644 --- a/tests/unit/records/test_dag_metadata_tagging.py +++ b/tests/unit/records/test_dag_metadata_tagging.py @@ -137,8 +137,6 @@ def test_branch_stats_defaults_all_zero(self): "joins_suppressed": 0, "children_truncated": 0, "children_delayed": 0, - "graphs_admitted": 0, - "graphs_completed_to_end": 0, } def test_branch_stats_dict_helper(self): @@ -159,8 +157,6 @@ def test_branch_stats_dict_helper(self): "joins_suppressed": 0, "children_truncated": 0, "children_delayed": 0, - "graphs_admitted": 0, - "graphs_completed_to_end": 0, } def test_branch_stats_roundtrip_through_profile_results(self): From 1cef575efc0a67dc415938515aa4cc7e1bae5d0c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:22:13 -0500 Subject: [PATCH 084/119] feat(agentx): keep 1.0.9 runtime defaults for upstream additions The agentx preset disables the progress-stall watchdog and the X-Session-Affinity header so 1.0.x requests and run outcomes match 1.0.9. --- src/aiperf/common/scenario/inferencex_agentx_mvp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/aiperf/common/scenario/inferencex_agentx_mvp.py b/src/aiperf/common/scenario/inferencex_agentx_mvp.py index 3f2981c1de..e6cd845173 100644 --- a/src/aiperf/common/scenario/inferencex_agentx_mvp.py +++ b/src/aiperf/common/scenario/inferencex_agentx_mvp.py @@ -64,7 +64,11 @@ "DATASET": {"CONFIGURATION_TIMEOUT": 1800}, "SERVICE": {"PROFILE_CONFIGURE_TIMEOUT": 1800}, "UI": {"REALTIME_METRICS_ENABLED": True}, - "HTTP": {"TCP_USER_TIMEOUT": 900000}, + # 1.0.x keeps 1.0.9's request headers: no X-Session-Affinity. + "HTTP": { + "TCP_USER_TIMEOUT": 900000, + "X_SESSION_AFFINITY_FROM_CORRELATION_ID": False, + }, # 1.0.x keeps the pre-watchdog behavior: one long request at low # concurrency can legitimately run past the 600s default. "RECORD": {"PROGRESS_STALL_TIMEOUT": 0}, From e084b2aab279728e3f12bb5cfa601ae3a2dba733 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:35:17 -0500 Subject: [PATCH 085/119] chore: regenerate config schema and CLI and environment docs --- docs/cli-options.md | 28 +- docs/environment-variables.md | 2 + .../config/schema/aiperf-config.schema.json | 23373 +--------------- 3 files changed, 16 insertions(+), 23387 deletions(-) diff --git a/docs/cli-options.md b/docs/cli-options.md index 314767cf91..90fa4e93c5 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -490,7 +490,7 @@ Path to file or directory containing benchmark dataset. Required when using `--c #### `--public-dataset` `` Pre-configured public dataset to download and use for benchmarking (e.g., `sharegpt`). AIPerf automatically downloads and parses these datasets. Mutually exclusive with `--custom-dataset-type`. Run `aiperf plugins public_dataset_loader` to list available datasets. Use `--hf-subset` to override the HuggingFace subset/config for HF-backed datasets. -
_Choices: [`exgentic_v2`, `exgentic`, `sharegpt`, `aimo`, `mmstar`, `mmvu`, `vision_arena`, `llava_onevision`, `aimo_aime`, `aimo_numina_cot`, `aimo_numina_1_5`, `spec_bench`, `spec_al_gsm8k`, `spec_al_math500`, `spec_al_humaneval`, `spec_al_mbpp`, `spec_al_mtbench`, `instruct_coder`, `blazedit_5k`, `blazedit_10k`, `librispeech`, `voxpopuli`, `gigaspeech`, `ami`, `spgispeech`, `semianalysis_cc_traces_weka`, `semianalysis_cc_traces_weka_no_subagents`, `semianalysis_cc_traces_weka_with_subagents`, `semianalysis_cc_traces_weka_with_subagents_256k`, `semianalysis_cc_traces_weka_with_subagents_060226`, `semianalysis_cc_traces_weka_with_subagents_060226_256k`, `semianalysis_cc_traces_weka_with_subagents_060526`, `semianalysis_cc_traces_weka_with_subagents_060526_256k`, `semianalysis_cc_traces_weka_with_subagents_060826`, `semianalysis_cc_traces_weka_with_subagents_060826_256k`, `semianalysis_cc_traces_weka_061326`, `semianalysis_cc_traces_weka_061326_256k`, `semianalysis_cc_traces_weka_061526`, `semianalysis_cc_traces_weka_061526_256k`, `semianalysis_cc_traces_weka_062126`, `semianalysis_cc_traces_weka_062126_256k`, `weka_hf`]_ +
_Choices: [`exgentic_v2`, `exgentic`, `sharegpt`, `aimo`, `mmstar`, `mmvu`, `vision_arena`, `llava_onevision`, `aimo_aime`, `aimo_numina_cot`, `aimo_numina_1_5`, `spec_bench`, `spec_al_gsm8k`, `spec_al_math500`, `spec_al_humaneval`, `spec_al_mbpp`, `spec_al_mtbench`, `instruct_coder`, `blazedit_5k`, `blazedit_10k`, `librispeech`, `voxpopuli`, `gigaspeech`, `ami`, `spgispeech`, `semianalysis_cc_traces_weka`, `semianalysis_cc_traces_weka_no_subagents`, `semianalysis_cc_traces_weka_with_subagents`, `semianalysis_cc_traces_weka_with_subagents_256k`, `semianalysis_cc_traces_weka_with_subagents_060226`, `semianalysis_cc_traces_weka_with_subagents_060226_256k`, `semianalysis_cc_traces_weka_with_subagents_060526`, `semianalysis_cc_traces_weka_with_subagents_060526_256k`, `semianalysis_cc_traces_weka_with_subagents_060826`, `semianalysis_cc_traces_weka_with_subagents_060826_256k`, `semianalysis_cc_traces_weka_061326`, `semianalysis_cc_traces_weka_061326_256k`, `semianalysis_cc_traces_weka_061526`, `semianalysis_cc_traces_weka_061526_256k`, `semianalysis_cc_traces_weka_062126`, `semianalysis_cc_traces_weka_062126_256k`, `weka_hf`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `h_cua_perf`]_ #### `--hf-subset` `` @@ -507,7 +507,7 @@ Dataset-specific filter in key=value form. Repeat for multiple filters. Only sup #### `--custom-dataset-type` `` Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), `tracelab` (TraceLab agentic-coding corpus, JSONL or gzipped JSONL), `mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; when using `random_pool`, `--conversation-num` defaults to 100 if not specified; batch sizes > 1 sample each modality independently from a flat pool and do not preserve per-entry associations - use `single_turn` if paired modalities must stay together). Requires `--input-file`. Mutually exclusive with `--public-dataset`. -
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `tracelab`, `weka_trace`]_ +
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `h_cua_perf`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `tracelab`, `weka_trace`]_ #### `--ignore-trace-delays` @@ -1244,11 +1244,11 @@ Output directory for all benchmark artifacts including metrics (`.csv`, `.json`, #### `--profile-export-prefix`, `--profile-export-file` `` -Base filename for ALL exported files. With prefix='foo' every output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, `foo_gpu_telemetry.jsonl`, and `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), historical per-file names are used: `profile_export_aiperf.{csv,json}` for the summary, `profile_export.jsonl` and `profile_export_raw.jsonl` for records, `gpu_telemetry_export.jsonl`, and `server_metrics_export.*`. Known suffixes (e.g. `_raw.jsonl`, `_timeslices.csv`, `_server_metrics.parquet`) are stripped from the supplied value. +Base filename for ALL exported files. With prefix='foo' every output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, `foo_outputs.json`, `foo_gpu_telemetry.jsonl`, and `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), historical per-file names are used: `profile_export_aiperf.{csv,json}` for the summary, `profile_export.jsonl` and `profile_export_raw.jsonl` for records, `outputs.json`, `gpu_telemetry_export.jsonl`, and `server_metrics_export.*`. Known suffixes (e.g. `_raw.jsonl`, `_outputs.json`, `_timeslices.csv`, `_server_metrics.parquet`) are stripped from the supplied value. #### `--export-level`, `--profile-export-level` `` -Controls which output files are generated. `summary`: Only aggregate metrics files (`.csv`, `.json`). `records`: Includes per-request metrics (`.jsonl`). `raw`: Includes raw request/response data (`_raw.jsonl`). +Controls which output files are generated. `summary`: Only aggregate metrics files (`.csv`, `.json`). `records`: Includes per-request metrics (`.jsonl`). `raw`: Includes raw request/response data (`_raw.jsonl`) and, unless `--no-export-outputs-json` is passed, the generated text (`_outputs.json`). **Choices:** @@ -1272,10 +1272,9 @@ Auto-invoke `aiperf plot` against the artifact directory after the benchmark com Treat auto-plot failures as fatal: re-raise so `aiperf profile` exits non-zero. Only meaningful when auto-plot is on. Default False = warn and continue.
_Flag (no value required)_ -#### `--export-outputs-json` +#### `--export-outputs-json`, `--no-export-outputs-json` -Export generated response text to outputs.json after the run. When enabled, the raw generated-text payload for each request is written to an outputs.json file in the artifact directory. -
_Flag (no value required)_ +Export generated response text after the run. When enabled, the raw generated-text payload for each request is written to `outputs.json` in the artifact directory, or to `_outputs.json` when `--profile-export-prefix` is set (prefix `foo` gives `foo_outputs.json`). Implied by `--export-level raw`; pass `--no-export-outputs-json` to opt out of the extra file while keeping raw export. #### `--otel-url` `` @@ -2054,7 +2053,7 @@ Path to file or directory containing benchmark dataset. Required when using `--c #### `--public-dataset` `` Pre-configured public dataset to download and use for benchmarking (e.g., `sharegpt`). AIPerf automatically downloads and parses these datasets. Mutually exclusive with `--custom-dataset-type`. Run `aiperf plugins public_dataset_loader` to list available datasets. Use `--hf-subset` to override the HuggingFace subset/config for HF-backed datasets. -
_Choices: [`exgentic_v2`, `exgentic`, `sharegpt`, `aimo`, `mmstar`, `mmvu`, `vision_arena`, `llava_onevision`, `aimo_aime`, `aimo_numina_cot`, `aimo_numina_1_5`, `spec_bench`, `spec_al_gsm8k`, `spec_al_math500`, `spec_al_humaneval`, `spec_al_mbpp`, `spec_al_mtbench`, `instruct_coder`, `blazedit_5k`, `blazedit_10k`, `librispeech`, `voxpopuli`, `gigaspeech`, `ami`, `spgispeech`, `semianalysis_cc_traces_weka`, `semianalysis_cc_traces_weka_no_subagents`, `semianalysis_cc_traces_weka_with_subagents`, `semianalysis_cc_traces_weka_with_subagents_256k`, `semianalysis_cc_traces_weka_with_subagents_060226`, `semianalysis_cc_traces_weka_with_subagents_060226_256k`, `semianalysis_cc_traces_weka_with_subagents_060526`, `semianalysis_cc_traces_weka_with_subagents_060526_256k`, `semianalysis_cc_traces_weka_with_subagents_060826`, `semianalysis_cc_traces_weka_with_subagents_060826_256k`, `semianalysis_cc_traces_weka_061326`, `semianalysis_cc_traces_weka_061326_256k`, `semianalysis_cc_traces_weka_061526`, `semianalysis_cc_traces_weka_061526_256k`, `semianalysis_cc_traces_weka_062126`, `semianalysis_cc_traces_weka_062126_256k`, `weka_hf`]_ +
_Choices: [`exgentic_v2`, `exgentic`, `sharegpt`, `aimo`, `mmstar`, `mmvu`, `vision_arena`, `llava_onevision`, `aimo_aime`, `aimo_numina_cot`, `aimo_numina_1_5`, `spec_bench`, `spec_al_gsm8k`, `spec_al_math500`, `spec_al_humaneval`, `spec_al_mbpp`, `spec_al_mtbench`, `instruct_coder`, `blazedit_5k`, `blazedit_10k`, `librispeech`, `voxpopuli`, `gigaspeech`, `ami`, `spgispeech`, `semianalysis_cc_traces_weka`, `semianalysis_cc_traces_weka_no_subagents`, `semianalysis_cc_traces_weka_with_subagents`, `semianalysis_cc_traces_weka_with_subagents_256k`, `semianalysis_cc_traces_weka_with_subagents_060226`, `semianalysis_cc_traces_weka_with_subagents_060226_256k`, `semianalysis_cc_traces_weka_with_subagents_060526`, `semianalysis_cc_traces_weka_with_subagents_060526_256k`, `semianalysis_cc_traces_weka_with_subagents_060826`, `semianalysis_cc_traces_weka_with_subagents_060826_256k`, `semianalysis_cc_traces_weka_061326`, `semianalysis_cc_traces_weka_061326_256k`, `semianalysis_cc_traces_weka_061526`, `semianalysis_cc_traces_weka_061526_256k`, `semianalysis_cc_traces_weka_062126`, `semianalysis_cc_traces_weka_062126_256k`, `weka_hf`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `h_cua_perf`]_ #### `--hf-subset` `` @@ -2071,7 +2070,7 @@ Dataset-specific filter in key=value form. Repeat for multiple filters. Only sup #### `--custom-dataset-type` `` Format specification for custom dataset provided via `--input-file`. Determines parsing logic and expected file structure. Options: `single_turn` (JSONL with single exchanges), `multi_turn` (JSONL with conversation history), `tracelab` (TraceLab agentic-coding corpus, JSONL or gzipped JSONL), `mooncake_trace`/`bailian_trace`/`baseten_trace` (timestamped trace files), `random_pool` (directory of reusable prompts; when using `random_pool`, `--conversation-num` defaults to 100 if not specified; batch sizes > 1 sample each modality independently from a flat pool and do not preserve per-entry associations - use `single_turn` if paired modalities must stay together). Requires `--input-file`. Mutually exclusive with `--public-dataset`. -
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_coding`, `speed_bench_humanities`, `speed_bench_math`, `speed_bench_multilingual`, `speed_bench_qa`, `speed_bench_rag`, `speed_bench_reasoning`, `speed_bench_roleplay`, `speed_bench_stem`, `speed_bench_summarization`, `speed_bench_writing`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `speed_bench_throughput_1k_low_entropy`, `speed_bench_throughput_1k_mixed`, `speed_bench_throughput_1k_high_entropy`, `speed_bench_throughput_2k_low_entropy`, `speed_bench_throughput_2k_mixed`, `speed_bench_throughput_2k_high_entropy`, `speed_bench_throughput_8k_low_entropy`, `speed_bench_throughput_8k_mixed`, `speed_bench_throughput_8k_high_entropy`, `speed_bench_throughput_16k_low_entropy`, `speed_bench_throughput_16k_mixed`, `speed_bench_throughput_16k_high_entropy`, `speed_bench_throughput_32k_low_entropy`, `speed_bench_throughput_32k_mixed`, `speed_bench_throughput_32k_high_entropy`, `tracelab`, `weka_trace`]_ +
_Choices: [`burst_gpt_trace`, `bailian_trace`, `baseten_trace`, `mooncake_trace`, `h_cua_perf`, `raw_payload`, `inputs_json`, `dag_jsonl`, `sagemaker_data_capture`, `multi_turn`, `random_pool`, `single_turn`, `speed_bench_qualitative`, `speed_bench_throughput_1k`, `speed_bench_throughput_2k`, `speed_bench_throughput_8k`, `speed_bench_throughput_16k`, `speed_bench_throughput_32k`, `tracelab`, `weka_trace`]_ #### `--ignore-trace-delays` @@ -2808,11 +2807,11 @@ Output directory for all benchmark artifacts including metrics (`.csv`, `.json`, #### `--profile-export-prefix`, `--profile-export-file` `` -Base filename for ALL exported files. With prefix='foo' every output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, `foo_gpu_telemetry.jsonl`, and `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), historical per-file names are used: `profile_export_aiperf.{csv,json}` for the summary, `profile_export.jsonl` and `profile_export_raw.jsonl` for records, `gpu_telemetry_export.jsonl`, and `server_metrics_export.*`. Known suffixes (e.g. `_raw.jsonl`, `_timeslices.csv`, `_server_metrics.parquet`) are stripped from the supplied value. +Base filename for ALL exported files. With prefix='foo' every output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, `foo_outputs.json`, `foo_gpu_telemetry.jsonl`, and `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), historical per-file names are used: `profile_export_aiperf.{csv,json}` for the summary, `profile_export.jsonl` and `profile_export_raw.jsonl` for records, `outputs.json`, `gpu_telemetry_export.jsonl`, and `server_metrics_export.*`. Known suffixes (e.g. `_raw.jsonl`, `_outputs.json`, `_timeslices.csv`, `_server_metrics.parquet`) are stripped from the supplied value. #### `--export-level`, `--profile-export-level` `` -Controls which output files are generated. `summary`: Only aggregate metrics files (`.csv`, `.json`). `records`: Includes per-request metrics (`.jsonl`). `raw`: Includes raw request/response data (`_raw.jsonl`). +Controls which output files are generated. `summary`: Only aggregate metrics files (`.csv`, `.json`). `records`: Includes per-request metrics (`.jsonl`). `raw`: Includes raw request/response data (`_raw.jsonl`) and, unless `--no-export-outputs-json` is passed, the generated text (`_outputs.json`). **Choices:** @@ -2836,10 +2835,9 @@ Auto-invoke `aiperf plot` against the artifact directory after the benchmark com Treat auto-plot failures as fatal: re-raise so `aiperf profile` exits non-zero. Only meaningful when auto-plot is on. Default False = warn and continue.
_Flag (no value required)_ -#### `--export-outputs-json` +#### `--export-outputs-json`, `--no-export-outputs-json` -Export generated response text to outputs.json after the run. When enabled, the raw generated-text payload for each request is written to an outputs.json file in the artifact directory. -
_Flag (no value required)_ +Export generated response text after the run. When enabled, the raw generated-text payload for each request is written to `outputs.json` in the artifact directory, or to `_outputs.json` when `--profile-export-prefix` is set (prefix `foo` gives `foo_outputs.json`). Implied by `--export-level raw`; pass `--no-export-outputs-json` to opt out of the extra file while keeping raw export. #### `--otel-url` `` @@ -3370,7 +3368,7 @@ Output format - 'csv', 'table', or 'both'. Defaults to 'both'. Which metric to report - 'accept_length', 'accept_rate', or 'throughput'. Defaults to 'accept_length'.
_Default: `accept_length`_ -### `--source` `` +#### `--source` `` Where acceptance numbers come from - 'records' (per-request ``profile_export.jsonl``), 'summary' (the same per-request data reduced to run-level scalars in ``profile_export_aiperf.json``, the only source left at ``--export-level summary``), 'server' (Prometheus scrape in ``server_metrics_export.json``), or 'auto' to try them in that order. Only 'records' can split one run into per-category columns; only 'server' reads server-side data. Defaults to 'auto'.
_Default: `auto`_ diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 9ae5c55407..fc4212e7f7 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -238,6 +238,8 @@ Record processing and export configuration. Controls batch sizes, processor scal | `AIPERF_RECORD_PROGRESS_REPORT_INTERVAL` | `2.0` | ≥ 0.1, ≤ 600.0 | Interval in seconds between records progress report messages | | `AIPERF_RECORD_PROCESS_RECORDS_TIMEOUT` | `300.0` | ≥ 1.0, ≤ 100000.0 | Timeout in seconds for processing record results | | `AIPERF_RECORD_STRIP_PAYLOAD_BYTES` | `None` | — | Tri-state control for omitting canonical request payload bytes from RecordContext after a request is sent, which substantially reduces record-pipeline memory for very large prompts. None (default) auto-detects: bytes are stripped only when no downstream record consumer needs them (client-side input tokenization disabled, no synthetic image/audio/video inputs, and raw payload export off). True forces stripping even when a consumer wants the bytes, disabling client-side input tokenization, media counting from request bodies, and raw request payload export. False always retains them. Auto-detection does not see media embedded in custom dataset payloads under server-token-count mode; set False explicitly for that case. | +| `AIPERF_RECORD_PROGRESS_STALL_TIMEOUT` | `600.0` | ≥ 0.0, ≤ 86400.0 | Seconds during which a profiling phase has requests in flight and none of them complete, after which the run fails instead of waiting indefinitely. A phase completes only once the credit phase reports its final count, so a credit that is never returned leaves that condition unevaluable and the run waits forever with no output. Per-request timeouts do not cover this: they default to 6 hours and never apply to a request that was not dispatched. Note that a single legitimately long request will fail the run if it exceeds this timeout while nothing else completes, so raise this value for workloads whose individual requests run longer. Each profiling phase is judged on its own counters, so an overlapping `seamless` phase that keeps completing work does not mask a stall in another. Set to 0 to disable and restore the previous unbounded wait. | +| `AIPERF_RECORD_PROGRESS_STALL_CHECK_INTERVAL` | `30.0` | ≥ 1.0, ≤ 600.0 | Seconds between stall checks. Each check with no completions also logs how many requests are still in flight and which phase they belong to, so a stalled run is diagnosable from the log rather than silent. | ## SEARCHPLANNER diff --git a/src/aiperf/config/schema/aiperf-config.schema.json b/src/aiperf/config/schema/aiperf-config.schema.json index 122cd9ac39..2544f03b1c 100644 --- a/src/aiperf/config/schema/aiperf-config.schema.json +++ b/src/aiperf/config/schema/aiperf-config.schema.json @@ -1,23372 +1 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://nvidia.github.io/aiperf/schemas/aiperf-config.schema.json", - "$defs": { - "AIPerfLogLevel": { - "description": "Logging levels for AIPerf output verbosity.", - "enum": [ - "TRACE", - "DEBUG", - "INFO", - "NOTICE", - "WARNING", - "SUCCESS", - "ERROR", - "CRITICAL" - ], - "title": "AIPerfLogLevel", - "type": "string" - }, - "AccuracyBenchmarkType": { - "enum": [ - "mmlu", - "mmlu_pro", - "aime", - "hellaswag", - "bigbench", - "aime24", - "aime25", - "math_500", - "gsm8k", - "gpqa_diamond", - "lcb_codegeneration" - ], - "title": "AccuracyBenchmarkType", - "type": "string" - }, - "AccuracyConfig": { - "additionalProperties": false, - "description": "Configuration for accuracy benchmarking mode.\n\nWhen benchmark is set, enables accuracy evaluation alongside\nperformance profiling using standard benchmarks (MMLU, AIME, etc.).", - "properties": { - "benchmark": { - "anyOf": [ - { - "$ref": "#/$defs/AccuracyBenchmarkType" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Accuracy benchmark to run. When set, enables accuracy benchmarking alongside performance profiling. AIME variants: 'aime' is the legacy combined set (deprecated for new runs); prefer the year-pinned 'aime24' or 'aime25' for reproducibility." - }, - "tasks": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Specific tasks or subtasks within the benchmark to evaluate (e.g., specific MMLU subjects). Accepts comma-separated values (e.g. abstract_algebra,anatomy) or repeated flags. If not set, all tasks are included.", - "title": "Tasks" - }, - "nShots": { - "anyOf": [ - { - "maximum": 32, - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Number of few-shot examples to include in the prompt. 0 means zero-shot evaluation, None uses the benchmark default (e.g. MMLU=5). Maximum 32.", - "title": "Nshots", - "x-jinja2-supported": true - }, - "enableCot": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Enable chain-of-thought prompting for accuracy evaluation. Adds reasoning instructions to the prompt. Defaults to the benchmark's ``default_enable_cot`` metadata when unset (e.g. AIME defaults to True).", - "title": "Enablecot" - }, - "grader": { - "anyOf": [ - { - "$ref": "#/$defs/AccuracyGraderType" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Override the default grader for the selected benchmark (e.g., exact_match, math, multiple_choice, code_execution). If not set, uses the benchmark's default grader." - }, - "systemPrompt": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Custom system prompt to use for accuracy evaluation. Overrides any benchmark-specific system prompt.", - "title": "Systemprompt" - }, - "verbose": { - "default": false, - "description": "Enable verbose output for accuracy evaluation, showing per-problem grading details.", - "title": "Verbose", - "type": "boolean" - } - }, - "title": "AccuracyConfig", - "type": "object" - }, - "AccuracyGraderType": { - "enum": [ - "exact_match", - "math", - "multiple_choice", - "code_execution", - "lighteval_expr", - "lighteval_latex", - "lighteval_gpqa", - "lighteval_gsm8k", - "mmlu_pro" - ], - "title": "AccuracyGraderType", - "type": "string" - }, - "AdaptiveSearchSweep": { - "additionalProperties": false, - "description": "Adaptive outer-loop search (Bayesian / monotonic / etc.).\n\nA third sweep variant: BO drives variation choice itself, so the\ngrid-style ``iteration_order`` and ``same_seed`` knobs do not apply.\nInherits cooldown/sla_filters/post_process from `_SweepBase` only.", - "properties": { - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaFilters": { - "description": "SLA constraints applied at trial scoring / aggregation time.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Slafilters", - "type": "array" - }, - "postProcess": { - "anyOf": [ - { - "$ref": "#/$defs/PostProcessSpec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional post-aggregation handler emitted by a search recipe." - }, - "recipeName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.", - "title": "Recipename" - }, - "type": { - "const": "adaptive_search", - "default": "adaptive_search", - "description": "Sweep type discriminator.", - "title": "Type", - "type": "string" - }, - "planner": { - "$ref": "#/$defs/SearchPlannerType", - "default": "bayesian", - "description": "Outer-loop planner plugin name (bayesian | optuna | monotonic_sla | smooth_isotonic)." - }, - "searchSpace": { - "description": "Dimensions to optimize over.", - "items": { - "$ref": "#/$defs/SearchSpaceDimension" - }, - "minItems": 1, - "title": "Searchspace", - "type": "array" - }, - "objectives": { - "description": "What to optimize. Length-1 = single-objective search; length-N = multi-objective search. BoTorch-backed Pareto BO requires --optuna-sampler botorch with a multi-objective acquisition like qlognehvi.", - "items": { - "$ref": "#/$defs/Objective" - }, - "minItems": 1, - "title": "Objectives", - "type": "array" - }, - "outcomeConstraints": { - "description": "Feasibility gates on non-optimized metrics. Empty = no constraints.", - "items": { - "$ref": "#/$defs/OutcomeConstraint" - }, - "title": "Outcomeconstraints", - "type": "array" - }, - "maxIterations": { - "description": "Maximum BO iterations.", - "title": "Maxiterations", - "oneOf": [ - { - "description": "Maximum BO iterations.", - "maximum": 200, - "minimum": 2, - "title": "Maxiterations", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "nInitialPoints": { - "default": 5, - "description": "Sobol-random points before the GP fits. Must be < max_iterations for `bayesian`/`optuna` planners; ignored by 1-D SLA planners.", - "title": "Ninitialpoints", - "oneOf": [ - { - "default": 5, - "description": "Sobol-random points before the GP fits. Must be < max_iterations for `bayesian`/`optuna` planners; ignored by 1-D SLA planners.", - "minimum": 1, - "title": "Ninitialpoints", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "plateauWindow": { - "default": 8, - "description": "Recent-iteration window for plateau detection. Default 8 keeps the window strictly larger than ``n_initial_points`` (default 5), so the plateau test always inspects at least a few GP-driven iterations rather than tripping on the Sobol-only prefix when the initial points happen to land in a flat region.", - "title": "Plateauwindow", - "oneOf": [ - { - "default": 8, - "description": "Recent-iteration window for plateau detection. Default 8 keeps the window strictly larger than ``n_initial_points`` (default 5), so the plateau test always inspects at least a few GP-driven iterations rather than tripping on the Sobol-only prefix when the initial points happen to land in a flat region.", - "minimum": 2, - "title": "Plateauwindow", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "plateauThreshold": { - "default": 0.01, - "description": "Coefficient-of-variation threshold for plateau stop.", - "title": "Plateauthreshold", - "oneOf": [ - { - "default": 0.01, - "description": "Coefficient-of-variation threshold for plateau stop.", - "exclusiveMinimum": 0, - "title": "Plateauthreshold", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "improvementPatience": { - "default": 10, - "description": "Stop after this many no-improvement iterations.", - "title": "Improvementpatience", - "oneOf": [ - { - "default": 10, - "description": "Stop after this many no-improvement iterations.", - "minimum": 2, - "title": "Improvementpatience", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "randomSeed": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "If set, passed as `random_state` to the planner backend.", - "title": "Randomseed", - "x-jinja2-supported": true - }, - "optunaSampler": { - "default": "botorch", - "description": "Optuna sampler when planner=optuna. Default ``botorch`` is the preferred Gaussian-process qLogNEI/qLogNEHVI path and requires the optional ``botorch`` extra; when this default is implicit and the optional stack is unavailable, the planner warns and falls back to ``tpe``. ``tpe`` is the dep-light Tree-Parzen estimator that ships with Optuna core. ``gp`` is Optuna's pure-GP sampler and requires torch.", - "enum": [ - "gp", - "tpe", - "botorch" - ], - "title": "Optunasampler", - "type": "string" - }, - "optunaAcquisition": { - "anyOf": [ - { - "enum": [ - "logei", - "qlogei", - "qnei", - "qlognei", - "qehvi", - "qnehvi", - "qlognehvi" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Acquisition override for the BoTorch sampler. None lets Optuna pick (single-objective LogEI). ``qnei`` = Letham 2017 noisy-EI; ``qlognei`` = Ament 2023 (BoTorch's strongly recommended modern default; requires botorch>=0.10). ``qehvi``/``qnehvi``/``qlognehvi`` are multi-objective hypervolume acquisitions for Pareto BO. Only meaningful when planner=optuna and optuna_sampler=botorch; ignored otherwise.", - "title": "Optunaacquisition" - }, - "optunaTerminator": { - "default": "none", - "description": "Optional posterior-regret stopping rule (Makarova 2022 / Ishibashi 2023; same family as Wilson 2024). Layered on top of three-signal convergence. Only meaningful when planner=optuna; ignored otherwise.", - "enum": [ - "regret", - "emmr", - "none" - ], - "title": "Optunaterminator", - "type": "string" - }, - "objectivePooling": { - "default": "mean", - "description": "Per-trial percentile aggregation. ``mean`` = arithmetic mean of per-trial percentiles across trials (current default). ``pooled`` = walks each trial's profile_export.jsonl and computes np.percentile over the pooled raw-sample bag (the statistic that satisfies SLO claims; requires --export-level records). Only meaningful when objective.stat is a percentile (p50/p90/p95/p99); a no-op when stat=avg.", - "enum": [ - "mean", - "pooled" - ], - "title": "Objectivepooling", - "type": "string" - }, - "monotonicStabilityTrials": { - "default": 2, - "description": "MonotonicSLA planner stability-window size.", - "title": "Monotonicstabilitytrials", - "oneOf": [ - { - "default": 2, - "description": "MonotonicSLA planner stability-window size.", - "minimum": 1, - "title": "Monotonicstabilitytrials", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaReplicates": { - "default": 0, - "description": "Replicate-step trial count override for the smooth_isotonic SLA planner. 0 = auto (Hyperband-flavored formula based on sigma_margin/threshold). >0 = fixed override. Ignored by other planners.", - "title": "Slareplicates", - "oneOf": [ - { - "default": 0, - "description": "Replicate-step trial count override for the smooth_isotonic SLA planner. 0 = auto (Hyperband-flavored formula based on sigma_margin/threshold). >0 = fixed override. Ignored by other planners.", - "minimum": 0, - "title": "Slareplicates", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaPrecision": { - "default": "normal", - "description": "Per-probe sample-budget knob for the smooth_isotonic SLA planner. tight=10000, normal=1000, coarse=300 requests. Drives p99 CI width. Ignored by other planners.", - "enum": [ - "tight", - "normal", - "coarse" - ], - "title": "Slaprecision", - "type": "string" - }, - "slaWarmupSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Discard this many seconds at the start of each probe before computing margins, to skip CUDA-graph warmup and KV-cache shape transients. None = auto (max(30s, 3 * inter-batch-time)). Ignored by other planners.", - "title": "Slawarmupseconds", - "x-jinja2-supported": true - }, - "constraintMode": { - "default": "penalty", - "description": "Constrained-BO acquisition mode (penalty | eic).", - "enum": [ - "penalty", - "eic" - ], - "title": "Constraintmode", - "type": "string" - }, - "slaTiers": { - "description": "Multi-tier SLO definitions parsed from --search-sla-tier flags. Each tier is a named group of SLA filters sharing a common boundary search. When non-empty (2-10 tiers), the MultiTierPlanner is activated instead of single-tier planners.", - "items": { - "$ref": "#/$defs/SLOTier" - }, - "title": "Slatiers", - "type": "array" - } - }, - "required": [ - "searchSpace", - "objectives", - "maxIterations" - ], - "title": "AdaptiveSearchSweep", - "type": "object" - }, - "ArtifactsConfig": { - "additionalProperties": false, - "description": "Artifacts configuration for benchmark output.\n\nControls where and how benchmark results are exported.\nUses flat structure with format arrays instead of nested export configs.", - "properties": { - "dir": { - "default": "artifacts", - "description": "Output directory for all benchmark artifacts. Created if it doesn't exist.", - "format": "path", - "title": "Dir", - "type": "string" - }, - "prefix": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Base filename override applied to ALL profile and server-metrics exports. With prefix='foo' every output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, `foo_gpu_telemetry.jsonl`, `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), historical per-file names are used: `profile_export_aiperf.csv/json`, `profile_export.jsonl`, `profile_export_raw.jsonl`, `gpu_telemetry_export.jsonl`, `server_metrics_export.{jsonl,json,csv,parquet}`. Known suffixes (`_raw.jsonl`, `_timeslices.{csv,json}`, `_gpu_telemetry.jsonl`, `_server_metrics.{jsonl,json,csv,parquet}`, `.csv`/`.json`/`.jsonl`/`.parquet`) are stripped from the supplied value so `--profile-export-prefix foo_raw.jsonl` still yields a clean `foo` base.", - "title": "Prefix" - }, - "summary": { - "anyOf": [ - { - "items": { - "const": "json", - "type": "string" - }, - "type": "array" - }, - { - "const": false, - "type": "boolean" - } - ], - "description": "Summary export formats. Only 'json' is wired up to this field; the CSV summary is emitted regardless. Set to false to disable the summary JSON file only.", - "title": "Summary" - }, - "records": { - "anyOf": [ - { - "items": { - "const": "jsonl", - "type": "string" - }, - "type": "array" - }, - { - "const": false, - "type": "boolean" - } - ], - "description": "Per-request records export formats. Only 'jsonl' is wired up today. Set to false to disable the per-record JSONL file.", - "title": "Records" - }, - "raw": { - "default": false, - "description": "Export raw request/response payloads as JSONL.", - "title": "Raw", - "type": "boolean" - }, - "trace": { - "default": false, - "description": "Export HTTP trace data for debugging.", - "title": "Trace", - "type": "boolean" - }, - "sliceDuration": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Time slice duration in seconds for trend analysis (must be > 0). Divides benchmark into windows for per-window statistics. Supports: 30, '30s', '5m', '2h'.", - "title": "Sliceduration", - "x-jinja2-supported": true - }, - "showTraceTiming": { - "default": false, - "description": "Display HTTP trace timing metrics in console output. Shows detailed timing breakdown: blocked, DNS, connecting, sending, waiting (TTFB), receiving, and total duration.", - "title": "Showtracetiming", - "type": "boolean" - }, - "userFiles": { - "description": "User-defined templated files materialized into the run directory before the benchmark begins.", - "items": { - "$ref": "#/$defs/UserFile" - }, - "title": "Userfiles", - "type": "array" - }, - "autoPlot": { - "default": false, - "description": "Auto-invoke `aiperf plot` against the artifact directory after the benchmark completes. Resolved by the CLI converter from the tri-state CLI flag and the active search recipe's auto_plot_default; by the time it lands here it is a plain bool.", - "title": "Autoplot", - "type": "boolean" - }, - "plotRequired": { - "default": false, - "description": "Treat auto-plot failures as fatal: re-raise so `aiperf profile` exits non-zero. Only meaningful when auto_plot is True. Default False = warn and continue.", - "title": "Plotrequired", - "type": "boolean" - }, - "exportOutputsJson": { - "default": false, - "description": "Export generated response text to outputs.json after the run.", - "title": "Exportoutputsjson", - "type": "boolean" - } - }, - "title": "ArtifactsConfig", - "type": "object" - }, - "AudioConfig": { - "additionalProperties": false, - "description": "Configuration for synthetic audio generation in multimodal datasets.\n\nControls the generation of synthetic audio for speech-to-text\nand audio-language model benchmarking. Generated audio is random\nnoise with specified sample rate, bit depth, and format.", - "properties": { - "batchSize": { - "default": 0, - "description": "Number of audio inputs to include in each multimodal request. Supported with chat endpoint type for multimodal models. Set to 0 to disable audio inputs.", - "title": "Batchsize", - "oneOf": [ - { - "default": 0, - "description": "Number of audio inputs to include in each multimodal request. Supported with chat endpoint type for multimodal models. Set to 0 to disable audio inputs.", - "minimum": 0, - "title": "Batchsize", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "length": { - "description": "Audio duration in seconds. Can be a fixed value or {mean, stddev} distribution. Used when batch_size > 0 for multimodal benchmarking.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Length", - "x-jinja2-supported": true - }, - "format": { - "$ref": "#/$defs/AudioFormat", - "default": "wav", - "description": "File format for generated audio files. wav: uncompressed PCM (larger files). mp3: compressed (smaller files). Format affects file size in multimodal requests but not audio characteristics." - }, - "sampleRates": { - "description": "List of audio sample rates in kHz to randomly select from. Common values: 8.0 (telephony), 16.0 (speech), 44.1 (CD quality), 48.0 (professional). Specify multiple values for mixed-quality testing.", - "items": { - "type": "number" - }, - "title": "Samplerates", - "type": "array" - }, - "depths": { - "description": "List of audio bit depths in bits to randomly select from. Each audio file is assigned a random depth from this list. Common values: 8 (low quality), 16 (CD quality), 24 (professional), 32 (high-end). Specify multiple values for mixed-quality testing.", - "items": { - "type": "integer" - }, - "title": "Depths", - "type": "array" - }, - "channels": { - "default": 1, - "description": "Number of audio channels. 1 = mono (single channel), 2 = stereo (left/right channels). Stereo doubles file size. Most speech models use mono.", - "title": "Channels", - "oneOf": [ - { - "default": 1, - "description": "Number of audio channels. 1 = mono (single channel), 2 = stereo (left/right channels). Stereo doubles file size. Most speech models use mono.", - "maximum": 2, - "minimum": 1, - "title": "Channels", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "title": "AudioConfig", - "type": "object" - }, - "AudioFormat": { - "description": "Audio file formats for synthetic audio generation.", - "enum": [ - "wav", - "mp3" - ], - "title": "AudioFormat", - "type": "string" - }, - "BenchmarkConfig": { - "additionalProperties": false, - "description": "Pure runtime configuration - what SystemController and services need.\n\nContains all fields required to execute a single benchmark run.\nDoes NOT include sweep, multi_run, variables, or random_seed settings\n(those live on AIPerfConfig as envelope-level cross-variation fields).\n\nRequired Sections:\n - models: Model(s) to benchmark\n - endpoint: Server connection settings\n - datasets: Single-element list of dataset configuration\n - phases: Ordered list of benchmark phases. A single-dict under\n ``phases:`` is rejected; use the envelope-level ``warmup:`` /\n ``profiling:`` shorthand or an explicit list.\n\nOptional Sections:\n - artifacts: Export and console settings\n - slos: SLO-based quality metrics (generic dict)\n - tokenizer: Token counting configuration\n - gpu_telemetry: GPU metrics from DCGM endpoints\n - server_metrics: Server metrics from Prometheus endpoints\n - runtime: Worker and communication settings\n - logging: Logging and debug settings\n - metrics: Metrics aggregation configuration\n - accuracy: Accuracy evaluation configuration\n\nValidation:\n The configuration is validated in several stages:\n 1. Individual field validation (types, ranges, formats)\n 2. Cross-field validation (mutual exclusivity, dependencies)\n\nEnvironment Variables:\n Values can reference environment variables using ${VAR} syntax.\n Optional defaults: ${VAR:default_value}\n\n Example:\n api_key: ${OPENAI_API_KEY}\n api_key: ${OPENAI_API_KEY:sk-default}", - "properties": { - "models": { - "description": "Model configuration. Accepts a single model name string, a list of model names, or an advanced configuration with strategy and weighted items. All forms are normalized to ModelsAdvanced. Accepts a model name string, list of model names, or full ModelsAdvanced config.", - "oneOf": [ - { - "type": "string", - "description": "Single model name (normalized to ModelsAdvanced)." - }, - { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "List of model names (normalized to ModelsAdvanced)." - }, - { - "$ref": "#/$defs/ModelsAdvanced" - } - ] - }, - "endpoint": { - "$ref": "#/$defs/EndpointConfig", - "description": "Endpoint configuration for connecting to inference server(s). Includes URLs, API type, authentication, timeout, and connection settings." - }, - "datasets": { - "description": "Dataset configuration as a single-element list. The list shape exists to share the schema between YAML and the AIPerfSweep CRD; the runtime currently loads exactly one dataset. Singular `dataset:` shorthand at the BenchmarkConfig top level is normalized to a one-entry list with name='default'.", - "items": { - "discriminator": { - "propertyName": "type", - "mapping": { - "synthetic": "#/$defs/SyntheticDataset", - "file": "#/$defs/FileDataset", - "public": "#/$defs/PublicDataset" - } - }, - "anyOf": [ - { - "$ref": "#/$defs/SyntheticDataset" - }, - { - "$ref": "#/$defs/FileDataset" - }, - { - "$ref": "#/$defs/PublicDataset" - } - ] - }, - "maxItems": 1, - "minItems": 1, - "title": "Datasets", - "type": "array" - }, - "phases": { - "description": "Ordered benchmark phases. Each entry must have a unique 'name' (e.g. 'warmup', 'profiling'). Order in the list IS the execution order; the first phase runs first. Single-config shorthand ({'type': 'concurrency', ...}) is normalized to a list of one. Top-level 'warmup:'/'profiling:' shorthand is normalized to a [warmup, profiling] list pre-validation. Shorthand: use a single phase config directly (with 'type' key) and it is normalized to a one-entry phases list.", - "oneOf": [ - { - "description": "Ordered benchmark phases. Each entry must have a unique 'name' (e.g. 'warmup', 'profiling'). Order in the list IS the execution order; the first phase runs first. Single-config shorthand ({'type': 'concurrency', ...}) is normalized to a list of one. Top-level 'warmup:'/'profiling:' shorthand is normalized to a [warmup, profiling] list pre-validation.", - "items": { - "discriminator": { - "propertyName": "type", - "mapping": { - "concurrency": "#/$defs/ConcurrencyPhase", - "poisson": "#/$defs/PoissonPhase", - "gamma": "#/$defs/GammaPhase", - "constant": "#/$defs/ConstantPhase", - "user_centric": "#/$defs/UserCentricPhase", - "fixed_schedule": "#/$defs/FixedSchedulePhase" - } - }, - "anyOf": [ - { - "$ref": "#/$defs/ConcurrencyPhase" - }, - { - "$ref": "#/$defs/PoissonPhase" - }, - { - "$ref": "#/$defs/GammaPhase" - }, - { - "$ref": "#/$defs/ConstantPhase" - }, - { - "$ref": "#/$defs/UserCentricPhase" - }, - { - "$ref": "#/$defs/FixedSchedulePhase" - } - ] - }, - "minItems": 1, - "title": "Phases", - "type": "array" - }, - { - "anyOf": [ - { - "additionalProperties": false, - "description": "Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "concurrency", - "description": "Concurrency-controlled immediate dispatch.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "default": 1, - "description": "Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.", - "minimum": 1, - "title": "Concurrency", - "type": "integer" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "ConcurrencyPhase", - "type": "object" - }, - { - "additionalProperties": false, - "description": "Poisson-distributed request arrivals at the target rate.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "poisson", - "description": "Poisson-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "PoissonPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "Gamma-distributed request arrivals with configurable smoothness.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "gamma", - "description": "Gamma-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "smoothness": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.", - "title": "Smoothness" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "GammaPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "Constant-rate request arrivals (fixed inter-arrival time).", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "constant", - "description": "Constant rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "ConstantPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "user_centric", - "description": "N users sharing a global request rate.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "users": { - "description": "Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.", - "minimum": 1, - "title": "Users", - "type": "integer" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type", - "users", - "rate" - ], - "title": "UserCentricPhase", - "type": "object", - "allOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - } - ] - }, - { - "additionalProperties": false, - "description": "Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "fixed_schedule", - "description": "Replay requests at trace timestamps.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "autoOffset": { - "default": true, - "description": "Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.", - "title": "Autooffset", - "type": "boolean" - }, - "startOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter out trace requests before this timestamp in ms (must be >= 0).", - "title": "Startoffset" - }, - "endOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter out trace requests after this timestamp in ms (must be >= 0).", - "title": "Endoffset" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "FixedSchedulePhase", - "type": "object" - } - ], - "description": "Single phase config (normalized to a one-entry phases list).", - "discriminator": { - "propertyName": "type", - "mapping": { - "concurrency": "#/$defs/ConcurrencyPhase", - "poisson": "#/$defs/PoissonPhase", - "gamma": "#/$defs/GammaPhase", - "constant": "#/$defs/ConstantPhase", - "user_centric": "#/$defs/UserCentricPhase", - "fixed_schedule": "#/$defs/FixedSchedulePhase" - } - } - } - ] - }, - "artifacts": { - "$ref": "#/$defs/ArtifactsConfig", - "description": "Artifacts configuration for benchmark output. Controls output directory and export formats." - }, - "slos": { - "anyOf": [ - { - "additionalProperties": { - "oneOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "SLO (Service Level Objectives) configuration as a generic dict. Maps metric names to threshold values. A request is counted as good only if it meets ALL specified thresholds.", - "title": "Slos", - "x-jinja2-supported": true - }, - "tokenizer": { - "anyOf": [ - { - "$ref": "#/$defs/TokenizerConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Tokenizer configuration for token counting. Used for ISL/OSL enforcement and accurate metrics. If not specified, uses the first model name." - }, - "gpuTelemetry": { - "$ref": "#/$defs/GpuTelemetryConfig", - "description": "GPU telemetry configuration for DCGM metrics collection. Collects GPU metrics (power, utilization, temperature) from DCGM endpoints. Enabled by default. Set enabled: false to disable." - }, - "serverMetrics": { - "$ref": "#/$defs/ServerMetricsConfig", - "description": "Server metrics configuration for Prometheus scraping. Collects operational metrics (queue depth, KV cache, batch sizes) from inference server Prometheus endpoints. Enabled by default. Set enabled: false to disable." - }, - "networkLatency": { - "$ref": "#/$defs/NetworkLatencyConfig", - "description": "Network latency calibration configuration. Probes the endpoint RTT during profiling and subtracts the mean from latency metrics, emitted as separate network_adjusted_* metrics. Disabled by default." - }, - "otel": { - "$ref": "#/$defs/OTelConfig" - }, - "mlflow": { - "$ref": "#/$defs/MLflowConfig" - }, - "wandb": { - "$ref": "#/$defs/WandbConfig" - }, - "runtime": { - "$ref": "#/$defs/RuntimeConfig", - "description": "Runtime configuration for worker processes and inter-process communication." - }, - "logging": { - "$ref": "#/$defs/LoggingConfig" - }, - "metrics": { - "$ref": "#/$defs/MetricsConfig", - "description": "Metrics aggregation configuration for benchmark summaries." - }, - "accuracy": { - "anyOf": [ - { - "$ref": "#/$defs/AccuracyConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Accuracy benchmarking configuration. When set, enables accuracy evaluation alongside performance profiling." - }, - "scenario": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Lock all benchmark invariants for a named scenario (e.g. 'inferencex-agentx-mvp'). Plain data here; the lock is applied by the ScenarioResolver step in the pre-bootstrap resolver chain (auto-fills defaults, validates, raises ScenarioLockError on conflict). Distinct from the sweep ``scenarios`` strategy (SweepConfig), which expands hand-picked named runs -- this field is a single invariant LOCK, not a sweep.", - "title": "Scenario" - }, - "unsafeOverride": { - "default": false, - "description": "Convert scenario lock errors to warnings; stamps submission_valid=false in the resolved scenario outcome. No-op without ``scenario``. Plain data; consumed by the ScenarioResolver.", - "title": "Unsafeoverride", - "type": "boolean" - }, - "model": { - "description": "Shorthand for 'models'. Accepts a single value which will be converted to a list/dict. Model configuration. Accepts a single model name string, a list of model names, or an advanced configuration with strategy and weighted items. All forms are normalized to ModelsAdvanced.", - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "$ref": "#/$defs/ModelsAdvanced", - "description": "Model configuration. Accepts a single model name string, a list of model names, or an advanced configuration with strategy and weighted items. All forms are normalized to ModelsAdvanced.", - "x-kubernetes-preserve-unknown-fields": true - } - ], - "x-singular-alias-of": "models" - }, - "dataset": { - "anyOf": [ - { - "additionalProperties": false, - "description": "Synthetic dataset configuration.\n\nGenerates prompts programmatically based on token length\nspecifications. Ideal for controlled experiments.", - "properties": { - "name": { - "description": "Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "type": { - "const": "synthetic", - "description": "Dataset type discriminator. Must be 'synthetic'.", - "title": "Type", - "type": "string" - }, - "entries": { - "default": 100, - "description": "Total number of unique entries to generate for the dataset. Each entry represents a unique prompt with sampled ISL/OSL. Entries are reused across conversations and turns according to the sampling strategy. Higher values provide more diversity.", - "minimum": 1, - "title": "Entries", - "type": "integer" - }, - "randomSeed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Random seed for deterministic dataset generation. When set, makes synthetic prompts, sampling, and other random operations reproducible across runs. Essential for A/B testing and debugging. Overrides global random_seed for this dataset.", - "title": "Randomseed" - }, - "sampling": { - "$ref": "#/$defs/DatasetSamplingStrategy", - "default": "sequential", - "description": "Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion." - }, - "prompts": { - "anyOf": [ - { - "$ref": "#/$defs/PromptConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Prompt/token length configuration specifying ISL, OSL, sequence distributions, and batch processing settings." - }, - "prefixPrompts": { - "anyOf": [ - { - "$ref": "#/$defs/PrefixPromptConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Shared prefix configuration for KV cache testing. Generates prefix prompts that are prepended to user prompts, simulating cached context scenarios." - }, - "turns": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Number of request-response turns per conversation. Can be a fixed integer or {mean, stddev} distribution. Each turn consists of a user message and model response. Set to 1 for single-turn interactions. Multi-turn conversations enable testing of context retention and conversation history handling.", - "title": "Turns" - }, - "turnDelay": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Delay in milliseconds between consecutive turns within a multi-turn conversation. Can be a fixed value or {mean, stddev} distribution. Simulates user think time between receiving a response and sending the next message. Only applies when turns > 1. Set to 0 for back-to-back turns.", - "title": "Turndelay" - }, - "turnDelayRatio": { - "default": 1.0, - "description": "Multiplier for scaling all turn delays. Applied after mean/stddev calculation: actual_delay = calculated_delay * ratio. Values < 1 speed up conversations, > 1 slow them down. Set to 0 to eliminate delays entirely.", - "minimum": 0.0, - "title": "Turndelayratio", - "type": "number" - }, - "images": { - "anyOf": [ - { - "$ref": "#/$defs/ImageConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Synthetic image configuration for multimodal vision-language testing." - }, - "audio": { - "anyOf": [ - { - "$ref": "#/$defs/AudioConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Synthetic audio configuration for multimodal speech/audio testing." - }, - "video": { - "anyOf": [ - { - "$ref": "#/$defs/VideoConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Synthetic video configuration for multimodal video understanding testing." - }, - "rankings": { - "anyOf": [ - { - "$ref": "#/$defs/RankingsConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Rankings/reranking configuration for generating query-passage pairs. Only relevant for rankings endpoint types." - }, - "isl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Shorthand for prompts.isl; hoisted into prompts before validation.", - "title": "Isl" - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Shorthand for prompts.osl; hoisted into prompts before validation.", - "title": "Osl" - } - }, - "required": [ - "type" - ], - "title": "SyntheticDataset", - "type": "object" - }, - { - "additionalProperties": false, - "description": "File-based dataset configuration.\n\nLoads prompts from a local file in various formats.\nSupports trace replay and custom sampling strategies.", - "properties": { - "name": { - "description": "Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "type": { - "const": "file", - "description": "Dataset type discriminator. Must be 'file'.", - "title": "Type", - "type": "string" - }, - "path": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to file or directory containing benchmark dataset. Can be absolute or relative. Mutually exclusive with `records:`. Supported formats depend on the format field: JSONL for single_turn/multi_turn, JSONL trace files for mooncake_trace/bailian_trace, Parquet for baseten_trace, directories for random_pool.", - "title": "Path" - }, - "records": { - "anyOf": [ - { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - { - "additionalProperties": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inline benchmark records, embedded directly in the YAML config. Mutually exclusive with `path:`. The element schema is determined by `format:` (same shape as one line of the equivalent JSONL file). For `format: random_pool`, may be either a flat list (single pool) or a dict-of-lists (multi-pool, mirrors the directory-of-JSONLs file mode). All other formats require a flat list.", - "title": "Records" - }, - "format": { - "$ref": "#/$defs/DatasetFormat", - "default": "single_turn", - "description": "Dataset file format determining parsing logic and expected file structure. single_turn: JSONL with single prompt-response exchanges. multi_turn: JSONL with conversation history. mooncake_trace / bailian_trace / baseten_trace / burst_gpt_trace: timestamped trace files for replay. sagemaker_data_capture: JSONL captured by SageMaker DataCapture. random_pool: directory of reusable prompts." - }, - "sampling": { - "$ref": "#/$defs/DatasetSamplingStrategy", - "default": "sequential", - "description": "Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion." - }, - "synthesis": { - "anyOf": [ - { - "$ref": "#/$defs/SynthesisConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Trace synthesis/transformation configuration. Allows scaling timestamps and token lengths before replay. Applies to trace formats such as mooncake_trace and baseten_trace, except speedup_ratio, which is rejected for baseten_trace (use replay_speedup / --replay-speedup to scale replay pacing there)." - }, - "entries": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Limit number of records to use from file. If not specified, uses all records in the file.", - "title": "Entries" - }, - "randomSeed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Random seed for deterministic sampling. When set, makes random/shuffle sampling reproducible across runs. Overrides global random_seed for this dataset.", - "title": "Randomseed" - }, - "traceSessionSampleRatio": { - "anyOf": [ - { - "exclusiveMinimum": 0.0, - "maximum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Fraction of trace sessions to keep for replay, sampled whole-session to preserve multi-turn integrity; deterministic when ``random_seed`` is set. Only supported by the baseten_trace loader.", - "title": "Tracesessionsampleratio" - }, - "interTurnDelayCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Clamp per-turn replay delays to at most this many seconds; ``None`` disables the cap. Honored by the DAG JSONL loader and the baseten_trace loader's closed-loop think-times; the clamp count is reported at end of load.", - "title": "Interturndelaycapseconds" - }, - "maxIdleGapCapSeconds": { - "anyOf": [ - { - "exclusiveMinimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Collapse idle gaps between consecutive requests (across all sessions) to at most this many seconds, so a sparse or session-sampled trace does not replay dead air; ``None`` disables the cap. The cap is in replay wall-clock seconds, applied after replay_speedup compression, so it bounds actual benchmark idle time regardless of speedup. Only supported by the baseten_trace loader.", - "title": "Maxidlegapcapseconds" - }, - "replaySpeedup": { - "anyOf": [ - { - "exclusiveMinimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Trace replay wall-clock compression (10 = 10x faster than recorded): divides normalized timestamps and inter-turn delays; ``None`` = real time. Unlike synthesis speedup_ratio, hash_ids stay untouched (KV-cache fidelity). Only supported by the baseten_trace loader.", - "title": "Replayspeedup" - }, - "openLoopReplay": { - "default": true, - "description": "Open-loop replay (the default): each session starts at its absolute, speedup-scaled recorded timestamp; continuation turns fire at max(recorded timestamp, prior-turn completion). Set ``False`` for closed-loop back-pressure: continuation turns fire a think-time (recorded start-to-start gap minus recorded e2e duration) after the prior turn completes, keeping sessions causally ordered when replayed service times differ from recorded (e.g. A/A comparisons). Only honored by the baseten_trace loader.", - "title": "Openloopreplay", - "type": "boolean" - }, - "openLoopStrict": { - "default": false, - "description": "In open-loop replay, fire every trace row at its absolute recorded timestamp as an independent single-turn session, trading away multi-turn grouping and session metrics. Only honored by the baseten_trace loader.", - "title": "Openloopstrict", - "type": "boolean" - }, - "omitKvHints": { - "default": false, - "description": "Drop recorded KV-cache hints (``hash_ids``, ``block_size``) from replayed request bodies, for strict frontends that reject unknown parameters. Only honored by the baseten_trace loader.", - "title": "Omitkvhints", - "type": "boolean" - }, - "forceMinTokens": { - "default": true, - "description": "Pin ``min_tokens`` to the recorded output length so replayed generations match recorded lengths; disable to let EOS end generations naturally (some servers reject ``min_tokens``). Only honored by the baseten_trace loader.", - "title": "Forcemintokens", - "type": "boolean" - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output sequence length to apply when records do not specify one. Can be a fixed integer or {mean, stddev} distribution. Per-line `output_length` values in the file always take precedence.", - "title": "Osl" - }, - "blockSize": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Hash-id block granularity for trace replay (--isl-block-size). hash-id trace loaders (mooncake_trace, bailian_trace, burst_gpt_trace, sagemaker_data_capture) decode each hash_id into a cached block of this many tokens; total ISL = (num_hash_ids - 1) * block_size + final_block_size. When unset, the loader's plugin-metadata default applies (e.g. 512 for mooncake_trace, 16 for bailian_trace). Not used by weka, which carries its own inline per-block sizes.", - "title": "Blocksize" - }, - "traceIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree. The idle clock covers initial future work and restarts when the final in-flight request across the root and all descendant streams completes. If no request from that tree reaches the wire before the cap, AIPerf uniformly advances only that tree's pending replay timers. Dataset timestamps are unchanged, and request order, spawn/join dependencies, and replay barriers remain authoritative. Defaults to None (no per-trace runtime idle cap).", - "title": "Traceidlegapcapseconds" - }, - "ignoreTraceDelays": { - "default": false, - "description": "Strip per-turn timestamps and inter-turn delays from trace datasets at load time. With this flag, Turn.timestamp and Turn.delay are emitted as None so concurrency / request-rate timing modes dispatch turns back-to-back instead of reproducing the recorded user think-time gaps. No effect under fixed-schedule (timestamps drive that mode before they could be ignored -- combine with --no-fixed-schedule if you want both behaviors). Mutually exclusive with use_think_time_only.", - "title": "Ignoretracedelays", - "type": "boolean" - }, - "useThinkTimeOnly": { - "default": false, - "description": "For weka_trace inputs, emit Turn.delay using only the recorded per-request `think_time` (client-side delay before each request) instead of the full `t_curr - t_prev` inter-request delta. Compresses replay wall time against zero-latency mocks because the recorded `api_time` portion of each gap is dropped. Mirrors kv-cache-tester's default `--timing-strategy think-only`. Falls back to the full delta for turns whose recorded `think_time` is null. Mutually exclusive with ignore_trace_delays. No effect on non-weka trace loaders.", - "title": "Usethinktimeonly", - "type": "boolean" - }, - "maxContextLength": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum peak prompt+output context length (tokens) per Weka root trace. Only honored by format weka_trace: the loader drops whole traces whose *recorded* peak exceeds this ceiling at load time (filter-then-cap before entries). Not a DatasetManager tokenize filter; rejected for non-Weka formats.", - "title": "Maxcontextlength" - }, - "cacheBust": { - "$ref": "#/$defs/CacheBustConfig", - "description": "Per-conversation cache-bust marker injection for trace replay (mirror of the synthetic dataset's prompts.cache_bust). The engine reads the active dataset's target via ``BenchmarkConfig.get_cache_bust_target()``; 'none' (default) disables it." - }, - "prompts": { - "anyOf": [ - { - "$ref": "#/$defs/PromptSelectionConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it." - } - }, - "required": [ - "type" - ], - "title": "FileDataset", - "type": "object" - }, - { - "additionalProperties": false, - "description": "Public dataset configuration.\n\nUses well-known public benchmarking datasets that are\nautomatically downloaded and processed by AIPerf.", - "properties": { - "name": { - "description": "Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "type": { - "const": "public", - "description": "Dataset type discriminator. Must be 'public'.", - "title": "Type", - "type": "string" - }, - "dataset": { - "$ref": "#/$defs/PublicDatasetType", - "description": "Pre-configured public dataset to download and use for benchmarking. Name of the HuggingFace public dataset enum (e.g. 'sharegpt', 'alpaca'). AIPerf automatically downloads and parses these datasets." - }, - "entries": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Limit number of records to use from the dataset. If not specified, uses all available records.", - "title": "Entries" - }, - "randomSeed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Random seed for deterministic sampling from the dataset. Overrides global random_seed for this dataset.", - "title": "Randomseed" - }, - "sampling": { - "$ref": "#/$defs/DatasetSamplingStrategy", - "default": "sequential", - "description": "Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion." - }, - "hfSubset": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace dataset subset/config name override (e.g. 'sharegpt4o'). Only applies for HuggingFace-backed public dataset loaders. Takes priority over the subset defined in the plugin registry.", - "title": "Hfsubset" - }, - "filters": { - "additionalProperties": { - "type": "string" - }, - "description": "Dataset-specific filters forwarded to public dataset loaders. Supported keys and values depend on the selected dataset.", - "title": "Filters", - "type": "object" - }, - "hfWekaDataset": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace dataset repo for the generic Weka loader (e.g. `semianalysisai/cc-traces-weka-061526`). Only applies with `dataset: weka_hf`; setting it with any other public dataset is an error. Passing `--hf-weka-dataset` on the CLI auto-selects `--public-dataset weka_hf`, so the repo flag works on its own. Pinned Weka public dataset aliases keep their registry-defined repo names.", - "title": "Hfwekadataset" - }, - "interTurnDelayCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Clamp per-turn replay delays to at most this many seconds for HF-backed Weka trace replay (mirror of ``FileDataset.inter_turn_delay_cap_seconds``). ``None`` disables the cap.", - "title": "Interturndelaycapseconds" - }, - "traceIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree for HF-backed Weka replay (mirror of ``FileDataset.trace_idle_gap_cap_seconds``). Dataset timestamps remain unchanged. Defaults to None (no per-trace runtime idle cap).", - "title": "Traceidlegapcapseconds" - }, - "ignoreTraceDelays": { - "default": false, - "description": "Strip per-turn timestamps and inter-turn delays from HF-backed Weka trace replay at load time (mirror of ``FileDataset.ignore_trace_delays``). Mutually exclusive with use_think_time_only.", - "title": "Ignoretracedelays", - "type": "boolean" - }, - "useThinkTimeOnly": { - "default": false, - "description": "For HF-backed Weka replay, emit Turn.delay using only the recorded per-request ``think_time`` instead of the full inter-request delta (mirror of ``FileDataset.use_think_time_only``). Mutually exclusive with ignore_trace_delays.", - "title": "Usethinktimeonly", - "type": "boolean" - }, - "maxContextLength": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum peak prompt+output context length (tokens) per conversation for HF-backed Weka replay; drops over-length traces at load using *recorded* lengths (mirror of ``FileDataset.max_context_length``). Rejected for non-Weka public datasets.", - "title": "Maxcontextlength" - }, - "cacheBust": { - "$ref": "#/$defs/CacheBustConfig", - "description": "Per-conversation cache-bust marker injection for HF-backed Weka trace replay (mirror of ``FileDataset.cache_bust``). 'none' (default) disables it." - }, - "prompts": { - "anyOf": [ - { - "$ref": "#/$defs/PromptSelectionConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it." - }, - "synthesis": { - "anyOf": [ - { - "$ref": "#/$defs/SynthesisConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Trace synthesis/transformation configuration for HF-backed Weka trace replay (mirror of ``FileDataset.synthesis``). ``--max-isl``/``--max-osl`` cap the per-conversation input/output lengths of the replayed HF Weka traces; without this they applied only to file-based traces. ``None`` disables synthesis." - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output sequence length to apply when records do not specify one (mirror of ``FileDataset.osl``). Can be a fixed integer or {mean, stddev} distribution. Per-record output lengths always take precedence.", - "title": "Osl" - } - }, - "required": [ - "type", - "dataset" - ], - "title": "PublicDataset", - "type": "object" - } - ], - "description": "Single dataset config (normalized to a one-entry datasets list).", - "x-singular-alias-of": "datasets" - }, - "warmup": { - "anyOf": [ - { - "additionalProperties": false, - "description": "Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "concurrency", - "description": "Concurrency-controlled immediate dispatch.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "default": 1, - "description": "Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.", - "minimum": 1, - "title": "Concurrency", - "type": "integer" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "ConcurrencyPhase", - "type": "object" - }, - { - "additionalProperties": false, - "description": "Poisson-distributed request arrivals at the target rate.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "poisson", - "description": "Poisson-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "PoissonPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "Gamma-distributed request arrivals with configurable smoothness.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "gamma", - "description": "Gamma-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "smoothness": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.", - "title": "Smoothness" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "GammaPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "Constant-rate request arrivals (fixed inter-arrival time).", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "constant", - "description": "Constant rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "ConstantPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "user_centric", - "description": "N users sharing a global request rate.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "users": { - "description": "Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.", - "minimum": 1, - "title": "Users", - "type": "integer" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type", - "users", - "rate" - ], - "title": "UserCentricPhase", - "type": "object", - "allOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - } - ] - }, - { - "additionalProperties": false, - "description": "Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "fixed_schedule", - "description": "Replay requests at trace timestamps.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "autoOffset": { - "default": true, - "description": "Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.", - "title": "Autooffset", - "type": "boolean" - }, - "startOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter out trace requests before this timestamp in ms (must be >= 0).", - "title": "Startoffset" - }, - "endOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter out trace requests after this timestamp in ms (must be >= 0).", - "title": "Endoffset" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "FixedSchedulePhase", - "type": "object" - } - ], - "description": "Warmup phase shorthand. Requires profiling when used.", - "discriminator": { - "propertyName": "type", - "mapping": { - "concurrency": "#/$defs/ConcurrencyPhase", - "poisson": "#/$defs/PoissonPhase", - "gamma": "#/$defs/GammaPhase", - "constant": "#/$defs/ConstantPhase", - "user_centric": "#/$defs/UserCentricPhase", - "fixed_schedule": "#/$defs/FixedSchedulePhase" - } - } - }, - "profiling": { - "anyOf": [ - { - "additionalProperties": false, - "description": "Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "concurrency", - "description": "Concurrency-controlled immediate dispatch.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "default": 1, - "description": "Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.", - "minimum": 1, - "title": "Concurrency", - "type": "integer" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "ConcurrencyPhase", - "type": "object" - }, - { - "additionalProperties": false, - "description": "Poisson-distributed request arrivals at the target rate.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "poisson", - "description": "Poisson-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "PoissonPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "Gamma-distributed request arrivals with configurable smoothness.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "gamma", - "description": "Gamma-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "smoothness": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.", - "title": "Smoothness" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "GammaPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "Constant-rate request arrivals (fixed inter-arrival time).", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "constant", - "description": "Constant rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "ConstantPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - { - "additionalProperties": false, - "description": "N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "user_centric", - "description": "N users sharing a global request rate.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate" - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}." - }, - "users": { - "description": "Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.", - "minimum": 1, - "title": "Users", - "type": "integer" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type", - "users", - "rate" - ], - "title": "UserCentricPhase", - "type": "object", - "allOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - } - ] - }, - { - "additionalProperties": false, - "description": "Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration" - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod" - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax" - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "fixed_schedule", - "description": "Replay requests at trace timestamps.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests" - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration" - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions" - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency" - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency" - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}." - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod" - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold" - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds" - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration" - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane" - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod" - }, - "autoOffset": { - "default": true, - "description": "Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.", - "title": "Autooffset", - "type": "boolean" - }, - "startOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter out trace requests before this timestamp in ms (must be >= 0).", - "title": "Startoffset" - }, - "endOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter out trace requests after this timestamp in ms (must be >= 0).", - "title": "Endoffset" - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "type" - ], - "title": "FixedSchedulePhase", - "type": "object" - } - ], - "description": "Profiling phase shorthand. Normalized to a phases entry named 'profiling'.", - "discriminator": { - "propertyName": "type", - "mapping": { - "concurrency": "#/$defs/ConcurrencyPhase", - "poisson": "#/$defs/PoissonPhase", - "gamma": "#/$defs/GammaPhase", - "constant": "#/$defs/ConstantPhase", - "user_centric": "#/$defs/UserCentricPhase", - "fixed_schedule": "#/$defs/FixedSchedulePhase" - } - } - } - }, - "required": [ - "endpoint" - ], - "title": "BenchmarkConfig", - "type": "object", - "allOf": [ - { - "anyOf": [ - { - "required": [ - "model" - ] - }, - { - "required": [ - "models" - ] - } - ] - }, - { - "anyOf": [ - { - "required": [ - "dataset" - ] - }, - { - "required": [ - "datasets" - ] - } - ] - }, - { - "anyOf": [ - { - "required": [ - "phases" - ] - }, - { - "required": [ - "profiling" - ] - } - ] - } - ] - }, - "CacheBustConfig": { - "additionalProperties": false, - "description": "Per-conversation cache-bust marker injected into the prompt.\n\nPrefix variants diverge at token 0 (defeats KV-cache prefix matching for\nthe entire prompt -- recommended when shared_system_length is large).\nSuffix variants append after existing content (lighter bust; preserves\nleading-prefix caching). Marker is deterministic from\n(benchmark_id, recycle_pass, trajectory_index) -- reproducible across\nreruns. Same marker for all turns within a conversation; fresh marker\non each recycle of a trace_id.", - "properties": { - "target": { - "$ref": "#/$defs/CacheBustTarget", - "default": "none", - "description": "Where (and how) to inject a per-conversation cache-bust marker. Prefix variants prepend at token 0 (most aggressive); suffix variants append after existing content. 'none' disables the feature (default)." - } - }, - "title": "CacheBustConfig", - "type": "object" - }, - "CacheBustTarget": { - "description": "Where (and how) to inject a per-conversation cache-bust marker.\n\nPrefix variants diverge at token 0 of the prompt (most aggressive -- defeats\nKV-cache prefix matching for the entire prompt). Suffix variants append\nafter existing content (preserves leading-prefix caching).", - "enum": [ - "none", - "system_prefix", - "system_suffix", - "first_turn_prefix", - "first_turn_suffix" - ], - "title": "CacheBustTarget", - "type": "string" - }, - "CancellationConfig": { - "additionalProperties": false, - "description": "Configuration for request cancellation testing.\n\nEnables testing server behavior when clients cancel requests mid-flight.", - "properties": { - "rate": { - "description": "Percentage of requests to cancel (0-100). 0.5 means half a percent; 10.0 means 10%% of requests will be cancelled.", - "title": "Rate", - "oneOf": [ - { - "description": "Percentage of requests to cancel (0-100). 0.5 means half a percent; 10.0 means 10%% of requests will be cancelled.", - "maximum": 100.0, - "minimum": 0.0, - "title": "Rate", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "delay": { - "default": 0.0, - "description": "Seconds to wait after sending before cancelling. 0.0 means cancel immediately after send.", - "title": "Delay", - "oneOf": [ - { - "default": 0.0, - "description": "Seconds to wait after sending before cancelling. 0.0 means cancel immediately after send.", - "minimum": 0.0, - "title": "Delay", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "rate" - ], - "title": "CancellationConfig", - "type": "object" - }, - "ConcurrencyPhase": { - "additionalProperties": false, - "description": "Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration", - "x-jinja2-supported": true - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod", - "x-jinja2-supported": true - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "title": "Adaptivemincompletedrequests", - "oneOf": [ - { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "title": "Adaptivecontrolmin", - "oneOf": [ - { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax", - "x-jinja2-supported": true - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "title": "Adaptivescalebasestep", - "oneOf": [ - { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "title": "Adaptivescalemaxstepmultiplier", - "oneOf": [ - { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "title": "Adaptivescalesteppercent", - "oneOf": [ - { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "concurrency", - "description": "Concurrency-controlled immediate dispatch.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests", - "x-jinja2-supported": true - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration", - "x-jinja2-supported": true - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions", - "x-jinja2-supported": true - }, - "concurrency": { - "default": 1, - "description": "Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.", - "title": "Concurrency", - "oneOf": [ - { - "default": 1, - "description": "Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.", - "minimum": 1, - "title": "Concurrency", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency", - "x-jinja2-supported": true - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod", - "x-jinja2-supported": true - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold", - "x-jinja2-supported": true - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "title": "Trajectorystartminratio", - "oneOf": [ - { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "title": "Trajectorystartmaxratio", - "oneOf": [ - { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds", - "x-jinja2-supported": true - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration", - "x-jinja2-supported": true - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane", - "x-jinja2-supported": true - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod", - "x-jinja2-supported": true - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "name", - "type" - ], - "title": "ConcurrencyPhase", - "type": "object" - }, - "ConnectionReuseStrategy": { - "description": "Transport connection reuse strategy. Controls how and when connections are reused across requests.", - "enum": [ - "pooled", - "never", - "sticky-user-sessions" - ], - "title": "ConnectionReuseStrategy", - "type": "string" - }, - "ConstantPhase": { - "additionalProperties": false, - "description": "Constant-rate request arrivals (fixed inter-arrival time).", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration", - "x-jinja2-supported": true - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod", - "x-jinja2-supported": true - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "title": "Adaptivemincompletedrequests", - "oneOf": [ - { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "title": "Adaptivecontrolmin", - "oneOf": [ - { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax", - "x-jinja2-supported": true - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "title": "Adaptivescalebasestep", - "oneOf": [ - { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "title": "Adaptivescalemaxstepmultiplier", - "oneOf": [ - { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "title": "Adaptivescalesteppercent", - "oneOf": [ - { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "constant", - "description": "Constant rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests", - "x-jinja2-supported": true - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration", - "x-jinja2-supported": true - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions", - "x-jinja2-supported": true - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency", - "x-jinja2-supported": true - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency", - "x-jinja2-supported": true - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod", - "x-jinja2-supported": true - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold", - "x-jinja2-supported": true - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "title": "Trajectorystartminratio", - "oneOf": [ - { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "title": "Trajectorystartmaxratio", - "oneOf": [ - { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds", - "x-jinja2-supported": true - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration", - "x-jinja2-supported": true - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane", - "x-jinja2-supported": true - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod", - "x-jinja2-supported": true - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate", - "x-jinja2-supported": true - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "name", - "type" - ], - "title": "ConstantPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - "ConvergenceConfig": { - "additionalProperties": false, - "description": "Adaptive trial-stopping criterion.\n\nPresence of this object on `MultiRunConfig.convergence` enables\nadaptive stopping: trials run until the criterion fires (or\n`MultiRunConfig.num_runs` is reached, whichever comes first).", - "properties": { - "metric": { - "description": "Metric tag to evaluate convergence on (e.g., 'ttft').", - "title": "Metric", - "type": "string" - }, - "stat": { - "$ref": "#/$defs/ConvergenceStat", - "default": "avg", - "description": "Statistic on the metric to inspect (avg/p50/p90/p95/p99)." - }, - "mode": { - "$ref": "#/$defs/ConvergenceCriterionType", - "default": "ci_width", - "description": "Convergence criterion plugin name (ci_width|cv|distribution)." - }, - "threshold": { - "anyOf": [ - { - "exclusiveMaximum": 1, - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Criterion threshold (interpretation depends on `mode`). When None (default), each criterion uses its own algorithm-specific default: ci_width=0.10, cv=0.05, distribution=0.05 (KS p-value).", - "title": "Threshold", - "x-jinja2-supported": true - }, - "minRuns": { - "default": 2, - "description": "Minimum trials before the criterion is checked.", - "title": "Minruns", - "oneOf": [ - { - "default": 2, - "description": "Minimum trials before the criterion is checked.", - "minimum": 2, - "title": "Minruns", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "metric" - ], - "title": "ConvergenceConfig", - "type": "object" - }, - "ConvergenceCriterionType": { - "enum": [ - "ci_width", - "cv", - "distribution" - ], - "title": "ConvergenceCriterionType", - "type": "string" - }, - "ConvergenceStat": { - "description": "Statistic to evaluate for convergence when using ci_width or cv mode.", - "enum": [ - "avg", - "p50", - "p90", - "p95", - "p99", - "min", - "max" - ], - "title": "ConvergenceStat", - "type": "string" - }, - "DatasetFormat": { - "enum": [ - "burst_gpt_trace", - "bailian_trace", - "baseten_trace", - "mooncake_trace", - "raw_payload", - "inputs_json", - "dag_jsonl", - "sagemaker_data_capture", - "multi_turn", - "random_pool", - "single_turn", - "speed_bench_qualitative", - "speed_bench_coding", - "speed_bench_humanities", - "speed_bench_math", - "speed_bench_multilingual", - "speed_bench_qa", - "speed_bench_rag", - "speed_bench_reasoning", - "speed_bench_roleplay", - "speed_bench_stem", - "speed_bench_summarization", - "speed_bench_writing", - "speed_bench_throughput_1k", - "speed_bench_throughput_2k", - "speed_bench_throughput_8k", - "speed_bench_throughput_16k", - "speed_bench_throughput_32k", - "speed_bench_throughput_1k_low_entropy", - "speed_bench_throughput_1k_mixed", - "speed_bench_throughput_1k_high_entropy", - "speed_bench_throughput_2k_low_entropy", - "speed_bench_throughput_2k_mixed", - "speed_bench_throughput_2k_high_entropy", - "speed_bench_throughput_8k_low_entropy", - "speed_bench_throughput_8k_mixed", - "speed_bench_throughput_8k_high_entropy", - "speed_bench_throughput_16k_low_entropy", - "speed_bench_throughput_16k_mixed", - "speed_bench_throughput_16k_high_entropy", - "speed_bench_throughput_32k_low_entropy", - "speed_bench_throughput_32k_mixed", - "speed_bench_throughput_32k_high_entropy", - "weka_trace" - ], - "title": "DatasetFormat", - "type": "string" - }, - "DatasetSamplingStrategy": { - "enum": [ - "random", - "sequential", - "shuffle" - ], - "title": "DatasetSamplingStrategy", - "type": "string" - }, - "DualBindCommunicationConfig": { - "additionalProperties": false, - "description": "Dual-bind (IPC + TCP) communication configuration.\n\nFor Kubernetes deployments where controller services connect via IPC\n(co-located in same pod) and worker pods connect via TCP.\n\nWhen controller_host is None, services use IPC (local mode).\nWhen controller_host is set, services use TCP to that host (remote mode).", - "properties": { - "type": { - "const": "dual", - "description": "Communication type. Must be 'dual'.", - "title": "Type", - "type": "string" - }, - "ipcPath": { - "default": "/tmp/aiperf", - "description": "Directory for IPC socket files (local services).", - "title": "Ipcpath", - "type": "string" - }, - "tcpHost": { - "default": "0.0.0.0", - "description": "TCP bind host for proxies. Use 0.0.0.0 to listen on all interfaces.", - "title": "Tcphost", - "type": "string" - }, - "controllerHost": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Controller host for remote TCP connections. When set, services connect via TCP to this host instead of IPC. In Kubernetes, set via JobSet DNS (e.g., controller.namespace.svc).", - "title": "Controllerhost" - }, - "recordsPort": { - "default": 5557, - "description": "Port for records push/pull communication.", - "title": "Recordsport", - "oneOf": [ - { - "default": 5557, - "description": "Port for records push/pull communication.", - "maximum": 65535, - "minimum": 1, - "title": "Recordsport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "creditRouterPort": { - "default": 5564, - "description": "Port for credit router (ROUTER-DEALER).", - "title": "Creditrouterport", - "oneOf": [ - { - "default": 5564, - "description": "Port for credit router (ROUTER-DEALER).", - "maximum": 65535, - "minimum": 1, - "title": "Creditrouterport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "controlPort": { - "default": 5667, - "description": "Port for control channel (ROUTER-DEALER).", - "title": "Controlport", - "oneOf": [ - { - "default": 5667, - "description": "Port for control channel (ROUTER-DEALER).", - "maximum": 65535, - "minimum": 1, - "title": "Controlport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "eventBusProxy": { - "$ref": "#/$defs/TcpProxyConfig", - "description": "Event bus proxy ports (XPUB/XSUB)." - }, - "datasetManagerProxy": { - "$ref": "#/$defs/TcpProxyConfig", - "description": "Dataset manager proxy ports (DEALER/ROUTER)." - }, - "rawInferenceProxy": { - "$ref": "#/$defs/TcpProxyConfig", - "description": "Raw inference proxy ports (PUSH/PULL)." - } - }, - "required": [ - "type" - ], - "title": "DualBindCommunicationConfig", - "type": "object" - }, - "EmpiricalDistribution": { - "additionalProperties": false, - "description": "Discrete distribution sampled from weighted values.\n\nYAML:\n isl:\n points:\n - {value: 128, weight: 40}\n - {value: 512, weight: 35}\n - {value: 2048, weight: 20}\n - {value: 8192, weight: 5}", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min", - "x-jinja2-supported": true - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max", - "x-jinja2-supported": true - }, - "points": { - "description": "Weighted discrete values. Weights are relative.", - "items": { - "$ref": "#/$defs/EmpiricalPoint" - }, - "title": "Points", - "type": "array" - }, - "type": { - "const": "empirical", - "description": "Optional distribution type marker; stripped before concrete validation." - } - }, - "required": [ - "points" - ], - "title": "EmpiricalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - "EmpiricalPoint": { - "additionalProperties": false, - "description": "A weighted value in an empirical distribution.", - "properties": { - "value": { - "description": "The discrete value.", - "title": "Value", - "oneOf": [ - { - "description": "The discrete value.", - "title": "Value", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalized internally).", - "title": "Weight", - "oneOf": [ - { - "default": 1.0, - "description": "Relative weight (normalized internally).", - "exclusiveMinimum": 0.0, - "title": "Weight", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "value" - ], - "title": "EmpiricalPoint", - "type": "object" - }, - "EndpointConfig": { - "additionalProperties": false, - "description": "Endpoint configuration for connecting to inference servers.\n\nThis section configures how AIPerf connects to and communicates\nwith the target inference server(s). It supports multiple URLs\nfor load-balanced deployments and various API types.", - "properties": { - "urls": { - "description": "List of server URLs to benchmark. Requests distributed according to url_strategy. URLs without a scheme have ``http://`` prepended automatically. Example: ['http://localhost:8000/v1/chat/completions']", - "items": { - "type": "string" - }, - "title": "Urls", - "type": "array", - "x-kubernetes-preserve-unknown-fields": true - }, - "urlStrategy": { - "$ref": "#/$defs/URLSelectionStrategy", - "default": "round_robin", - "description": "Strategy for distributing requests across multiple URLs. round_robin cycles through URLs in order." - }, - "type": { - "$ref": "#/$defs/EndpointType", - "default": "chat", - "description": "API endpoint type determining request/response format. chat: OpenAI chat completions, completions: OpenAI completions, embeddings: vector embeddings, rankings: reranking, template: custom format, and others — see `aiperf plugins` for the full list." - }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Override default endpoint path. Use for servers with non-standard API paths. Example: '/custom/v2/generate'", - "title": "Path" - }, - "apiKey": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "API authentication key. Supports environment variable substitution: ${OPENAI_API_KEY}. Can also use ${VAR:default} syntax for defaults.", - "title": "Apikey" - }, - "timeout": { - "default": 21600, - "description": "Request timeout in seconds (0 = no timeout). Requests exceeding this duration are marked as failed. Should exceed expected max response time.", - "title": "Timeout", - "oneOf": [ - { - "default": 21600, - "description": "Request timeout in seconds (0 = no timeout). Requests exceeding this duration are marked as failed. Should exceed expected max response time.", - "minimum": 0.0, - "title": "Timeout", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "streaming": { - "default": false, - "description": "Enable streaming (Server-Sent Events) responses. Required for accurate TTFT (time to first token) measurement. Server must support streaming for this to work.", - "title": "Streaming", - "type": "boolean" - }, - "transport": { - "anyOf": [ - { - "$ref": "#/$defs/TransportType" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Transport plugin name. Currently only 'http' (aiohttp-based HTTP/1.1) is shipped. Auto-detected from URL when unset; explicit setting overrides auto-detection." - }, - "connectionReuse": { - "$ref": "#/$defs/ConnectionReuseStrategy", - "default": "pooled", - "description": "HTTP connection management strategy. pooled: shared connection pool (fastest), never: new connection per request (includes TCP overhead), sticky-user-sessions: dedicated connection per user session." - }, - "useLegacyMaxTokens": { - "default": false, - "description": "Use 'max_tokens' field instead of 'max_completion_tokens'. Enable for compatibility with older OpenAI API versions.", - "title": "Uselegacymaxtokens", - "type": "boolean" - }, - "useServerTokenCount": { - "default": false, - "description": "Use server-reported token counts from response usage field. When true, trusts usage.prompt_tokens and usage.completion_tokens. When false, counts tokens locally using configured tokenizer.", - "title": "Useservertokencount", - "type": "boolean" - }, - "template": { - "anyOf": [ - { - "$ref": "#/$defs/TemplateConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Custom template configuration for template endpoint type. Only used when type='template'. Defines request body format and response parsing." - }, - "headers": { - "additionalProperties": { - "type": "string" - }, - "description": "Custom HTTP headers to include in all requests. Useful for authentication, tracing, or routing. Values support environment variable substitution.", - "title": "Headers", - "type": "object" - }, - "extra": { - "additionalProperties": true, - "description": "Additional fields to include in request body. Merged into every request. Common fields: temperature, top_p, top_k, stop.", - "title": "Extra", - "type": "object" - }, - "downloadVideoContent": { - "default": false, - "description": "For video generation endpoints, download the video content after generation completes. Adds a content download step to the async polling flow.", - "title": "Downloadvideocontent", - "type": "boolean" - }, - "requestContentType": { - "anyOf": [ - { - "$ref": "#/$defs/RequestContentType" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Content type for request body serialization. Default is 'application/json'. Set to 'multipart/form-data' for servers that require form-encoded requests (e.g. vLLM video generation)." - }, - "sessionHeader": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HTTP header name used to carry the per-session affinity identifier. When set, replaces the default `X-Correlation-ID` header. Useful when the inference server expects a custom session-affinity header (e.g. `--session-header X-Session-ID`).", - "title": "Sessionheader" - }, - "uuidAndStrip": { - "default": false, - "description": "Enable AIPerf-managed image stripping for vLLM's multimodal processor cache. Dataset-authored image UUIDs, including UUID-only cache references, always pass through on the chat endpoint; this flag only strips repeated content after AIPerf observes it in the same session. Automatic stripping supports only single_turn datasets with session_id-grouped rows; multi_turn is rejected. The server cache must cover the working set, and requests in a session must reach a replica that retains earlier UUIDs.", - "title": "Uuidandstrip", - "type": "boolean" - }, - "waitForModelTimeout": { - "default": 0.0, - "description": "Enable a pre-flight readiness probe by setting this to a positive value (seconds). aiperf applies this timeout to each URL/model probe before starting the benchmark, aborting with a non-zero exit if any probe times out. For multiple URLs or models, worst-case wall-clock time can be roughly this timeout multiplied by the number of URL/model probes. The probe strategy is controlled by `--wait-for-model-mode`, which defaults to sending a 1-token inference request. 0 (default) disables the probe. Eliminates the need for external shell-based readiness loops in containers and Kubernetes recipes.", - "title": "Waitformodeltimeout", - "oneOf": [ - { - "default": 0.0, - "description": "Enable a pre-flight readiness probe by setting this to a positive value (seconds). aiperf applies this timeout to each URL/model probe before starting the benchmark, aborting with a non-zero exit if any probe times out. For multiple URLs or models, worst-case wall-clock time can be roughly this timeout multiplied by the number of URL/model probes. The probe strategy is controlled by `--wait-for-model-mode`, which defaults to sending a 1-token inference request. 0 (default) disables the probe. Eliminates the need for external shell-based readiness loops in containers and Kubernetes recipes.", - "minimum": 0.0, - "title": "Waitformodeltimeout", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "waitForModelInterval": { - "default": 5.0, - "description": "Seconds between readiness probe attempts. Only consulted when `--wait-for-model-timeout` is positive.", - "title": "Waitformodelinterval", - "oneOf": [ - { - "default": 5.0, - "description": "Seconds between readiness probe attempts. Only consulted when `--wait-for-model-timeout` is positive.", - "exclusiveMinimum": 0.0, - "title": "Waitformodelinterval", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "waitForModelMode": { - "default": "inference", - "description": "Strategy for the readiness probe. 'inference' (default): POST a 1-token inference request to the configured endpoint; this is the strongest signal — it proves the full stack (frontend, scheduler, worker, forward pass) is live. Any HTTP status < 500 counts as ready. 'models': GET `/v1/models` and verify the model id appears in `data[]` (cheaper, no tokens consumed; falls back to a plain GET on the base URL on 404). 'both': run 'models' first, then 'inference'. Only consulted when `--wait-for-model-timeout` is positive.", - "enum": [ - "models", - "inference", - "both" - ], - "title": "Waitformodelmode", - "type": "string" - }, - "url": { - "description": "Shorthand for 'urls'. Accepts a single value which will be converted to a list/dict. List of server URLs to benchmark. Requests distributed according to url_strategy. URLs without a scheme have ``http://`` prepended automatically. Example: ['http://localhost:8000/v1/chat/completions']", - "oneOf": [ - { - "type": "string" - }, - { - "description": "List of server URLs to benchmark. Requests distributed according to url_strategy. URLs without a scheme have ``http://`` prepended automatically. Example: ['http://localhost:8000/v1/chat/completions']", - "items": { - "type": "string" - }, - "title": "Urls", - "type": "array", - "x-kubernetes-preserve-unknown-fields": true - } - ], - "x-singular-alias-of": "urls" - } - }, - "required": [], - "title": "EndpointConfig", - "type": "object", - "allOf": [ - { - "anyOf": [ - { - "required": [ - "url" - ] - }, - { - "required": [ - "urls" - ] - } - ] - } - ] - }, - "EndpointType": { - "enum": [ - "chat", - "cohere_rankings", - "completions", - "responses", - "messages", - "chat_embeddings", - "embeddings", - "hf_tei_rankings", - "huggingface_generate", - "image_generation", - "image_edit", - "video_generation", - "image_retrieval", - "nim_embeddings", - "nim_rankings", - "solido_rag", - "raw", - "template" - ], - "title": "EndpointType", - "type": "string" - }, - "FileDataset": { - "additionalProperties": false, - "description": "File-based dataset configuration.\n\nLoads prompts from a local file in various formats.\nSupports trace replay and custom sampling strategies.", - "properties": { - "name": { - "description": "Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "type": { - "const": "file", - "description": "Dataset type discriminator. Must be 'file'.", - "title": "Type", - "type": "string" - }, - "path": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to file or directory containing benchmark dataset. Can be absolute or relative. Mutually exclusive with `records:`. Supported formats depend on the format field: JSONL for single_turn/multi_turn, JSONL trace files for mooncake_trace/bailian_trace, Parquet for baseten_trace, directories for random_pool.", - "title": "Path" - }, - "records": { - "anyOf": [ - { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - { - "additionalProperties": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inline benchmark records, embedded directly in the YAML config. Mutually exclusive with `path:`. The element schema is determined by `format:` (same shape as one line of the equivalent JSONL file). For `format: random_pool`, may be either a flat list (single pool) or a dict-of-lists (multi-pool, mirrors the directory-of-JSONLs file mode). All other formats require a flat list.", - "title": "Records" - }, - "format": { - "$ref": "#/$defs/DatasetFormat", - "default": "single_turn", - "description": "Dataset file format determining parsing logic and expected file structure. single_turn: JSONL with single prompt-response exchanges. multi_turn: JSONL with conversation history. mooncake_trace / bailian_trace / baseten_trace / burst_gpt_trace: timestamped trace files for replay. sagemaker_data_capture: JSONL captured by SageMaker DataCapture. random_pool: directory of reusable prompts." - }, - "sampling": { - "$ref": "#/$defs/DatasetSamplingStrategy", - "default": "sequential", - "description": "Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion." - }, - "synthesis": { - "anyOf": [ - { - "$ref": "#/$defs/SynthesisConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Trace synthesis/transformation configuration. Allows scaling timestamps and token lengths before replay. Applies to trace formats such as mooncake_trace and baseten_trace, except speedup_ratio, which is rejected for baseten_trace (use replay_speedup / --replay-speedup to scale replay pacing there)." - }, - "entries": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Limit number of records to use from file. If not specified, uses all records in the file.", - "title": "Entries", - "x-jinja2-supported": true - }, - "randomSeed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Random seed for deterministic sampling. When set, makes random/shuffle sampling reproducible across runs. Overrides global random_seed for this dataset.", - "title": "Randomseed", - "x-jinja2-supported": true - }, - "traceSessionSampleRatio": { - "anyOf": [ - { - "exclusiveMinimum": 0.0, - "maximum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Fraction of trace sessions to keep for replay, sampled whole-session to preserve multi-turn integrity; deterministic when ``random_seed`` is set. Only supported by the baseten_trace loader.", - "title": "Tracesessionsampleratio", - "x-jinja2-supported": true - }, - "interTurnDelayCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Clamp per-turn replay delays to at most this many seconds; ``None`` disables the cap. Honored by the DAG JSONL loader and the baseten_trace loader's closed-loop think-times; the clamp count is reported at end of load.", - "title": "Interturndelaycapseconds", - "x-jinja2-supported": true - }, - "maxIdleGapCapSeconds": { - "anyOf": [ - { - "exclusiveMinimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Collapse idle gaps between consecutive requests (across all sessions) to at most this many seconds, so a sparse or session-sampled trace does not replay dead air; ``None`` disables the cap. The cap is in replay wall-clock seconds, applied after replay_speedup compression, so it bounds actual benchmark idle time regardless of speedup. Only supported by the baseten_trace loader.", - "title": "Maxidlegapcapseconds", - "x-jinja2-supported": true - }, - "replaySpeedup": { - "anyOf": [ - { - "exclusiveMinimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Trace replay wall-clock compression (10 = 10x faster than recorded): divides normalized timestamps and inter-turn delays; ``None`` = real time. Unlike synthesis speedup_ratio, hash_ids stay untouched (KV-cache fidelity). Only supported by the baseten_trace loader.", - "title": "Replayspeedup", - "x-jinja2-supported": true - }, - "openLoopReplay": { - "default": true, - "description": "Open-loop replay (the default): each session starts at its absolute, speedup-scaled recorded timestamp; continuation turns fire at max(recorded timestamp, prior-turn completion). Set ``False`` for closed-loop back-pressure: continuation turns fire a think-time (recorded start-to-start gap minus recorded e2e duration) after the prior turn completes, keeping sessions causally ordered when replayed service times differ from recorded (e.g. A/A comparisons). Only honored by the baseten_trace loader.", - "title": "Openloopreplay", - "type": "boolean" - }, - "openLoopStrict": { - "default": false, - "description": "In open-loop replay, fire every trace row at its absolute recorded timestamp as an independent single-turn session, trading away multi-turn grouping and session metrics. Only honored by the baseten_trace loader.", - "title": "Openloopstrict", - "type": "boolean" - }, - "omitKvHints": { - "default": false, - "description": "Drop recorded KV-cache hints (``hash_ids``, ``block_size``) from replayed request bodies, for strict frontends that reject unknown parameters. Only honored by the baseten_trace loader.", - "title": "Omitkvhints", - "type": "boolean" - }, - "forceMinTokens": { - "default": true, - "description": "Pin ``min_tokens`` to the recorded output length so replayed generations match recorded lengths; disable to let EOS end generations naturally (some servers reject ``min_tokens``). Only honored by the baseten_trace loader.", - "title": "Forcemintokens", - "type": "boolean" - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output sequence length to apply when records do not specify one. Can be a fixed integer or {mean, stddev} distribution. Per-line `output_length` values in the file always take precedence.", - "title": "Osl" - }, - "blockSize": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Hash-id block granularity for trace replay (--isl-block-size). hash-id trace loaders (mooncake_trace, bailian_trace, burst_gpt_trace, sagemaker_data_capture) decode each hash_id into a cached block of this many tokens; total ISL = (num_hash_ids - 1) * block_size + final_block_size. When unset, the loader's plugin-metadata default applies (e.g. 512 for mooncake_trace, 16 for bailian_trace). Not used by weka, which carries its own inline per-block sizes.", - "title": "Blocksize", - "x-jinja2-supported": true - }, - "traceIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree. The idle clock covers initial future work and restarts when the final in-flight request across the root and all descendant streams completes. If no request from that tree reaches the wire before the cap, AIPerf uniformly advances only that tree's pending replay timers. Dataset timestamps are unchanged, and request order, spawn/join dependencies, and replay barriers remain authoritative. Defaults to None (no per-trace runtime idle cap).", - "title": "Traceidlegapcapseconds", - "x-jinja2-supported": true - }, - "ignoreTraceDelays": { - "default": false, - "description": "Strip per-turn timestamps and inter-turn delays from trace datasets at load time. With this flag, Turn.timestamp and Turn.delay are emitted as None so concurrency / request-rate timing modes dispatch turns back-to-back instead of reproducing the recorded user think-time gaps. No effect under fixed-schedule (timestamps drive that mode before they could be ignored -- combine with --no-fixed-schedule if you want both behaviors). Mutually exclusive with use_think_time_only.", - "title": "Ignoretracedelays", - "type": "boolean" - }, - "useThinkTimeOnly": { - "default": false, - "description": "For weka_trace inputs, emit Turn.delay using only the recorded per-request `think_time` (client-side delay before each request) instead of the full `t_curr - t_prev` inter-request delta. Compresses replay wall time against zero-latency mocks because the recorded `api_time` portion of each gap is dropped. Mirrors kv-cache-tester's default `--timing-strategy think-only`. Falls back to the full delta for turns whose recorded `think_time` is null. Mutually exclusive with ignore_trace_delays. No effect on non-weka trace loaders.", - "title": "Usethinktimeonly", - "type": "boolean" - }, - "maxContextLength": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum peak prompt+output context length (tokens) per Weka root trace. Only honored by format weka_trace: the loader drops whole traces whose *recorded* peak exceeds this ceiling at load time (filter-then-cap before entries). Not a DatasetManager tokenize filter; rejected for non-Weka formats.", - "title": "Maxcontextlength", - "x-jinja2-supported": true - }, - "cacheBust": { - "$ref": "#/$defs/CacheBustConfig", - "description": "Per-conversation cache-bust marker injection for trace replay (mirror of the synthetic dataset's prompts.cache_bust). The engine reads the active dataset's target via ``BenchmarkConfig.get_cache_bust_target()``; 'none' (default) disables it." - }, - "prompts": { - "anyOf": [ - { - "$ref": "#/$defs/PromptSelectionConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it." - } - }, - "required": [ - "name", - "type" - ], - "title": "FileDataset", - "type": "object" - }, - "FixedDistribution": { - "title": "FixedDistribution", - "description": "Returns a constant value on every sample. Scalars coerce to this. Can be a number (interpreted as a fixed value) or {value} object.", - "anyOf": [ - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable resolving to a fixed distribution value." - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template resolving to a fixed distribution value." - }, - { - "additionalProperties": false, - "description": "Returns a constant value on every sample. Scalars coerce to this.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min" - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max" - }, - "value": { - "description": "The constant value returned on every sample.", - "title": "Value", - "type": "number" - }, - "type": { - "const": "fixed", - "description": "Optional distribution type marker; stripped before concrete validation." - } - }, - "required": [ - "value" - ], - "title": "FixedDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "type": "integer" - }, - { - "type": "number" - } - ] - }, - "FixedSchedulePhase": { - "additionalProperties": false, - "description": "Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration", - "x-jinja2-supported": true - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod", - "x-jinja2-supported": true - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "title": "Adaptivemincompletedrequests", - "oneOf": [ - { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "title": "Adaptivecontrolmin", - "oneOf": [ - { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax", - "x-jinja2-supported": true - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "title": "Adaptivescalebasestep", - "oneOf": [ - { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "title": "Adaptivescalemaxstepmultiplier", - "oneOf": [ - { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "title": "Adaptivescalesteppercent", - "oneOf": [ - { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "fixed_schedule", - "description": "Replay requests at trace timestamps.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests", - "x-jinja2-supported": true - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration", - "x-jinja2-supported": true - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions", - "x-jinja2-supported": true - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency", - "x-jinja2-supported": true - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency", - "x-jinja2-supported": true - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod", - "x-jinja2-supported": true - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold", - "x-jinja2-supported": true - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "title": "Trajectorystartminratio", - "oneOf": [ - { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "title": "Trajectorystartmaxratio", - "oneOf": [ - { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds", - "x-jinja2-supported": true - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration", - "x-jinja2-supported": true - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane", - "x-jinja2-supported": true - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod", - "x-jinja2-supported": true - }, - "autoOffset": { - "default": true, - "description": "Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.", - "title": "Autooffset", - "type": "boolean" - }, - "startOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Filter out trace requests before this timestamp in ms (must be >= 0).", - "title": "Startoffset", - "x-jinja2-supported": true - }, - "endOffset": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Filter out trace requests after this timestamp in ms (must be >= 0).", - "title": "Endoffset", - "x-jinja2-supported": true - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.", - "title": "Adaptivescale", - "anyOf": [ - { - "const": false - }, - { - "$ref": "#/$defs/DisabledAdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "name", - "type" - ], - "title": "FixedSchedulePhase", - "type": "object" - }, - "GPUTelemetryCollectorType": { - "enum": [ - "dcgm", - "pynvml", - "amdsmi" - ], - "title": "GPUTelemetryCollectorType", - "type": "string" - }, - "GPUTelemetryMode": { - "description": "GPU telemetry display mode.", - "enum": [ - "summary", - "realtime_dashboard" - ], - "title": "GPUTelemetryMode", - "type": "string" - }, - "GammaPhase": { - "additionalProperties": false, - "description": "Gamma-distributed request arrivals with configurable smoothness.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration", - "x-jinja2-supported": true - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod", - "x-jinja2-supported": true - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "title": "Adaptivemincompletedrequests", - "oneOf": [ - { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "title": "Adaptivecontrolmin", - "oneOf": [ - { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax", - "x-jinja2-supported": true - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "title": "Adaptivescalebasestep", - "oneOf": [ - { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "title": "Adaptivescalemaxstepmultiplier", - "oneOf": [ - { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "title": "Adaptivescalesteppercent", - "oneOf": [ - { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "gamma", - "description": "Gamma-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests", - "x-jinja2-supported": true - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration", - "x-jinja2-supported": true - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions", - "x-jinja2-supported": true - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency", - "x-jinja2-supported": true - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency", - "x-jinja2-supported": true - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod", - "x-jinja2-supported": true - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold", - "x-jinja2-supported": true - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "title": "Trajectorystartminratio", - "oneOf": [ - { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "title": "Trajectorystartmaxratio", - "oneOf": [ - { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds", - "x-jinja2-supported": true - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration", - "x-jinja2-supported": true - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane", - "x-jinja2-supported": true - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod", - "x-jinja2-supported": true - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate", - "x-jinja2-supported": true - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "smoothness": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.", - "title": "Smoothness", - "x-jinja2-supported": true - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "name", - "type" - ], - "title": "GammaPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - "GpuTelemetryConfig": { - "title": "GpuTelemetryConfig", - "description": "GPU telemetry configuration for live or replayed GPU metrics collection.\n\nCollects GPU metrics through DCGM exporter endpoints by default. The\n``collector`` field can switch to a local backend (``pynvml`` for NVIDIA,\n``amdsmi`` for AMD ROCm); ``mode`` controls summary vs. realtime dashboard\ndisplay.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9400/metrics\"\n → GpuTelemetryConfig(enabled=True, urls=[\"http://localhost:9400/metrics\"])\n - Singular url field: {url: \"...\"}\n → GpuTelemetryConfig(urls=[\"...\"]) Can be a URL string or full config object.", - "anyOf": [ - { - "additionalProperties": false, - "description": "GPU telemetry configuration for live or replayed GPU metrics collection.\n\nCollects GPU metrics through DCGM exporter endpoints by default. The\n``collector`` field can switch to a local backend (``pynvml`` for NVIDIA,\n``amdsmi`` for AMD ROCm); ``mode`` controls summary vs. realtime dashboard\ndisplay.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9400/metrics\"\n → GpuTelemetryConfig(enabled=True, urls=[\"http://localhost:9400/metrics\"])\n - Singular url field: {url: \"...\"}\n → GpuTelemetryConfig(urls=[\"...\"])", - "properties": { - "enabled": { - "default": true, - "description": "Enable GPU telemetry collection. Set to false to disable.", - "title": "Enabled", - "type": "boolean" - }, - "urls": { - "description": "DCGM exporter endpoint URLs. Example: http://localhost:9400/metrics", - "items": { - "type": "string" - }, - "title": "Urls", - "type": "array" - }, - "metricsFile": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CSV file with pre-recorded GPU metrics. Alternative to live DCGM collection.", - "title": "Metricsfile" - }, - "collector": { - "$ref": "#/$defs/GPUTelemetryCollectorType", - "default": "dcgm", - "description": "GPU telemetry collector backend. Use 'dcgm' for DCGM exporter endpoints or a local collector (e.g. 'pynvml' for NVIDIA, 'amdsmi' for AMD ROCm) for on-host metrics collection." - }, - "mode": { - "$ref": "#/$defs/GPUTelemetryMode", - "default": "summary", - "description": "GPU telemetry display mode. Summary emits aggregate console output; realtime_dashboard enables live dashboard updates." - }, - "url": { - "description": "Shorthand for 'urls'. Accepts a single value which will be converted to a list/dict. DCGM exporter endpoint URLs. Example: http://localhost:9400/metrics", - "oneOf": [ - { - "type": "string" - }, - { - "description": "DCGM exporter endpoint URLs. Example: http://localhost:9400/metrics", - "items": { - "type": "string" - }, - "title": "Urls", - "type": "array" - } - ], - "x-singular-alias-of": "urls" - } - }, - "title": "GpuTelemetryConfig", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "type": "string" - } - ] - }, - "GridSweep": { - "additionalProperties": false, - "description": "Grid sweep - all combinations of parameters (Cartesian product).", - "properties": { - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaFilters": { - "description": "SLA constraints applied at trial scoring / aggregation time.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Slafilters", - "type": "array" - }, - "postProcess": { - "anyOf": [ - { - "$ref": "#/$defs/PostProcessSpec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional post-aggregation handler emitted by a search recipe." - }, - "recipeName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.", - "title": "Recipename" - }, - "iterationOrder": { - "$ref": "#/$defs/SweepMode", - "default": "repeated", - "description": "Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search." - }, - "sameSeed": { - "default": false, - "description": "If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.", - "title": "Sameseed", - "type": "boolean" - }, - "type": { - "const": "grid", - "default": "grid", - "description": "Sweep type discriminator.", - "title": "Type", - "type": "string" - }, - "parameters": { - "additionalProperties": { - "items": {}, - "type": "array" - }, - "description": "Parameters to sweep: dot-notation path -> list of values.", - "minProperties": 1, - "title": "Parameters", - "type": "object" - } - }, - "required": [ - "parameters" - ], - "title": "GridSweep", - "type": "object" - }, - "ImageConfig": { - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "source": { - "const": "noise" - } - } - }, - "then": { - "properties": { - "sourceSampling": { - "const": "random-with-replacement" - } - } - } - } - ], - "description": "Configuration for synthetic image generation in multimodal datasets.\n\nControls the generation of synthetic images for vision-language\nmodel benchmarking. Images are generated by randomly sampling and\nresizing source images to specified dimensions.", - "properties": { - "batchSize": { - "default": 0, - "description": "Number of images to include in each multimodal request. Supported with chat endpoint type for vision-language models. Set to 0 to disable image inputs. Higher batch sizes test multi-image understanding and increase request payload size.", - "title": "Batchsize", - "oneOf": [ - { - "default": 0, - "description": "Number of images to include in each multimodal request. Supported with chat endpoint type for vision-language models. Set to 0 to disable image inputs. Higher batch sizes test multi-image understanding and increase request payload size.", - "minimum": 0, - "title": "Batchsize", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "width": { - "description": "Image width in pixels. Can be a fixed integer or {mean, stddev} distribution. Combined with height to determine image dimensions and file sizes for multimodal benchmarking.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Width", - "x-jinja2-supported": true - }, - "height": { - "description": "Image height in pixels. Can be a fixed integer or {mean, stddev} distribution. Used when batch_size > 0 for multimodal vision benchmarking.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Height", - "x-jinja2-supported": true - }, - "format": { - "$ref": "#/$defs/ImageFormat", - "default": "jpeg", - "description": "Image file format for generated images. png: lossless compression (larger files, best quality). jpeg: lossy compression (smaller files, good quality). random: randomly select between PNG and JPEG per image. Format affects file size in multimodal requests and encoding overhead." - }, - "source": { - "anyOf": [ - { - "$ref": "#/$defs/ImageSource" - }, - { - "format": "path", - "type": "string" - } - ], - "default": "noise", - "description": "Source image generation mode (default: noise). noise: generate random noise images on the fly — no files on disk, effectively unbounded variety so servers cannot dedupe identical inputs. assets: index images from the bundled assets/source_images directory and lazily load them at the requested dimensions. A path string indexes images from the given directory (e.g. ./source_images). Random-noise images are roughly incompressible, so payload bytes are larger than equivalent natural images.", - "title": "Source" - }, - "sourceSampling": { - "$ref": "#/$defs/ImageSourceSamplingStrategy", - "default": "random-with-replacement", - "description": "How source images are selected from finite image sources selected by source='assets' or a directory path. random-with-replacement: draw each source image independently; repeats may occur immediately. shuffle-cycle: draw every source image once per shuffled cycle, reshuffling after exhaustion. sequential-cycle: walk source images in sorted load order and wrap after exhaustion. For noise mode, only random-with-replacement is valid because there is no finite source pool." - } - }, - "title": "ImageConfig", - "type": "object" - }, - "ImageFormat": { - "description": "Image file formats for synthetic image generation.", - "enum": [ - "png", - "jpeg", - "random" - ], - "title": "ImageFormat", - "type": "string" - }, - "ImageSource": { - "description": "Source image generation mode for multimodal benchmarking.", - "enum": [ - "assets", - "noise" - ], - "title": "ImageSource", - "type": "string" - }, - "ImageSourceSamplingStrategy": { - "description": "How source images are selected from a finite source-image pool.", - "enum": [ - "random-with-replacement", - "shuffle-cycle", - "sequential-cycle" - ], - "title": "ImageSourceSamplingStrategy", - "type": "string" - }, - "IpcCommunicationConfig": { - "additionalProperties": false, - "description": "IPC (Unix socket) communication configuration.\n\nFor single-machine deployments with lowest latency.\nUses Unix domain sockets for all inter-service communication.", - "properties": { - "type": { - "const": "ipc", - "description": "Communication type. Must be 'ipc'.", - "title": "Type", - "type": "string" - }, - "path": { - "default": "/tmp/aiperf", - "description": "Directory for IPC socket files. AIPerf creates multiple socket files in this directory.", - "title": "Path", - "type": "string" - } - }, - "required": [ - "type" - ], - "title": "IpcCommunicationConfig", - "type": "object" - }, - "LatinHypercubeSweep": { - "additionalProperties": false, - "description": "Latin hypercube sampling — perfect marginal stratification per axis.", - "properties": { - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaFilters": { - "description": "SLA constraints applied at trial scoring / aggregation time.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Slafilters", - "type": "array" - }, - "postProcess": { - "anyOf": [ - { - "$ref": "#/$defs/PostProcessSpec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional post-aggregation handler emitted by a search recipe." - }, - "recipeName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.", - "title": "Recipename" - }, - "iterationOrder": { - "$ref": "#/$defs/SweepMode", - "default": "repeated", - "description": "Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search." - }, - "sameSeed": { - "default": false, - "description": "If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.", - "title": "Sameseed", - "type": "boolean" - }, - "samples": { - "description": "Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.", - "title": "Samples", - "oneOf": [ - { - "description": "Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.", - "maximum": 1048576, - "minimum": 2, - "title": "Samples", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "seed": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "RNG seed for reproducibility. Must be non-negative; scipy's QMC engines reject negative seeds with an opaque low-level error.", - "title": "Seed", - "x-jinja2-supported": true - }, - "dimensions": { - "description": "Dimensions to sample over.", - "items": { - "$ref": "#/$defs/SamplingDimension" - }, - "minItems": 1, - "title": "Dimensions", - "type": "array" - }, - "labelFormat": { - "default": "index", - "description": "Variation-label format. 'index' => '_NNNN' (short, sortable). 'kv' => 'k1=v1, k2=v2' (grid-style).", - "enum": [ - "index", - "kv" - ], - "title": "Labelformat", - "type": "string" - }, - "type": { - "const": "latin_hypercube", - "default": "latin_hypercube", - "description": "Sweep type discriminator.", - "title": "Type", - "type": "string" - }, - "optimization": { - "default": "random-cd", - "description": "LHS post-processing to reduce centered discrepancy. None = vanilla LHS.", - "enum": [ - null, - "random-cd", - "lloyd" - ], - "title": "Optimization" - } - }, - "required": [ - "samples", - "dimensions" - ], - "title": "LatinHypercubeSweep", - "type": "object" - }, - "LogNormalDistribution": { - "additionalProperties": false, - "description": "Log-normal parameterized by mean and median (right-skewed, always positive).\n\nSkew is controlled by the mean/median ratio: larger ratio = heavier right tail.\nWhen mean == median the distribution is deterministic.\n\nInternally: sigma = sqrt(2 * log(mean / median)), mu = log(median).", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min", - "x-jinja2-supported": true - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max", - "x-jinja2-supported": true - }, - "mean": { - "description": "Desired mean of the output distribution.", - "title": "Mean", - "oneOf": [ - { - "description": "Desired mean of the output distribution.", - "exclusiveMinimum": 0.0, - "title": "Mean", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "median": { - "description": "Desired median. Must be <= mean. Lower median = more right skew.", - "title": "Median", - "oneOf": [ - { - "description": "Desired median. Must be <= mean. Lower median = more right skew.", - "exclusiveMinimum": 0.0, - "title": "Median", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "type": { - "const": "lognormal", - "description": "Optional distribution type marker; stripped before concrete validation." - } - }, - "required": [ - "mean", - "median" - ], - "title": "LogNormalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - "LoggingConfig": { - "additionalProperties": false, - "description": "Logging configuration for verbosity and debug settings.", - "properties": { - "level": { - "$ref": "#/$defs/AIPerfLogLevel", - "default": "INFO", - "description": "Global logging verbosity level (loguru-style severity ladder; TRACE most verbose, CRITICAL least). NOTICE and SUCCESS are loguru-specific intermediate levels (between INFO and WARNING) and have no equivalent in the Python stdlib `logging` module — pick INFO/WARNING/ERROR if you want stdlib-compatible severities." - } - }, - "title": "LoggingConfig", - "type": "object" - }, - "MLflowConfig": { - "additionalProperties": false, - "description": "MLflow tracking and artifact-upload configuration.", - "properties": { - "trackingUri": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "MLflow tracking URI.", - "title": "Trackinguri" - }, - "experiment": { - "default": "aiperf", - "description": "MLflow experiment name.", - "title": "Experiment", - "type": "string" - }, - "runName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "MLflow run name.", - "title": "Runname" - }, - "tags": { - "anyOf": [ - { - "type": "array", - "items": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "minItems": 2, - "maxItems": 2 - } - ] - } - }, - { - "type": "string" - }, - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Additional MLflow run tags to attach on upload.", - "title": "Tags" - }, - "parentRunId": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional MLflow parent run ID.", - "title": "Parentrunid" - }, - "artifactGlobs": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Artifact glob overrides for MLflow upload.", - "title": "Artifactglobs" - } - }, - "title": "MLflowConfig", - "type": "object" - }, - "MetricsConfig": { - "additionalProperties": false, - "description": "Configuration for benchmark metric aggregation behavior.\n\nCurrently empty — list-valued record metrics aggregate via t-digest at\nfirst-touch (see :mod:`aiperf.metrics.list_metric_aggregation`); the\nsketch compression is tunable via ``AIPERF_METRICS_TDIGEST_COMPRESSION``.\nReserved as the home for any future user-facing metric-aggregation knobs.", - "properties": {}, - "title": "MetricsConfig", - "type": "object" - }, - "ModelItem": { - "additionalProperties": false, - "description": "Configuration for a single model in advanced models configuration.\n\nUsed when the models section uses the advanced format with\nexplicit items, weights, and per-model settings.", - "properties": { - "name": { - "description": "Model name or identifier as known to the inference server.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "weight": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Selection weight for weighted strategy (0.0-1.0). Weights must sum to 1.0 (+/-0.01) across all models; they are validated, not auto-normalized. Example: weight=0.7 means ~70%% of requests to this model.", - "title": "Weight", - "x-jinja2-supported": true - }, - "lora": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "LoRA adapter name to load with this model. Server must support dynamic LoRA adapter loading.", - "title": "Lora" - }, - "modalities": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "List of input modalities this model supports. [Currently a no-op: no selection strategy consumes this field. Accepted for forward-compatibility / declarative documentation.]", - "title": "Modalities" - }, - "tokenizer": { - "anyOf": [ - { - "$ref": "#/$defs/TokenizerOverride" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Per-model tokenizer override. Use when this model requires a different tokenizer than global config." - } - }, - "required": [ - "name" - ], - "title": "ModelItem", - "type": "object" - }, - "ModelSelectionStrategy": { - "description": "Strategy for selecting the model to use for the request.", - "enum": [ - "round_robin", - "random", - "weighted" - ], - "title": "ModelSelectionStrategy", - "type": "string" - }, - "ModelsAdvanced": { - "additionalProperties": false, - "description": "Advanced models configuration with selection strategy and item details.\n\nUse this format when you need weighted routing, LoRA adapters,\nor per-model tokenizer overrides.", - "properties": { - "strategy": { - "$ref": "#/$defs/ModelSelectionStrategy", - "default": "round_robin", - "description": "Strategy for selecting models when multiple are configured. round_robin cycles through models, random selects randomly, weighted uses configured weights." - }, - "items": { - "description": "List of model configurations. At least one model required.", - "items": { - "$ref": "#/$defs/ModelItem" - }, - "minItems": 1, - "title": "Items", - "type": "array" - } - }, - "required": [ - "items" - ], - "title": "ModelsAdvanced", - "type": "object" - }, - "MultiRunConfig": { - "additionalProperties": false, - "description": "Trial mechanics within a single sweep variation.\n\nWhen `num_runs > 1` (or `convergence` is set), AIPerf runs multiple\ntrials per variation and aggregates statistics. When `num_runs == 1`\nand `convergence is None`, exactly one trial runs per variation.", - "properties": { - "numRuns": { - "default": 1, - "description": "Upper bound on trials per variation. When `convergence` is set, trials may stop earlier; this is the hard ceiling. Cap matches `BenchmarkPlan.trials` so programmatic plan assembly cannot exceed what user-facing config validation allows.", - "title": "Numruns", - "oneOf": [ - { - "default": 1, - "description": "Upper bound on trials per variation. When `convergence` is set, trials may stop earlier; this is the hard ceiling. Cap matches `BenchmarkPlan.trials` so programmatic plan assembly cannot exceed what user-facing config validation allows.", - "maximum": 10, - "minimum": 1, - "title": "Numruns", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between trials within a single variation. Capped at 24h to surface typos like `1e18` at config-load time rather than blocking the orchestrator inside ``asyncio.sleep``.", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between trials within a single variation. Capped at 24h to surface typos like `1e18` at config-load time rather than blocking the orchestrator inside ``asyncio.sleep``.", - "maximum": 86400, - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "confidenceLevel": { - "default": 0.95, - "description": "Confidence level for aggregate statistics (0.90, 0.95, 0.99).", - "title": "Confidencelevel", - "oneOf": [ - { - "default": 0.95, - "description": "Confidence level for aggregate statistics (0.90, 0.95, 0.99).", - "exclusiveMaximum": 1, - "exclusiveMinimum": 0, - "title": "Confidencelevel", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "setConsistentSeed": { - "default": true, - "description": "Auto-set a random seed when none is given for cross-trial consistency.", - "title": "Setconsistentseed", - "type": "boolean" - }, - "varySeedPerTrial": { - "default": false, - "description": "When True, derive a distinct seed for each trial of a variation via SHA-256 over `(envelope_seed, variation.label, trial)`. When False (default), all trials of a variation share the variation's seed. Off gives pure-runtime variance for confidence statistics; on captures end-to-end variance including per-trial workload sampling.", - "title": "Varyseedpertrial", - "type": "boolean" - }, - "disableWarmupAfterFirst": { - "default": true, - "description": "Skip warmup on trials 2..N of a variation (steady-state measurement).", - "title": "Disablewarmupafterfirst", - "type": "boolean" - }, - "convergence": { - "anyOf": [ - { - "$ref": "#/$defs/ConvergenceConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional adaptive early-stop. Absent = fixed-N (run `num_runs` trials)." - } - }, - "title": "MultiRunConfig", - "type": "object" - }, - "MultimodalDistribution": { - "additionalProperties": false, - "description": "Weighted mixture of N peaks (N >= 2).\n\nYAML:\n isl:\n peaks:\n - {mean: 128, stddev: 20, weight: 60}\n - {mean: 2048, median: 1800, weight: 40}\n # Equal split — omit weight:\n isl:\n peaks:\n - {mean: 128, stddev: 20}\n - {mean: 2048, median: 1800}\n - {mean: 8192, median: 4096}", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min", - "x-jinja2-supported": true - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max", - "x-jinja2-supported": true - }, - "peaks": { - "description": "Two or more weighted sub-distributions.", - "items": { - "$ref": "#/$defs/PeakEntry" - }, - "minItems": 2, - "title": "Peaks", - "type": "array" - }, - "type": { - "const": "multimodal", - "description": "Optional distribution type marker; stripped before concrete validation." - } - }, - "required": [ - "peaks" - ], - "title": "MultimodalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - "NetworkLatencyConfig": { - "additionalProperties": false, - "description": "Network latency calibration configuration.\n\nWhen ``enabled``, AIPerf opens a fresh TCP connection to the endpoint\nhost:port on an interval during profiling and records the handshake RTT.\nThe mean RTT is subtracted from the request-start-anchored latency metrics\n(request_latency, time_to_first_token, time_to_first_output_token) to produce\nnon-destructive ``network_adjusted_*``\nvariants. ``mean_ms`` supplies a fixed mean RTT and skips active probing.", - "properties": { - "enabled": { - "default": false, - "description": "Enable network latency calibration. When true, AIPerf measures the endpoint RTT (or uses mean_ms) and subtracts it from latency metrics, emitted as separate network_adjusted_* metrics.", - "title": "Enabled", - "type": "boolean" - }, - "pingInterval": { - "description": "Seconds between TCP-handshake RTT probes during profiling. Ignored when mean_ms is set.", - "title": "Pinginterval", - "oneOf": [ - { - "description": "Seconds between TCP-handshake RTT probes during profiling. Ignored when mean_ms is set.", - "exclusiveMinimum": 0.0, - "title": "Pinginterval", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "meanMs": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Fixed mean network RTT in milliseconds to subtract, bypassing active probing. When set, no probes are sent and this value is used directly.", - "title": "Meanms", - "x-jinja2-supported": true - } - }, - "title": "NetworkLatencyConfig", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - "NormalDistribution": { - "additionalProperties": false, - "description": "Gaussian (truncated at 0) parameterized by mean and stddev.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min", - "x-jinja2-supported": true - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max", - "x-jinja2-supported": true - }, - "mean": { - "description": "Mean value. Must be >= 0; samples below 0 are truncated, so a negative mean would yield a degenerate distribution. Zero is allowed (e.g. OSL=0 disables output, turn_delay mean=0 disables inter-turn delay).", - "title": "Mean", - "oneOf": [ - { - "description": "Mean value. Must be >= 0; samples below 0 are truncated, so a negative mean would yield a degenerate distribution. Zero is allowed (e.g. OSL=0 disables output, turn_delay mean=0 disables inter-turn delay).", - "minimum": 0.0, - "title": "Mean", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "stddev": { - "default": 0.0, - "description": "Standard deviation. 0 = deterministic.", - "title": "Stddev", - "oneOf": [ - { - "default": 0.0, - "description": "Standard deviation. 0 = deterministic.", - "minimum": 0.0, - "title": "Stddev", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "type": { - "const": "normal", - "description": "Optional distribution type marker; stripped before concrete validation." - } - }, - "required": [ - "mean" - ], - "title": "NormalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - "OTelConfig": { - "additionalProperties": false, - "description": "OpenTelemetry metrics streaming configuration.", - "properties": { - "metricsUrl": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "OTLP/HTTP metrics endpoint URL.", - "title": "Metricsurl" - }, - "streamMetricsEnabled": { - "default": true, - "description": "Stream metric records to OTel.", - "title": "Streammetricsenabled", - "type": "boolean" - }, - "streamTimingEnabled": { - "default": true, - "description": "Stream timing records to OTel.", - "title": "Streamtimingenabled", - "type": "boolean" - }, - "customResourceAttributes": { - "additionalProperties": { - "type": "string" - }, - "description": "Custom OTel resource attributes.", - "title": "Customresourceattributes", - "type": "object" - }, - "genAiProvider": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "GenAI semantic convention provider override.", - "title": "Genaiprovider" - } - }, - "title": "OTelConfig", - "type": "object" - }, - "Objective": { - "additionalProperties": false, - "description": "Single optimization objective.\n\nMultiple ``Objective`` entries on an ``AdaptiveSearchSweep`` enable\nPareto Bayesian optimization (e.g. qlognehvi / qnehvi / qehvi). A\nlength-1 list is single-objective BO.", - "properties": { - "metric": { - "description": "Metric tag to optimize (e.g. 'output_token_throughput').", - "title": "Metric", - "type": "string" - }, - "stat": { - "default": "avg", - "description": "Statistic on the metric.", - "enum": [ - "avg", - "p50", - "p90", - "p95", - "p99" - ], - "title": "Stat", - "type": "string" - }, - "direction": { - "$ref": "#/$defs/OptimizationDirection", - "description": "MAXIMIZE or MINIMIZE." - }, - "threshold": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Pareto reference point for hypervolume computation. Trials worse than this on this objective do not contribute to hypervolume. When None, auto-derived from the worst observed value among Sobol initial points. Ignored for single-objective runs.", - "title": "Threshold", - "x-jinja2-supported": true - } - }, - "required": [ - "metric", - "direction" - ], - "title": "Objective", - "type": "object" - }, - "OptimizationDirection": { - "description": "Direction of optimization for a metric.\n\nMAXIMIZE: Higher values are preferred (e.g. throughput, goodput).\nMINIMIZE: Lower values are preferred (e.g. latency, TTFT, p99).", - "enum": [ - "maximize", - "minimize" - ], - "title": "OptimizationDirection", - "type": "string" - }, - "OutcomeConstraint": { - "additionalProperties": false, - "description": "Feasibility gate on a metric the optimizer is *not* optimizing.\n\nDistinct from ``Objective.threshold`` (Pareto reference point) and from\n``SLAFilter`` (post-hoc filter on benchmark eligibility). An outcome\nconstraint masks BoTorch acquisition: candidates predicted to violate\nit are downweighted.", - "properties": { - "metric": { - "description": "Metric tag to constrain.", - "title": "Metric", - "type": "string" - }, - "op": { - "description": "Comparison operator.", - "enum": [ - "<=", - ">=", - "==" - ], - "title": "Op", - "type": "string" - }, - "bound": { - "description": "Threshold value.", - "title": "Bound", - "oneOf": [ - { - "description": "Threshold value.", - "title": "Bound", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "metric", - "op", - "bound" - ], - "title": "OutcomeConstraint", - "type": "object" - }, - "PeakEntry": { - "title": "PeakEntry", - "description": "A weighted component in a multimodal distribution.\n\nThe weight and distribution fields are written inline in YAML:\n {mean: 128, stddev: 20, weight: 60}\n\nThe ``weight`` key is extracted before the remaining fields are parsed\nas a SamplingDistribution. Defaults to 1.0 (equal split when omitted).", - "anyOf": [ - { - "additionalProperties": false, - "description": "A weighted component in a multimodal distribution.\n\nThe weight and distribution fields are written inline in YAML:\n {mean: 128, stddev: 20, weight: 60}\n\nThe ``weight`` key is extracted before the remaining fields are parsed\nas a SamplingDistribution. Defaults to 1.0 (equal split when omitted).", - "properties": { - "distribution": { - "description": "The sub-distribution for this peak.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ], - "title": "Distribution" - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalised internally).", - "minimum": 0.0, - "title": "Weight", - "type": "number" - } - }, - "required": [ - "distribution" - ], - "title": "PeakEntry", - "type": "object" - }, - { - "additionalProperties": false, - "description": "Inline FixedDistribution peak with optional weight.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min" - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max" - }, - "value": { - "description": "The constant value returned on every sample.", - "title": "Value", - "type": "number" - }, - "type": { - "const": "fixed", - "description": "Optional distribution type marker; stripped before concrete validation." - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalised internally).", - "minimum": 0.0, - "title": "Weight", - "type": "number" - } - }, - "required": [ - "value" - ], - "title": "FixedDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "additionalProperties": false, - "description": "Inline NormalDistribution peak with optional weight.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min" - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max" - }, - "mean": { - "description": "Mean value. Must be >= 0; samples below 0 are truncated, so a negative mean would yield a degenerate distribution. Zero is allowed (e.g. OSL=0 disables output, turn_delay mean=0 disables inter-turn delay).", - "minimum": 0.0, - "title": "Mean", - "type": "number" - }, - "stddev": { - "default": 0.0, - "description": "Standard deviation. 0 = deterministic.", - "minimum": 0.0, - "title": "Stddev", - "type": "number" - }, - "type": { - "const": "normal", - "description": "Optional distribution type marker; stripped before concrete validation." - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalised internally).", - "minimum": 0.0, - "title": "Weight", - "type": "number" - } - }, - "required": [ - "mean" - ], - "title": "NormalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "additionalProperties": false, - "description": "Inline LogNormalDistribution peak with optional weight.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min" - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max" - }, - "mean": { - "description": "Desired mean of the output distribution.", - "exclusiveMinimum": 0.0, - "title": "Mean", - "type": "number" - }, - "median": { - "description": "Desired median. Must be <= mean. Lower median = more right skew.", - "exclusiveMinimum": 0.0, - "title": "Median", - "type": "number" - }, - "type": { - "const": "lognormal", - "description": "Optional distribution type marker; stripped before concrete validation." - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalised internally).", - "minimum": 0.0, - "title": "Weight", - "type": "number" - } - }, - "required": [ - "mean", - "median" - ], - "title": "LogNormalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "additionalProperties": false, - "description": "Inline MultimodalDistribution peak with optional weight.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min" - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max" - }, - "peaks": { - "description": "Two or more weighted sub-distributions.", - "items": { - "$ref": "#/$defs/PeakEntry" - }, - "minItems": 2, - "title": "Peaks", - "type": "array" - }, - "type": { - "const": "multimodal", - "description": "Optional distribution type marker; stripped before concrete validation." - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalised internally).", - "minimum": 0.0, - "title": "Weight", - "type": "number" - } - }, - "required": [ - "peaks" - ], - "title": "MultimodalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "additionalProperties": false, - "description": "Inline EmpiricalDistribution peak with optional weight.", - "properties": { - "min": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.", - "title": "Min" - }, - "max": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inclusive upper bound; samples above are clamped down.", - "title": "Max" - }, - "points": { - "description": "Weighted discrete values. Weights are relative.", - "items": { - "$ref": "#/$defs/EmpiricalPoint" - }, - "title": "Points", - "type": "array" - }, - "type": { - "const": "empirical", - "description": "Optional distribution type marker; stripped before concrete validation." - }, - "weight": { - "default": 1.0, - "description": "Relative weight (normalised internally).", - "minimum": 0.0, - "title": "Weight", - "type": "number" - } - }, - "required": [ - "points" - ], - "title": "EmpiricalDistribution", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - } - ] - }, - "PoissonPhase": { - "additionalProperties": false, - "description": "Poisson-distributed request arrivals at the target rate.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration", - "x-jinja2-supported": true - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod", - "x-jinja2-supported": true - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "title": "Adaptivemincompletedrequests", - "oneOf": [ - { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "title": "Adaptivecontrolmin", - "oneOf": [ - { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax", - "x-jinja2-supported": true - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "title": "Adaptivescalebasestep", - "oneOf": [ - { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "title": "Adaptivescalemaxstepmultiplier", - "oneOf": [ - { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "title": "Adaptivescalesteppercent", - "oneOf": [ - { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "poisson", - "description": "Poisson-distributed rate-controlled arrivals.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests", - "x-jinja2-supported": true - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration", - "x-jinja2-supported": true - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions", - "x-jinja2-supported": true - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency", - "x-jinja2-supported": true - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency", - "x-jinja2-supported": true - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod", - "x-jinja2-supported": true - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold", - "x-jinja2-supported": true - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "title": "Trajectorystartminratio", - "oneOf": [ - { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "title": "Trajectorystartmaxratio", - "oneOf": [ - { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds", - "x-jinja2-supported": true - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration", - "x-jinja2-supported": true - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane", - "x-jinja2-supported": true - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod", - "x-jinja2-supported": true - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate", - "x-jinja2-supported": true - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "rateSeries": { - "anyOf": [ - { - "$ref": "#/$defs/RateSeriesConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Piecewise-linear request-rate schedule." - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "name", - "type" - ], - "title": "PoissonPhase", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "rateSeries" - ], - "properties": { - "rateSeries": { - "not": { - "type": "null" - } - } - } - } - ] - } - ] - }, - "PostProcessSpec": { - "additionalProperties": false, - "description": "Post-aggregation hook spec emitted by a search recipe.\n\nResolved through the ``search_recipe_post_process`` plugin category and\ninvoked by ``aggregate_sweep_and_export`` after per-variation aggregation.\n\nExample: ``PostProcessSpec(handler=\"ttft_sla_curve\", params={\"sla_ms\": 200},\noutput_filename=\"ttft_sla_curve.json\")``.", - "properties": { - "handler": { - "description": "Registered post-process handler name. Looked up in the ``search_recipe_post_process`` plugin category.", - "title": "Handler", - "type": "string" - }, - "params": { - "additionalProperties": true, - "description": "Free-form parameters forwarded to the handler. Shape is handler-defined; Any is intentional because handlers ingest dynamic recipe-derived inputs.", - "title": "Params", - "type": "object" - }, - "output_filename": { - "description": "Filename (relative to ``sweep_aggregate/`` under the artifact dir) to write the handler's output to. Must end in ``.json`` and contain no path separators -- the hook joins it with ``sweep_aggregate/`` and refuses to escape that directory. NUL bytes and dot-only stems are also rejected.", - "title": "Output Filename", - "type": "string" - } - }, - "required": [ - "handler", - "output_filename" - ], - "title": "PostProcessSpec", - "type": "object" - }, - "PrefixPromptConfig": { - "additionalProperties": false, - "description": "Configuration for prefix prompts (KV cache testing).\n\nPrefix prompts allow testing KV cache efficiency by generating\nrequests that share common prefixes. This simulates scenarios\nlike system prompts or shared context that can be cached.\n\nNote: pool_size/length are mutually exclusive with shared_system_length\nand user_context_length.", - "properties": { - "poolSize": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Number of distinct prefix prompts to generate for KV cache testing. Each prefix is prepended to user prompts, simulating cached context scenarios. Prefixes are randomly selected from pool per request. Mutually exclusive with shared_system_length/user_context_length.", - "title": "Poolsize", - "x-jinja2-supported": true - }, - "length": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Token length for each prefix prompt in the pool. Only used when pool_size is set. Note: due to prefix and user prompts being concatenated, the final prompt token count may be off by one. Mutually exclusive with shared_system_length/user_context_length.", - "title": "Length", - "x-jinja2-supported": true - }, - "sharedSystemLength": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Length of shared system prompt in tokens. This prompt is identical across all sessions and appears as a system message. First part of a two-part prefix structure with high cache hit rate expected. Mutually exclusive with pool_size/length.", - "title": "Sharedsystemlength", - "x-jinja2-supported": true - }, - "userContextLength": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Length of per-session user context prompt in tokens. Each dataset entry gets a unique user context prompt. Second part of two-part prefix structure with lower cache hit rate expected. Mutually exclusive with pool_size/length.", - "title": "Usercontextlength", - "x-jinja2-supported": true - } - }, - "title": "PrefixPromptConfig", - "type": "object" - }, - "PromptConfig": { - "additionalProperties": false, - "description": "Configuration for prompt/token specifications in synthetic datasets.\n\nThis is the core configuration for controlling input sequence length (ISL)\nand output sequence length (OSL) in synthetic data generation.", - "properties": { - "isl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Input sequence length in tokens. Can be a fixed integer (e.g., 512) or distribution {mean: 512, stddev: 50}. AIPerf generates prompts with lengths following a normal distribution around the mean (±stddev). Ignored when sequence_distribution is specified.", - "title": "Isl" - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output sequence length (max tokens to request via max_completion_tokens). Can be a fixed integer or distribution {mean, stddev}. Controls response length for synthetic datasets. When not set, the model determines output length. Ignored when sequence_distribution is specified.", - "title": "Osl" - }, - "blockSize": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Token block size for hash-based prompt caching in mooncake_trace datasets. When hash_ids are provided in trace entries, prompts are divided into blocks of this size. Each hash_id maps to a cached block, enabling simulation of KV-cache sharing patterns from production workloads. Total prompt length = (num_hash_ids - 1) * block_size + final_block_size.", - "title": "Blocksize", - "x-jinja2-supported": true - }, - "batchSize": { - "default": 1, - "description": "Number of text inputs to include in each request for batch processing endpoints. Supported by embeddings and rankings endpoint types where models can process multiple inputs simultaneously. Set to 1 for single-input requests. Not applicable to chat or completions endpoints.", - "title": "Batchsize", - "oneOf": [ - { - "default": 1, - "description": "Number of text inputs to include in each request for batch processing endpoints. Supported by embeddings and rankings endpoint types where models can process multiple inputs simultaneously. Set to 1 for single-input requests. Not applicable to chat or completions endpoints.", - "minimum": 1, - "title": "Batchsize", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "cacheBust": { - "$ref": "#/$defs/CacheBustConfig", - "description": "Per-conversation cache-bust marker configuration. Injects a deterministic marker into prompts to defeat KV-cache prefix matching. Disabled by default (target='none')." - }, - "corpus": { - "anyOf": [ - { - "$ref": "#/$defs/PromptCorpus" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare sonnets. 'coding' uses realistic coding content (code, bash output, JSON, error tracebacks, git diffs). When unset, the active dataset loader's default applies (most loaders default to 'sonnet'; agentic-coding loaders such as weka_trace default to 'coding')." - }, - "sequenceDistribution": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/SequenceDistributionEntry" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Distribution of (ISL, OSL) pairs with probabilities for mixed workload simulation. Each entry specifies {isl, osl, probability}. Probabilities are percentages (0-100) and must sum to 100. When specified, requests are sampled from this distribution instead of using isl/osl fields.", - "title": "Sequencedistribution" - } - }, - "title": "PromptConfig", - "type": "object" - }, - "PromptCorpus": { - "description": "Corpus used for synthetic prompt text generation.", - "enum": [ - "sonnet", - "coding" - ], - "title": "PromptCorpus", - "type": "string" - }, - "PromptSelectionConfig": { - "additionalProperties": false, - "description": "Slim prompts block for file/public datasets (corpus selection only).", - "properties": { - "corpus": { - "anyOf": [ - { - "$ref": "#/$defs/PromptCorpus" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source corpus for synthesized prompt text. 'sonnet' uses Shakespeare sonnets. 'coding' uses realistic coding content. When unset, the active dataset loader's default applies. Honored only where content is synthesized (synthetic + hash/trace replay); verbatim loaders ignore it." - } - }, - "title": "PromptSelectionConfig", - "type": "object" - }, - "PublicDataset": { - "additionalProperties": false, - "description": "Public dataset configuration.\n\nUses well-known public benchmarking datasets that are\nautomatically downloaded and processed by AIPerf.", - "properties": { - "name": { - "description": "Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "type": { - "const": "public", - "description": "Dataset type discriminator. Must be 'public'.", - "title": "Type", - "type": "string" - }, - "dataset": { - "$ref": "#/$defs/PublicDatasetType", - "description": "Pre-configured public dataset to download and use for benchmarking. Name of the HuggingFace public dataset enum (e.g. 'sharegpt', 'alpaca'). AIPerf automatically downloads and parses these datasets." - }, - "entries": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Limit number of records to use from the dataset. If not specified, uses all available records.", - "title": "Entries", - "x-jinja2-supported": true - }, - "randomSeed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Random seed for deterministic sampling from the dataset. Overrides global random_seed for this dataset.", - "title": "Randomseed", - "x-jinja2-supported": true - }, - "sampling": { - "$ref": "#/$defs/DatasetSamplingStrategy", - "default": "sequential", - "description": "Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion." - }, - "hfSubset": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace dataset subset/config name override (e.g. 'sharegpt4o'). Only applies for HuggingFace-backed public dataset loaders. Takes priority over the subset defined in the plugin registry.", - "title": "Hfsubset" - }, - "filters": { - "additionalProperties": { - "type": "string" - }, - "description": "Dataset-specific filters forwarded to public dataset loaders. Supported keys and values depend on the selected dataset.", - "title": "Filters", - "type": "object" - }, - "hfWekaDataset": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace dataset repo for the generic Weka loader (e.g. `semianalysisai/cc-traces-weka-061526`). Only applies with `dataset: weka_hf`; setting it with any other public dataset is an error. Passing `--hf-weka-dataset` on the CLI auto-selects `--public-dataset weka_hf`, so the repo flag works on its own. Pinned Weka public dataset aliases keep their registry-defined repo names.", - "title": "Hfwekadataset" - }, - "interTurnDelayCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Clamp per-turn replay delays to at most this many seconds for HF-backed Weka trace replay (mirror of ``FileDataset.inter_turn_delay_cap_seconds``). ``None`` disables the cap.", - "title": "Interturndelaycapseconds", - "x-jinja2-supported": true - }, - "traceIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree for HF-backed Weka replay (mirror of ``FileDataset.trace_idle_gap_cap_seconds``). Dataset timestamps remain unchanged. Defaults to None (no per-trace runtime idle cap).", - "title": "Traceidlegapcapseconds", - "x-jinja2-supported": true - }, - "ignoreTraceDelays": { - "default": false, - "description": "Strip per-turn timestamps and inter-turn delays from HF-backed Weka trace replay at load time (mirror of ``FileDataset.ignore_trace_delays``). Mutually exclusive with use_think_time_only.", - "title": "Ignoretracedelays", - "type": "boolean" - }, - "useThinkTimeOnly": { - "default": false, - "description": "For HF-backed Weka replay, emit Turn.delay using only the recorded per-request ``think_time`` instead of the full inter-request delta (mirror of ``FileDataset.use_think_time_only``). Mutually exclusive with ignore_trace_delays.", - "title": "Usethinktimeonly", - "type": "boolean" - }, - "maxContextLength": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum peak prompt+output context length (tokens) per conversation for HF-backed Weka replay; drops over-length traces at load using *recorded* lengths (mirror of ``FileDataset.max_context_length``). Rejected for non-Weka public datasets.", - "title": "Maxcontextlength", - "x-jinja2-supported": true - }, - "cacheBust": { - "$ref": "#/$defs/CacheBustConfig", - "description": "Per-conversation cache-bust marker injection for HF-backed Weka trace replay (mirror of ``FileDataset.cache_bust``). 'none' (default) disables it." - }, - "prompts": { - "anyOf": [ - { - "$ref": "#/$defs/PromptSelectionConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it." - }, - "synthesis": { - "anyOf": [ - { - "$ref": "#/$defs/SynthesisConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Trace synthesis/transformation configuration for HF-backed Weka trace replay (mirror of ``FileDataset.synthesis``). ``--max-isl``/``--max-osl`` cap the per-conversation input/output lengths of the replayed HF Weka traces; without this they applied only to file-based traces. ``None`` disables synthesis." - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output sequence length to apply when records do not specify one (mirror of ``FileDataset.osl``). Can be a fixed integer or {mean, stddev} distribution. Per-record output lengths always take precedence.", - "title": "Osl" - } - }, - "required": [ - "name", - "type", - "dataset" - ], - "title": "PublicDataset", - "type": "object" - }, - "PublicDatasetType": { - "enum": [ - "exgentic_v2", - "exgentic", - "sharegpt", - "aimo", - "mmstar", - "mmvu", - "vision_arena", - "llava_onevision", - "aimo_aime", - "aimo_numina_cot", - "aimo_numina_1_5", - "spec_bench", - "spec_al_gsm8k", - "spec_al_math500", - "spec_al_humaneval", - "spec_al_mbpp", - "spec_al_mtbench", - "instruct_coder", - "blazedit_5k", - "blazedit_10k", - "librispeech", - "voxpopuli", - "gigaspeech", - "ami", - "spgispeech", - "semianalysis_cc_traces_weka", - "semianalysis_cc_traces_weka_no_subagents", - "semianalysis_cc_traces_weka_with_subagents", - "semianalysis_cc_traces_weka_with_subagents_256k", - "semianalysis_cc_traces_weka_with_subagents_060226", - "semianalysis_cc_traces_weka_with_subagents_060226_256k", - "semianalysis_cc_traces_weka_with_subagents_060526", - "semianalysis_cc_traces_weka_with_subagents_060526_256k", - "semianalysis_cc_traces_weka_with_subagents_060826", - "semianalysis_cc_traces_weka_with_subagents_060826_256k", - "semianalysis_cc_traces_weka_061326", - "semianalysis_cc_traces_weka_061326_256k", - "semianalysis_cc_traces_weka_061526", - "semianalysis_cc_traces_weka_061526_256k", - "semianalysis_cc_traces_weka_062126", - "semianalysis_cc_traces_weka_062126_256k", - "weka_hf" - ], - "title": "PublicDatasetType", - "type": "string" - }, - "RampConfig": { - "title": "RampConfig", - "description": "Configuration for gradual value ramping.\n\nControls how a value (concurrency, rate, etc.) transitions from\nstart to target over time. Can be a duration (number in seconds or string like '30s', '5m', '2h') or {duration, strategy} object.", - "anyOf": [ - { - "additionalProperties": false, - "description": "Configuration for gradual value ramping.\n\nControls how a value (concurrency, rate, etc.) transitions from\nstart to target over time.", - "properties": { - "duration": { - "description": "Seconds to ramp from start to target value.", - "exclusiveMinimum": 0.0, - "title": "Duration", - "type": "number" - }, - "strategy": { - "$ref": "#/$defs/RampType", - "default": "linear", - "description": "Ramp curve shape: linear (constant rate), exponential (slow start, fast finish), poisson (stochastic with guaranteed completion)." - } - }, - "required": [ - "duration" - ], - "title": "RampConfig", - "type": "object" - }, - { - "type": "integer" - }, - { - "type": "number" - }, - { - "type": "string" - } - ] - }, - "RampType": { - "enum": [ - "exponential", - "linear", - "poisson" - ], - "title": "RampType", - "type": "string" - }, - "RankingsConfig": { - "additionalProperties": false, - "description": "Configuration for rankings/reranking endpoint datasets.\n\nControls the generation of query-passage pairs for benchmarking\nreranking and ranking models. Each request contains one query\nand multiple passages to rank.", - "properties": { - "passages": { - "description": "Number of passages per ranking request. Can be a fixed integer or {mean, stddev} distribution. Higher values test ranking at scale but increase request payload size and processing time.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Passages", - "x-jinja2-supported": true - }, - "passageTokens": { - "description": "Token length for each passage in ranking requests. Can be a fixed integer or {mean, stddev} distribution. Passages are synthetically generated text. Longer passages increase input processing demands and request size.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Passagetokens", - "x-jinja2-supported": true - }, - "queryTokens": { - "description": "Token length for the query text in ranking requests. Can be a fixed integer or {mean, stddev} distribution. Each ranking request contains one query and multiple passages.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Querytokens", - "x-jinja2-supported": true - } - }, - "title": "RankingsConfig", - "type": "object" - }, - "RateSeriesConfig": { - "additionalProperties": false, - "description": "Piecewise-linear request-rate schedule.", - "properties": { - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "JSON file path containing request-rate control points.", - "title": "Path" - }, - "points": { - "description": "Strictly increasing request-rate control points.", - "items": { - "$ref": "#/$defs/RateSeriesPoint" - }, - "title": "Points", - "type": "array", - "minItems": 2 - } - }, - "title": "RateSeriesConfig", - "type": "object", - "allOf": [ - { - "oneOf": [ - { - "required": [ - "path" - ], - "properties": { - "path": { - "not": { - "type": "null" - } - } - } - }, - { - "required": [ - "points" - ], - "properties": { - "points": { - "minItems": 2 - } - } - } - ] - } - ] - }, - "RateSeriesPoint": { - "additionalProperties": false, - "description": "One request-rate control point.", - "properties": { - "timeS": { - "description": "Elapsed time in seconds from the start of the request-rate series, after any request-rate ramp completes.", - "title": "Times", - "oneOf": [ - { - "description": "Elapsed time in seconds from the start of the request-rate series, after any request-rate ramp completes.", - "minimum": 0.0, - "title": "Times", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "qps": { - "description": "Request rate in queries per second at this point.", - "title": "Qps", - "oneOf": [ - { - "description": "Request rate in queries per second at this point.", - "exclusiveMinimum": 0.0, - "title": "Qps", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "timeS", - "qps" - ], - "title": "RateSeriesPoint", - "type": "object" - }, - "RequestContentType": { - "description": "Content type for HTTP request body serialization.", - "enum": [ - "application/json", - "multipart/form-data" - ], - "title": "RequestContentType", - "type": "string" - }, - "RuntimeConfig": { - "additionalProperties": false, - "description": "Runtime configuration for benchmark execution.", - "properties": { - "ui": { - "$ref": "#/$defs/UIType", - "default": "dashboard", - "description": "User interface mode. dashboard: rich interactive UI, simple: text progress, none: silent operation." - }, - "workers": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Total worker processes across the deployment (the maximum, ramped up to from `workers_min`). null = auto-detect based on CPU cores. Distinct from `workers_per_pod` (per-Kubernetes-pod fan-out) and `workers_min` (lower bound for adaptive ramp).", - "title": "Workers", - "x-jinja2-supported": true - }, - "recordProcessors": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Total parallel record processors across the deployment. null = auto-detect based on CPU cores. Distinct from `record_processors_per_pod` (per-Kubernetes-pod fan-out).", - "title": "Recordprocessors", - "x-jinja2-supported": true - }, - "serviceRunType": { - "$ref": "#/$defs/ServiceRunType", - "default": "multiprocessing", - "description": "Execution mode. multiprocessing: local multi-process (default for `aiperf profile`). kubernetes: distributed across pods. [operator-managed under AIPerfJob — the operator forces 'kubernetes' on the controller pod; do not set this in a CR spec.]" - }, - "communication": { - "anyOf": [ - { - "discriminator": { - "propertyName": "type", - "mapping": { - "ipc": "#/$defs/IpcCommunicationConfig", - "tcp": "#/$defs/TcpCommunicationConfig", - "dual": "#/$defs/DualBindCommunicationConfig" - } - }, - "anyOf": [ - { - "$ref": "#/$defs/IpcCommunicationConfig" - }, - { - "$ref": "#/$defs/TcpCommunicationConfig" - }, - { - "$ref": "#/$defs/DualBindCommunicationConfig" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Inter-process communication configuration. Defaults to IPC for single-machine operation.", - "title": "Communication" - }, - "apiPort": { - "anyOf": [ - { - "maximum": 65535, - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AIPerf API server port. Enables HTTP and WebSocket endpoints for real-time metrics and control.", - "title": "Apiport", - "x-jinja2-supported": true - }, - "apiHost": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "AIPerf API server host. Requires api_port to be set.", - "title": "Apihost" - }, - "datasetApiBaseUrl": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Base URL the operator injects into worker pods so they can fetch datasets from the controller-pod sidecar in Kubernetes mode. [operator-managed; do not set in a CR spec.]", - "title": "Datasetapibaseurl" - }, - "workersPerPod": { - "anyOf": [ - { - "maximum": 100, - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Worker containers packed into each Kubernetes worker pod (Kubernetes mode only). The total number of worker containers cluster-wide is `workers` (or auto-detected); this knob controls how that total is fanned across pods. Ignored in multiprocessing mode.", - "title": "Workersperpod", - "x-jinja2-supported": true - }, - "recordProcessorsPerPod": { - "anyOf": [ - { - "maximum": 100, - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Record-processor containers packed into each Kubernetes worker pod (Kubernetes mode only). Sibling of `workers_per_pod`; controls per-pod fan-out, not the cluster-wide total. Ignored in multiprocessing mode.", - "title": "Recordprocessorsperpod", - "x-jinja2-supported": true - }, - "workersMin": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Lower bound on worker processes for the adaptive ramp. Distinct from `workers` (the upper bound / target total).", - "title": "Workersmin", - "x-jinja2-supported": true - }, - "statsInterval": { - "anyOf": [ - { - "maximum": 1000.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Interval in seconds between realtime stats publishes (dashboards and the per-tick log block). 0 disables the log block while dashboards continue to poll. Defaults to 5s under --ui dashboard, 30s otherwise. Overrides AIPERF_UI_REALTIME_METRICS_INTERVAL.", - "title": "Statsinterval", - "x-jinja2-supported": true - } - }, - "title": "RuntimeConfig", - "type": "object" - }, - "SLAFilter": { - "additionalProperties": false, - "description": "SLA constraint applied to BO scoring or grid filtering.\n\nA trial is considered feasible iff ``stat(metric_tag) op threshold`` holds.\n``BayesianSearchPlanner`` consumes these for soft-penalty BO scoring and\n``write_search_history`` consumes them for lexicographic\nfeasibility-first best-result selection.\n\nLives in ``aiperf.config`` rather than ``aiperf.search_recipes`` because\n``AdaptiveSearchSweep`` carries a ``list[SLAFilter]`` field; the inverse\nlocation would force a circular import. Re-exported from\n``aiperf.search_recipes._base`` for ergonomic recipe authoring.\n\nExample: enforce p95 TTFT under 200 ms via\n``SLAFilter(metric_tag=\"time_to_first_token\", stat=\"p95\", op=\"lt\", threshold=200.0)``.", - "properties": { - "metricTag": { - "description": "Metric tag to filter on, e.g. 'time_to_first_token'. Must match a key in RunResult.summary_metrics produced by the run.", - "title": "Metrictag", - "type": "string" - }, - "stat": { - "default": "p95", - "description": "Statistic on the metric to compare against the threshold.", - "enum": [ - "avg", - "p1", - "p5", - "p10", - "p25", - "p50", - "p75", - "p90", - "p95", - "p99", - "min", - "max" - ], - "title": "Stat", - "type": "string" - }, - "op": { - "description": "Comparison operator. Filter passes when stat(metric) op threshold is true.", - "enum": [ - "lt", - "le", - "gt", - "ge" - ], - "title": "Op", - "type": "string" - }, - "threshold": { - "description": "Numeric threshold the metric statistic is compared against.", - "title": "Threshold", - "oneOf": [ - { - "description": "Numeric threshold the metric statistic is compared against.", - "title": "Threshold", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "metricTag", - "op", - "threshold" - ], - "title": "SLAFilter", - "type": "object" - }, - "SLOTier": { - "additionalProperties": false, - "description": "Named group of SLA filters representing one service-level objective.\n\nDefines a single tier for multi-tier SLO boundary search. Each tier\ncontains one or more SLA filters that must ALL pass for the tier to be\nconsidered feasible at a given concurrency level.\n\nLives in the config layer (alongside SLAFilter) to avoid circular imports\nbetween config and orchestrator packages. Re-exported from\n``aiperf.orchestrator.search_planner.multi_tier_models`` for ergonomic\norchestrator usage.", - "properties": { - "label": { - "description": "Unique tier identifier for output artifacts", - "title": "Label", - "type": "string" - }, - "filters": { - "description": "SLA filters that must ALL pass for this tier to be feasible", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "minItems": 1, - "title": "Filters", - "type": "array" - }, - "orderingRank": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Rank in detected monotonic ordering (0=strictest); None if unordered", - "title": "Orderingrank", - "x-jinja2-supported": true - } - }, - "required": [ - "label", - "filters" - ], - "title": "SLOTier", - "type": "object" - }, - "SamplingDimension": { - "additionalProperties": false, - "description": "One dimension of a QMC sampling space.\n\nUsed by `SobolSweep` and `LatinHypercubeSweep`. Distinct from\n`SearchSpaceDimension` (BO) — same path/lo/hi grammar, but native\n`choices` support and explicit `scale` validation.\n\nPath is a dotted path of the form `phases.profiling.concurrency` —\nthe same grammar accepted by `_set_nested_value`.", - "properties": { - "path": { - "description": "Dotted-path into BenchmarkConfig.", - "title": "Path", - "type": "string" - }, - "lo": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive lower bound (with hi).", - "title": "Lo", - "x-jinja2-supported": true - }, - "hi": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Inclusive upper bound (with lo).", - "title": "Hi", - "x-jinja2-supported": true - }, - "scale": { - "default": "linear", - "description": "Mapping from unit cube to value.", - "enum": [ - "linear", - "log" - ], - "title": "Scale", - "type": "string" - }, - "kind": { - "default": "real", - "description": "Output type after mapping.", - "enum": [ - "int", - "real" - ], - "title": "Kind", - "type": "string" - }, - "choices": { - "anyOf": [ - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Discrete value list. Mutually exclusive with (lo, hi).", - "title": "Choices" - } - }, - "required": [ - "path" - ], - "title": "SamplingDimension", - "type": "object" - }, - "ScenarioSweep": { - "additionalProperties": false, - "description": "Scenario sweep - hand-picked configurations deep-merged with base.", - "properties": { - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaFilters": { - "description": "SLA constraints applied at trial scoring / aggregation time.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Slafilters", - "type": "array" - }, - "postProcess": { - "anyOf": [ - { - "$ref": "#/$defs/PostProcessSpec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional post-aggregation handler emitted by a search recipe." - }, - "recipeName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.", - "title": "Recipename" - }, - "iterationOrder": { - "$ref": "#/$defs/SweepMode", - "default": "repeated", - "description": "Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search." - }, - "sameSeed": { - "default": false, - "description": "If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.", - "title": "Sameseed", - "type": "boolean" - }, - "type": { - "const": "scenarios", - "default": "scenarios", - "description": "Sweep type discriminator.", - "title": "Type", - "type": "string" - }, - "runs": { - "description": "List of scenario dicts to deep-merge with base config.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "minItems": 1, - "title": "Runs", - "type": "array" - } - }, - "required": [ - "runs" - ], - "title": "ScenarioSweep", - "type": "object" - }, - "SearchPlannerType": { - "enum": [ - "bayesian", - "monotonic_sla", - "smooth_isotonic", - "optuna" - ], - "title": "SearchPlannerType", - "type": "string" - }, - "SearchSpaceDimension": { - "additionalProperties": false, - "description": "One dimension of the BO search space.\n\n`path` is a dotted path of the form `phases.profiling.concurrency` —\nthe same grammar accepted by `aiperf.config.sweep._set_nested_value`.", - "properties": { - "path": { - "description": "Dotted-path into BenchmarkConfig (e.g. 'phases.profiling.concurrency').", - "title": "Path", - "type": "string" - }, - "lo": { - "description": "Inclusive lower bound.", - "title": "Lo", - "oneOf": [ - { - "description": "Inclusive lower bound.", - "title": "Lo", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "hi": { - "description": "Inclusive upper bound.", - "title": "Hi", - "oneOf": [ - { - "description": "Inclusive upper bound.", - "title": "Hi", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "kind": { - "default": "real", - "description": "Dimension type. 'int' rounds planner suggestions to integers; 'real' keeps floats.", - "enum": [ - "int", - "real" - ], - "title": "Kind", - "type": "string" - }, - "prior": { - "default": "uniform", - "description": "Sampling prior for BO/Optuna planners. 'uniform' draws Sobol/random samples linearly across [lo, hi]; 'log-uniform' draws them log-spaced. Use 'log-uniform' when a parameter spans orders of magnitude (e.g. concurrency in [1, 1000]) so the initial-points phase covers low and high decades evenly. Requires lo > 0. Ignored by planners that do not draw initial points (monotonic, smooth_isotonic) where the search procedure is structurally adaptive rather than sampling-based.", - "enum": [ - "uniform", - "log-uniform" - ], - "title": "Prior", - "type": "string" - } - }, - "required": [ - "path", - "lo", - "hi" - ], - "title": "SearchSpaceDimension", - "type": "object" - }, - "SequenceDistributionEntry": { - "additionalProperties": false, - "description": "Defines a single entry in an ISL/OSL probability distribution.\n\nAIPerf supports multi-modal token length distributions, allowing\ndifferent ISL/OSL combinations with relative frequencies for\nrealistic workload modeling.\n\nYAML Representation:\n sequence_distribution:\n - {isl: 128, osl: 64, probability: 40}\n - {isl: {mean: 512, stddev: 50}, osl: 256, probability: 35}\n - {isl: {mean: 2048, median: 1800}, osl: 512, probability: 25}", - "examples": [ - { - "isl": 128, - "osl": 64, - "probability": 40 - }, - { - "isl": 512, - "isl_stddev": 50, - "osl": 256, - "osl_stddev": 25, - "probability": 35 - } - ], - "properties": { - "isl": { - "description": "Input sequence length (tokens). Can be a fixed integer, a {mean, stddev} distribution, or any distribution like {mean: 512, median: 400} for lognormal.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Isl", - "x-jinja2-supported": true - }, - "islStddev": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Shorthand for ISL standard deviation. If provided when isl is an integer, creates a normal distribution. Cannot be used when isl is already a distribution dict.", - "title": "Islstddev", - "x-jinja2-supported": true - }, - "osl": { - "description": "Output sequence length (tokens). Can be a fixed integer, a {mean, stddev} distribution, or any typed distribution.", - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "title": "Osl", - "x-jinja2-supported": true - }, - "oslStddev": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Shorthand for OSL standard deviation. If provided when osl is an integer, creates a normal distribution. Cannot be used when osl is already a distribution dict.", - "title": "Oslstddev", - "x-jinja2-supported": true - }, - "probability": { - "description": "Relative probability weight for this distribution bucket (0-100). Weights are normalized across all entries. Example: probability=40 means 40%% of requests use this ISL/OSL.", - "title": "Probability", - "oneOf": [ - { - "description": "Relative probability weight for this distribution bucket (0-100). Weights are normalized across all entries. Example: probability=40 means 40%% of requests use this ISL/OSL.", - "maximum": 100.0, - "minimum": 0.0, - "title": "Probability", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "isl", - "osl", - "probability" - ], - "title": "SequenceDistributionEntry", - "type": "object" - }, - "ServerMetricsConfig": { - "title": "ServerMetricsConfig", - "description": "Server metrics configuration for Prometheus scraping.\n\nCollects server-side operational metrics (queue depth, KV cache utilization,\nbatch sizes, GPU memory) from Prometheus endpoints exposed by inference servers\nlike vLLM, TensorRT-LLM, or Triton.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9090/metrics\"\n → ServerMetricsConfig(enabled=True, urls=[\"http://localhost:9090/metrics\"])\n - Singular url field: {url: \"...\"}\n → ServerMetricsConfig(urls=[\"...\"]) Can be a URL string or full config object.", - "anyOf": [ - { - "additionalProperties": false, - "description": "Server metrics configuration for Prometheus scraping.\n\nCollects server-side operational metrics (queue depth, KV cache utilization,\nbatch sizes, GPU memory) from Prometheus endpoints exposed by inference servers\nlike vLLM, TensorRT-LLM, or Triton.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9090/metrics\"\n → ServerMetricsConfig(enabled=True, urls=[\"http://localhost:9090/metrics\"])\n - Singular url field: {url: \"...\"}\n → ServerMetricsConfig(urls=[\"...\"])", - "properties": { - "enabled": { - "default": true, - "description": "Enable Prometheus metrics scraping. Set to false to disable.", - "title": "Enabled", - "type": "boolean" - }, - "urls": { - "description": "Prometheus metrics endpoint URLs to scrape. Typically the /metrics endpoint on inference servers.", - "items": { - "type": "string" - }, - "title": "Urls", - "type": "array" - }, - "formats": { - "description": "Export formats for scraped metrics. Options: json, csv, parquet, jsonl.", - "items": { - "$ref": "#/$defs/ServerMetricsFormat" - }, - "title": "Formats", - "type": "array" - }, - "discovery": { - "$ref": "#/$defs/ServerMetricsDiscoveryConfig", - "description": "Auto-discovery of Prometheus endpoints in Kubernetes. Discovers pods via Dynamo labels, Prometheus annotations, or custom label selectors." - }, - "url": { - "description": "Shorthand for 'urls'. Accepts a single value which will be converted to a list/dict. Prometheus metrics endpoint URLs to scrape. Typically the /metrics endpoint on inference servers.", - "oneOf": [ - { - "type": "string" - }, - { - "description": "Prometheus metrics endpoint URLs to scrape. Typically the /metrics endpoint on inference servers.", - "items": { - "type": "string" - }, - "title": "Urls", - "type": "array" - } - ], - "x-singular-alias-of": "urls" - } - }, - "title": "ServerMetricsConfig", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - }, - { - "type": "string" - } - ] - }, - "ServerMetricsDiscoveryConfig": { - "additionalProperties": false, - "description": "Kubernetes-based auto-discovery of inference-server /metrics endpoints.\n\nWhen mode is 'auto' or 'kubernetes', queries the K8s API for pods that\nare recognizable inference servers (vLLM, SGLang, Triton Inference Server,\nTensorRT-LLM, NVIDIA Dynamo). Eligibility (any one is enough):\n1. Dynamo opt-in label: nvidia.com/metrics-enabled=true\n2. AIPerf opt-in annotation: aiperf.nvidia.com/metrics-paths=...\n3. A container image matching a known inference-server signature\n4. User-provided label_selector (server-side filter)\n\nThe broad ``prometheus.io/scrape=true`` annotation is intentionally NOT a\ntrigger: Loki, Grafana, kube-state-metrics, and many platform components\nset it without being inference servers. ``prometheus.io/{port,path,scheme}``\nare still honored to construct the scrape URL when an eligible pod sets them.", - "properties": { - "mode": { - "$ref": "#/$defs/ServerMetricsDiscoveryMode", - "default": "auto", - "description": "Discovery mode: 'auto' detects environment and tries K8s if in-cluster, 'kubernetes' forces K8s API discovery, 'disabled' uses only explicit URLs." - }, - "labelSelector": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Kubernetes label selector for discovery. Example: 'app=vllm,env=prod'. Applied in addition to built-in Dynamo and Prometheus discovery.", - "title": "Labelselector" - }, - "namespace": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Kubernetes namespace to search. If not specified, searches all namespaces.", - "title": "Namespace" - } - }, - "title": "ServerMetricsDiscoveryConfig", - "type": "object" - }, - "ServerMetricsDiscoveryMode": { - "description": "Mode for discovering server metrics endpoints in distributed environments.", - "enum": [ - "auto", - "kubernetes", - "disabled" - ], - "title": "ServerMetricsDiscoveryMode", - "type": "string" - }, - "ServerMetricsFormat": { - "description": "Format options for server metrics export.\n\nControls which output files are generated for server metrics data.\nDefault selection is JSON + CSV (JSONL excluded to avoid large files).", - "enum": [ - "json", - "csv", - "jsonl", - "parquet" - ], - "title": "ServerMetricsFormat", - "type": "string" - }, - "ServiceRunType": { - "enum": [ - "multiprocessing" - ], - "title": "ServiceRunType", - "type": "string" - }, - "SobolSweep": { - "additionalProperties": false, - "description": "Sobol quasi-Monte-Carlo sweep — low-discrepancy joint coverage.\n\nNote on ``scramble=False``: scipy's raw Sobol sequence starts at the\norigin (the first row is exactly all-zeros), so the first variant\nwill land on ``(lo, lo, ...)`` for every dimension. With small\n``samples`` this produces degenerate corner-only coverage. Prefer\nthe default ``scramble=True`` (Owen-scrambled) unless you have a\nspecific reason to want raw Sobol points.", - "properties": { - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaFilters": { - "description": "SLA constraints applied at trial scoring / aggregation time.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Slafilters", - "type": "array" - }, - "postProcess": { - "anyOf": [ - { - "$ref": "#/$defs/PostProcessSpec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional post-aggregation handler emitted by a search recipe." - }, - "recipeName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.", - "title": "Recipename" - }, - "iterationOrder": { - "$ref": "#/$defs/SweepMode", - "default": "repeated", - "description": "Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search." - }, - "sameSeed": { - "default": false, - "description": "If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.", - "title": "Sameseed", - "type": "boolean" - }, - "samples": { - "description": "Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.", - "title": "Samples", - "oneOf": [ - { - "description": "Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.", - "maximum": 1048576, - "minimum": 2, - "title": "Samples", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "seed": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "RNG seed for reproducibility. Must be non-negative; scipy's QMC engines reject negative seeds with an opaque low-level error.", - "title": "Seed", - "x-jinja2-supported": true - }, - "dimensions": { - "description": "Dimensions to sample over.", - "items": { - "$ref": "#/$defs/SamplingDimension" - }, - "minItems": 1, - "title": "Dimensions", - "type": "array" - }, - "labelFormat": { - "default": "index", - "description": "Variation-label format. 'index' => '_NNNN' (short, sortable). 'kv' => 'k1=v1, k2=v2' (grid-style).", - "enum": [ - "index", - "kv" - ], - "title": "Labelformat", - "type": "string" - }, - "type": { - "const": "sobol", - "default": "sobol", - "description": "Sweep type discriminator.", - "title": "Type", - "type": "string" - }, - "scramble": { - "default": true, - "description": "Owen-scrambled Sobol (recommended). False = raw Sobol sequence.", - "title": "Scramble", - "type": "boolean" - } - }, - "required": [ - "samples", - "dimensions" - ], - "title": "SobolSweep", - "type": "object" - }, - "SweepMode": { - "description": "Execution order for sweep + confidence composition.\n\nREPEATED: for trial in trials: for value in values\n All values tested per trial before moving to next trial.\n\nINDEPENDENT: for value in values: for trial in trials\n All trials completed per value before moving to next value.", - "enum": [ - "independent", - "repeated" - ], - "title": "SweepMode", - "type": "string" - }, - "SynthesisConfig": { - "additionalProperties": false, - "description": "Configuration for trace synthesis/transformation.\n\nUsed with mooncake_trace format to transform production trace\ndata before replay. Allows scaling timestamps, token lengths,\nand radix tree structure.", - "properties": { - "speedupRatio": { - "default": 1.0, - "description": "Multiplier for timestamp scaling in synthesized traces. 1.0 = real-time, 2.0 = 2x faster, 0.5 = 2x slower.", - "title": "Speedupratio", - "oneOf": [ - { - "default": 1.0, - "description": "Multiplier for timestamp scaling in synthesized traces. 1.0 = real-time, 2.0 = 2x faster, 0.5 = 2x slower.", - "exclusiveMinimum": 0.0, - "title": "Speedupratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "prefixLenMultiplier": { - "default": 1.0, - "description": "Multiplier for core prefix branch lengths in the radix tree. 1.5 means prefix branches are 50%% longer.", - "title": "Prefixlenmultiplier", - "oneOf": [ - { - "default": 1.0, - "description": "Multiplier for core prefix branch lengths in the radix tree. 1.5 means prefix branches are 50%% longer.", - "exclusiveMinimum": 0.0, - "title": "Prefixlenmultiplier", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "prefixRootMultiplier": { - "default": 1, - "description": "Number of independent radix trees to distribute traces across. Higher values increase prefix diversity.", - "title": "Prefixrootmultiplier", - "oneOf": [ - { - "default": 1, - "description": "Number of independent radix trees to distribute traces across. Higher values increase prefix diversity.", - "minimum": 1, - "title": "Prefixrootmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "promptLenMultiplier": { - "default": 1.0, - "description": "Multiplier for leaf path (unique prompt) lengths. 2.0 means prompts are 2x longer.", - "title": "Promptlenmultiplier", - "oneOf": [ - { - "default": 1.0, - "description": "Multiplier for leaf path (unique prompt) lengths. 2.0 means prompts are 2x longer.", - "exclusiveMinimum": 0.0, - "title": "Promptlenmultiplier", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "outputLenMultiplier": { - "default": 1.0, - "description": "Multiplier for output lengths in synthesized traces.", - "title": "Outputlenmultiplier", - "oneOf": [ - { - "default": 1.0, - "description": "Multiplier for output lengths in synthesized traces.", - "minimum": 0.0, - "title": "Outputlenmultiplier", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "maxIsl": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum input sequence length filter. Traces with input_length > max_isl are skipped entirely.", - "title": "Maxisl", - "x-jinja2-supported": true - }, - "maxOsl": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum output sequence length cap. Traces with output_length > max_osl are capped to this value (not filtered).", - "title": "Maxosl", - "x-jinja2-supported": true - }, - "allowDatasetWrap": { - "default": false, - "description": "Allow the finite weka/agentic trace pool to wrap (reuse distinct roots across concurrency lanes) when concurrency exceeds the number of eligible traces. Defaults to False: over-subscription fails unless this is explicitly enabled or an active cache-bust target already keeps repeated-trace traffic distinct.", - "title": "Allowdatasetwrap", - "type": "boolean" - } - }, - "title": "SynthesisConfig", - "type": "object" - }, - "SyntheticDataset": { - "additionalProperties": false, - "description": "Synthetic dataset configuration.\n\nGenerates prompts programmatically based on token length\nspecifications. Ideal for controlled experiments.", - "properties": { - "name": { - "description": "Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.", - "minLength": 1, - "title": "Name", - "type": "string" - }, - "type": { - "const": "synthetic", - "description": "Dataset type discriminator. Must be 'synthetic'.", - "title": "Type", - "type": "string" - }, - "entries": { - "default": 100, - "description": "Total number of unique entries to generate for the dataset. Each entry represents a unique prompt with sampled ISL/OSL. Entries are reused across conversations and turns according to the sampling strategy. Higher values provide more diversity.", - "title": "Entries", - "oneOf": [ - { - "default": 100, - "description": "Total number of unique entries to generate for the dataset. Each entry represents a unique prompt with sampled ISL/OSL. Entries are reused across conversations and turns according to the sampling strategy. Higher values provide more diversity.", - "minimum": 1, - "title": "Entries", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "randomSeed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Random seed for deterministic dataset generation. When set, makes synthetic prompts, sampling, and other random operations reproducible across runs. Essential for A/B testing and debugging. Overrides global random_seed for this dataset.", - "title": "Randomseed", - "x-jinja2-supported": true - }, - "sampling": { - "$ref": "#/$defs/DatasetSamplingStrategy", - "default": "sequential", - "description": "Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion." - }, - "prompts": { - "anyOf": [ - { - "$ref": "#/$defs/PromptConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Prompt/token length configuration specifying ISL, OSL, sequence distributions, and batch processing settings." - }, - "prefixPrompts": { - "anyOf": [ - { - "$ref": "#/$defs/PrefixPromptConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Shared prefix configuration for KV cache testing. Generates prefix prompts that are prepended to user prompts, simulating cached context scenarios." - }, - "turns": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Number of request-response turns per conversation. Can be a fixed integer or {mean, stddev} distribution. Each turn consists of a user message and model response. Set to 1 for single-turn interactions. Multi-turn conversations enable testing of context retention and conversation history handling.", - "title": "Turns" - }, - "turnDelay": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Delay in milliseconds between consecutive turns within a multi-turn conversation. Can be a fixed value or {mean, stddev} distribution. Simulates user think time between receiving a response and sending the next message. Only applies when turns > 1. Set to 0 for back-to-back turns.", - "title": "Turndelay" - }, - "turnDelayRatio": { - "default": 1.0, - "description": "Multiplier for scaling all turn delays. Applied after mean/stddev calculation: actual_delay = calculated_delay * ratio. Values < 1 speed up conversations, > 1 slow them down. Set to 0 to eliminate delays entirely.", - "title": "Turndelayratio", - "oneOf": [ - { - "default": 1.0, - "description": "Multiplier for scaling all turn delays. Applied after mean/stddev calculation: actual_delay = calculated_delay * ratio. Values < 1 speed up conversations, > 1 slow them down. Set to 0 to eliminate delays entirely.", - "minimum": 0.0, - "title": "Turndelayratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "images": { - "anyOf": [ - { - "$ref": "#/$defs/ImageConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Synthetic image configuration for multimodal vision-language testing." - }, - "audio": { - "anyOf": [ - { - "$ref": "#/$defs/AudioConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Synthetic audio configuration for multimodal speech/audio testing." - }, - "video": { - "anyOf": [ - { - "$ref": "#/$defs/VideoConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Synthetic video configuration for multimodal video understanding testing." - }, - "rankings": { - "anyOf": [ - { - "$ref": "#/$defs/RankingsConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Rankings/reranking configuration for generating query-passage pairs. Only relevant for rankings endpoint types." - }, - "isl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Shorthand for prompts.isl; hoisted into prompts before validation.", - "title": "Isl" - }, - "osl": { - "anyOf": [ - { - "oneOf": [ - { - "$ref": "#/$defs/FixedDistribution" - }, - { - "$ref": "#/$defs/NormalDistribution" - }, - { - "$ref": "#/$defs/LogNormalDistribution" - }, - { - "$ref": "#/$defs/MultimodalDistribution" - }, - { - "$ref": "#/$defs/EmpiricalDistribution" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Shorthand for prompts.osl; hoisted into prompts before validation.", - "title": "Osl" - } - }, - "required": [ - "name", - "type" - ], - "title": "SyntheticDataset", - "type": "object" - }, - "TcpCommunicationConfig": { - "additionalProperties": false, - "description": "TCP socket communication configuration.\n\nFor distributed deployments across machines.\nProvides detailed port configuration for all ZMQ proxies.", - "properties": { - "type": { - "const": "tcp", - "description": "Communication type. Must be 'tcp'.", - "title": "Type", - "type": "string" - }, - "host": { - "default": "127.0.0.1", - "description": "Host address for TCP communication. Use 0.0.0.0 to listen on all interfaces.", - "title": "Host", - "type": "string" - }, - "recordsPort": { - "default": 5557, - "description": "Port for records push/pull communication.", - "title": "Recordsport", - "oneOf": [ - { - "default": 5557, - "description": "Port for records push/pull communication.", - "maximum": 65535, - "minimum": 1, - "title": "Recordsport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "creditRouterPort": { - "default": 5564, - "description": "Port for credit router (ROUTER-DEALER).", - "title": "Creditrouterport", - "oneOf": [ - { - "default": 5564, - "description": "Port for credit router (ROUTER-DEALER).", - "maximum": 65535, - "minimum": 1, - "title": "Creditrouterport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "controlPort": { - "default": 5667, - "description": "Port for control channel (ROUTER-DEALER).", - "title": "Controlport", - "oneOf": [ - { - "default": 5667, - "description": "Port for control channel (ROUTER-DEALER).", - "maximum": 65535, - "minimum": 1, - "title": "Controlport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "eventBusProxy": { - "$ref": "#/$defs/TcpProxyConfig", - "description": "Event bus proxy ports (XPUB/XSUB)." - }, - "datasetManagerProxy": { - "$ref": "#/$defs/TcpProxyConfig", - "description": "Dataset manager proxy ports (DEALER/ROUTER)." - }, - "rawInferenceProxy": { - "$ref": "#/$defs/TcpProxyConfig", - "description": "Raw inference proxy ports (PUSH/PULL)." - } - }, - "required": [ - "type" - ], - "title": "TcpCommunicationConfig", - "type": "object" - }, - "TcpProxyConfig": { - "additionalProperties": false, - "description": "TCP proxy port configuration for a single ZMQ proxy.", - "properties": { - "frontendPort": { - "description": "TCP port for proxy frontend.", - "title": "Frontendport", - "oneOf": [ - { - "description": "TCP port for proxy frontend.", - "maximum": 65535, - "minimum": 1, - "title": "Frontendport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "backendPort": { - "description": "TCP port for proxy backend.", - "title": "Backendport", - "oneOf": [ - { - "description": "TCP port for proxy backend.", - "maximum": 65535, - "minimum": 1, - "title": "Backendport", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "required": [ - "frontendPort", - "backendPort" - ], - "title": "TcpProxyConfig", - "type": "object" - }, - "TemplateConfig": { - "additionalProperties": false, - "description": "Configuration for custom template-based endpoints.\n\nWhen endpoint type is \"template\", this configures how requests\nare formatted and responses are parsed.", - "properties": { - "body": { - "description": "Jinja2 template string for request body. Variables: {{prompt}}, {{max_tokens}}, {{model}}, {{messages}}, etc.", - "title": "Body", - "type": "string" - }, - "responseField": { - "default": "text", - "description": "JSON path to extract response text from API response. Use dot notation for nested fields: 'choices.0.message.content'.", - "title": "Responsefield", - "type": "string" - } - }, - "required": [ - "body" - ], - "title": "TemplateConfig", - "type": "object" - }, - "TimingMode": { - "enum": [ - "fixed_schedule", - "request_rate", - "adaptive_scale", - "user_centric_rate", - "agentic_replay" - ], - "title": "TimingMode", - "type": "string" - }, - "TokenizerConfig": { - "additionalProperties": false, - "description": "Tokenizer configuration for token counting and prompt generation.\n\nAIPerf uses a HuggingFace tokenizer for accurate token counting,\nwhich is essential for ISL/OSL enforcement and metrics calculation.", - "properties": { - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "HuggingFace tokenizer identifier, local filesystem path, or `builtin` for a zero-network-access tokenizer backed by tiktoken (o200k_base encoding). Should match the model's tokenizer for accurate token counts. If `--tokenizer` is not set and the model name looks like an obvious placeholder (e.g. `mock-model`, `test-model`, `fake-model`), AIPerf substitutes `builtin` automatically and emits a warning. Example: 'meta-llama/Llama-3.1-8B-Instruct'", - "title": "Name" - }, - "revision": { - "default": "main", - "description": "Model revision to use: branch name, tag, or commit hash. Use for version pinning to ensure reproducibility.", - "title": "Revision", - "type": "string" - }, - "trustRemoteCode": { - "default": false, - "description": "Allow execution of custom tokenizer code from the repository. Required for some models but poses security risk. Only enable for trusted sources.", - "title": "Trustremotecode", - "type": "boolean" - }, - "applyChatTemplate": { - "default": false, - "description": "Apply the HuggingFace tokenizer's chat template when counting input tokens. When enabled: synthetic ISL is compensated for chat-template wrapping (BOS, role headers, EOT, generation-prompt suffix) and the record processor reports ISL using `apply_chat_template(tokenize=True, add_generation_prompt=True)` for chat-shape payloads. When disabled (default), both paths use bare-text encoding, so reported ISL matches the prompt content the user asked for and ignores template overhead. Requires an HF tokenizer with a chat template configured; no-ops on tiktoken / un-templated models.", - "title": "Applychattemplate", - "type": "boolean" - } - }, - "title": "TokenizerConfig", - "type": "object" - }, - "TokenizerOverride": { - "additionalProperties": false, - "description": "Per-model tokenizer override configuration.\n\nAllows specifying a different tokenizer for a specific model,\nuseful when models require specialized tokenization.", - "properties": { - "name": { - "description": "HuggingFace tokenizer identifier or local filesystem path.", - "title": "Name", - "type": "string" - } - }, - "required": [ - "name" - ], - "title": "TokenizerOverride", - "type": "object" - }, - "TransportType": { - "enum": [ - "http" - ], - "title": "TransportType", - "type": "string" - }, - "UIType": { - "enum": [ - "dashboard", - "none", - "simple" - ], - "title": "UIType", - "type": "string" - }, - "URLSelectionStrategy": { - "enum": [ - "round_robin" - ], - "title": "URLSelectionStrategy", - "type": "string" - }, - "UserCentricPhase": { - "additionalProperties": false, - "description": "N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.", - "properties": { - "adaptiveScale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - }, - "adaptiveSustainDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds to sustain load near the discovered adaptive scale boundary.", - "title": "Adaptivesustainduration", - "x-jinja2-supported": true - }, - "adaptiveAssessmentPeriod": { - "anyOf": [ - { - "minimum": 1.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Duration in seconds for each adaptive scale SLA assessment window.", - "title": "Adaptiveassessmentperiod", - "x-jinja2-supported": true - }, - "adaptiveMinCompletedRequests": { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "title": "Adaptivemincompletedrequests", - "oneOf": [ - { - "default": 1, - "description": "Minimum completed requests needed before an adaptive SLA window can make a decision.", - "minimum": 1, - "title": "Adaptivemincompletedrequests", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlVariable": { - "default": "concurrency", - "description": "Named adaptive control variable.", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ], - "title": "Adaptivecontrolvariable", - "type": "string" - }, - "adaptiveControlMin": { - "default": 1, - "description": "Minimum adaptive control value.", - "title": "Adaptivecontrolmin", - "oneOf": [ - { - "default": 1, - "description": "Minimum adaptive control value.", - "exclusiveMinimum": 0, - "title": "Adaptivecontrolmin", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveControlMax": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Maximum adaptive control value. Inferred from the phase target when omitted.", - "title": "Adaptivecontrolmax", - "x-jinja2-supported": true - }, - "adaptiveScaleStrategyType": { - "const": "ramp_until_fail", - "default": "ramp_until_fail", - "description": "Adaptive scale controller strategy. v1 supports ramp_until_fail.", - "title": "Adaptivescalestrategytype", - "type": "string" - }, - "adaptiveScaleStepPolicy": { - "default": "sla_margin", - "description": "Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.", - "enum": [ - "sla_margin", - "fixed_percent_step" - ], - "title": "Adaptivescalesteppolicy", - "type": "string" - }, - "adaptiveScaleBaseStep": { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "title": "Adaptivescalebasestep", - "oneOf": [ - { - "default": 10, - "description": "Minimum adaptive scale step for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalebasestep", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleMaxStepMultiplier": { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "title": "Adaptivescalemaxstepmultiplier", - "oneOf": [ - { - "default": 4, - "description": "Maximum base-step multiplier for SLA-margin policy.", - "minimum": 1, - "title": "Adaptivescalemaxstepmultiplier", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptiveScaleStepPercent": { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "title": "Adaptivescalesteppercent", - "oneOf": [ - { - "default": 25.0, - "description": "Percent of current concurrency used by fixed-percent adaptive scaling.", - "exclusiveMinimum": 0, - "title": "Adaptivescalesteppercent", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "name": { - "description": "Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.", - "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$", - "title": "Name", - "type": "string" - }, - "kind": { - "anyOf": [ - { - "enum": [ - "warmup", - "profiling" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.", - "title": "Kind" - }, - "type": { - "const": "user_centric", - "description": "N users sharing a global request rate.", - "title": "Type", - "type": "string" - }, - "excludeFromResults": { - "default": false, - "description": "Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.", - "title": "Excludefromresults", - "type": "boolean" - }, - "requests": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many requests sent (must be >= 1).", - "title": "Requests", - "x-jinja2-supported": true - }, - "duration": { - "anyOf": [ - { - "type": "string", - "pattern": "^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$", - "description": "Duration string (e.g., '30s', '5M', '2h', 'inf')." - }, - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.", - "gt": 0, - "title": "Duration", - "x-jinja2-supported": true - }, - "sessions": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Stop after this many sessions completed (must be >= 1).", - "title": "Sessions", - "x-jinja2-supported": true - }, - "concurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.", - "title": "Concurrency", - "x-jinja2-supported": true - }, - "concurrencyRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "prefillConcurrency": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.", - "title": "Prefillconcurrency", - "x-jinja2-supported": true - }, - "prefillRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "gracePeriod": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.", - "ge": 0, - "title": "Graceperiod", - "x-jinja2-supported": true - }, - "cancellation": { - "anyOf": [ - { - "$ref": "#/$defs/CancellationConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Request cancellation testing configuration." - }, - "sla": { - "description": "SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.", - "title": "Sla", - "anyOf": [ - { - "description": "SLA filters evaluated by adaptive load controllers.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Sla", - "type": "array" - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "seamless": { - "default": false, - "description": "Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.", - "title": "Seamless", - "type": "boolean" - }, - "timingMode": { - "anyOf": [ - { - "$ref": "#/$defs/TimingMode" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy." - }, - "failedRequestThreshold": { - "anyOf": [ - { - "maximum": 1.0, - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.", - "title": "Failedrequestthreshold", - "x-jinja2-supported": true - }, - "trajectoryStartMinRatio": { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "title": "Trajectorystartminratio", - "oneOf": [ - { - "default": 0.25, - "description": "AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartminratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "trajectoryStartMaxRatio": { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "title": "Trajectorystartmaxratio", - "oneOf": [ - { - "default": 0.75, - "description": "AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.", - "maximum": 1.0, - "minimum": 0.0, - "title": "Trajectorystartmaxratio", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "burstPhaseStarts": { - "default": false, - "description": "AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.", - "title": "Burstphasestarts", - "type": "boolean" - }, - "systemIdleGapCapSeconds": { - "anyOf": [ - { - "minimum": 0.0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.", - "title": "Systemidlegapcapseconds", - "x-jinja2-supported": true - }, - "agenticCacheWarmupDuration": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.", - "title": "Agenticcachewarmupduration", - "x-jinja2-supported": true - }, - "warmupRequestsPerLane": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.", - "title": "Warmuprequestsperlane", - "x-jinja2-supported": true - }, - "agenticWarmupGracePeriod": { - "anyOf": [ - { - "minimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).", - "title": "Agenticwarmupgraceperiod", - "x-jinja2-supported": true - }, - "rate": { - "anyOf": [ - { - "exclusiveMinimum": 0, - "type": "number" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Target request rate in requests per second. Required unless rate_series is set.", - "title": "Rate", - "x-jinja2-supported": true - }, - "rateRamp": { - "anyOf": [ - { - "$ref": "#/$defs/RampConfig" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.", - "x-jinja2-supported": true - }, - "users": { - "description": "Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.", - "title": "Users", - "oneOf": [ - { - "description": "Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.", - "minimum": 1, - "title": "Users", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "adaptive_scale": { - "description": "Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.", - "title": "Adaptivescale", - "anyOf": [ - { - "default": false, - "description": "Enable single-run adaptive scale control for this phase.", - "title": "Adaptivescale", - "type": "boolean" - }, - { - "$ref": "#/$defs/AdaptiveScaleNestedBlock" - } - ] - } - }, - "required": [ - "name", - "type", - "users", - "rate" - ], - "title": "UserCentricPhase", - "type": "object", - "allOf": [ - { - "required": [ - "rate" - ], - "properties": { - "rate": { - "not": { - "type": "null" - } - } - } - } - ] - }, - "UserFile": { - "description": "One user-declared output file rendered into the run directory before benchmark start.\n\nPath is relative to the run directory; subdirectories are allowed; absolute\npaths and any segment equal to '..' are rejected. Content is rendered with\njinja2 against a documented context (variables: + system-injected names).", - "properties": { - "path": { - "description": "Output path relative to the run directory. Subdirectories allowed. Absolute paths and any segment equal to '..' are rejected.", - "title": "Path", - "type": "string" - }, - "format": { - "anyOf": [ - { - "enum": [ - "json", - "yaml", - "text" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Serialization format. If omitted: 'text' when content is a string, 'json' otherwise.", - "title": "Format" - }, - "content": { - "description": "Templated value. Structured (dict/list/scalar) for json/yaml; string for text. Jinja2 expressions in any string leaf are rendered with the user_files context.", - "title": "Content" - } - }, - "required": [ - "path", - "content" - ], - "title": "UserFile", - "type": "object" - }, - "VideoAudioCodec": { - "description": "Audio codecs for embedding audio in synthetic video files.", - "enum": [ - "aac", - "libvorbis", - "libopus" - ], - "title": "VideoAudioCodec", - "type": "string" - }, - "VideoAudioConfig": { - "additionalProperties": false, - "description": "Configuration for embedding an audio track in synthetic video files.", - "properties": { - "sampleRate": { - "default": 44.1, - "description": "Audio sample rate in kHz for the embedded audio track. Common values: 8 (telephony), 16 (speech), 44.1 (CD quality), 48 (professional). Higher sample rates increase audio fidelity and file size.", - "title": "Samplerate", - "oneOf": [ - { - "default": 44.1, - "description": "Audio sample rate in kHz for the embedded audio track. Common values: 8 (telephony), 16 (speech), 44.1 (CD quality), 48 (professional). Higher sample rates increase audio fidelity and file size.", - "maximum": 96.0, - "minimum": 8.0, - "title": "Samplerate", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "channels": { - "default": 0, - "description": "Number of audio channels to embed in generated video files. 0 = disabled (no audio track, default), 1 = mono, 2 = stereo. When set to 1 or 2, a Gaussian noise audio track matching the video duration is muxed into each video via FFmpeg.", - "title": "Channels", - "oneOf": [ - { - "default": 0, - "description": "Number of audio channels to embed in generated video files. 0 = disabled (no audio track, default), 1 = mono, 2 = stereo. When set to 1 or 2, a Gaussian noise audio track matching the video duration is muxed into each video via FFmpeg.", - "maximum": 2, - "minimum": 0, - "title": "Channels", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "codec": { - "anyOf": [ - { - "$ref": "#/$defs/VideoAudioCodec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Audio codec for the embedded audio track. If not specified, auto-selects based on video format: aac for MP4, libvorbis for WebM. Options: aac, libvorbis, libopus." - }, - "depth": { - "description": "Audio bit depth for the embedded audio track. Supported values: 8, 16, 24, or 32 bits. Higher bit depths provide greater dynamic range but increase file size.", - "default": 16, - "oneOf": [ - { - "default": 16, - "description": "Audio bit depth for the embedded audio track. Supported values: 8, 16, 24, or 32 bits. Higher bit depths provide greater dynamic range but increase file size.", - "enum": [ - 8, - 16, - 24, - 32 - ], - "title": "Depth", - "type": "integer" - }, - { - "type": "string", - "enum": [ - "8", - "16", - "24", - "32" - ] - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - } - }, - "title": "VideoAudioConfig", - "type": "object" - }, - "VideoConfig": { - "additionalProperties": false, - "description": "Configuration for synthetic video generation in multimodal datasets.\n\nControls the generation of synthetic videos for video understanding\nmodel benchmarking. Requires FFmpeg for video generation.", - "properties": { - "batchSize": { - "default": 0, - "description": "Number of video files to include in each multimodal request. Supported with chat endpoint type for video understanding models. Set to 0 to disable video inputs. Higher batch sizes significantly increase request payload size.", - "title": "Batchsize", - "oneOf": [ - { - "default": 0, - "description": "Number of video files to include in each multimodal request. Supported with chat endpoint type for video understanding models. Set to 0 to disable video inputs. Higher batch sizes significantly increase request payload size.", - "minimum": 0, - "title": "Batchsize", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "duration": { - "default": 1.0, - "description": "Duration in seconds for each generated video clip. Combined with fps, determines total frame count (frames = duration * fps). Longer durations increase file size and processing time. Typical values: 1-10 seconds for testing.", - "title": "Duration", - "oneOf": [ - { - "default": 1.0, - "description": "Duration in seconds for each generated video clip. Combined with fps, determines total frame count (frames = duration * fps). Longer durations increase file size and processing time. Typical values: 1-10 seconds for testing.", - "exclusiveMinimum": 0.0, - "title": "Duration", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "fps": { - "default": 4, - "description": "Frames per second for generated video. Higher FPS creates smoother video but increases frame count and file size. Common values: 4 (minimal, recommended for Cosmos models), 24 (cinematic), 30 (standard), 60 (high frame rate). Total frames = duration * fps.", - "title": "Fps", - "oneOf": [ - { - "default": 4, - "description": "Frames per second for generated video. Higher FPS creates smoother video but increases frame count and file size. Common values: 4 (minimal, recommended for Cosmos models), 24 (cinematic), 30 (standard), 60 (high frame rate). Total frames = duration * fps.", - "minimum": 1, - "title": "Fps", - "type": "integer" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "width": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Video frame width in pixels. Determines video resolution and file size. Common values: 640 (SD), 1280 (HD), 1920 (Full HD). If not specified, uses codec/format defaults.", - "title": "Width", - "x-jinja2-supported": true - }, - "height": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Video frame height in pixels. Combined with width determines aspect ratio and total pixel count per frame. Common values: 480 (SD), 720 (HD), 1080 (Full HD). If not specified, uses codec/format defaults.", - "title": "Height", - "x-jinja2-supported": true - }, - "format": { - "$ref": "#/$defs/VideoFormat", - "default": "webm", - "description": "Container format for generated video files. webm: VP9 codec, BSD-licensed, recommended for open-source workflows. mp4: H.264/H.265, widely compatible. Format affects compatibility, file size, and encoding options." - }, - "codec": { - "default": "libvpx-vp9", - "description": "Video codec for encoding. Common options: libvpx-vp9 (CPU, BSD-licensed, default for WebM), libx264 (CPU, GPL, widely compatible), libx265 (CPU, GPL, smaller files), h264_nvenc (NVIDIA GPU), hevc_nvenc (NVIDIA GPU, smaller files). Any FFmpeg-supported codec can be used.", - "title": "Codec", - "type": "string" - }, - "synthType": { - "$ref": "#/$defs/VideoSynthType", - "default": "moving_shapes", - "description": "Algorithm for generating synthetic video content. Different types produce different visual patterns for testing. Content doesn't affect semantic meaning but may impact encoding efficiency and file size." - }, - "audio": { - "$ref": "#/$defs/VideoAudioConfig", - "description": "Audio track configuration for embedding audio in generated videos." - } - }, - "title": "VideoConfig", - "type": "object" - }, - "VideoFormat": { - "description": "Video container formats for synthetic video generation.", - "enum": [ - "mp4", - "webm" - ], - "title": "VideoFormat", - "type": "string" - }, - "VideoSynthType": { - "enum": [ - "moving_shapes", - "grid_clock", - "noise" - ], - "title": "VideoSynthType", - "type": "string" - }, - "WandbConfig": { - "additionalProperties": false, - "dependentRequired": { - "entity": [ - "project" - ], - "runName": [ - "project" - ], - "tags": [ - "project" - ] - }, - "description": "Weights & Biases run-upload configuration.", - "properties": { - "project": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Weights & Biases project name. Setting this enables wandb export.", - "title": "Project" - }, - "entity": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Weights & Biases entity (team or user). Defaults to the API key's default entity.", - "title": "Entity" - }, - "runName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Weights & Biases run name.", - "title": "Runname" - }, - "tags": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Additional Weights & Biases run tags to attach on upload.", - "title": "Tags" - } - }, - "title": "WandbConfig", - "type": "object" - }, - "ZipSweep": { - "additionalProperties": false, - "description": "Zip sweep — parameters paired element-wise (lockstep).\n\nLike ``GridSweep`` but uses ``zip(strict=True)`` instead of Cartesian\nproduct. All parameter lists must have identical length. Use when you\nwant N runs each setting a coordinated tuple of fields, without the\nNxM blow-up of grid. Canonical use case: paired ISL/OSL.", - "properties": { - "cooldownSeconds": { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "title": "Cooldownseconds", - "oneOf": [ - { - "default": 0.0, - "description": "Cooldown between sweep variations (or BO iterations).", - "minimum": 0, - "title": "Cooldownseconds", - "type": "number" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "x-jinja2-supported": true - }, - "slaFilters": { - "description": "SLA constraints applied at trial scoring / aggregation time.", - "items": { - "$ref": "#/$defs/SLAFilter" - }, - "title": "Slafilters", - "type": "array" - }, - "postProcess": { - "anyOf": [ - { - "$ref": "#/$defs/PostProcessSpec" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional post-aggregation handler emitted by a search recipe." - }, - "recipeName": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.", - "title": "Recipename" - }, - "iterationOrder": { - "$ref": "#/$defs/SweepMode", - "default": "repeated", - "description": "Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search." - }, - "sameSeed": { - "default": false, - "description": "If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.", - "title": "Sameseed", - "type": "boolean" - }, - "type": { - "const": "zip", - "default": "zip", - "description": "Sweep type discriminator.", - "title": "Type", - "type": "string" - }, - "parameters": { - "additionalProperties": { - "items": {}, - "type": "array" - }, - "description": "Parameters to sweep in lockstep: dot-notation path -> list of values. All lists must have equal length.", - "minProperties": 1, - "title": "Parameters", - "type": "object" - } - }, - "required": [ - "parameters" - ], - "title": "ZipSweep", - "type": "object" - }, - "AdaptiveScaleNestedBlock": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "enum": [ - "true", - "false", - "yes", - "no", - "on", - "off", - "1", - "0" - ] - } - ] - }, - "control": { - "type": "object", - "additionalProperties": false, - "properties": { - "variable": { - "type": "string", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ] - }, - "min": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "max": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - } - } - }, - "strategy": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "ramp_until_fail" - ] - }, - "stepPolicy": { - "type": "string", - "enum": [ - "sla_margin", - "fixed_percent_step" - ] - }, - "step_policy": { - "type": "string", - "enum": [ - "sla_margin", - "fixed_percent_step" - ] - }, - "baseStep": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "base_step": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "maxStepMultiplier": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "max_step_multiplier": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "stepPercent": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "step_percent": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - } - } - }, - "sla": { - "anyOf": [ - { - "type": "array", - "items": { - "$ref": "#/$defs/SLAFilter" - } - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "controlVariable": { - "type": "string", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ] - }, - "control_variable": { - "type": "string", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ] - }, - "controlMin": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "control_min": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "controlMax": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "control_max": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "minConcurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "min_concurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "maxConcurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "max_concurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "window": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "number", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "assessmentPeriod": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "number", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "assessment_period": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "number", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "minCompletedRequests": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "min_completed_requests": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "sustainDuration": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "sustain_duration": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - } - } - }, - "DisabledAdaptiveScaleNestedBlock": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "anyOf": [ - { - "const": false - }, - { - "type": "string", - "enum": [ - "false", - "no", - "off", - "0" - ] - } - ] - }, - "control": { - "type": "object", - "additionalProperties": false, - "properties": { - "variable": { - "type": "string", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ] - }, - "min": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "max": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - } - } - }, - "strategy": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "ramp_until_fail" - ] - }, - "stepPolicy": { - "type": "string", - "enum": [ - "sla_margin", - "fixed_percent_step" - ] - }, - "step_policy": { - "type": "string", - "enum": [ - "sla_margin", - "fixed_percent_step" - ] - }, - "baseStep": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "base_step": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "maxStepMultiplier": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "max_step_multiplier": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "stepPercent": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "step_percent": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - } - } - }, - "sla": { - "anyOf": [ - { - "type": "array", - "items": { - "$ref": "#/$defs/SLAFilter" - } - }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "type": [ - "number", - "string" - ] - } - } - }, - "description": "Compact SLA mapping: metric -> stat -> operator -> threshold." - } - ] - }, - "controlVariable": { - "type": "string", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ] - }, - "control_variable": { - "type": "string", - "enum": [ - "concurrency", - "prefill_concurrency", - "request_rate", - "users" - ] - }, - "controlMin": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "control_min": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "controlMax": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "control_max": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "minConcurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "min_concurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "maxConcurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "max_concurrency": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "window": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "number", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "assessmentPeriod": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "number", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "assessment_period": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "number", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "minCompletedRequests": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "min_completed_requests": { - "oneOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "sustainDuration": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - }, - "sustain_duration": { - "anyOf": [ - { - "type": "integer", - "exclusiveMinimum": 0 - }, - { - "type": "number", - "exclusiveMinimum": 0 - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ] - } - }, - "required": [ - "enabled" - ] - } - }, - "additionalProperties": false, - "description": "JSON Schema for AIPerf YAML configuration files. This schema enables IDE autocompletion and validation for AIPerf benchmark configurations. Schema version: 2.0.0", - "properties": { - "schemaVersion": { - "const": "2.0", - "default": "2.0", - "description": "AIPerf config schema version. Pinned to '2.0' for the envelope/body shape (envelope keys at top level, swept body under ``benchmark``).", - "title": "Schemaversion", - "type": "string" - }, - "benchmark": { - "$ref": "#/$defs/BenchmarkConfig", - "description": "Benchmark workload (the swept body)." - }, - "sweep": { - "anyOf": [ - { - "discriminator": { - "propertyName": "type", - "mapping": { - "grid": "#/$defs/GridSweep", - "zip": "#/$defs/ZipSweep", - "scenarios": "#/$defs/ScenarioSweep", - "adaptive_search": "#/$defs/AdaptiveSearchSweep", - "sobol": "#/$defs/SobolSweep", - "latin_hypercube": "#/$defs/LatinHypercubeSweep" - } - }, - "anyOf": [ - { - "$ref": "#/$defs/GridSweep" - }, - { - "$ref": "#/$defs/ZipSweep" - }, - { - "$ref": "#/$defs/ScenarioSweep" - }, - { - "$ref": "#/$defs/AdaptiveSearchSweep" - }, - { - "$ref": "#/$defs/SobolSweep" - }, - { - "$ref": "#/$defs/LatinHypercubeSweep" - } - ] - }, - { - "type": "null" - } - ], - "default": null, - "description": "Sweep configuration for parameter exploration. Supports grid (Cartesian product), zip (lock-stepped axes), scenarios (hand-picked), sobol / latin_hypercube (QMC sampling), and adaptive_search (Bayesian / monotonic outer-loop) strategies.", - "title": "Sweep" - }, - "multiRun": { - "$ref": "#/$defs/MultiRunConfig", - "description": "Multi-run benchmarking configuration for statistical reporting. When num_runs > 1, executes multiple runs and computes aggregate statistics." - }, - "plot": { - "default": null, - "description": "Plot configuration for this run/sweep. When set, ``~/.aiperf/plot_config.yaml`` is ignored. May be a bare-string path to a plot YAML (resolved relative to the AIPerf config file's directory, or absolute), or an inline mapping mirroring ``default_plot_config.yaml``. Presence implies ``artifacts.auto_plot=True`` unless explicitly set False.", - "title": "Plot" - }, - "variables": { - "additionalProperties": true, - "description": "User-defined values exposed to Jinja2 in `{{ ... }}` expressions during config load. Cross-variation: scenario `runs[i].variables:` deep-merge over this base. Preserved on the resolved config so run-time renderers can resolve them again.", - "title": "Variables", - "type": "object" - }, - "randomSeed": { - "anyOf": [ - { - "minimum": 0, - "type": "integer" - }, - { - "type": "null" - }, - { - "type": "string", - "pattern": ".*\\{\\{.*\\}\\}.*", - "description": "Jinja2 template (e.g., '{{ variable }}')." - }, - { - "type": "string", - "pattern": ".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*", - "description": "Environment variable (e.g., '${VAR}' or '${VAR:default}')." - } - ], - "default": null, - "description": "Global random seed for reproducibility. Base seed for per-variation derivation in sweep mode (variation N gets base + N). Must be non-negative; numpy and scipy reject negative seeds.", - "title": "Randomseed", - "x-jinja2-supported": true - }, - "noSweepTable": { - "default": false, - "description": "Suppress the per-cell streaming sweep table during multi-variation sweeps. Auto-suppressed when stdout is non-interactive, when the dashboard UI is active, or for single-cell sweeps.", - "title": "Nosweeptable", - "type": "boolean" - } - }, - "required": [ - "benchmark" - ], - "title": "AIPerf Configuration Schema", - "type": "object", - "$comment": "Auto-generated by tools/generate_config_schema.py. Do not edit manually.", - "x-taplo": { - "initKeys": [ - "benchmark", - "sweep", - "multiRun", - "variables", - "randomSeed" - ] - } -} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://nvidia.github.io/aiperf/schemas/aiperf-config.schema.json","$defs":{"AIPerfLogLevel":{"description":"Logging levels for AIPerf output verbosity.","enum":["TRACE","DEBUG","INFO","NOTICE","WARNING","SUCCESS","ERROR","CRITICAL"],"title":"AIPerfLogLevel","type":"string"},"AccuracyBenchmarkType":{"enum":["mmlu","mmlu_pro","aime","hellaswag","bigbench","aime24","aime25","math_500","gsm8k","gpqa_diamond","lcb_codegeneration"],"title":"AccuracyBenchmarkType","type":"string"},"AccuracyConfig":{"additionalProperties":false,"description":"Configuration for accuracy benchmarking mode.\n\nWhen benchmark is set, enables accuracy evaluation alongside\nperformance profiling using standard benchmarks (MMLU, AIME, etc.).","properties":{"benchmark":{"anyOf":[{"$ref":"#/$defs/AccuracyBenchmarkType"},{"type":"null"}],"default":null,"description":"Accuracy benchmark to run. When set, enables accuracy benchmarking alongside performance profiling. AIME variants: 'aime' is the legacy combined set (deprecated for new runs); prefer the year-pinned 'aime24' or 'aime25' for reproducibility."},"tasks":{"anyOf":[{"type":"string"},{"items":{"type":"string"},"type":"array"},{"type":"null"}],"default":null,"description":"Specific tasks or subtasks within the benchmark to evaluate (e.g., specific MMLU subjects). Accepts comma-separated values (e.g. abstract_algebra,anatomy) or repeated flags. If not set, all tasks are included.","title":"Tasks"},"nShots":{"anyOf":[{"maximum":32,"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Number of few-shot examples to include in the prompt. 0 means zero-shot evaluation, None uses the benchmark default (e.g. MMLU=5). Maximum 32.","title":"Nshots","x-jinja2-supported":true},"enableCot":{"anyOf":[{"type":"boolean"},{"type":"null"}],"default":null,"description":"Enable chain-of-thought prompting for accuracy evaluation. Adds reasoning instructions to the prompt. Defaults to the benchmark's ``default_enable_cot`` metadata when unset (e.g. AIME defaults to True).","title":"Enablecot"},"grader":{"anyOf":[{"$ref":"#/$defs/AccuracyGraderType"},{"type":"null"}],"default":null,"description":"Override the default grader for the selected benchmark (e.g., exact_match, math, multiple_choice, code_execution). If not set, uses the benchmark's default grader."},"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Custom system prompt to use for accuracy evaluation. Overrides any benchmark-specific system prompt.","title":"Systemprompt"},"verbose":{"default":false,"description":"Enable verbose output for accuracy evaluation, showing per-problem grading details.","title":"Verbose","type":"boolean"}},"title":"AccuracyConfig","type":"object"},"AccuracyGraderType":{"enum":["exact_match","math","multiple_choice","code_execution","lighteval_expr","lighteval_latex","lighteval_gpqa","lighteval_gsm8k","mmlu_pro"],"title":"AccuracyGraderType","type":"string"},"AdaptiveSearchSweep":{"additionalProperties":false,"description":"Adaptive outer-loop search (Bayesian / monotonic / etc.).\n\nA third sweep variant: BO drives variation choice itself, so the\ngrid-style ``iteration_order`` and ``same_seed`` knobs do not apply.\nInherits cooldown/sla_filters/post_process from `_SweepBase` only.","properties":{"cooldownSeconds":{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaFilters":{"description":"SLA constraints applied at trial scoring / aggregation time.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Slafilters","type":"array"},"postProcess":{"anyOf":[{"$ref":"#/$defs/PostProcessSpec"},{"type":"null"}],"default":null,"description":"Optional post-aggregation handler emitted by a search recipe."},"recipeName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.","title":"Recipename"},"type":{"const":"adaptive_search","default":"adaptive_search","description":"Sweep type discriminator.","title":"Type","type":"string"},"planner":{"$ref":"#/$defs/SearchPlannerType","default":"bayesian","description":"Outer-loop planner plugin name (bayesian | optuna | monotonic_sla | smooth_isotonic)."},"searchSpace":{"description":"Dimensions to optimize over.","items":{"$ref":"#/$defs/SearchSpaceDimension"},"minItems":1,"title":"Searchspace","type":"array"},"objectives":{"description":"What to optimize. Length-1 = single-objective search; length-N = multi-objective search. BoTorch-backed Pareto BO requires --optuna-sampler botorch with a multi-objective acquisition like qlognehvi.","items":{"$ref":"#/$defs/Objective"},"minItems":1,"title":"Objectives","type":"array"},"outcomeConstraints":{"description":"Feasibility gates on non-optimized metrics. Empty = no constraints.","items":{"$ref":"#/$defs/OutcomeConstraint"},"title":"Outcomeconstraints","type":"array"},"maxIterations":{"description":"Maximum BO iterations.","title":"Maxiterations","oneOf":[{"description":"Maximum BO iterations.","maximum":200,"minimum":2,"title":"Maxiterations","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"nInitialPoints":{"default":5,"description":"Sobol-random points before the GP fits. Must be < max_iterations for `bayesian`/`optuna` planners; ignored by 1-D SLA planners.","title":"Ninitialpoints","oneOf":[{"default":5,"description":"Sobol-random points before the GP fits. Must be < max_iterations for `bayesian`/`optuna` planners; ignored by 1-D SLA planners.","minimum":1,"title":"Ninitialpoints","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"plateauWindow":{"default":8,"description":"Recent-iteration window for plateau detection. Default 8 keeps the window strictly larger than ``n_initial_points`` (default 5), so the plateau test always inspects at least a few GP-driven iterations rather than tripping on the Sobol-only prefix when the initial points happen to land in a flat region.","title":"Plateauwindow","oneOf":[{"default":8,"description":"Recent-iteration window for plateau detection. Default 8 keeps the window strictly larger than ``n_initial_points`` (default 5), so the plateau test always inspects at least a few GP-driven iterations rather than tripping on the Sobol-only prefix when the initial points happen to land in a flat region.","minimum":2,"title":"Plateauwindow","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"plateauThreshold":{"default":0.01,"description":"Coefficient-of-variation threshold for plateau stop.","title":"Plateauthreshold","oneOf":[{"default":0.01,"description":"Coefficient-of-variation threshold for plateau stop.","exclusiveMinimum":0,"title":"Plateauthreshold","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"improvementPatience":{"default":10,"description":"Stop after this many no-improvement iterations.","title":"Improvementpatience","oneOf":[{"default":10,"description":"Stop after this many no-improvement iterations.","minimum":2,"title":"Improvementpatience","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"randomSeed":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"If set, passed as `random_state` to the planner backend.","title":"Randomseed","x-jinja2-supported":true},"optunaSampler":{"default":"botorch","description":"Optuna sampler when planner=optuna. Default ``botorch`` is the preferred Gaussian-process qLogNEI/qLogNEHVI path and requires the optional ``botorch`` extra; when this default is implicit and the optional stack is unavailable, the planner warns and falls back to ``tpe``. ``tpe`` is the dep-light Tree-Parzen estimator that ships with Optuna core. ``gp`` is Optuna's pure-GP sampler and requires torch.","enum":["gp","tpe","botorch"],"title":"Optunasampler","type":"string"},"optunaAcquisition":{"anyOf":[{"enum":["logei","qlogei","qnei","qlognei","qehvi","qnehvi","qlognehvi"],"type":"string"},{"type":"null"}],"default":null,"description":"Acquisition override for the BoTorch sampler. None lets Optuna pick (single-objective LogEI). ``qnei`` = Letham 2017 noisy-EI; ``qlognei`` = Ament 2023 (BoTorch's strongly recommended modern default; requires botorch>=0.10). ``qehvi``/``qnehvi``/``qlognehvi`` are multi-objective hypervolume acquisitions for Pareto BO. Only meaningful when planner=optuna and optuna_sampler=botorch; ignored otherwise.","title":"Optunaacquisition"},"optunaTerminator":{"default":"none","description":"Optional posterior-regret stopping rule (Makarova 2022 / Ishibashi 2023; same family as Wilson 2024). Layered on top of three-signal convergence. Only meaningful when planner=optuna; ignored otherwise.","enum":["regret","emmr","none"],"title":"Optunaterminator","type":"string"},"objectivePooling":{"default":"mean","description":"Per-trial percentile aggregation. ``mean`` = arithmetic mean of per-trial percentiles across trials (current default). ``pooled`` = walks each trial's profile_export.jsonl and computes np.percentile over the pooled raw-sample bag (the statistic that satisfies SLO claims; requires --export-level records). Only meaningful when objective.stat is a percentile (p50/p90/p95/p99); a no-op when stat=avg.","enum":["mean","pooled"],"title":"Objectivepooling","type":"string"},"monotonicStabilityTrials":{"default":2,"description":"MonotonicSLA planner stability-window size.","title":"Monotonicstabilitytrials","oneOf":[{"default":2,"description":"MonotonicSLA planner stability-window size.","minimum":1,"title":"Monotonicstabilitytrials","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaReplicates":{"default":0,"description":"Replicate-step trial count override for the smooth_isotonic SLA planner. 0 = auto (Hyperband-flavored formula based on sigma_margin/threshold). >0 = fixed override. Ignored by other planners.","title":"Slareplicates","oneOf":[{"default":0,"description":"Replicate-step trial count override for the smooth_isotonic SLA planner. 0 = auto (Hyperband-flavored formula based on sigma_margin/threshold). >0 = fixed override. Ignored by other planners.","minimum":0,"title":"Slareplicates","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaPrecision":{"default":"normal","description":"Per-probe sample-budget knob for the smooth_isotonic SLA planner. tight=10000, normal=1000, coarse=300 requests. Drives p99 CI width. Ignored by other planners.","enum":["tight","normal","coarse"],"title":"Slaprecision","type":"string"},"slaWarmupSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Discard this many seconds at the start of each probe before computing margins, to skip CUDA-graph warmup and KV-cache shape transients. None = auto (max(30s, 3 * inter-batch-time)). Ignored by other planners.","title":"Slawarmupseconds","x-jinja2-supported":true},"constraintMode":{"default":"penalty","description":"Constrained-BO acquisition mode (penalty | eic).","enum":["penalty","eic"],"title":"Constraintmode","type":"string"},"slaTiers":{"description":"Multi-tier SLO definitions parsed from --search-sla-tier flags. Each tier is a named group of SLA filters sharing a common boundary search. When non-empty (2-10 tiers), the MultiTierPlanner is activated instead of single-tier planners.","items":{"$ref":"#/$defs/SLOTier"},"title":"Slatiers","type":"array"}},"required":["searchSpace","objectives","maxIterations"],"title":"AdaptiveSearchSweep","type":"object"},"ArtifactsConfig":{"additionalProperties":false,"description":"Artifacts configuration for benchmark output.\n\nControls where and how benchmark results are exported.\nUses flat structure with format arrays instead of nested export configs.","properties":{"dir":{"default":"artifacts","description":"Output directory for all benchmark artifacts. Created if it doesn't exist.","format":"path","title":"Dir","type":"string"},"prefix":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Base filename override applied to ALL profile and server-metrics exports. With prefix='foo' every output becomes `foo.csv`, `foo.json`, `foo_timeslices.{csv,json}`, `foo.jsonl`, `foo_raw.jsonl`, `foo_outputs.json`, `foo_gpu_telemetry.jsonl`, `foo_server_metrics.{jsonl,json,csv,parquet}`. When unset (the default), historical per-file names are used: `profile_export_aiperf.csv/json`, `profile_export.jsonl`, `profile_export_raw.jsonl`, `outputs.json`, `gpu_telemetry_export.jsonl`, `server_metrics_export.{jsonl,json,csv,parquet}`. Known suffixes (`_raw.jsonl`, `_outputs.json`, `_timeslices.{csv,json}`, `_gpu_telemetry.jsonl`, `_server_metrics.{jsonl,json,csv,parquet}`, `.csv`/`.json`/`.jsonl`/`.parquet`) are stripped from the supplied value so `--profile-export-prefix foo_raw.jsonl` still yields a clean `foo` base.","title":"Prefix"},"summary":{"anyOf":[{"items":{"const":"json","type":"string"},"type":"array"},{"const":false,"type":"boolean"}],"description":"Summary export formats. Only 'json' is wired up to this field; the CSV summary is emitted regardless. Set to false to disable the summary JSON file only.","title":"Summary"},"records":{"anyOf":[{"items":{"const":"jsonl","type":"string"},"type":"array"},{"const":false,"type":"boolean"}],"description":"Per-request records export formats. Only 'jsonl' is wired up today. Set to false to disable the per-record JSONL file.","title":"Records"},"raw":{"default":false,"description":"Export raw request/response payloads as JSONL.","title":"Raw","type":"boolean"},"trace":{"default":false,"description":"Export HTTP trace data for debugging.","title":"Trace","type":"boolean"},"sliceDuration":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Time slice duration in seconds for trend analysis (must be > 0). Divides benchmark into windows for per-window statistics. Supports: 30, '30s', '5m', '2h'.","title":"Sliceduration","x-jinja2-supported":true},"showTraceTiming":{"default":false,"description":"Display HTTP trace timing metrics in console output. Shows detailed timing breakdown: blocked, DNS, connecting, sending, waiting (TTFB), receiving, and total duration.","title":"Showtracetiming","type":"boolean"},"userFiles":{"description":"User-defined templated files materialized into the run directory before the benchmark begins.","items":{"$ref":"#/$defs/UserFile"},"title":"Userfiles","type":"array"},"autoPlot":{"anyOf":[{"type":"boolean"},{"type":"null"}],"default":null,"description":"Auto-invoke `aiperf plot` against the artifact directory after the benchmark completes. Unset (the default) means a `plot:` section implies True; set False to suppress plots even with one, True to force them. `AIPerfConfig` resolves this to a concrete bool, so any config that came through the loader carries a plain bool.","title":"Autoplot"},"plotRequired":{"default":false,"description":"Treat auto-plot failures as fatal: re-raise so `aiperf profile` exits non-zero. Only meaningful when auto_plot is True. Default False = warn and continue.","title":"Plotrequired","type":"boolean"},"exportOutputsJson":{"anyOf":[{"type":"boolean"},{"type":"null"}],"default":null,"description":"Export generated response text after the run, to `outputs.json` or `_outputs.json` when `prefix` is set. Unset (the default) means the `raw` export level implies it; set False to opt out at raw, True to force it at any level. Always a concrete bool once validation has run.","title":"Exportoutputsjson"}},"title":"ArtifactsConfig","type":"object"},"AudioConfig":{"additionalProperties":false,"description":"Configuration for synthetic audio generation in multimodal datasets.\n\nControls the generation of synthetic audio for speech-to-text\nand audio-language model benchmarking. Generated audio is random\nnoise with specified sample rate, bit depth, and format.","properties":{"batchSize":{"default":0,"description":"Number of audio inputs to include in each multimodal request. Supported with chat endpoint type for multimodal models. Set to 0 to disable audio inputs.","title":"Batchsize","oneOf":[{"default":0,"description":"Number of audio inputs to include in each multimodal request. Supported with chat endpoint type for multimodal models. Set to 0 to disable audio inputs.","minimum":0,"title":"Batchsize","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"length":{"description":"Audio duration in seconds. Can be a fixed value or {mean, stddev} distribution. Used when batch_size > 0 for multimodal benchmarking.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Length","x-jinja2-supported":true},"format":{"$ref":"#/$defs/AudioFormat","default":"wav","description":"File format for generated audio files. wav: uncompressed PCM (larger files). mp3: compressed (smaller files). Format affects file size in multimodal requests but not audio characteristics."},"sampleRates":{"description":"List of audio sample rates in kHz to randomly select from. Common values: 8.0 (telephony), 16.0 (speech), 44.1 (CD quality), 48.0 (professional). Specify multiple values for mixed-quality testing.","items":{"type":"number"},"title":"Samplerates","type":"array"},"depths":{"description":"List of audio bit depths in bits to randomly select from. Each audio file is assigned a random depth from this list. Common values: 8 (low quality), 16 (CD quality), 24 (professional), 32 (high-end). Specify multiple values for mixed-quality testing.","items":{"type":"integer"},"title":"Depths","type":"array"},"channels":{"default":1,"description":"Number of audio channels. 1 = mono (single channel), 2 = stereo (left/right channels). Stereo doubles file size. Most speech models use mono.","title":"Channels","oneOf":[{"default":1,"description":"Number of audio channels. 1 = mono (single channel), 2 = stereo (left/right channels). Stereo doubles file size. Most speech models use mono.","maximum":2,"minimum":1,"title":"Channels","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"title":"AudioConfig","type":"object"},"AudioFormat":{"description":"Audio file formats for synthetic audio generation.","enum":["wav","mp3"],"title":"AudioFormat","type":"string"},"BenchmarkConfig":{"additionalProperties":false,"description":"Pure runtime configuration - what SystemController and services need.\n\nContains all fields required to execute a single benchmark run.\nDoes NOT include sweep, multi_run, variables, or random_seed settings\n(those live on AIPerfConfig as envelope-level cross-variation fields).\n\nRequired Sections:\n - models: Model(s) to benchmark\n - endpoint: Server connection settings\n - datasets: Single-element list of dataset configuration\n - phases: Ordered list of benchmark phases. A single-dict under\n ``phases:`` is rejected; use the envelope-level ``warmup:`` /\n ``profiling:`` shorthand or an explicit list.\n\nOptional Sections:\n - artifacts: Export and console settings\n - slos: SLO-based quality metrics (generic dict)\n - tokenizer: Token counting configuration\n - gpu_telemetry: GPU metrics from DCGM endpoints\n - server_metrics: Server metrics from Prometheus endpoints\n - runtime: Worker and communication settings\n - logging: Logging and debug settings\n - metrics: Metrics aggregation configuration\n - accuracy: Accuracy evaluation configuration\n\nValidation:\n The configuration is validated in several stages:\n 1. Individual field validation (types, ranges, formats)\n 2. Cross-field validation (mutual exclusivity, dependencies)\n\nEnvironment Variables:\n Values can reference environment variables using ${VAR} syntax.\n Optional defaults: ${VAR:default_value}\n\n Example:\n api_key: ${OPENAI_API_KEY}\n api_key: ${OPENAI_API_KEY:sk-default}","properties":{"models":{"description":"Model configuration. Accepts a single model name string, a list of model names, or an advanced configuration with strategy and weighted items. All forms are normalized to ModelsAdvanced. Accepts a model name string, list of model names, or full ModelsAdvanced config.","oneOf":[{"type":"string","description":"Single model name (normalized to ModelsAdvanced)."},{"type":"array","items":{"type":"string"},"minItems":1,"description":"List of model names (normalized to ModelsAdvanced)."},{"$ref":"#/$defs/ModelsAdvanced"}]},"endpoint":{"$ref":"#/$defs/EndpointConfig","description":"Endpoint configuration for connecting to inference server(s). Includes URLs, API type, authentication, timeout, and connection settings."},"datasets":{"description":"Dataset configuration as a single-element list. The list shape exists to share the schema between YAML and the AIPerfSweep CRD; the runtime currently loads exactly one dataset. Singular `dataset:` shorthand at the BenchmarkConfig top level is normalized to a one-entry list with name='default'.","items":{"discriminator":{"propertyName":"type","mapping":{"synthetic":"#/$defs/SyntheticDataset","file":"#/$defs/FileDataset","public":"#/$defs/PublicDataset"}},"anyOf":[{"$ref":"#/$defs/SyntheticDataset"},{"$ref":"#/$defs/FileDataset"},{"$ref":"#/$defs/PublicDataset"}]},"maxItems":1,"minItems":1,"title":"Datasets","type":"array"},"phases":{"description":"Ordered benchmark phases. Each entry must have a unique 'name' (e.g. 'warmup', 'profiling'). Order in the list IS the execution order; the first phase runs first. Single-config shorthand ({'type': 'concurrency', ...}) is normalized to a list of one. Top-level 'warmup:'/'profiling:' shorthand is normalized to a [warmup, profiling] list pre-validation. Shorthand: use a single phase config directly (with 'type' key) and it is normalized to a one-entry phases list.","oneOf":[{"description":"Ordered benchmark phases. Each entry must have a unique 'name' (e.g. 'warmup', 'profiling'). Order in the list IS the execution order; the first phase runs first. Single-config shorthand ({'type': 'concurrency', ...}) is normalized to a list of one. Top-level 'warmup:'/'profiling:' shorthand is normalized to a [warmup, profiling] list pre-validation.","items":{"discriminator":{"propertyName":"type","mapping":{"concurrency":"#/$defs/ConcurrencyPhase","poisson":"#/$defs/PoissonPhase","gamma":"#/$defs/GammaPhase","constant":"#/$defs/ConstantPhase","user_centric":"#/$defs/UserCentricPhase","fixed_schedule":"#/$defs/FixedSchedulePhase"}},"anyOf":[{"$ref":"#/$defs/ConcurrencyPhase"},{"$ref":"#/$defs/PoissonPhase"},{"$ref":"#/$defs/GammaPhase"},{"$ref":"#/$defs/ConstantPhase"},{"$ref":"#/$defs/UserCentricPhase"},{"$ref":"#/$defs/FixedSchedulePhase"}]},"minItems":1,"title":"Phases","type":"array"},{"anyOf":[{"additionalProperties":false,"description":"Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"concurrency","description":"Concurrency-controlled immediate dispatch.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"default":1,"description":"Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.","minimum":1,"title":"Concurrency","type":"integer"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"ConcurrencyPhase","type":"object"},{"additionalProperties":false,"description":"Poisson-distributed request arrivals at the target rate.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"poisson","description":"Poisson-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"PoissonPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"Gamma-distributed request arrivals with configurable smoothness.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"gamma","description":"Gamma-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"smoothness":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.","title":"Smoothness"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"GammaPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"Constant-rate request arrivals (fixed inter-arrival time).","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"constant","description":"Constant rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"ConstantPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"user_centric","description":"N users sharing a global request rate.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"users":{"description":"Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.","minimum":1,"title":"Users","type":"integer"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type","users","rate"],"title":"UserCentricPhase","type":"object","allOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}}]},{"additionalProperties":false,"description":"Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"fixed_schedule","description":"Replay requests at trace timestamps.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"autoOffset":{"default":true,"description":"Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.","title":"Autooffset","type":"boolean"},"startOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Filter out trace requests before this timestamp in ms (must be >= 0).","title":"Startoffset"},"endOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Filter out trace requests after this timestamp in ms (must be >= 0).","title":"Endoffset"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"FixedSchedulePhase","type":"object"}],"description":"Single phase config (normalized to a one-entry phases list).","discriminator":{"propertyName":"type","mapping":{"concurrency":"#/$defs/ConcurrencyPhase","poisson":"#/$defs/PoissonPhase","gamma":"#/$defs/GammaPhase","constant":"#/$defs/ConstantPhase","user_centric":"#/$defs/UserCentricPhase","fixed_schedule":"#/$defs/FixedSchedulePhase"}}}]},"artifacts":{"$ref":"#/$defs/ArtifactsConfig","description":"Artifacts configuration for benchmark output. Controls output directory and export formats."},"slos":{"anyOf":[{"additionalProperties":{"oneOf":[{"type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"type":"object"},{"type":"null"}],"default":null,"description":"SLO (Service Level Objectives) configuration as a generic dict. Maps metric names to threshold values. A request is counted as good only if it meets ALL specified thresholds.","title":"Slos","x-jinja2-supported":true},"tokenizer":{"anyOf":[{"$ref":"#/$defs/TokenizerConfig"},{"type":"null"}],"default":null,"description":"Tokenizer configuration for token counting. Used for ISL/OSL enforcement and accurate metrics. If not specified, uses the first model name."},"gpuTelemetry":{"$ref":"#/$defs/GpuTelemetryConfig","description":"GPU telemetry configuration for DCGM metrics collection. Collects GPU metrics (power, utilization, temperature) from DCGM endpoints. Enabled by default. Set enabled: false to disable."},"serverMetrics":{"$ref":"#/$defs/ServerMetricsConfig","description":"Server metrics configuration for Prometheus scraping. Collects operational metrics (queue depth, KV cache, batch sizes) from inference server Prometheus endpoints. Enabled by default. Set enabled: false to disable."},"networkLatency":{"$ref":"#/$defs/NetworkLatencyConfig","description":"Network latency calibration configuration. Probes the endpoint RTT during profiling and subtracts the mean from latency metrics, emitted as separate network_adjusted_* metrics. Disabled by default."},"otel":{"$ref":"#/$defs/OTelConfig"},"mlflow":{"$ref":"#/$defs/MLflowConfig"},"wandb":{"$ref":"#/$defs/WandbConfig"},"runtime":{"$ref":"#/$defs/RuntimeConfig","description":"Runtime configuration for worker processes and inter-process communication."},"logging":{"$ref":"#/$defs/LoggingConfig"},"metrics":{"$ref":"#/$defs/MetricsConfig","description":"Metrics aggregation configuration for benchmark summaries."},"accuracy":{"anyOf":[{"$ref":"#/$defs/AccuracyConfig"},{"type":"null"}],"default":null,"description":"Accuracy benchmarking configuration. When set, enables accuracy evaluation alongside performance profiling."},"scenario":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Lock all benchmark invariants for a named scenario (e.g. 'inferencex-agentx-mvp'). Plain data here; the lock is applied by the ScenarioResolver step in the pre-bootstrap resolver chain (auto-fills defaults, validates, raises ScenarioLockError on conflict). Distinct from the sweep ``scenarios`` strategy (SweepConfig), which expands hand-picked named runs -- this field is a single invariant LOCK, not a sweep.","title":"Scenario"},"unsafeOverride":{"default":false,"description":"Convert scenario lock errors to warnings; stamps submission_valid=false in the resolved scenario outcome. No-op without ``scenario``. Plain data; consumed by the ScenarioResolver.","title":"Unsafeoverride","type":"boolean"},"model":{"description":"Shorthand for 'models'. Accepts a single value which will be converted to a list/dict. Model configuration. Accepts a single model name string, a list of model names, or an advanced configuration with strategy and weighted items. All forms are normalized to ModelsAdvanced.","oneOf":[{"type":"string"},{"type":"array","items":{"type":"string"}},{"$ref":"#/$defs/ModelsAdvanced","description":"Model configuration. Accepts a single model name string, a list of model names, or an advanced configuration with strategy and weighted items. All forms are normalized to ModelsAdvanced.","x-kubernetes-preserve-unknown-fields":true}],"x-singular-alias-of":"models"},"dataset":{"anyOf":[{"additionalProperties":false,"description":"Synthetic dataset configuration.\n\nGenerates prompts programmatically based on token length\nspecifications. Ideal for controlled experiments.","properties":{"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Verbatim system prompt text, identical across every conversation. Emitted as a system-role message ahead of all turns. When the dataset already carries its own system message, this text is prepended to it rather than replacing it. Tokens are additive: --isl continues to size the generated user prompt only. Mutually exclusive with system_prompt_file and with prefix_prompts.shared_system_length/pool_size/length.","title":"Systemprompt"},"systemPromptFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to a UTF-8 text file holding the verbatim system prompt. Preferred over system_prompt for real production prompts, which are long enough that shell quoting mangles them. Read once at config-validation time so a missing or unreadable file fails at startup. Mutually exclusive with system_prompt.","title":"Systempromptfile"},"name":{"description":"Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.","minLength":1,"title":"Name","type":"string"},"type":{"const":"synthetic","description":"Dataset type discriminator. Must be 'synthetic'.","title":"Type","type":"string"},"entries":{"default":100,"description":"Total number of unique entries to generate for the dataset. Each entry represents a unique prompt with sampled ISL/OSL. Entries are reused across conversations and turns according to the sampling strategy. Higher values provide more diversity.","minimum":1,"title":"Entries","type":"integer"},"randomSeed":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":null,"description":"Random seed for deterministic dataset generation. When set, makes synthetic prompts, sampling, and other random operations reproducible across runs. Essential for A/B testing and debugging. Overrides global random_seed for this dataset.","title":"Randomseed"},"sampling":{"$ref":"#/$defs/DatasetSamplingStrategy","default":"sequential","description":"Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion."},"prompts":{"anyOf":[{"$ref":"#/$defs/PromptConfig"},{"type":"null"}],"default":null,"description":"Prompt/token length configuration specifying ISL, OSL, sequence distributions, and batch processing settings."},"prefixPrompts":{"anyOf":[{"$ref":"#/$defs/PrefixPromptConfig"},{"type":"null"}],"default":null,"description":"Shared prefix configuration for KV cache testing. Generates prefix prompts that are prepended to user prompts, simulating cached context scenarios."},"turns":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Number of request-response turns per conversation. Can be a fixed integer or {mean, stddev} distribution. Each turn consists of a user message and model response. Set to 1 for single-turn interactions. Multi-turn conversations enable testing of context retention and conversation history handling.","title":"Turns"},"turnDelay":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Delay in milliseconds between consecutive turns within a multi-turn conversation. Can be a fixed value or {mean, stddev} distribution. Simulates user think time between receiving a response and sending the next message. Only applies when turns > 1. Set to 0 for back-to-back turns.","title":"Turndelay"},"turnDelayRatio":{"default":1.0,"description":"Multiplier for scaling all turn delays. Applied after mean/stddev calculation: actual_delay = calculated_delay * ratio. Values < 1 speed up conversations, > 1 slow them down. Set to 0 to eliminate delays entirely.","minimum":0.0,"title":"Turndelayratio","type":"number"},"images":{"anyOf":[{"$ref":"#/$defs/ImageConfig"},{"type":"null"}],"default":null,"description":"Synthetic image configuration for multimodal vision-language testing."},"audio":{"anyOf":[{"$ref":"#/$defs/AudioConfig"},{"type":"null"}],"default":null,"description":"Synthetic audio configuration for multimodal speech/audio testing."},"video":{"anyOf":[{"$ref":"#/$defs/VideoConfig"},{"type":"null"}],"default":null,"description":"Synthetic video configuration for multimodal video understanding testing."},"rankings":{"anyOf":[{"$ref":"#/$defs/RankingsConfig"},{"type":"null"}],"default":null,"description":"Rankings/reranking configuration for generating query-passage pairs. Only relevant for rankings endpoint types."},"isl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Shorthand for prompts.isl; hoisted into prompts before validation.","title":"Isl"},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Shorthand for prompts.osl; hoisted into prompts before validation.","title":"Osl"}},"required":["type"],"title":"SyntheticDataset","type":"object"},{"additionalProperties":false,"description":"File-based dataset configuration.\n\nLoads prompts from a local file in various formats.\nSupports trace replay and custom sampling strategies.","properties":{"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Verbatim system prompt text, identical across every conversation. Emitted as a system-role message ahead of all turns. When the dataset already carries its own system message, this text is prepended to it rather than replacing it. Tokens are additive: --isl continues to size the generated user prompt only. Mutually exclusive with system_prompt_file and with prefix_prompts.shared_system_length/pool_size/length.","title":"Systemprompt"},"systemPromptFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to a UTF-8 text file holding the verbatim system prompt. Preferred over system_prompt for real production prompts, which are long enough that shell quoting mangles them. Read once at config-validation time so a missing or unreadable file fails at startup. Mutually exclusive with system_prompt.","title":"Systempromptfile"},"name":{"description":"Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.","minLength":1,"title":"Name","type":"string"},"type":{"const":"file","description":"Dataset type discriminator. Must be 'file'.","title":"Type","type":"string"},"path":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to file or directory containing benchmark dataset. Can be absolute or relative. Mutually exclusive with `records:`. Supported formats depend on the format field: JSONL for single_turn/multi_turn, JSONL (optionally gzipped) for tracelab, JSONL trace files for mooncake_trace/bailian_trace, Parquet for baseten_trace, directories for random_pool.","title":"Path"},"records":{"anyOf":[{"items":{"additionalProperties":true,"type":"object"},"type":"array"},{"additionalProperties":{"items":{"additionalProperties":true,"type":"object"},"type":"array"},"type":"object"},{"type":"null"}],"default":null,"description":"Inline benchmark records, embedded directly in the YAML config. Mutually exclusive with `path:`. The element schema is determined by `format:` (same shape as one line of the equivalent JSONL file). For `format: random_pool`, may be either a flat list (single pool) or a dict-of-lists (multi-pool, mirrors the directory-of-JSONLs file mode). All other formats require a flat list.","title":"Records"},"format":{"$ref":"#/$defs/DatasetFormat","default":"single_turn","description":"Dataset file format determining parsing logic and expected file structure. single_turn: JSONL with single prompt-response exchanges. multi_turn: JSONL with conversation history. tracelab: TraceLab agentic-coding corpus, JSONL or gzipped JSONL. mooncake_trace / bailian_trace / baseten_trace / burst_gpt_trace: timestamped trace files for replay. sagemaker_data_capture: JSONL captured by SageMaker DataCapture. random_pool: directory of reusable prompts."},"sampling":{"$ref":"#/$defs/DatasetSamplingStrategy","default":"sequential","description":"Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion."},"synthesis":{"anyOf":[{"$ref":"#/$defs/SynthesisConfig"},{"type":"null"}],"default":null,"description":"Trace synthesis/transformation configuration. Allows scaling timestamps and token lengths before replay. Applies to trace formats such as mooncake_trace and baseten_trace, except speedup_ratio, which is rejected for baseten_trace (use replay_speedup / --replay-speedup to scale replay pacing there)."},"entries":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Limit number of records to use from file. If not specified, uses all records in the file.","title":"Entries"},"randomSeed":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":null,"description":"Random seed for deterministic sampling. When set, makes random/shuffle sampling reproducible across runs. Overrides global random_seed for this dataset.","title":"Randomseed"},"traceSessionSampleRatio":{"anyOf":[{"exclusiveMinimum":0.0,"maximum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Fraction of trace sessions to keep for replay, sampled whole-session to preserve multi-turn integrity; deterministic when ``random_seed`` is set. Only supported by the baseten_trace loader.","title":"Tracesessionsampleratio"},"interTurnDelayCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Clamp per-turn replay delays to at most this many seconds; ``None`` disables the cap. Honored by the DAG JSONL loader and the baseten_trace loader's closed-loop think-times; the clamp count is reported at end of load.","title":"Interturndelaycapseconds"},"maxIdleGapCapSeconds":{"anyOf":[{"exclusiveMinimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Collapse idle gaps between consecutive requests (across all sessions) to at most this many seconds, so a sparse or session-sampled trace does not replay dead air; ``None`` disables the cap. The cap is in replay wall-clock seconds, applied after replay_speedup compression, so it bounds actual benchmark idle time regardless of speedup. Only supported by the baseten_trace loader.","title":"Maxidlegapcapseconds"},"replaySpeedup":{"anyOf":[{"exclusiveMinimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Trace replay wall-clock compression (10 = 10x faster than recorded): divides normalized timestamps and inter-turn delays; ``None`` = real time. Unlike synthesis speedup_ratio, hash_ids stay untouched (KV-cache fidelity). Only supported by the baseten_trace loader.","title":"Replayspeedup"},"openLoopReplay":{"default":true,"description":"Open-loop replay (the default): each session starts at its absolute, speedup-scaled recorded timestamp; continuation turns fire at max(recorded timestamp, prior-turn completion). Set ``False`` for closed-loop back-pressure: continuation turns fire a think-time (recorded start-to-start gap minus recorded e2e duration) after the prior turn completes, keeping sessions causally ordered when replayed service times differ from recorded (e.g. A/A comparisons). Only honored by the baseten_trace loader.","title":"Openloopreplay","type":"boolean"},"openLoopStrict":{"default":false,"description":"In open-loop replay, fire every trace row at its absolute recorded timestamp as an independent single-turn session, trading away multi-turn grouping and session metrics. Only honored by the baseten_trace loader.","title":"Openloopstrict","type":"boolean"},"omitKvHints":{"default":false,"description":"Drop recorded KV-cache hints (``hash_ids``, ``block_size``) from replayed request bodies, for strict frontends that reject unknown parameters. Only honored by the baseten_trace loader.","title":"Omitkvhints","type":"boolean"},"forceMinTokens":{"default":true,"description":"Pin ``min_tokens`` to the recorded output length so replayed generations match recorded lengths; disable to let EOS end generations naturally (some servers reject ``min_tokens``). Only honored by the baseten_trace loader.","title":"Forcemintokens","type":"boolean"},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Output sequence length to apply when records do not specify one. Can be a fixed integer or {mean, stddev} distribution. Per-line `output_length` values in the file always take precedence.","title":"Osl"},"promptBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Number of text items per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable text inputs entirely (e.g. image/audio/video-only workloads).","title":"Promptbatchsize"},"imageBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Number of images per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable image inputs entirely.","title":"Imagebatchsize"},"audioBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Number of audio items per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable audio inputs entirely.","title":"Audiobatchsize"},"videoBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Number of video items per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable video inputs entirely.","title":"Videobatchsize"},"blockSize":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Hash-id block granularity for trace replay (--isl-block-size). hash-id trace loaders (mooncake_trace, bailian_trace, baseten_trace, tracelab) decode each hash_id into a cached block of this many tokens; total ISL = (num_hash_ids - 1) * block_size + final_block_size. When unset, the loader's plugin-metadata default applies (e.g. 512 for mooncake_trace, 16 for bailian_trace). Not used by weka, which carries its own inline per-block sizes.","title":"Blocksize"},"traceIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree. The idle clock covers initial future work and restarts when the final in-flight request across the root and all descendant streams completes. If no request from that tree reaches the wire before the cap, AIPerf uniformly advances only that tree's pending replay timers. Dataset timestamps are unchanged, and request order, spawn/join dependencies, and replay barriers remain authoritative. Defaults to None (no per-trace runtime idle cap).","title":"Traceidlegapcapseconds"},"ignoreTraceDelays":{"default":false,"description":"Strip per-turn timestamps and inter-turn delays from trace datasets at load time. With this flag, Turn.timestamp and Turn.delay are emitted as None so concurrency / request-rate timing modes dispatch turns back-to-back instead of reproducing the recorded user think-time gaps. No effect under fixed-schedule (timestamps drive that mode before they could be ignored -- combine with --no-fixed-schedule if you want both behaviors). Mutually exclusive with use_think_time_only.","title":"Ignoretracedelays","type":"boolean"},"useThinkTimeOnly":{"default":false,"description":"For weka_trace inputs, emit Turn.delay using only the recorded per-request `think_time` (client-side delay before each request) instead of the full `t_curr - t_prev` inter-request delta. Compresses replay wall time against zero-latency mocks because the recorded `api_time` portion of each gap is dropped. Mirrors kv-cache-tester's default `--timing-strategy think-only`. Falls back to the full delta for turns whose recorded `think_time` is null. Mutually exclusive with ignore_trace_delays. No effect on non-weka trace loaders.","title":"Usethinktimeonly","type":"boolean"},"maxContextLength":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Maximum peak prompt+output context length (tokens) per Weka root trace. Only honored by format weka_trace: the loader drops whole traces whose *recorded* peak exceeds this ceiling at load time (filter-then-cap before entries). Not a DatasetManager tokenize filter; rejected for non-Weka formats.","title":"Maxcontextlength"},"cacheBust":{"$ref":"#/$defs/CacheBustConfig","description":"Per-conversation cache-bust marker injection for trace replay (mirror of the synthetic dataset's prompts.cache_bust). The engine reads the active dataset's target via ``BenchmarkConfig.get_cache_bust_target()``; 'none' (default) disables it."},"prompts":{"anyOf":[{"$ref":"#/$defs/PromptSelectionConfig"},{"type":"null"}],"default":null,"description":"Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it."}},"required":["type"],"title":"FileDataset","type":"object"},{"additionalProperties":false,"description":"Public dataset configuration.\n\nUses well-known public benchmarking datasets that are\nautomatically downloaded and processed by AIPerf.","properties":{"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Verbatim system prompt text, identical across every conversation. Emitted as a system-role message ahead of all turns. When the dataset already carries its own system message, this text is prepended to it rather than replacing it. Tokens are additive: --isl continues to size the generated user prompt only. Mutually exclusive with system_prompt_file and with prefix_prompts.shared_system_length/pool_size/length.","title":"Systemprompt"},"systemPromptFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to a UTF-8 text file holding the verbatim system prompt. Preferred over system_prompt for real production prompts, which are long enough that shell quoting mangles them. Read once at config-validation time so a missing or unreadable file fails at startup. Mutually exclusive with system_prompt.","title":"Systempromptfile"},"name":{"description":"Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.","minLength":1,"title":"Name","type":"string"},"type":{"const":"public","description":"Dataset type discriminator. Must be 'public'.","title":"Type","type":"string"},"dataset":{"$ref":"#/$defs/PublicDatasetType","description":"Pre-configured public dataset to download and use for benchmarking. Name of the HuggingFace public dataset enum (e.g. 'sharegpt', 'alpaca'). AIPerf automatically downloads and parses these datasets."},"entries":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Limit number of records to use from the dataset. If not specified, uses all available records.","title":"Entries"},"randomSeed":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":null,"description":"Random seed for deterministic sampling from the dataset. Overrides global random_seed for this dataset.","title":"Randomseed"},"sampling":{"$ref":"#/$defs/DatasetSamplingStrategy","default":"sequential","description":"Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion."},"hfSubset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name override (e.g. 'sharegpt4o'). Only applies for HuggingFace-backed public dataset loaders. Takes priority over the subset defined in the plugin registry.","title":"Hfsubset"},"filters":{"additionalProperties":{"type":"string"},"description":"Dataset-specific filters forwarded to public dataset loaders. Supported keys and values depend on the selected dataset.","title":"Filters","type":"object"},"hfWekaDataset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset repo for the generic Weka loader (e.g. `semianalysisai/cc-traces-weka-061526`). Only applies with `dataset: weka_hf`; setting it with any other public dataset is an error. Passing `--hf-weka-dataset` on the CLI auto-selects `--public-dataset weka_hf`, so the repo flag works on its own. Pinned Weka public dataset aliases keep their registry-defined repo names.","title":"Hfwekadataset"},"interTurnDelayCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Clamp per-turn replay delays to at most this many seconds for HF-backed Weka trace replay (mirror of ``FileDataset.inter_turn_delay_cap_seconds``). ``None`` disables the cap.","title":"Interturndelaycapseconds"},"traceIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree for HF-backed Weka replay (mirror of ``FileDataset.trace_idle_gap_cap_seconds``). Dataset timestamps remain unchanged. Defaults to None (no per-trace runtime idle cap).","title":"Traceidlegapcapseconds"},"ignoreTraceDelays":{"default":false,"description":"Strip per-turn timestamps and inter-turn delays from HF-backed Weka trace replay at load time (mirror of ``FileDataset.ignore_trace_delays``). Mutually exclusive with use_think_time_only.","title":"Ignoretracedelays","type":"boolean"},"useThinkTimeOnly":{"default":false,"description":"For HF-backed Weka replay, emit Turn.delay using only the recorded per-request ``think_time`` instead of the full inter-request delta (mirror of ``FileDataset.use_think_time_only``). Mutually exclusive with ignore_trace_delays.","title":"Usethinktimeonly","type":"boolean"},"maxContextLength":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Maximum peak prompt+output context length (tokens) per conversation for HF-backed Weka replay; drops over-length traces at load using *recorded* lengths (mirror of ``FileDataset.max_context_length``). Rejected for non-Weka public datasets.","title":"Maxcontextlength"},"cacheBust":{"$ref":"#/$defs/CacheBustConfig","description":"Per-conversation cache-bust marker injection for HF-backed Weka trace replay (mirror of ``FileDataset.cache_bust``). 'none' (default) disables it."},"prompts":{"anyOf":[{"$ref":"#/$defs/PromptSelectionConfig"},{"type":"null"}],"default":null,"description":"Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it."},"synthesis":{"anyOf":[{"$ref":"#/$defs/SynthesisConfig"},{"type":"null"}],"default":null,"description":"Trace synthesis/transformation configuration for HF-backed Weka trace replay (mirror of ``FileDataset.synthesis``). ``--max-isl``/``--max-osl`` cap the per-conversation input/output lengths of the replayed HF Weka traces; without this they applied only to file-based traces. ``None`` disables synthesis."},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Output sequence length to apply when records do not specify one (mirror of ``FileDataset.osl``). Can be a fixed integer or {mean, stddev} distribution. Per-record output lengths always take precedence.","title":"Osl"}},"required":["type","dataset"],"title":"PublicDataset","type":"object"}],"description":"Single dataset config (normalized to a one-entry datasets list).","x-singular-alias-of":"datasets"},"warmup":{"anyOf":[{"additionalProperties":false,"description":"Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"concurrency","description":"Concurrency-controlled immediate dispatch.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"default":1,"description":"Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.","minimum":1,"title":"Concurrency","type":"integer"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"ConcurrencyPhase","type":"object"},{"additionalProperties":false,"description":"Poisson-distributed request arrivals at the target rate.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"poisson","description":"Poisson-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"PoissonPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"Gamma-distributed request arrivals with configurable smoothness.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"gamma","description":"Gamma-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"smoothness":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.","title":"Smoothness"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"GammaPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"Constant-rate request arrivals (fixed inter-arrival time).","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"constant","description":"Constant rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"ConstantPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"user_centric","description":"N users sharing a global request rate.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"users":{"description":"Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.","minimum":1,"title":"Users","type":"integer"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type","users","rate"],"title":"UserCentricPhase","type":"object","allOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}}]},{"additionalProperties":false,"description":"Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"fixed_schedule","description":"Replay requests at trace timestamps.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"autoOffset":{"default":true,"description":"Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.","title":"Autooffset","type":"boolean"},"startOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Filter out trace requests before this timestamp in ms (must be >= 0).","title":"Startoffset"},"endOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Filter out trace requests after this timestamp in ms (must be >= 0).","title":"Endoffset"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"FixedSchedulePhase","type":"object"}],"description":"Warmup phase shorthand. Requires profiling when used.","discriminator":{"propertyName":"type","mapping":{"concurrency":"#/$defs/ConcurrencyPhase","poisson":"#/$defs/PoissonPhase","gamma":"#/$defs/GammaPhase","constant":"#/$defs/ConstantPhase","user_centric":"#/$defs/UserCentricPhase","fixed_schedule":"#/$defs/FixedSchedulePhase"}}},"profiling":{"anyOf":[{"additionalProperties":false,"description":"Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"concurrency","description":"Concurrency-controlled immediate dispatch.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"default":1,"description":"Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.","minimum":1,"title":"Concurrency","type":"integer"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"ConcurrencyPhase","type":"object"},{"additionalProperties":false,"description":"Poisson-distributed request arrivals at the target rate.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"poisson","description":"Poisson-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"PoissonPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"Gamma-distributed request arrivals with configurable smoothness.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"gamma","description":"Gamma-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"smoothness":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.","title":"Smoothness"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"GammaPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"Constant-rate request arrivals (fixed inter-arrival time).","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"constant","description":"Constant rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"ConstantPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},{"additionalProperties":false,"description":"N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"user_centric","description":"N users sharing a global request rate.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate"},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}."},"users":{"description":"Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.","minimum":1,"title":"Users","type":"integer"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["type","users","rate"],"title":"UserCentricPhase","type":"object","allOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}}]},{"additionalProperties":false,"description":"Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration"},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod"},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax"},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"fixed_schedule","description":"Replay requests at trace timestamps.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests"},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration"},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions"},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency"},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency"},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}."},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod"},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold"},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds"},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration"},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane"},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod"},"autoOffset":{"default":true,"description":"Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.","title":"Autooffset","type":"boolean"},"startOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Filter out trace requests before this timestamp in ms (must be >= 0).","title":"Startoffset"},"endOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":null,"description":"Filter out trace requests after this timestamp in ms (must be >= 0).","title":"Endoffset"},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]}},"required":["type"],"title":"FixedSchedulePhase","type":"object"}],"description":"Profiling phase shorthand. Normalized to a phases entry named 'profiling'.","discriminator":{"propertyName":"type","mapping":{"concurrency":"#/$defs/ConcurrencyPhase","poisson":"#/$defs/PoissonPhase","gamma":"#/$defs/GammaPhase","constant":"#/$defs/ConstantPhase","user_centric":"#/$defs/UserCentricPhase","fixed_schedule":"#/$defs/FixedSchedulePhase"}}}},"required":["endpoint"],"title":"BenchmarkConfig","type":"object","allOf":[{"anyOf":[{"required":["model"]},{"required":["models"]}]},{"anyOf":[{"required":["dataset"]},{"required":["datasets"]}]},{"anyOf":[{"required":["phases"]},{"required":["profiling"]}]}]},"CacheBustConfig":{"additionalProperties":false,"description":"Per-run cache-bust marker configuration.\n\nTwo families of targets:\n\nRID targets (``system_prefix``, ``system_suffix``, ``first_turn_prefix``,\n``first_turn_suffix``): inject a per-trajectory unique marker derived from a\nSHA-256 digest of (benchmark_id, recycle_pass, trajectory_index, trace_id).\nThe digest is phase-agnostic — warmup and profiling turns for the same\ntrajectory share the same marker — so warmup KV-cache work transfers to\nprofiling. Different trajectories (different lanes or recycle passes) receive\ndistinct markers. Prefix variants diverge at token 0; suffix variants append\nafter existing content.\n\nWarmup-isolation targets (``warmup_isolation_system``,\n``warmup_isolation_first_turn``): inject a constant ``[warmup]`` marker only\nduring the WARMUP phase; during PROFILING the marker is ``None`` so the\nprofiling request arrives clean. These targets are phase-aware and do NOT use\na per-trajectory digest. Incompatible with ``agentic_replay`` timing mode.","properties":{"target":{"$ref":"#/$defs/CacheBustTarget","default":"none","description":"Where (and how) to inject a per-conversation cache-bust marker. Prefix variants prepend at token 0 (most aggressive); suffix variants append after existing content. 'none' disables the feature (default)."}},"title":"CacheBustConfig","type":"object"},"CacheBustTarget":{"description":"Where (and how) to inject a per-conversation cache-bust marker.\n\nPrefix variants diverge at token 0 of the prompt (most aggressive -- defeats\nKV-cache prefix matching for the entire prompt). Suffix variants append\nafter existing content (preserves leading-prefix caching).","enum":["none","system_prefix","system_suffix","first_turn_prefix","first_turn_suffix","warmup_isolation_system","warmup_isolation_first_turn"],"title":"CacheBustTarget","type":"string"},"CancellationConfig":{"additionalProperties":false,"description":"Configuration for request cancellation testing.\n\nEnables testing server behavior when clients cancel requests mid-flight.","properties":{"rate":{"description":"Percentage of requests to cancel (0-100). 0.5 means half a percent; 10.0 means 10%% of requests will be cancelled.","title":"Rate","oneOf":[{"description":"Percentage of requests to cancel (0-100). 0.5 means half a percent; 10.0 means 10%% of requests will be cancelled.","maximum":100.0,"minimum":0.0,"title":"Rate","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"delay":{"default":0.0,"description":"Seconds to wait after sending before cancelling. 0.0 means cancel immediately after send.","title":"Delay","oneOf":[{"default":0.0,"description":"Seconds to wait after sending before cancelling. 0.0 means cancel immediately after send.","minimum":0.0,"title":"Delay","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["rate"],"title":"CancellationConfig","type":"object"},"ConcurrencyPhase":{"additionalProperties":false,"description":"Concurrency-controlled load: dispatch immediately when a slot opens.\n\nPrimary control is ``concurrency`` (defaults to 1).\nNo rate limiting -- pure concurrency-based throughput.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration","x-jinja2-supported":true},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod","x-jinja2-supported":true},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","title":"Adaptivemincompletedrequests","oneOf":[{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","title":"Adaptivecontrolmin","oneOf":[{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax","x-jinja2-supported":true},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","title":"Adaptivescalebasestep","oneOf":[{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","title":"Adaptivescalemaxstepmultiplier","oneOf":[{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","title":"Adaptivescalesteppercent","oneOf":[{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"concurrency","description":"Concurrency-controlled immediate dispatch.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests","x-jinja2-supported":true},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration","x-jinja2-supported":true},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions","x-jinja2-supported":true},"concurrency":{"default":1,"description":"Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.","title":"Concurrency","oneOf":[{"default":1,"description":"Max concurrent in-flight requests (must be >= 1). Primary control for concurrency phases.","minimum":1,"title":"Concurrency","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency","x-jinja2-supported":true},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod","x-jinja2-supported":true},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold","x-jinja2-supported":true},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","title":"Trajectorystartminratio","oneOf":[{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","title":"Trajectorystartmaxratio","oneOf":[{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds","x-jinja2-supported":true},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration","x-jinja2-supported":true},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane","x-jinja2-supported":true},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod","x-jinja2-supported":true},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["name","type"],"title":"ConcurrencyPhase","type":"object"},"ConnectionReuseStrategy":{"description":"Transport connection reuse strategy. Controls how and when connections are reused across requests.","enum":["pooled","never","sticky-user-sessions"],"title":"ConnectionReuseStrategy","type":"string"},"ConstantPhase":{"additionalProperties":false,"description":"Constant-rate request arrivals (fixed inter-arrival time).","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration","x-jinja2-supported":true},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod","x-jinja2-supported":true},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","title":"Adaptivemincompletedrequests","oneOf":[{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","title":"Adaptivecontrolmin","oneOf":[{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax","x-jinja2-supported":true},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","title":"Adaptivescalebasestep","oneOf":[{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","title":"Adaptivescalemaxstepmultiplier","oneOf":[{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","title":"Adaptivescalesteppercent","oneOf":[{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"constant","description":"Constant rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests","x-jinja2-supported":true},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration","x-jinja2-supported":true},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions","x-jinja2-supported":true},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency","x-jinja2-supported":true},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency","x-jinja2-supported":true},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod","x-jinja2-supported":true},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold","x-jinja2-supported":true},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","title":"Trajectorystartminratio","oneOf":[{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","title":"Trajectorystartmaxratio","oneOf":[{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds","x-jinja2-supported":true},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration","x-jinja2-supported":true},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane","x-jinja2-supported":true},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod","x-jinja2-supported":true},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate","x-jinja2-supported":true},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["name","type"],"title":"ConstantPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},"ConvergenceConfig":{"additionalProperties":false,"description":"Adaptive trial-stopping criterion.\n\nPresence of this object on `MultiRunConfig.convergence` enables\nadaptive stopping: trials run until the criterion fires (or\n`MultiRunConfig.num_runs` is reached, whichever comes first).","properties":{"metric":{"description":"Metric tag to evaluate convergence on (e.g., 'ttft').","title":"Metric","type":"string"},"stat":{"$ref":"#/$defs/ConvergenceStat","default":"avg","description":"Statistic on the metric to inspect (avg/p50/p90/p95/p99)."},"mode":{"$ref":"#/$defs/ConvergenceCriterionType","default":"ci_width","description":"Convergence criterion plugin name (ci_width|cv|distribution)."},"threshold":{"anyOf":[{"exclusiveMaximum":1,"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Criterion threshold (interpretation depends on `mode`). When None (default), each criterion uses its own algorithm-specific default: ci_width=0.10, cv=0.05, distribution=0.05 (KS p-value).","title":"Threshold","x-jinja2-supported":true},"minRuns":{"default":2,"description":"Minimum trials before the criterion is checked.","title":"Minruns","oneOf":[{"default":2,"description":"Minimum trials before the criterion is checked.","minimum":2,"title":"Minruns","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["metric"],"title":"ConvergenceConfig","type":"object"},"ConvergenceCriterionType":{"enum":["ci_width","cv","distribution"],"title":"ConvergenceCriterionType","type":"string"},"ConvergenceStat":{"description":"Statistic to evaluate for convergence when using ci_width or cv mode.","enum":["avg","p50","p90","p95","p99","min","max"],"title":"ConvergenceStat","type":"string"},"DatasetFormat":{"enum":["burst_gpt_trace","bailian_trace","baseten_trace","mooncake_trace","h_cua_perf","raw_payload","inputs_json","dag_jsonl","sagemaker_data_capture","multi_turn","random_pool","single_turn","speed_bench_qualitative","speed_bench_throughput_1k","speed_bench_throughput_2k","speed_bench_throughput_8k","speed_bench_throughput_16k","speed_bench_throughput_32k","tracelab","weka_trace"],"title":"DatasetFormat","type":"string"},"DatasetSamplingStrategy":{"enum":["random","sequential","shuffle"],"title":"DatasetSamplingStrategy","type":"string"},"DualBindCommunicationConfig":{"additionalProperties":false,"description":"Dual-bind (IPC + TCP) communication configuration.\n\nFor Kubernetes deployments where controller services connect via IPC\n(co-located in same pod) and worker pods connect via TCP.\n\nWhen controller_host is None, services use IPC (local mode).\nWhen controller_host is set, services use TCP to that host (remote mode).","properties":{"type":{"const":"dual","description":"Communication type. Must be 'dual'.","title":"Type","type":"string"},"ipcPath":{"default":"/tmp/aiperf","description":"Directory for IPC socket files (local services).","title":"Ipcpath","type":"string"},"tcpHost":{"default":"0.0.0.0","description":"TCP bind host for proxies. Use 0.0.0.0 to listen on all interfaces.","title":"Tcphost","type":"string"},"controllerHost":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Controller host for remote TCP connections. When set, services connect via TCP to this host instead of IPC. In Kubernetes, set via JobSet DNS (e.g., controller.namespace.svc).","title":"Controllerhost"},"recordsPort":{"default":5557,"description":"Port for records push/pull communication.","title":"Recordsport","oneOf":[{"default":5557,"description":"Port for records push/pull communication.","maximum":65535,"minimum":1,"title":"Recordsport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"creditRouterPort":{"default":5564,"description":"Port for credit router (ROUTER-DEALER).","title":"Creditrouterport","oneOf":[{"default":5564,"description":"Port for credit router (ROUTER-DEALER).","maximum":65535,"minimum":1,"title":"Creditrouterport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"controlPort":{"default":5667,"description":"Port for control channel (ROUTER-DEALER).","title":"Controlport","oneOf":[{"default":5667,"description":"Port for control channel (ROUTER-DEALER).","maximum":65535,"minimum":1,"title":"Controlport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"eventBusProxy":{"$ref":"#/$defs/TcpProxyConfig","description":"Event bus proxy ports (XPUB/XSUB)."},"datasetManagerProxy":{"$ref":"#/$defs/TcpProxyConfig","description":"Dataset manager proxy ports (DEALER/ROUTER)."},"rawInferenceProxy":{"$ref":"#/$defs/TcpProxyConfig","description":"Raw inference proxy ports (PUSH/PULL)."}},"required":["type"],"title":"DualBindCommunicationConfig","type":"object"},"EmpiricalDistribution":{"additionalProperties":false,"description":"Discrete distribution sampled from weighted values.\n\nYAML:\n isl:\n points:\n - {value: 128, weight: 40}\n - {value: 512, weight: 35}\n - {value: 2048, weight: 20}\n - {value: 8192, weight: 5}","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min","x-jinja2-supported":true},"max":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max","x-jinja2-supported":true},"points":{"description":"Weighted discrete values. Weights are relative.","items":{"$ref":"#/$defs/EmpiricalPoint"},"title":"Points","type":"array"},"type":{"const":"empirical","description":"Optional distribution type marker; stripped before concrete validation."}},"required":["points"],"title":"EmpiricalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},"EmpiricalPoint":{"additionalProperties":false,"description":"A weighted value in an empirical distribution.","properties":{"value":{"description":"The discrete value.","title":"Value","oneOf":[{"description":"The discrete value.","title":"Value","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"weight":{"default":1.0,"description":"Relative weight (normalized internally).","title":"Weight","oneOf":[{"default":1.0,"description":"Relative weight (normalized internally).","exclusiveMinimum":0.0,"title":"Weight","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["value"],"title":"EmpiricalPoint","type":"object"},"EndpointConfig":{"additionalProperties":false,"description":"Endpoint configuration for connecting to inference servers.\n\nThis section configures how AIPerf connects to and communicates\nwith the target inference server(s). It supports multiple URLs\nfor load-balanced deployments and various API types.","properties":{"urls":{"description":"List of server URLs to benchmark. Requests distributed according to url_strategy. URLs without a scheme have ``http://`` prepended automatically. Example: ['http://localhost:8000/v1/chat/completions']","items":{"type":"string"},"title":"Urls","type":"array","x-kubernetes-preserve-unknown-fields":true},"urlStrategy":{"$ref":"#/$defs/URLSelectionStrategy","default":"round_robin","description":"Strategy for distributing requests across multiple URLs. round_robin cycles through URLs in order."},"type":{"$ref":"#/$defs/EndpointType","default":"chat","description":"API endpoint type determining request/response format. chat: OpenAI chat completions, completions: OpenAI completions, embeddings: vector embeddings, rankings: reranking, template: custom format, and others — see `aiperf plugins` for the full list."},"path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Override default endpoint path. Use for servers with non-standard API paths. Example: '/custom/v2/generate'","title":"Path"},"apiKey":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"API authentication key. Supports environment variable substitution: ${OPENAI_API_KEY}. Can also use ${VAR:default} syntax for defaults.","title":"Apikey"},"timeout":{"default":21600,"description":"Request timeout in seconds (0 = no timeout). Requests exceeding this duration are marked as failed. Should exceed expected max response time.","title":"Timeout","oneOf":[{"default":21600,"description":"Request timeout in seconds (0 = no timeout). Requests exceeding this duration are marked as failed. Should exceed expected max response time.","minimum":0.0,"title":"Timeout","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"streaming":{"default":false,"description":"Enable streaming (Server-Sent Events) responses. Required for accurate TTFT (time to first token) measurement. Server must support streaming for this to work.","title":"Streaming","type":"boolean"},"transport":{"anyOf":[{"$ref":"#/$defs/TransportType"},{"type":"null"}],"default":null,"description":"Transport plugin name. Currently only 'http' (aiohttp-based HTTP/1.1) is shipped. Auto-detected from URL when unset; explicit setting overrides auto-detection."},"connectionReuse":{"$ref":"#/$defs/ConnectionReuseStrategy","default":"pooled","description":"HTTP connection management strategy. pooled: shared connection pool (fastest), never: new connection per request (includes TCP overhead), sticky-user-sessions: dedicated connection per user session."},"useLegacyMaxTokens":{"default":false,"description":"Use 'max_tokens' field instead of 'max_completion_tokens'. Enable for compatibility with older OpenAI API versions.","title":"Uselegacymaxtokens","type":"boolean"},"useServerTokenCount":{"default":false,"description":"Use server-reported token counts from response usage field. When true, trusts usage.prompt_tokens and usage.completion_tokens. When false, counts tokens locally using configured tokenizer.","title":"Useservertokencount","type":"boolean"},"perChunkUsage":{"default":false,"description":"Request per-chunk token usage on streaming responses by setting stream_options.continuous_usage_stats. When true, the server reports cumulative usage on every chunk (not just the final one), which lets inter-token latency subtract the first content chunk's real token count instead of assuming one token. Requires a server that supports continuous_usage_stats (e.g. vLLM, TRT-LLM); strict OpenAI rejects it. Requires use_server_token_count so OSL and the first-chunk count share the server source.","title":"Perchunkusage","type":"boolean"},"template":{"anyOf":[{"$ref":"#/$defs/TemplateConfig"},{"type":"null"}],"default":null,"description":"Custom template configuration for template endpoint type. Only used when type='template'. Defines request body format and response parsing."},"headers":{"additionalProperties":{"type":"string"},"description":"Custom HTTP headers to include in all requests. Useful for authentication, tracing, or routing. Values support environment variable substitution.","title":"Headers","type":"object"},"extra":{"additionalProperties":true,"description":"Additional fields to include in request body. Merged into every request. Common fields: temperature, top_p, top_k, stop.","title":"Extra","type":"object"},"downloadVideoContent":{"default":false,"description":"For video generation endpoints, download the video content after generation completes. Adds a content download step to the async polling flow.","title":"Downloadvideocontent","type":"boolean"},"requestContentType":{"anyOf":[{"$ref":"#/$defs/RequestContentType"},{"type":"null"}],"default":null,"description":"Content type for request body serialization. Default is 'application/json'. Set to 'multipart/form-data' for servers that require form-encoded requests (e.g. vLLM video generation)."},"sessionHeader":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HTTP header name used to carry the per-session affinity identifier. When set, replaces the default `X-Correlation-ID` header. Useful when the inference server expects a custom session-affinity header (e.g. `--session-header X-Session-ID`).","title":"Sessionheader"},"uuidAndStrip":{"default":false,"description":"Enable AIPerf-managed image stripping for vLLM's multimodal processor cache. Dataset-authored image UUIDs, including UUID-only cache references, always pass through on the chat endpoint; this flag only strips repeated content after AIPerf observes it in the same session. Automatic stripping supports only single_turn datasets with session_id-grouped rows; multi_turn is rejected. The server cache must cover the working set, and requests in a session must reach a replica that retains earlier UUIDs.","title":"Uuidandstrip","type":"boolean"},"waitForModelTimeout":{"default":0.0,"description":"Enable a pre-flight readiness probe by setting this to a positive value (seconds). aiperf applies this timeout to each URL/model probe before starting the benchmark, aborting with a non-zero exit if any probe times out. For multiple URLs or models, worst-case wall-clock time can be roughly this timeout multiplied by the number of URL/model probes. The probe strategy is controlled by `--wait-for-model-mode`, which defaults to sending a 1-token inference request. 0 (default) disables the probe. Eliminates the need for external shell-based readiness loops in containers and Kubernetes recipes.","title":"Waitformodeltimeout","oneOf":[{"default":0.0,"description":"Enable a pre-flight readiness probe by setting this to a positive value (seconds). aiperf applies this timeout to each URL/model probe before starting the benchmark, aborting with a non-zero exit if any probe times out. For multiple URLs or models, worst-case wall-clock time can be roughly this timeout multiplied by the number of URL/model probes. The probe strategy is controlled by `--wait-for-model-mode`, which defaults to sending a 1-token inference request. 0 (default) disables the probe. Eliminates the need for external shell-based readiness loops in containers and Kubernetes recipes.","minimum":0.0,"title":"Waitformodeltimeout","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"waitForModelInterval":{"default":5.0,"description":"Seconds between readiness probe attempts. Only consulted when `--wait-for-model-timeout` is positive.","title":"Waitformodelinterval","oneOf":[{"default":5.0,"description":"Seconds between readiness probe attempts. Only consulted when `--wait-for-model-timeout` is positive.","exclusiveMinimum":0.0,"title":"Waitformodelinterval","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"waitForModelMode":{"default":"inference","description":"Strategy for the readiness probe. 'inference' (default): POST a 1-token inference request to the configured endpoint; this is the strongest signal — it proves the full stack (frontend, scheduler, worker, forward pass) is live. Any HTTP status < 500 counts as ready. 'models': GET `/v1/models` and verify the model id appears in `data[]` (cheaper, no tokens consumed; falls back to a plain GET on the base URL on 404). 'both': run 'models' first, then 'inference'. Only consulted when `--wait-for-model-timeout` is positive.","enum":["models","inference","both"],"title":"Waitformodelmode","type":"string"},"url":{"description":"Shorthand for 'urls'. Accepts a single value which will be converted to a list/dict. List of server URLs to benchmark. Requests distributed according to url_strategy. URLs without a scheme have ``http://`` prepended automatically. Example: ['http://localhost:8000/v1/chat/completions']","oneOf":[{"type":"string"},{"description":"List of server URLs to benchmark. Requests distributed according to url_strategy. URLs without a scheme have ``http://`` prepended automatically. Example: ['http://localhost:8000/v1/chat/completions']","items":{"type":"string"},"title":"Urls","type":"array","x-kubernetes-preserve-unknown-fields":true}],"x-singular-alias-of":"urls"}},"required":[],"title":"EndpointConfig","type":"object","allOf":[{"anyOf":[{"required":["url"]},{"required":["urls"]}]}]},"EndpointType":{"enum":["audio_transcription","chat","cohere_rankings","completions","responses","messages","chat_embeddings","embeddings","hf_tei_rankings","huggingface_generate","image_generation","image_edit","video_generation","image_retrieval","nim_embeddings","nim_rankings","solido_rag","raw","template"],"title":"EndpointType","type":"string"},"FileDataset":{"additionalProperties":false,"description":"File-based dataset configuration.\n\nLoads prompts from a local file in various formats.\nSupports trace replay and custom sampling strategies.","properties":{"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Verbatim system prompt text, identical across every conversation. Emitted as a system-role message ahead of all turns. When the dataset already carries its own system message, this text is prepended to it rather than replacing it. Tokens are additive: --isl continues to size the generated user prompt only. Mutually exclusive with system_prompt_file and with prefix_prompts.shared_system_length/pool_size/length.","title":"Systemprompt"},"systemPromptFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to a UTF-8 text file holding the verbatim system prompt. Preferred over system_prompt for real production prompts, which are long enough that shell quoting mangles them. Read once at config-validation time so a missing or unreadable file fails at startup. Mutually exclusive with system_prompt.","title":"Systempromptfile"},"name":{"description":"Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.","minLength":1,"title":"Name","type":"string"},"type":{"const":"file","description":"Dataset type discriminator. Must be 'file'.","title":"Type","type":"string"},"path":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to file or directory containing benchmark dataset. Can be absolute or relative. Mutually exclusive with `records:`. Supported formats depend on the format field: JSONL for single_turn/multi_turn, JSONL (optionally gzipped) for tracelab, JSONL trace files for mooncake_trace/bailian_trace, Parquet for baseten_trace, directories for random_pool.","title":"Path"},"records":{"anyOf":[{"items":{"additionalProperties":true,"type":"object"},"type":"array"},{"additionalProperties":{"items":{"additionalProperties":true,"type":"object"},"type":"array"},"type":"object"},{"type":"null"}],"default":null,"description":"Inline benchmark records, embedded directly in the YAML config. Mutually exclusive with `path:`. The element schema is determined by `format:` (same shape as one line of the equivalent JSONL file). For `format: random_pool`, may be either a flat list (single pool) or a dict-of-lists (multi-pool, mirrors the directory-of-JSONLs file mode). All other formats require a flat list.","title":"Records"},"format":{"$ref":"#/$defs/DatasetFormat","default":"single_turn","description":"Dataset file format determining parsing logic and expected file structure. single_turn: JSONL with single prompt-response exchanges. multi_turn: JSONL with conversation history. tracelab: TraceLab agentic-coding corpus, JSONL or gzipped JSONL. mooncake_trace / bailian_trace / baseten_trace / burst_gpt_trace: timestamped trace files for replay. sagemaker_data_capture: JSONL captured by SageMaker DataCapture. random_pool: directory of reusable prompts."},"sampling":{"$ref":"#/$defs/DatasetSamplingStrategy","default":"sequential","description":"Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion."},"synthesis":{"anyOf":[{"$ref":"#/$defs/SynthesisConfig"},{"type":"null"}],"default":null,"description":"Trace synthesis/transformation configuration. Allows scaling timestamps and token lengths before replay. Applies to trace formats such as mooncake_trace and baseten_trace, except speedup_ratio, which is rejected for baseten_trace (use replay_speedup / --replay-speedup to scale replay pacing there)."},"entries":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Limit number of records to use from file. If not specified, uses all records in the file.","title":"Entries","x-jinja2-supported":true},"randomSeed":{"anyOf":[{"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Random seed for deterministic sampling. When set, makes random/shuffle sampling reproducible across runs. Overrides global random_seed for this dataset.","title":"Randomseed","x-jinja2-supported":true},"traceSessionSampleRatio":{"anyOf":[{"exclusiveMinimum":0.0,"maximum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Fraction of trace sessions to keep for replay, sampled whole-session to preserve multi-turn integrity; deterministic when ``random_seed`` is set. Only supported by the baseten_trace loader.","title":"Tracesessionsampleratio","x-jinja2-supported":true},"interTurnDelayCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Clamp per-turn replay delays to at most this many seconds; ``None`` disables the cap. Honored by the DAG JSONL loader and the baseten_trace loader's closed-loop think-times; the clamp count is reported at end of load.","title":"Interturndelaycapseconds","x-jinja2-supported":true},"maxIdleGapCapSeconds":{"anyOf":[{"exclusiveMinimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Collapse idle gaps between consecutive requests (across all sessions) to at most this many seconds, so a sparse or session-sampled trace does not replay dead air; ``None`` disables the cap. The cap is in replay wall-clock seconds, applied after replay_speedup compression, so it bounds actual benchmark idle time regardless of speedup. Only supported by the baseten_trace loader.","title":"Maxidlegapcapseconds","x-jinja2-supported":true},"replaySpeedup":{"anyOf":[{"exclusiveMinimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Trace replay wall-clock compression (10 = 10x faster than recorded): divides normalized timestamps and inter-turn delays; ``None`` = real time. Unlike synthesis speedup_ratio, hash_ids stay untouched (KV-cache fidelity). Only supported by the baseten_trace loader.","title":"Replayspeedup","x-jinja2-supported":true},"openLoopReplay":{"default":true,"description":"Open-loop replay (the default): each session starts at its absolute, speedup-scaled recorded timestamp; continuation turns fire at max(recorded timestamp, prior-turn completion). Set ``False`` for closed-loop back-pressure: continuation turns fire a think-time (recorded start-to-start gap minus recorded e2e duration) after the prior turn completes, keeping sessions causally ordered when replayed service times differ from recorded (e.g. A/A comparisons). Only honored by the baseten_trace loader.","title":"Openloopreplay","type":"boolean"},"openLoopStrict":{"default":false,"description":"In open-loop replay, fire every trace row at its absolute recorded timestamp as an independent single-turn session, trading away multi-turn grouping and session metrics. Only honored by the baseten_trace loader.","title":"Openloopstrict","type":"boolean"},"omitKvHints":{"default":false,"description":"Drop recorded KV-cache hints (``hash_ids``, ``block_size``) from replayed request bodies, for strict frontends that reject unknown parameters. Only honored by the baseten_trace loader.","title":"Omitkvhints","type":"boolean"},"forceMinTokens":{"default":true,"description":"Pin ``min_tokens`` to the recorded output length so replayed generations match recorded lengths; disable to let EOS end generations naturally (some servers reject ``min_tokens``). Only honored by the baseten_trace loader.","title":"Forcemintokens","type":"boolean"},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Output sequence length to apply when records do not specify one. Can be a fixed integer or {mean, stddev} distribution. Per-line `output_length` values in the file always take precedence.","title":"Osl"},"promptBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Number of text items per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable text inputs entirely (e.g. image/audio/video-only workloads).","title":"Promptbatchsize","x-jinja2-supported":true},"imageBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Number of images per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable image inputs entirely.","title":"Imagebatchsize","x-jinja2-supported":true},"audioBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Number of audio items per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable audio inputs entirely.","title":"Audiobatchsize","x-jinja2-supported":true},"videoBatchSize":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Number of video items per request. Only applies to format: random_pool; rejected on other formats. Set to 0 to disable video inputs entirely.","title":"Videobatchsize","x-jinja2-supported":true},"blockSize":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Hash-id block granularity for trace replay (--isl-block-size). hash-id trace loaders (mooncake_trace, bailian_trace, baseten_trace, tracelab) decode each hash_id into a cached block of this many tokens; total ISL = (num_hash_ids - 1) * block_size + final_block_size. When unset, the loader's plugin-metadata default applies (e.g. 512 for mooncake_trace, 16 for bailian_trace). Not used by weka, which carries its own inline per-block sizes.","title":"Blocksize","x-jinja2-supported":true},"traceIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree. The idle clock covers initial future work and restarts when the final in-flight request across the root and all descendant streams completes. If no request from that tree reaches the wire before the cap, AIPerf uniformly advances only that tree's pending replay timers. Dataset timestamps are unchanged, and request order, spawn/join dependencies, and replay barriers remain authoritative. Defaults to None (no per-trace runtime idle cap).","title":"Traceidlegapcapseconds","x-jinja2-supported":true},"ignoreTraceDelays":{"default":false,"description":"Strip per-turn timestamps and inter-turn delays from trace datasets at load time. With this flag, Turn.timestamp and Turn.delay are emitted as None so concurrency / request-rate timing modes dispatch turns back-to-back instead of reproducing the recorded user think-time gaps. No effect under fixed-schedule (timestamps drive that mode before they could be ignored -- combine with --no-fixed-schedule if you want both behaviors). Mutually exclusive with use_think_time_only.","title":"Ignoretracedelays","type":"boolean"},"useThinkTimeOnly":{"default":false,"description":"For weka_trace inputs, emit Turn.delay using only the recorded per-request `think_time` (client-side delay before each request) instead of the full `t_curr - t_prev` inter-request delta. Compresses replay wall time against zero-latency mocks because the recorded `api_time` portion of each gap is dropped. Mirrors kv-cache-tester's default `--timing-strategy think-only`. Falls back to the full delta for turns whose recorded `think_time` is null. Mutually exclusive with ignore_trace_delays. No effect on non-weka trace loaders.","title":"Usethinktimeonly","type":"boolean"},"maxContextLength":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum peak prompt+output context length (tokens) per Weka root trace. Only honored by format weka_trace: the loader drops whole traces whose *recorded* peak exceeds this ceiling at load time (filter-then-cap before entries). Not a DatasetManager tokenize filter; rejected for non-Weka formats.","title":"Maxcontextlength","x-jinja2-supported":true},"cacheBust":{"$ref":"#/$defs/CacheBustConfig","description":"Per-conversation cache-bust marker injection for trace replay (mirror of the synthetic dataset's prompts.cache_bust). The engine reads the active dataset's target via ``BenchmarkConfig.get_cache_bust_target()``; 'none' (default) disables it."},"prompts":{"anyOf":[{"$ref":"#/$defs/PromptSelectionConfig"},{"type":"null"}],"default":null,"description":"Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it."}},"required":["name","type"],"title":"FileDataset","type":"object"},"FixedDistribution":{"title":"FixedDistribution","description":"Returns a constant value on every sample. Scalars coerce to this. Can be a number (interpreted as a fixed value) or {value} object.","anyOf":[{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable resolving to a fixed distribution value."},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template resolving to a fixed distribution value."},{"additionalProperties":false,"description":"Returns a constant value on every sample. Scalars coerce to this.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min"},"max":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max"},"value":{"description":"The constant value returned on every sample.","title":"Value","type":"number"},"type":{"const":"fixed","description":"Optional distribution type marker; stripped before concrete validation."}},"required":["value"],"title":"FixedDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},{"type":"integer"},{"type":"number"}]},"FixedSchedulePhase":{"additionalProperties":false,"description":"Replay requests at predetermined timestamps from a trace dataset.\n\nStop condition not required -- the trace dataset determines when the\nphase ends.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration","x-jinja2-supported":true},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod","x-jinja2-supported":true},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","title":"Adaptivemincompletedrequests","oneOf":[{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","title":"Adaptivecontrolmin","oneOf":[{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax","x-jinja2-supported":true},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","title":"Adaptivescalebasestep","oneOf":[{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","title":"Adaptivescalemaxstepmultiplier","oneOf":[{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","title":"Adaptivescalesteppercent","oneOf":[{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"fixed_schedule","description":"Replay requests at trace timestamps.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests","x-jinja2-supported":true},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration","x-jinja2-supported":true},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions","x-jinja2-supported":true},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency","x-jinja2-supported":true},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency","x-jinja2-supported":true},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod","x-jinja2-supported":true},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold","x-jinja2-supported":true},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","title":"Trajectorystartminratio","oneOf":[{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","title":"Trajectorystartmaxratio","oneOf":[{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds","x-jinja2-supported":true},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration","x-jinja2-supported":true},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane","x-jinja2-supported":true},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod","x-jinja2-supported":true},"autoOffset":{"default":true,"description":"Normalize trace timestamps to start at 0. Subtracts minimum timestamp from all entries.","title":"Autooffset","type":"boolean"},"startOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Filter out trace requests before this timestamp in ms (must be >= 0).","title":"Startoffset","x-jinja2-supported":true},"endOffset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Filter out trace requests after this timestamp in ms (must be >= 0).","title":"Endoffset","x-jinja2-supported":true},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Fixed-schedule phases only accept disabled adaptive-scale no-op blocks.","title":"Adaptivescale","anyOf":[{"const":false},{"$ref":"#/$defs/DisabledAdaptiveScaleNestedBlock"}]}},"required":["name","type"],"title":"FixedSchedulePhase","type":"object"},"GPUTelemetryCollectorType":{"enum":["dcgm","pynvml","amdsmi"],"title":"GPUTelemetryCollectorType","type":"string"},"GPUTelemetryMode":{"description":"GPU telemetry display mode.","enum":["summary","realtime_dashboard"],"title":"GPUTelemetryMode","type":"string"},"GammaPhase":{"additionalProperties":false,"description":"Gamma-distributed request arrivals with configurable smoothness.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration","x-jinja2-supported":true},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod","x-jinja2-supported":true},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","title":"Adaptivemincompletedrequests","oneOf":[{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","title":"Adaptivecontrolmin","oneOf":[{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax","x-jinja2-supported":true},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","title":"Adaptivescalebasestep","oneOf":[{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","title":"Adaptivescalemaxstepmultiplier","oneOf":[{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","title":"Adaptivescalesteppercent","oneOf":[{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"gamma","description":"Gamma-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests","x-jinja2-supported":true},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration","x-jinja2-supported":true},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions","x-jinja2-supported":true},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency","x-jinja2-supported":true},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency","x-jinja2-supported":true},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod","x-jinja2-supported":true},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold","x-jinja2-supported":true},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","title":"Trajectorystartminratio","oneOf":[{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","title":"Trajectorystartmaxratio","oneOf":[{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds","x-jinja2-supported":true},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration","x-jinja2-supported":true},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane","x-jinja2-supported":true},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod","x-jinja2-supported":true},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate","x-jinja2-supported":true},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"smoothness":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Gamma distribution shape parameter (must be > 0). 1.0 = Poisson, <1 = bursty, >1 = regular.","title":"Smoothness","x-jinja2-supported":true},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["name","type"],"title":"GammaPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},"GpuTelemetryConfig":{"title":"GpuTelemetryConfig","description":"GPU telemetry configuration for live or replayed GPU metrics collection.\n\nCollects GPU metrics through DCGM exporter endpoints by default. The\n``collector`` field can switch to a local backend (``pynvml`` for NVIDIA,\n``amdsmi`` for AMD ROCm); ``mode`` controls summary vs. realtime dashboard\ndisplay.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9400/metrics\"\n → GpuTelemetryConfig(enabled=True, urls=[\"http://localhost:9400/metrics\"])\n - Singular url field: {url: \"...\"}\n → GpuTelemetryConfig(urls=[\"...\"]) Can be a URL string or full config object.","anyOf":[{"additionalProperties":false,"description":"GPU telemetry configuration for live or replayed GPU metrics collection.\n\nCollects GPU metrics through DCGM exporter endpoints by default. The\n``collector`` field can switch to a local backend (``pynvml`` for NVIDIA,\n``amdsmi`` for AMD ROCm); ``mode`` controls summary vs. realtime dashboard\ndisplay.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9400/metrics\"\n → GpuTelemetryConfig(enabled=True, urls=[\"http://localhost:9400/metrics\"])\n - Singular url field: {url: \"...\"}\n → GpuTelemetryConfig(urls=[\"...\"])","properties":{"enabled":{"default":true,"description":"Enable GPU telemetry collection. Set to false to disable.","title":"Enabled","type":"boolean"},"urls":{"description":"DCGM exporter endpoint URLs. Example: http://localhost:9400/metrics","items":{"type":"string"},"title":"Urls","type":"array"},"metricsFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to CSV file with pre-recorded GPU metrics. Alternative to live DCGM collection.","title":"Metricsfile"},"collector":{"$ref":"#/$defs/GPUTelemetryCollectorType","default":"dcgm","description":"GPU telemetry collector backend. Use 'dcgm' for DCGM exporter endpoints or a local collector (e.g. 'pynvml' for NVIDIA, 'amdsmi' for AMD ROCm) for on-host metrics collection."},"mode":{"$ref":"#/$defs/GPUTelemetryMode","default":"summary","description":"GPU telemetry display mode. Summary emits aggregate console output; realtime_dashboard enables live dashboard updates."},"url":{"description":"Shorthand for 'urls'. Accepts a single value which will be converted to a list/dict. DCGM exporter endpoint URLs. Example: http://localhost:9400/metrics","oneOf":[{"type":"string"},{"description":"DCGM exporter endpoint URLs. Example: http://localhost:9400/metrics","items":{"type":"string"},"title":"Urls","type":"array"}],"x-singular-alias-of":"urls"}},"title":"GpuTelemetryConfig","type":"object","x-kubernetes-preserve-unknown-fields":true},{"type":"string"}]},"GridSweep":{"additionalProperties":false,"description":"Grid sweep - all combinations of parameters (Cartesian product).","properties":{"cooldownSeconds":{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaFilters":{"description":"SLA constraints applied at trial scoring / aggregation time.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Slafilters","type":"array"},"postProcess":{"anyOf":[{"$ref":"#/$defs/PostProcessSpec"},{"type":"null"}],"default":null,"description":"Optional post-aggregation handler emitted by a search recipe."},"recipeName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.","title":"Recipename"},"iterationOrder":{"$ref":"#/$defs/SweepMode","default":"repeated","description":"Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search."},"sameSeed":{"default":false,"description":"If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.","title":"Sameseed","type":"boolean"},"type":{"const":"grid","default":"grid","description":"Sweep type discriminator.","title":"Type","type":"string"},"parameters":{"additionalProperties":{"items":{},"type":"array"},"description":"Parameters to sweep: dot-notation path -> list of values.","minProperties":1,"title":"Parameters","type":"object"}},"required":["parameters"],"title":"GridSweep","type":"object"},"ImageConfig":{"additionalProperties":false,"allOf":[{"if":{"properties":{"source":{"const":"noise"}}},"then":{"properties":{"sourceSampling":{"const":"random-with-replacement"}}}}],"description":"Configuration for synthetic image generation in multimodal datasets.\n\nControls the generation of synthetic images for vision-language\nmodel benchmarking. Images are generated by randomly sampling and\nresizing source images to specified dimensions.","properties":{"batchSize":{"default":0,"description":"Number of images to include in each multimodal request. Supported with chat endpoint type for vision-language models. Set to 0 to disable image inputs. Higher batch sizes test multi-image understanding and increase request payload size.","title":"Batchsize","oneOf":[{"default":0,"description":"Number of images to include in each multimodal request. Supported with chat endpoint type for vision-language models. Set to 0 to disable image inputs. Higher batch sizes test multi-image understanding and increase request payload size.","minimum":0,"title":"Batchsize","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"width":{"description":"Image width in pixels. Can be a fixed integer or {mean, stddev} distribution. Combined with height to determine image dimensions and file sizes for multimodal benchmarking.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Width","x-jinja2-supported":true},"height":{"description":"Image height in pixels. Can be a fixed integer or {mean, stddev} distribution. Used when batch_size > 0 for multimodal vision benchmarking.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Height","x-jinja2-supported":true},"format":{"$ref":"#/$defs/ImageFormat","default":"jpeg","description":"Image file format for generated images. png: lossless compression (larger files, best quality). jpeg: lossy compression (smaller files, good quality). random: randomly select between PNG and JPEG per image. Format affects file size in multimodal requests and encoding overhead."},"source":{"anyOf":[{"$ref":"#/$defs/ImageSource"},{"format":"path","type":"string"}],"default":"noise","description":"Source image generation mode (default: noise). noise: generate random noise images on the fly — no files on disk, effectively unbounded variety so servers cannot dedupe identical inputs. assets: index images from the bundled assets/source_images directory and lazily load them at the requested dimensions. A path string indexes images from the given directory (e.g. ./source_images). Random-noise images are roughly incompressible, so payload bytes are larger than equivalent natural images.","title":"Source"},"sourceSampling":{"$ref":"#/$defs/ImageSourceSamplingStrategy","default":"random-with-replacement","description":"How source images are selected from finite image sources selected by source='assets' or a directory path. random-with-replacement: draw each source image independently; repeats may occur immediately. shuffle-cycle: draw every source image once per shuffled cycle, reshuffling after exhaustion. sequential-cycle: walk source images in sorted load order and wrap after exhaustion. For noise mode, only random-with-replacement is valid because there is no finite source pool."}},"title":"ImageConfig","type":"object"},"ImageFormat":{"description":"Image file formats for synthetic image generation.","enum":["png","jpeg","random"],"title":"ImageFormat","type":"string"},"ImageSource":{"description":"Source image generation mode for multimodal benchmarking.","enum":["assets","noise"],"title":"ImageSource","type":"string"},"ImageSourceSamplingStrategy":{"description":"How source images are selected from a finite source-image pool.","enum":["random-with-replacement","shuffle-cycle","sequential-cycle"],"title":"ImageSourceSamplingStrategy","type":"string"},"IpcCommunicationConfig":{"additionalProperties":false,"description":"IPC (Unix socket) communication configuration.\n\nFor single-machine deployments with lowest latency.\nUses Unix domain sockets for all inter-service communication.","properties":{"type":{"const":"ipc","description":"Communication type. Must be 'ipc'.","title":"Type","type":"string"},"path":{"default":"/tmp/aiperf","description":"Directory for IPC socket files. AIPerf creates multiple socket files in this directory.","title":"Path","type":"string"}},"required":["type"],"title":"IpcCommunicationConfig","type":"object"},"LatinHypercubeSweep":{"additionalProperties":false,"description":"Latin hypercube sampling — perfect marginal stratification per axis.","properties":{"cooldownSeconds":{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaFilters":{"description":"SLA constraints applied at trial scoring / aggregation time.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Slafilters","type":"array"},"postProcess":{"anyOf":[{"$ref":"#/$defs/PostProcessSpec"},{"type":"null"}],"default":null,"description":"Optional post-aggregation handler emitted by a search recipe."},"recipeName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.","title":"Recipename"},"iterationOrder":{"$ref":"#/$defs/SweepMode","default":"repeated","description":"Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search."},"sameSeed":{"default":false,"description":"If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.","title":"Sameseed","type":"boolean"},"samples":{"description":"Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.","title":"Samples","oneOf":[{"description":"Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.","maximum":1048576,"minimum":2,"title":"Samples","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"seed":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"RNG seed for reproducibility. Must be non-negative; scipy's QMC engines reject negative seeds with an opaque low-level error.","title":"Seed","x-jinja2-supported":true},"dimensions":{"description":"Dimensions to sample over.","items":{"$ref":"#/$defs/SamplingDimension"},"minItems":1,"title":"Dimensions","type":"array"},"labelFormat":{"default":"index","description":"Variation-label format. 'index' => '_NNNN' (short, sortable). 'kv' => 'k1=v1, k2=v2' (grid-style).","enum":["index","kv"],"title":"Labelformat","type":"string"},"type":{"const":"latin_hypercube","default":"latin_hypercube","description":"Sweep type discriminator.","title":"Type","type":"string"},"optimization":{"default":"random-cd","description":"LHS post-processing to reduce centered discrepancy. None = vanilla LHS.","enum":[null,"random-cd","lloyd"],"title":"Optimization"}},"required":["samples","dimensions"],"title":"LatinHypercubeSweep","type":"object"},"LogNormalDistribution":{"additionalProperties":false,"description":"Log-normal parameterized by mean and median (right-skewed, always positive).\n\nSkew is controlled by the mean/median ratio: larger ratio = heavier right tail.\nWhen mean == median the distribution is deterministic.\n\nInternally: sigma = sqrt(2 * log(mean / median)), mu = log(median).","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min","x-jinja2-supported":true},"max":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max","x-jinja2-supported":true},"mean":{"description":"Desired mean of the output distribution.","title":"Mean","oneOf":[{"description":"Desired mean of the output distribution.","exclusiveMinimum":0.0,"title":"Mean","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"median":{"description":"Desired median. Must be <= mean. Lower median = more right skew.","title":"Median","oneOf":[{"description":"Desired median. Must be <= mean. Lower median = more right skew.","exclusiveMinimum":0.0,"title":"Median","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"type":{"const":"lognormal","description":"Optional distribution type marker; stripped before concrete validation."}},"required":["mean","median"],"title":"LogNormalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},"LoggingConfig":{"additionalProperties":false,"description":"Logging configuration for verbosity and debug settings.","properties":{"level":{"$ref":"#/$defs/AIPerfLogLevel","default":"INFO","description":"Global logging verbosity level (loguru-style severity ladder; TRACE most verbose, CRITICAL least). NOTICE and SUCCESS are loguru-specific intermediate levels (between INFO and WARNING) and have no equivalent in the Python stdlib `logging` module — pick INFO/WARNING/ERROR if you want stdlib-compatible severities."}},"title":"LoggingConfig","type":"object"},"MLflowConfig":{"additionalProperties":false,"description":"MLflow tracking and artifact-upload configuration.","properties":{"trackingUri":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"MLflow tracking URI.","title":"Trackinguri"},"experiment":{"default":"aiperf","description":"MLflow experiment name.","title":"Experiment","type":"string"},"runName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"MLflow run name.","title":"Runname"},"tags":{"anyOf":[{"type":"array","items":{"oneOf":[{"type":"string"},{"type":"array","minItems":2,"maxItems":2}]}},{"type":"string"},{"additionalProperties":{"type":"string"},"type":"object"},{"type":"null"}],"default":null,"description":"Additional MLflow run tags to attach on upload.","title":"Tags"},"parentRunId":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Optional MLflow parent run ID.","title":"Parentrunid"},"artifactGlobs":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"default":null,"description":"Artifact glob overrides for MLflow upload.","title":"Artifactglobs"}},"title":"MLflowConfig","type":"object"},"MetricsConfig":{"additionalProperties":false,"description":"Configuration for benchmark metric aggregation behavior.\n\nCurrently empty — list-valued record metrics aggregate via t-digest at\nfirst-touch (see :mod:`aiperf.metrics.list_metric_aggregation`); the\nsketch compression is tunable via ``AIPERF_METRICS_TDIGEST_COMPRESSION``.\nReserved as the home for any future user-facing metric-aggregation knobs.","properties":{},"title":"MetricsConfig","type":"object"},"ModelItem":{"additionalProperties":false,"description":"Configuration for a single model in advanced models configuration.\n\nUsed when the models section uses the advanced format with\nexplicit items, weights, and per-model settings.","properties":{"name":{"description":"Model name or identifier as known to the inference server.","minLength":1,"title":"Name","type":"string"},"weight":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Selection weight for weighted strategy (0.0-1.0). Weights must sum to 1.0 (+/-0.01) across all models; they are validated, not auto-normalized. Example: weight=0.7 means ~70%% of requests to this model.","title":"Weight","x-jinja2-supported":true},"lora":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"LoRA adapter name to load with this model. Server must support dynamic LoRA adapter loading.","title":"Lora"},"modalities":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"default":null,"description":"List of input modalities this model supports. [Currently a no-op: no selection strategy consumes this field. Accepted for forward-compatibility / declarative documentation.]","title":"Modalities"},"tokenizer":{"anyOf":[{"$ref":"#/$defs/TokenizerOverride"},{"type":"null"}],"default":null,"description":"Per-model tokenizer override. Use when this model requires a different tokenizer than global config."}},"required":["name"],"title":"ModelItem","type":"object"},"ModelSelectionStrategy":{"description":"Strategy for selecting the model to use for the request.","enum":["round_robin","random","weighted"],"title":"ModelSelectionStrategy","type":"string"},"ModelsAdvanced":{"additionalProperties":false,"description":"Advanced models configuration with selection strategy and item details.\n\nUse this format when you need weighted routing, LoRA adapters,\nor per-model tokenizer overrides.","properties":{"strategy":{"$ref":"#/$defs/ModelSelectionStrategy","default":"round_robin","description":"Strategy for selecting models when multiple are configured. round_robin cycles through models, random selects randomly, weighted uses configured weights."},"items":{"description":"List of model configurations. At least one model required.","items":{"$ref":"#/$defs/ModelItem"},"minItems":1,"title":"Items","type":"array"}},"required":["items"],"title":"ModelsAdvanced","type":"object"},"MultiRunConfig":{"additionalProperties":false,"description":"Trial mechanics within a single sweep variation.\n\nWhen `num_runs > 1` (or `convergence` is set), AIPerf runs multiple\ntrials per variation and aggregates statistics. When `num_runs == 1`\nand `convergence is None`, exactly one trial runs per variation.","properties":{"numRuns":{"default":1,"description":"Upper bound on trials per variation. When `convergence` is set, trials may stop earlier; this is the hard ceiling. Cap matches `BenchmarkPlan.trials` so programmatic plan assembly cannot exceed what user-facing config validation allows.","title":"Numruns","oneOf":[{"default":1,"description":"Upper bound on trials per variation. When `convergence` is set, trials may stop earlier; this is the hard ceiling. Cap matches `BenchmarkPlan.trials` so programmatic plan assembly cannot exceed what user-facing config validation allows.","maximum":10,"minimum":1,"title":"Numruns","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"cooldownSeconds":{"default":0.0,"description":"Cooldown between trials within a single variation. Capped at 24h to surface typos like `1e18` at config-load time rather than blocking the orchestrator inside ``asyncio.sleep``.","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between trials within a single variation. Capped at 24h to surface typos like `1e18` at config-load time rather than blocking the orchestrator inside ``asyncio.sleep``.","maximum":86400,"minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"confidenceLevel":{"default":0.95,"description":"Confidence level for aggregate statistics (0.90, 0.95, 0.99).","title":"Confidencelevel","oneOf":[{"default":0.95,"description":"Confidence level for aggregate statistics (0.90, 0.95, 0.99).","exclusiveMaximum":1,"exclusiveMinimum":0,"title":"Confidencelevel","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"setConsistentSeed":{"default":true,"description":"Auto-set a random seed when none is given for cross-trial consistency.","title":"Setconsistentseed","type":"boolean"},"varySeedPerTrial":{"default":false,"description":"When True, derive a distinct seed for each trial of a variation via SHA-256 over `(envelope_seed, variation.label, trial)`. When False (default), all trials of a variation share the variation's seed. Off gives pure-runtime variance for confidence statistics; on captures end-to-end variance including per-trial workload sampling.","title":"Varyseedpertrial","type":"boolean"},"disableWarmupAfterFirst":{"default":true,"description":"Skip warmup on trials 2..N of a variation (steady-state measurement).","title":"Disablewarmupafterfirst","type":"boolean"},"convergence":{"anyOf":[{"$ref":"#/$defs/ConvergenceConfig"},{"type":"null"}],"default":null,"description":"Optional adaptive early-stop. Absent = fixed-N (run `num_runs` trials)."}},"title":"MultiRunConfig","type":"object"},"MultimodalDistribution":{"additionalProperties":false,"description":"Weighted mixture of N peaks (N >= 2).\n\nYAML:\n isl:\n peaks:\n - {mean: 128, stddev: 20, weight: 60}\n - {mean: 2048, median: 1800, weight: 40}\n # Equal split — omit weight:\n isl:\n peaks:\n - {mean: 128, stddev: 20}\n - {mean: 2048, median: 1800}\n - {mean: 8192, median: 4096}","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min","x-jinja2-supported":true},"max":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max","x-jinja2-supported":true},"peaks":{"description":"Two or more weighted sub-distributions.","items":{"$ref":"#/$defs/PeakEntry"},"minItems":2,"title":"Peaks","type":"array"},"type":{"const":"multimodal","description":"Optional distribution type marker; stripped before concrete validation."}},"required":["peaks"],"title":"MultimodalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},"NetworkLatencyConfig":{"additionalProperties":false,"description":"Network latency calibration configuration.\n\nWhen ``enabled``, AIPerf opens a fresh TCP connection to the endpoint\nhost:port on an interval during profiling and records the handshake RTT.\nThe mean RTT is subtracted from the request-start-anchored latency metrics\n(request_latency, time_to_first_token, time_to_first_output_token) to produce\nnon-destructive ``network_adjusted_*``\nvariants. ``mean_ms`` supplies a fixed mean RTT and skips active probing.","properties":{"enabled":{"default":false,"description":"Enable network latency calibration. When true, AIPerf measures the endpoint RTT (or uses mean_ms) and subtracts it from latency metrics, emitted as separate network_adjusted_* metrics.","title":"Enabled","type":"boolean"},"pingInterval":{"description":"Seconds between TCP-handshake RTT probes during profiling. Ignored when mean_ms is set.","title":"Pinginterval","oneOf":[{"description":"Seconds between TCP-handshake RTT probes during profiling. Ignored when mean_ms is set.","exclusiveMinimum":0.0,"title":"Pinginterval","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"meanMs":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Fixed mean network RTT in milliseconds to subtract, bypassing active probing. When set, no probes are sent and this value is used directly.","title":"Meanms","x-jinja2-supported":true}},"title":"NetworkLatencyConfig","type":"object","x-kubernetes-preserve-unknown-fields":true},"NormalDistribution":{"additionalProperties":false,"description":"Gaussian (truncated at 0) parameterized by mean and stddev.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min","x-jinja2-supported":true},"max":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max","x-jinja2-supported":true},"mean":{"description":"Mean value. Must be >= 0; samples below 0 are truncated, so a negative mean would yield a degenerate distribution. Zero is allowed (e.g. OSL=0 disables output, turn_delay mean=0 disables inter-turn delay).","title":"Mean","oneOf":[{"description":"Mean value. Must be >= 0; samples below 0 are truncated, so a negative mean would yield a degenerate distribution. Zero is allowed (e.g. OSL=0 disables output, turn_delay mean=0 disables inter-turn delay).","minimum":0.0,"title":"Mean","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"stddev":{"default":0.0,"description":"Standard deviation. 0 = deterministic.","title":"Stddev","oneOf":[{"default":0.0,"description":"Standard deviation. 0 = deterministic.","minimum":0.0,"title":"Stddev","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"type":{"const":"normal","description":"Optional distribution type marker; stripped before concrete validation."}},"required":["mean"],"title":"NormalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},"OTelConfig":{"additionalProperties":false,"description":"OpenTelemetry metrics streaming configuration.","properties":{"metricsUrl":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"OTLP/HTTP metrics endpoint URL.","title":"Metricsurl"},"streamMetricsEnabled":{"default":true,"description":"Stream metric records to OTel.","title":"Streammetricsenabled","type":"boolean"},"streamTimingEnabled":{"default":true,"description":"Stream timing records to OTel.","title":"Streamtimingenabled","type":"boolean"},"customResourceAttributes":{"additionalProperties":{"type":"string"},"description":"Custom OTel resource attributes.","title":"Customresourceattributes","type":"object"},"genAiProvider":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"GenAI semantic convention provider override.","title":"Genaiprovider"}},"title":"OTelConfig","type":"object"},"Objective":{"additionalProperties":false,"description":"Single optimization objective.\n\nMultiple ``Objective`` entries on an ``AdaptiveSearchSweep`` enable\nPareto Bayesian optimization (e.g. qlognehvi / qnehvi / qehvi). A\nlength-1 list is single-objective BO.","properties":{"metric":{"description":"Metric tag to optimize (e.g. 'output_token_throughput').","title":"Metric","type":"string"},"stat":{"default":"avg","description":"Statistic on the metric.","enum":["avg","p50","p90","p95","p99"],"title":"Stat","type":"string"},"direction":{"$ref":"#/$defs/OptimizationDirection","description":"MAXIMIZE or MINIMIZE."},"threshold":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Pareto reference point for hypervolume computation. Trials worse than this on this objective do not contribute to hypervolume. When None, auto-derived from the worst observed value among Sobol initial points. Ignored for single-objective runs.","title":"Threshold","x-jinja2-supported":true}},"required":["metric","direction"],"title":"Objective","type":"object"},"OptimizationDirection":{"description":"Direction of optimization for a metric.\n\nMAXIMIZE: Higher values are preferred (e.g. throughput, goodput).\nMINIMIZE: Lower values are preferred (e.g. latency, TTFT, p99).","enum":["maximize","minimize"],"title":"OptimizationDirection","type":"string"},"OutcomeConstraint":{"additionalProperties":false,"description":"Feasibility gate on a metric the optimizer is *not* optimizing.\n\nDistinct from ``Objective.threshold`` (Pareto reference point) and from\n``SLAFilter`` (post-hoc filter on benchmark eligibility). An outcome\nconstraint masks BoTorch acquisition: candidates predicted to violate\nit are downweighted.","properties":{"metric":{"description":"Metric tag to constrain.","title":"Metric","type":"string"},"op":{"description":"Comparison operator.","enum":["<=",">=","=="],"title":"Op","type":"string"},"bound":{"description":"Threshold value.","title":"Bound","oneOf":[{"description":"Threshold value.","title":"Bound","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["metric","op","bound"],"title":"OutcomeConstraint","type":"object"},"PeakEntry":{"title":"PeakEntry","description":"A weighted component in a multimodal distribution.\n\nThe weight and distribution fields are written inline in YAML:\n {mean: 128, stddev: 20, weight: 60}\n\nThe ``weight`` key is extracted before the remaining fields are parsed\nas a SamplingDistribution. Defaults to 1.0 (equal split when omitted).","anyOf":[{"additionalProperties":false,"description":"A weighted component in a multimodal distribution.\n\nThe weight and distribution fields are written inline in YAML:\n {mean: 128, stddev: 20, weight: 60}\n\nThe ``weight`` key is extracted before the remaining fields are parsed\nas a SamplingDistribution. Defaults to 1.0 (equal split when omitted).","properties":{"distribution":{"description":"The sub-distribution for this peak.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}],"title":"Distribution"},"weight":{"default":1.0,"description":"Relative weight (normalised internally).","minimum":0.0,"title":"Weight","type":"number"}},"required":["distribution"],"title":"PeakEntry","type":"object"},{"additionalProperties":false,"description":"Inline FixedDistribution peak with optional weight.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min"},"max":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max"},"value":{"description":"The constant value returned on every sample.","title":"Value","type":"number"},"type":{"const":"fixed","description":"Optional distribution type marker; stripped before concrete validation."},"weight":{"default":1.0,"description":"Relative weight (normalised internally).","minimum":0.0,"title":"Weight","type":"number"}},"required":["value"],"title":"FixedDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},{"additionalProperties":false,"description":"Inline NormalDistribution peak with optional weight.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min"},"max":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max"},"mean":{"description":"Mean value. Must be >= 0; samples below 0 are truncated, so a negative mean would yield a degenerate distribution. Zero is allowed (e.g. OSL=0 disables output, turn_delay mean=0 disables inter-turn delay).","minimum":0.0,"title":"Mean","type":"number"},"stddev":{"default":0.0,"description":"Standard deviation. 0 = deterministic.","minimum":0.0,"title":"Stddev","type":"number"},"type":{"const":"normal","description":"Optional distribution type marker; stripped before concrete validation."},"weight":{"default":1.0,"description":"Relative weight (normalised internally).","minimum":0.0,"title":"Weight","type":"number"}},"required":["mean"],"title":"NormalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},{"additionalProperties":false,"description":"Inline LogNormalDistribution peak with optional weight.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min"},"max":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max"},"mean":{"description":"Desired mean of the output distribution.","exclusiveMinimum":0.0,"title":"Mean","type":"number"},"median":{"description":"Desired median. Must be <= mean. Lower median = more right skew.","exclusiveMinimum":0.0,"title":"Median","type":"number"},"type":{"const":"lognormal","description":"Optional distribution type marker; stripped before concrete validation."},"weight":{"default":1.0,"description":"Relative weight (normalised internally).","minimum":0.0,"title":"Weight","type":"number"}},"required":["mean","median"],"title":"LogNormalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},{"additionalProperties":false,"description":"Inline MultimodalDistribution peak with optional weight.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min"},"max":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max"},"peaks":{"description":"Two or more weighted sub-distributions.","items":{"$ref":"#/$defs/PeakEntry"},"minItems":2,"title":"Peaks","type":"array"},"type":{"const":"multimodal","description":"Optional distribution type marker; stripped before concrete validation."},"weight":{"default":1.0,"description":"Relative weight (normalised internally).","minimum":0.0,"title":"Weight","type":"number"}},"required":["peaks"],"title":"MultimodalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true},{"additionalProperties":false,"description":"Inline EmpiricalDistribution peak with optional weight.","properties":{"min":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive lower bound; samples below are clamped up. Applies to every distribution type — composes with mean/stddev/median/peaks/points/value.","title":"Min"},"max":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Inclusive upper bound; samples above are clamped down.","title":"Max"},"points":{"description":"Weighted discrete values. Weights are relative.","items":{"$ref":"#/$defs/EmpiricalPoint"},"title":"Points","type":"array"},"type":{"const":"empirical","description":"Optional distribution type marker; stripped before concrete validation."},"weight":{"default":1.0,"description":"Relative weight (normalised internally).","minimum":0.0,"title":"Weight","type":"number"}},"required":["points"],"title":"EmpiricalDistribution","type":"object","x-kubernetes-preserve-unknown-fields":true}]},"PoissonPhase":{"additionalProperties":false,"description":"Poisson-distributed request arrivals at the target rate.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration","x-jinja2-supported":true},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod","x-jinja2-supported":true},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","title":"Adaptivemincompletedrequests","oneOf":[{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","title":"Adaptivecontrolmin","oneOf":[{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax","x-jinja2-supported":true},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","title":"Adaptivescalebasestep","oneOf":[{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","title":"Adaptivescalemaxstepmultiplier","oneOf":[{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","title":"Adaptivescalesteppercent","oneOf":[{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"poisson","description":"Poisson-distributed rate-controlled arrivals.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests","x-jinja2-supported":true},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration","x-jinja2-supported":true},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions","x-jinja2-supported":true},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency","x-jinja2-supported":true},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency","x-jinja2-supported":true},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod","x-jinja2-supported":true},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold","x-jinja2-supported":true},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","title":"Trajectorystartminratio","oneOf":[{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","title":"Trajectorystartmaxratio","oneOf":[{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds","x-jinja2-supported":true},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration","x-jinja2-supported":true},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane","x-jinja2-supported":true},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod","x-jinja2-supported":true},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate","x-jinja2-supported":true},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"rateSeries":{"anyOf":[{"$ref":"#/$defs/RateSeriesConfig"},{"type":"null"}],"default":null,"description":"Piecewise-linear request-rate schedule."},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["name","type"],"title":"PoissonPhase","type":"object","allOf":[{"oneOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}},{"required":["rateSeries"],"properties":{"rateSeries":{"not":{"type":"null"}}}}]}]},"PostProcessSpec":{"additionalProperties":false,"description":"Post-aggregation hook spec emitted by a search recipe.\n\nResolved through the ``search_recipe_post_process`` plugin category and\ninvoked by ``aggregate_sweep_and_export`` after per-variation aggregation.\n\nExample: ``PostProcessSpec(handler=\"ttft_sla_curve\", params={\"sla_ms\": 200},\noutput_filename=\"ttft_sla_curve.json\")``.","properties":{"handler":{"description":"Registered post-process handler name. Looked up in the ``search_recipe_post_process`` plugin category.","title":"Handler","type":"string"},"params":{"additionalProperties":true,"description":"Free-form parameters forwarded to the handler. Shape is handler-defined; Any is intentional because handlers ingest dynamic recipe-derived inputs.","title":"Params","type":"object"},"output_filename":{"description":"Filename (relative to ``sweep_aggregate/`` under the artifact dir) to write the handler's output to. Must end in ``.json`` and contain no path separators -- the hook joins it with ``sweep_aggregate/`` and refuses to escape that directory. NUL bytes and dot-only stems are also rejected.","title":"Output Filename","type":"string"}},"required":["handler","output_filename"],"title":"PostProcessSpec","type":"object"},"PrefixPromptConfig":{"additionalProperties":false,"description":"Configuration for prefix prompts (KV cache testing).\n\nPrefix prompts allow testing KV cache efficiency by generating\nrequests that share common prefixes. This simulates scenarios\nlike system prompts or shared context that can be cached.\n\nNote: pool_size/length are mutually exclusive with shared_system_length\nand user_context_length.","properties":{"poolSize":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Number of distinct prefix prompts to generate for KV cache testing. Each prefix is prepended to user prompts, simulating cached context scenarios. Prefixes are randomly selected from pool per request. Mutually exclusive with shared_system_length/user_context_length.","title":"Poolsize","x-jinja2-supported":true},"length":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Token length for each prefix prompt in the pool. Only used when pool_size is set. The prefix is concatenated to the body at the token level, so the final prompt contains exactly prefix length + input length tokens. Mutually exclusive with shared_system_length/user_context_length.","title":"Length","x-jinja2-supported":true},"sharedSystemLength":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Length of shared system prompt in tokens. This prompt is identical across all sessions and appears as a system message. First part of a two-part prefix structure with high cache hit rate expected. Mutually exclusive with pool_size/length.","title":"Sharedsystemlength","x-jinja2-supported":true},"userContextLength":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Length of per-session user context prompt in tokens. Each dataset entry gets a unique user context prompt. Second part of two-part prefix structure with lower cache hit rate expected. Mutually exclusive with pool_size/length.","title":"Usercontextlength","x-jinja2-supported":true}},"title":"PrefixPromptConfig","type":"object"},"PromptConfig":{"additionalProperties":false,"description":"Configuration for prompt/token specifications in synthetic datasets.\n\nThis is the core configuration for controlling input sequence length (ISL)\nand output sequence length (OSL) in synthetic data generation.","properties":{"isl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Input sequence length in tokens. Can be a fixed integer (e.g., 512) or distribution {mean: 512, stddev: 50}. AIPerf generates prompts with lengths following a normal distribution around the mean (±stddev). Ignored when sequence_distribution is specified.","title":"Isl"},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Output sequence length (max tokens to request via max_completion_tokens). Can be a fixed integer or distribution {mean, stddev}. Controls response length for synthetic datasets. When not set, the model determines output length. Ignored when sequence_distribution is specified.","title":"Osl"},"blockSize":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Token block size for hash-based prompt caching in mooncake_trace datasets. When hash_ids are provided in trace entries, prompts are divided into blocks of this size. Each hash_id maps to a cached block, enabling simulation of KV-cache sharing patterns from production workloads. Total prompt length = (num_hash_ids - 1) * block_size + final_block_size.","title":"Blocksize","x-jinja2-supported":true},"batchSize":{"default":1,"description":"Number of text inputs to include in each request for batch processing endpoints. Supported by embeddings and rankings endpoint types where models can process multiple inputs simultaneously. Set to 1 for single-input requests. Not applicable to chat or completions endpoints.","title":"Batchsize","oneOf":[{"default":1,"description":"Number of text inputs to include in each request for batch processing endpoints. Supported by embeddings and rankings endpoint types where models can process multiple inputs simultaneously. Set to 1 for single-input requests. Not applicable to chat or completions endpoints.","minimum":1,"title":"Batchsize","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"cacheBust":{"$ref":"#/$defs/CacheBustConfig","description":"Per-conversation cache-bust marker configuration. Injects a deterministic marker into prompts to defeat KV-cache prefix matching. Disabled by default (target='none')."},"corpus":{"anyOf":[{"$ref":"#/$defs/PromptCorpus"},{"type":"null"}],"default":null,"description":"Source corpus for synthetic prompt text generation. 'sonnet' uses Shakespeare sonnets. 'coding' uses realistic coding content (code, bash output, JSON, error tracebacks, git diffs). When unset, the active dataset loader's default applies (most loaders default to 'sonnet'; agentic-coding loaders such as weka_trace default to 'coding')."},"sequenceDistribution":{"anyOf":[{"items":{"$ref":"#/$defs/SequenceDistributionEntry"},"type":"array"},{"type":"null"}],"default":null,"description":"Distribution of (ISL, OSL) pairs with probabilities for mixed workload simulation. Each entry specifies {isl, osl, probability}. Probabilities are percentages (0-100) and must sum to 100. When specified, requests are sampled from this distribution instead of using isl/osl fields.","title":"Sequencedistribution"},"randomRangeRatio":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Sample ISL and OSL uniformly from a ratio-defined integer window around the configured means. Accepts a single float (applied to both ISL and OSL) or a JSON object {\"input\": 0.3, \"output\": 0.5} for independent values. Mutually exclusive with sequence_distribution. When a tokenizer is configured, the ISL mean is automatically reduced by tokenizer.num_special_tokens_to_add(pair=False) so --isl represents total server-side input tokens.","title":"Randomrangeratio"},"randomCorpusStyle":{"$ref":"#/$defs/RandomCorpusStyle","default":"vllm","description":"Sampling formula for random_range_ratio. vllm (default): symmetric window [floor(mean*(1-r)), ceil(mean*(1+r))], ratio in [0, 1). sglang: lower-bounded window [max(1, int(mean*r)), mean], ratio in [0, 1]."}},"title":"PromptConfig","type":"object"},"PromptCorpus":{"description":"Corpus used for synthetic prompt text generation.","enum":["sonnet","coding","random"],"title":"PromptCorpus","type":"string"},"PromptSelectionConfig":{"additionalProperties":false,"description":"Slim prompts block for file/public datasets (corpus selection only).","properties":{"corpus":{"anyOf":[{"$ref":"#/$defs/PromptCorpus"},{"type":"null"}],"default":null,"description":"Source corpus for synthesized prompt text. 'sonnet' uses Shakespeare sonnets. 'coding' uses realistic coding content. When unset, the active dataset loader's default applies. Honored only where content is synthesized (synthetic + hash/trace replay); verbatim loaders ignore it."}},"title":"PromptSelectionConfig","type":"object"},"PublicDataset":{"additionalProperties":false,"description":"Public dataset configuration.\n\nUses well-known public benchmarking datasets that are\nautomatically downloaded and processed by AIPerf.","properties":{"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Verbatim system prompt text, identical across every conversation. Emitted as a system-role message ahead of all turns. When the dataset already carries its own system message, this text is prepended to it rather than replacing it. Tokens are additive: --isl continues to size the generated user prompt only. Mutually exclusive with system_prompt_file and with prefix_prompts.shared_system_length/pool_size/length.","title":"Systemprompt"},"systemPromptFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to a UTF-8 text file holding the verbatim system prompt. Preferred over system_prompt for real production prompts, which are long enough that shell quoting mangles them. Read once at config-validation time so a missing or unreadable file fails at startup. Mutually exclusive with system_prompt.","title":"Systempromptfile"},"name":{"description":"Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.","minLength":1,"title":"Name","type":"string"},"type":{"const":"public","description":"Dataset type discriminator. Must be 'public'.","title":"Type","type":"string"},"dataset":{"$ref":"#/$defs/PublicDatasetType","description":"Pre-configured public dataset to download and use for benchmarking. Name of the HuggingFace public dataset enum (e.g. 'sharegpt', 'alpaca'). AIPerf automatically downloads and parses these datasets."},"entries":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Limit number of records to use from the dataset. If not specified, uses all available records.","title":"Entries","x-jinja2-supported":true},"randomSeed":{"anyOf":[{"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Random seed for deterministic sampling from the dataset. Overrides global random_seed for this dataset.","title":"Randomseed","x-jinja2-supported":true},"sampling":{"$ref":"#/$defs/DatasetSamplingStrategy","default":"sequential","description":"Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion."},"hfSubset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset subset/config name override (e.g. 'sharegpt4o'). Only applies for HuggingFace-backed public dataset loaders. Takes priority over the subset defined in the plugin registry.","title":"Hfsubset"},"filters":{"additionalProperties":{"type":"string"},"description":"Dataset-specific filters forwarded to public dataset loaders. Supported keys and values depend on the selected dataset.","title":"Filters","type":"object"},"hfWekaDataset":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace dataset repo for the generic Weka loader (e.g. `semianalysisai/cc-traces-weka-061526`). Only applies with `dataset: weka_hf`; setting it with any other public dataset is an error. Passing `--hf-weka-dataset` on the CLI auto-selects `--public-dataset weka_hf`, so the repo flag works on its own. Pinned Weka public dataset aliases keep their registry-defined repo names.","title":"Hfwekadataset"},"interTurnDelayCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Clamp per-turn replay delays to at most this many seconds for HF-backed Weka trace replay (mirror of ``FileDataset.inter_turn_delay_cap_seconds``). ``None`` disables the cap.","title":"Interturndelaycapseconds","x-jinja2-supported":true},"traceIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Hard ceiling (seconds) for observed runtime idle time within each AgentX trajectory tree for HF-backed Weka replay (mirror of ``FileDataset.trace_idle_gap_cap_seconds``). Dataset timestamps remain unchanged. Defaults to None (no per-trace runtime idle cap).","title":"Traceidlegapcapseconds","x-jinja2-supported":true},"ignoreTraceDelays":{"default":false,"description":"Strip per-turn timestamps and inter-turn delays from HF-backed Weka trace replay at load time (mirror of ``FileDataset.ignore_trace_delays``). Mutually exclusive with use_think_time_only.","title":"Ignoretracedelays","type":"boolean"},"useThinkTimeOnly":{"default":false,"description":"For HF-backed Weka replay, emit Turn.delay using only the recorded per-request ``think_time`` instead of the full inter-request delta (mirror of ``FileDataset.use_think_time_only``). Mutually exclusive with ignore_trace_delays.","title":"Usethinktimeonly","type":"boolean"},"maxContextLength":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum peak prompt+output context length (tokens) per conversation for HF-backed Weka replay; drops over-length traces at load using *recorded* lengths (mirror of ``FileDataset.max_context_length``). Rejected for non-Weka public datasets.","title":"Maxcontextlength","x-jinja2-supported":true},"cacheBust":{"$ref":"#/$defs/CacheBustConfig","description":"Per-conversation cache-bust marker injection for HF-backed Weka trace replay (mirror of ``FileDataset.cache_bust``). 'none' (default) disables it."},"prompts":{"anyOf":[{"$ref":"#/$defs/PromptSelectionConfig"},{"type":"null"}],"default":null,"description":"Prompt synthesis selection for this dataset. Set ``corpus`` to choose sonnet vs coding when content is synthesized (trace hash_id reconstruction). Verbatim formats ignore it."},"synthesis":{"anyOf":[{"$ref":"#/$defs/SynthesisConfig"},{"type":"null"}],"default":null,"description":"Trace synthesis/transformation configuration for HF-backed Weka trace replay (mirror of ``FileDataset.synthesis``). ``--max-isl``/``--max-osl`` cap the per-conversation input/output lengths of the replayed HF Weka traces; without this they applied only to file-based traces. ``None`` disables synthesis."},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Output sequence length to apply when records do not specify one (mirror of ``FileDataset.osl``). Can be a fixed integer or {mean, stddev} distribution. Per-record output lengths always take precedence.","title":"Osl"}},"required":["name","type","dataset"],"title":"PublicDataset","type":"object"},"PublicDatasetType":{"enum":["exgentic_v2","exgentic","sharegpt","aimo","mmstar","mmvu","vision_arena","llava_onevision","aimo_aime","aimo_numina_cot","aimo_numina_1_5","spec_bench","spec_al_gsm8k","spec_al_math500","spec_al_humaneval","spec_al_mbpp","spec_al_mtbench","instruct_coder","blazedit_5k","blazedit_10k","librispeech","voxpopuli","gigaspeech","ami","spgispeech","semianalysis_cc_traces_weka","semianalysis_cc_traces_weka_no_subagents","semianalysis_cc_traces_weka_with_subagents","semianalysis_cc_traces_weka_with_subagents_256k","semianalysis_cc_traces_weka_with_subagents_060226","semianalysis_cc_traces_weka_with_subagents_060226_256k","semianalysis_cc_traces_weka_with_subagents_060526","semianalysis_cc_traces_weka_with_subagents_060526_256k","semianalysis_cc_traces_weka_with_subagents_060826","semianalysis_cc_traces_weka_with_subagents_060826_256k","semianalysis_cc_traces_weka_061326","semianalysis_cc_traces_weka_061326_256k","semianalysis_cc_traces_weka_061526","semianalysis_cc_traces_weka_061526_256k","semianalysis_cc_traces_weka_062126","semianalysis_cc_traces_weka_062126_256k","weka_hf","speed_bench_qualitative","speed_bench_coding","speed_bench_humanities","speed_bench_math","speed_bench_multilingual","speed_bench_qa","speed_bench_rag","speed_bench_reasoning","speed_bench_roleplay","speed_bench_stem","speed_bench_summarization","speed_bench_writing","speed_bench_throughput_1k","speed_bench_throughput_1k_low_entropy","speed_bench_throughput_1k_mixed","speed_bench_throughput_1k_high_entropy","speed_bench_throughput_2k","speed_bench_throughput_2k_low_entropy","speed_bench_throughput_2k_mixed","speed_bench_throughput_2k_high_entropy","speed_bench_throughput_8k","speed_bench_throughput_8k_low_entropy","speed_bench_throughput_8k_mixed","speed_bench_throughput_8k_high_entropy","speed_bench_throughput_16k","speed_bench_throughput_16k_low_entropy","speed_bench_throughput_16k_mixed","speed_bench_throughput_16k_high_entropy","speed_bench_throughput_32k","speed_bench_throughput_32k_low_entropy","speed_bench_throughput_32k_mixed","speed_bench_throughput_32k_high_entropy","h_cua_perf"],"title":"PublicDatasetType","type":"string"},"RampConfig":{"title":"RampConfig","description":"Configuration for gradual value ramping.\n\nControls how a value (concurrency, rate, etc.) transitions from\nstart to target over time. Can be a duration (number in seconds or string like '30s', '5m', '2h') or {duration, strategy} object.","anyOf":[{"additionalProperties":false,"description":"Configuration for gradual value ramping.\n\nControls how a value (concurrency, rate, etc.) transitions from\nstart to target over time.","properties":{"duration":{"description":"Seconds to ramp from start to target value.","exclusiveMinimum":0.0,"title":"Duration","type":"number"},"strategy":{"$ref":"#/$defs/RampType","default":"linear","description":"Ramp curve shape: linear (constant rate), exponential (slow start, fast finish), poisson (stochastic with guaranteed completion)."}},"required":["duration"],"title":"RampConfig","type":"object"},{"type":"integer"},{"type":"number"},{"type":"string"}]},"RampType":{"enum":["exponential","linear","poisson"],"title":"RampType","type":"string"},"RandomCorpusStyle":{"description":"Benchmark style for RANDOM corpus generation.\n\nControls the full set of behaviors that vary between tools. Each style\nis a bundle of decisions that together reproduce the statistical\ndistribution of a specific benchmarking tool:\n\n.. list-table::\n :header-rows: 1\n :widths: 30 35 35\n\n * - Concern\n - VLLM\n - SGLANG\n * - Token pool\n - ``valid_token_ids`` (special tokens excluded via ``all_special_ids``)\n - ``all_token_ids`` (full ``range(vocab_size)``, no exclusion)\n * - BOS / special-token adjustment\n - ``max(0, mean - num_special_tokens)``, applied to the mean before\n the window bounds are computed\n - ``max(1, drawn - num_special_tokens)``, applied per-request after\n sampling so the window keeps its raw-mean shape\n * - ISL/OSL range formula\n - Symmetric: ``[floor(mean*(1-r)), ceil(mean*(1+r))]``\n - Lower-bounded: ``[max(1, int(mean*r)), mean]``\n * - RNG algorithm (preseed)\n - PCG64 via ``numpy.random.default_rng(seed)``\n - MT19937 via a private ``numpy.random.RandomState(seed)``\n * - RNG draw order (with preseed)\n - All ISLs → all OSLs → all offsets\n - All ISLs → all OSLs → all offsets (same order, different algorithm)\n * - Top-up RNG\n - Continues from the same ``default_rng`` stream (``_preseed_rng``)\n - Continues from the same ``RandomState`` stream (``_preseed_rng``)\n\nBoth styles are preseeded whenever the corpus is\n:attr:`PromptCorpus.RANDOM` — the composer gates on\n``isinstance(dist, RangeRatioDistribution)``, which the SGLANG subclass\nsatisfies. ``_corpus_rng`` is the fallback for the non-preseeded paths\n(non-RANDOM corpora, and draws past cache exhaustion), not a per-style\ndifference.\n\n.. note::\n The two styles subtract special tokens at different points on purpose,\n each following its own upstream. VLLM folds the count into the bounds;\n SGLANG shifts each drawn length, mirroring ``sample_random_requests``\n under ``return_text`` (the default, and what aiperf sends)::\n\n input_lens[i] = max(1, input_lens[i] - num_special_tokens)\n\n Both land the same wire ISL on the configured value; only the resulting\n window shape differs.","enum":["vllm","sglang"],"title":"RandomCorpusStyle","type":"string"},"RankingsConfig":{"additionalProperties":false,"description":"Configuration for rankings/reranking endpoint datasets.\n\nControls the generation of query-passage pairs for benchmarking\nreranking and ranking models. Each request contains one query\nand multiple passages to rank.","properties":{"passages":{"description":"Number of passages per ranking request. Can be a fixed integer or {mean, stddev} distribution. Higher values test ranking at scale but increase request payload size and processing time.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Passages","x-jinja2-supported":true},"passageTokens":{"description":"Token length for each passage in ranking requests. Can be a fixed integer or {mean, stddev} distribution. Passages are synthetically generated text. Longer passages increase input processing demands and request size.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Passagetokens","x-jinja2-supported":true},"queryTokens":{"description":"Token length for the query text in ranking requests. Can be a fixed integer or {mean, stddev} distribution. Each ranking request contains one query and multiple passages.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Querytokens","x-jinja2-supported":true}},"title":"RankingsConfig","type":"object"},"RateSeriesConfig":{"additionalProperties":false,"description":"Piecewise-linear request-rate schedule.","properties":{"path":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"JSON file path containing request-rate control points.","title":"Path"},"points":{"description":"Strictly increasing request-rate control points.","items":{"$ref":"#/$defs/RateSeriesPoint"},"title":"Points","type":"array","minItems":2}},"title":"RateSeriesConfig","type":"object","allOf":[{"oneOf":[{"required":["path"],"properties":{"path":{"not":{"type":"null"}}}},{"required":["points"],"properties":{"points":{"minItems":2}}}]}]},"RateSeriesPoint":{"additionalProperties":false,"description":"One request-rate control point.","properties":{"timeS":{"description":"Elapsed time in seconds from the start of the request-rate series, after any request-rate ramp completes.","title":"Times","oneOf":[{"description":"Elapsed time in seconds from the start of the request-rate series, after any request-rate ramp completes.","minimum":0.0,"title":"Times","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"qps":{"description":"Request rate in queries per second at this point.","title":"Qps","oneOf":[{"description":"Request rate in queries per second at this point.","exclusiveMinimum":0.0,"title":"Qps","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["timeS","qps"],"title":"RateSeriesPoint","type":"object"},"RequestContentType":{"description":"Content type for HTTP request body serialization.","enum":["application/json","multipart/form-data"],"title":"RequestContentType","type":"string"},"RuntimeConfig":{"additionalProperties":false,"description":"Runtime configuration for benchmark execution.","properties":{"ui":{"$ref":"#/$defs/UIType","default":"dashboard","description":"User interface mode. dashboard: rich interactive UI, simple: text progress, none: silent operation."},"workers":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Total worker processes across the deployment (the maximum, ramped up to from `workers_min`). null = auto-detect based on CPU cores. Distinct from `workers_per_pod` (per-Kubernetes-pod fan-out) and `workers_min` (lower bound for adaptive ramp).","title":"Workers","x-jinja2-supported":true},"recordProcessors":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Total parallel record processors across the deployment. null = auto-detect based on CPU cores. Distinct from `record_processors_per_pod` (per-Kubernetes-pod fan-out).","title":"Recordprocessors","x-jinja2-supported":true},"serviceRunType":{"$ref":"#/$defs/ServiceRunType","default":"multiprocessing","description":"Execution mode. multiprocessing: local multi-process (default for `aiperf profile`). kubernetes: distributed across pods. [operator-managed under AIPerfJob — the operator forces 'kubernetes' on the controller pod; do not set this in a CR spec.]"},"communication":{"anyOf":[{"discriminator":{"propertyName":"type","mapping":{"ipc":"#/$defs/IpcCommunicationConfig","tcp":"#/$defs/TcpCommunicationConfig","dual":"#/$defs/DualBindCommunicationConfig"}},"anyOf":[{"$ref":"#/$defs/IpcCommunicationConfig"},{"$ref":"#/$defs/TcpCommunicationConfig"},{"$ref":"#/$defs/DualBindCommunicationConfig"}]},{"type":"null"}],"default":null,"description":"Inter-process communication configuration. Defaults to IPC for single-machine operation.","title":"Communication"},"apiPort":{"anyOf":[{"maximum":65535,"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AIPerf API server port. Enables HTTP and WebSocket endpoints for real-time metrics and control.","title":"Apiport","x-jinja2-supported":true},"apiHost":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"AIPerf API server host. Requires api_port to be set.","title":"Apihost"},"datasetApiBaseUrl":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Base URL the operator injects into worker pods so they can fetch datasets from the controller-pod sidecar in Kubernetes mode. [operator-managed; do not set in a CR spec.]","title":"Datasetapibaseurl"},"workersPerPod":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Worker containers packed into each Kubernetes worker pod (Kubernetes mode only). The total number of worker containers cluster-wide is `workers` (or auto-detected); this knob controls how that total is fanned across pods. Ignored in multiprocessing mode.","title":"Workersperpod","x-jinja2-supported":true},"recordProcessorsPerPod":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Record-processor containers packed into each Kubernetes worker pod (Kubernetes mode only). Sibling of `workers_per_pod`; controls per-pod fan-out, not the cluster-wide total. Ignored in multiprocessing mode.","title":"Recordprocessorsperpod","x-jinja2-supported":true},"workersMin":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Lower bound on worker processes for the adaptive ramp. Distinct from `workers` (the upper bound / target total).","title":"Workersmin","x-jinja2-supported":true},"statsInterval":{"anyOf":[{"maximum":1000.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Interval in seconds between realtime stats publishes (dashboards and the per-tick log block). 0 disables the log block while dashboards continue to poll. Defaults to 5s under --ui dashboard, 30s otherwise. Overrides AIPERF_UI_REALTIME_METRICS_INTERVAL.","title":"Statsinterval","x-jinja2-supported":true}},"title":"RuntimeConfig","type":"object"},"SLAFilter":{"additionalProperties":false,"description":"SLA constraint applied to BO scoring or grid filtering.\n\nA trial is considered feasible iff ``stat(metric_tag) op threshold`` holds.\n``BayesianSearchPlanner`` consumes these for soft-penalty BO scoring and\n``write_search_history`` consumes them for lexicographic\nfeasibility-first best-result selection.\n\nLives in ``aiperf.config`` rather than ``aiperf.search_recipes`` because\n``AdaptiveSearchSweep`` carries a ``list[SLAFilter]`` field; the inverse\nlocation would force a circular import. Re-exported from\n``aiperf.search_recipes._base`` for ergonomic recipe authoring.\n\nExample: enforce p95 TTFT under 200 ms via\n``SLAFilter(metric_tag=\"time_to_first_token\", stat=\"p95\", op=\"lt\", threshold=200.0)``.","properties":{"metricTag":{"description":"Metric tag to filter on, e.g. 'time_to_first_token'. Must match a key in RunResult.summary_metrics produced by the run.","title":"Metrictag","type":"string"},"stat":{"default":"p95","description":"Statistic on the metric to compare against the threshold.","enum":["avg","p1","p5","p10","p25","p50","p75","p90","p95","p99","min","max"],"title":"Stat","type":"string"},"op":{"description":"Comparison operator. Filter passes when stat(metric) op threshold is true.","enum":["lt","le","gt","ge"],"title":"Op","type":"string"},"threshold":{"description":"Numeric threshold the metric statistic is compared against.","title":"Threshold","oneOf":[{"description":"Numeric threshold the metric statistic is compared against.","title":"Threshold","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["metricTag","op","threshold"],"title":"SLAFilter","type":"object"},"SLOTier":{"additionalProperties":false,"description":"Named group of SLA filters representing one service-level objective.\n\nDefines a single tier for multi-tier SLO boundary search. Each tier\ncontains one or more SLA filters that must ALL pass for the tier to be\nconsidered feasible at a given concurrency level.\n\nLives in the config layer (alongside SLAFilter) to avoid circular imports\nbetween config and orchestrator packages. Re-exported from\n``aiperf.orchestrator.search_planner.multi_tier_models`` for ergonomic\norchestrator usage.","properties":{"label":{"description":"Unique tier identifier for output artifacts","title":"Label","type":"string"},"filters":{"description":"SLA filters that must ALL pass for this tier to be feasible","items":{"$ref":"#/$defs/SLAFilter"},"minItems":1,"title":"Filters","type":"array"},"orderingRank":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Rank in detected monotonic ordering (0=strictest); None if unordered","title":"Orderingrank","x-jinja2-supported":true}},"required":["label","filters"],"title":"SLOTier","type":"object"},"SamplingDimension":{"additionalProperties":false,"description":"One dimension of a QMC sampling space.\n\nUsed by `SobolSweep` and `LatinHypercubeSweep`. Distinct from\n`SearchSpaceDimension` (BO) — same path/lo/hi grammar, but native\n`choices` support and explicit `scale` validation.\n\nPath is a dotted path of the form `phases.profiling.concurrency` —\nthe same grammar accepted by `_set_nested_value`.","properties":{"path":{"description":"Dotted-path into BenchmarkConfig.","title":"Path","type":"string"},"lo":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive lower bound (with hi).","title":"Lo","x-jinja2-supported":true},"hi":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Inclusive upper bound (with lo).","title":"Hi","x-jinja2-supported":true},"scale":{"default":"linear","description":"Mapping from unit cube to value.","enum":["linear","log"],"title":"Scale","type":"string"},"kind":{"default":"real","description":"Output type after mapping.","enum":["int","real"],"title":"Kind","type":"string"},"choices":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}],"default":null,"description":"Discrete value list. Mutually exclusive with (lo, hi).","title":"Choices"}},"required":["path"],"title":"SamplingDimension","type":"object"},"ScenarioSweep":{"additionalProperties":false,"description":"Scenario sweep - hand-picked configurations deep-merged with base.","properties":{"cooldownSeconds":{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaFilters":{"description":"SLA constraints applied at trial scoring / aggregation time.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Slafilters","type":"array"},"postProcess":{"anyOf":[{"$ref":"#/$defs/PostProcessSpec"},{"type":"null"}],"default":null,"description":"Optional post-aggregation handler emitted by a search recipe."},"recipeName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.","title":"Recipename"},"iterationOrder":{"$ref":"#/$defs/SweepMode","default":"repeated","description":"Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search."},"sameSeed":{"default":false,"description":"If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.","title":"Sameseed","type":"boolean"},"type":{"const":"scenarios","default":"scenarios","description":"Sweep type discriminator.","title":"Type","type":"string"},"runs":{"description":"List of scenario dicts to deep-merge with base config.","items":{"additionalProperties":true,"type":"object"},"minItems":1,"title":"Runs","type":"array"}},"required":["runs"],"title":"ScenarioSweep","type":"object"},"SearchPlannerType":{"enum":["bayesian","monotonic_sla","smooth_isotonic","optuna"],"title":"SearchPlannerType","type":"string"},"SearchSpaceDimension":{"additionalProperties":false,"description":"One dimension of the BO search space.\n\n`path` is a dotted path of the form `phases.profiling.concurrency` —\nthe same grammar accepted by `aiperf.config.sweep._set_nested_value`.","properties":{"path":{"description":"Dotted-path into BenchmarkConfig (e.g. 'phases.profiling.concurrency').","title":"Path","type":"string"},"lo":{"description":"Inclusive lower bound.","title":"Lo","oneOf":[{"description":"Inclusive lower bound.","title":"Lo","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"hi":{"description":"Inclusive upper bound.","title":"Hi","oneOf":[{"description":"Inclusive upper bound.","title":"Hi","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"kind":{"default":"real","description":"Dimension type. 'int' rounds planner suggestions to integers; 'real' keeps floats.","enum":["int","real"],"title":"Kind","type":"string"},"prior":{"default":"uniform","description":"Sampling prior for BO/Optuna planners. 'uniform' draws Sobol/random samples linearly across [lo, hi]; 'log-uniform' draws them log-spaced. Use 'log-uniform' when a parameter spans orders of magnitude (e.g. concurrency in [1, 1000]) so the initial-points phase covers low and high decades evenly. Requires lo > 0. Ignored by planners that do not draw initial points (monotonic, smooth_isotonic) where the search procedure is structurally adaptive rather than sampling-based.","enum":["uniform","log-uniform"],"title":"Prior","type":"string"}},"required":["path","lo","hi"],"title":"SearchSpaceDimension","type":"object"},"SequenceDistributionEntry":{"additionalProperties":false,"description":"Defines a single entry in an ISL/OSL probability distribution.\n\nAIPerf supports multi-modal token length distributions, allowing\ndifferent ISL/OSL combinations with relative frequencies for\nrealistic workload modeling.\n\nYAML Representation:\n sequence_distribution:\n - {isl: 128, osl: 64, probability: 40}\n - {isl: {mean: 512, stddev: 50}, osl: 256, probability: 35}\n - {isl: {mean: 2048, median: 1800}, osl: 512, probability: 25}","examples":[{"isl":128,"osl":64,"probability":40},{"isl":512,"isl_stddev":50,"osl":256,"osl_stddev":25,"probability":35}],"properties":{"isl":{"description":"Input sequence length (tokens). Can be a fixed integer, a {mean, stddev} distribution, or any distribution like {mean: 512, median: 400} for lognormal.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Isl","x-jinja2-supported":true},"islStddev":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Shorthand for ISL standard deviation. If provided when isl is an integer, creates a normal distribution. Cannot be used when isl is already a distribution dict.","title":"Islstddev","x-jinja2-supported":true},"osl":{"description":"Output sequence length (tokens). Can be a fixed integer, a {mean, stddev} distribution, or any typed distribution.","oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"title":"Osl","x-jinja2-supported":true},"oslStddev":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Shorthand for OSL standard deviation. If provided when osl is an integer, creates a normal distribution. Cannot be used when osl is already a distribution dict.","title":"Oslstddev","x-jinja2-supported":true},"probability":{"description":"Relative probability weight for this distribution bucket (0-100). Weights are normalized across all entries. Example: probability=40 means 40%% of requests use this ISL/OSL.","title":"Probability","oneOf":[{"description":"Relative probability weight for this distribution bucket (0-100). Weights are normalized across all entries. Example: probability=40 means 40%% of requests use this ISL/OSL.","maximum":100.0,"minimum":0.0,"title":"Probability","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["isl","osl","probability"],"title":"SequenceDistributionEntry","type":"object"},"ServerMetricsConfig":{"title":"ServerMetricsConfig","description":"Server metrics configuration for Prometheus scraping.\n\nCollects server-side operational metrics (queue depth, KV cache utilization,\nbatch sizes, GPU memory) from Prometheus endpoints exposed by inference servers\nlike vLLM, TensorRT-LLM, or Triton.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9090/metrics\"\n → ServerMetricsConfig(enabled=True, urls=[\"http://localhost:9090/metrics\"])\n - Singular url field: {url: \"...\"}\n → ServerMetricsConfig(urls=[\"...\"]) Can be a URL string or full config object.","anyOf":[{"additionalProperties":false,"description":"Server metrics configuration for Prometheus scraping.\n\nCollects server-side operational metrics (queue depth, KV cache utilization,\nbatch sizes, GPU memory) from Prometheus endpoints exposed by inference servers\nlike vLLM, TensorRT-LLM, or Triton.\n\nAccepts shorthand forms:\n - String URL: \"http://localhost:9090/metrics\"\n → ServerMetricsConfig(enabled=True, urls=[\"http://localhost:9090/metrics\"])\n - Singular url field: {url: \"...\"}\n → ServerMetricsConfig(urls=[\"...\"])","properties":{"enabled":{"default":true,"description":"Enable Prometheus metrics scraping. Set to false to disable.","title":"Enabled","type":"boolean"},"urls":{"description":"Prometheus metrics endpoint URLs to scrape. Typically the /metrics endpoint on inference servers.","items":{"type":"string"},"title":"Urls","type":"array"},"formats":{"description":"Export formats for scraped metrics. Options: json, csv, parquet, jsonl.","items":{"$ref":"#/$defs/ServerMetricsFormat"},"title":"Formats","type":"array"},"discovery":{"$ref":"#/$defs/ServerMetricsDiscoveryConfig","description":"Auto-discovery of Prometheus endpoints in Kubernetes. Discovers pods via Dynamo labels, Prometheus annotations, or custom label selectors."},"url":{"description":"Shorthand for 'urls'. Accepts a single value which will be converted to a list/dict. Prometheus metrics endpoint URLs to scrape. Typically the /metrics endpoint on inference servers.","oneOf":[{"type":"string"},{"description":"Prometheus metrics endpoint URLs to scrape. Typically the /metrics endpoint on inference servers.","items":{"type":"string"},"title":"Urls","type":"array"}],"x-singular-alias-of":"urls"}},"title":"ServerMetricsConfig","type":"object","x-kubernetes-preserve-unknown-fields":true},{"type":"string"}]},"ServerMetricsDiscoveryConfig":{"additionalProperties":false,"description":"Kubernetes-based auto-discovery of inference-server /metrics endpoints.\n\nWhen mode is 'auto' or 'kubernetes', queries the K8s API for pods that\nare recognizable inference servers (vLLM, SGLang, Triton Inference Server,\nTensorRT-LLM, NVIDIA Dynamo). Eligibility (any one is enough):\n1. Dynamo opt-in label: nvidia.com/metrics-enabled=true\n2. AIPerf opt-in annotation: aiperf.nvidia.com/metrics-paths=...\n3. A container image matching a known inference-server signature\n4. User-provided label_selector (server-side filter)\n\nThe broad ``prometheus.io/scrape=true`` annotation is intentionally NOT a\ntrigger: Loki, Grafana, kube-state-metrics, and many platform components\nset it without being inference servers. ``prometheus.io/{port,path,scheme}``\nare still honored to construct the scrape URL when an eligible pod sets them.","properties":{"mode":{"$ref":"#/$defs/ServerMetricsDiscoveryMode","default":"auto","description":"Discovery mode: 'auto' detects environment and tries K8s if in-cluster, 'kubernetes' forces K8s API discovery, 'disabled' uses only explicit URLs."},"labelSelector":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Kubernetes label selector for discovery. Example: 'app=vllm,env=prod'. Applied in addition to built-in Dynamo and Prometheus discovery.","title":"Labelselector"},"namespace":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Kubernetes namespace to search. If not specified, searches all namespaces.","title":"Namespace"}},"title":"ServerMetricsDiscoveryConfig","type":"object"},"ServerMetricsDiscoveryMode":{"description":"Mode for discovering server metrics endpoints in distributed environments.","enum":["auto","kubernetes","disabled"],"title":"ServerMetricsDiscoveryMode","type":"string"},"ServerMetricsFormat":{"description":"Format options for server metrics export.\n\nControls which output files are generated for server metrics data.\nDefault selection is JSON + CSV (JSONL excluded to avoid large files).","enum":["json","csv","jsonl","parquet"],"title":"ServerMetricsFormat","type":"string"},"ServiceRunType":{"enum":["multiprocessing"],"title":"ServiceRunType","type":"string"},"SobolSweep":{"additionalProperties":false,"description":"Sobol quasi-Monte-Carlo sweep — low-discrepancy joint coverage.\n\nNote on ``scramble=False``: scipy's raw Sobol sequence starts at the\norigin (the first row is exactly all-zeros), so the first variant\nwill land on ``(lo, lo, ...)`` for every dimension. With small\n``samples`` this produces degenerate corner-only coverage. Prefer\nthe default ``scramble=True`` (Owen-scrambled) unless you have a\nspecific reason to want raw Sobol points.","properties":{"cooldownSeconds":{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaFilters":{"description":"SLA constraints applied at trial scoring / aggregation time.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Slafilters","type":"array"},"postProcess":{"anyOf":[{"$ref":"#/$defs/PostProcessSpec"},{"type":"null"}],"default":null,"description":"Optional post-aggregation handler emitted by a search recipe."},"recipeName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.","title":"Recipename"},"iterationOrder":{"$ref":"#/$defs/SweepMode","default":"repeated","description":"Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search."},"sameSeed":{"default":false,"description":"If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.","title":"Sameseed","type":"boolean"},"samples":{"description":"Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.","title":"Samples","oneOf":[{"description":"Number of variations to draw. Capped at 2**20 (1,048,576) — scipy's Sobol engine fails at 2**30 anyway, and full materialization beyond ~1M variants risks orchestrator OOM.","maximum":1048576,"minimum":2,"title":"Samples","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"seed":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"RNG seed for reproducibility. Must be non-negative; scipy's QMC engines reject negative seeds with an opaque low-level error.","title":"Seed","x-jinja2-supported":true},"dimensions":{"description":"Dimensions to sample over.","items":{"$ref":"#/$defs/SamplingDimension"},"minItems":1,"title":"Dimensions","type":"array"},"labelFormat":{"default":"index","description":"Variation-label format. 'index' => '_NNNN' (short, sortable). 'kv' => 'k1=v1, k2=v2' (grid-style).","enum":["index","kv"],"title":"Labelformat","type":"string"},"type":{"const":"sobol","default":"sobol","description":"Sweep type discriminator.","title":"Type","type":"string"},"scramble":{"default":true,"description":"Owen-scrambled Sobol (recommended). False = raw Sobol sequence.","title":"Scramble","type":"boolean"}},"required":["samples","dimensions"],"title":"SobolSweep","type":"object"},"SweepMode":{"description":"Execution order for sweep + confidence composition.\n\nREPEATED: for trial in trials: for value in values\n All values tested per trial before moving to next trial.\n\nINDEPENDENT: for value in values: for trial in trials\n All trials completed per value before moving to next value.","enum":["independent","repeated"],"title":"SweepMode","type":"string"},"SynthesisConfig":{"additionalProperties":false,"description":"Configuration for trace synthesis/transformation.\n\nUsed with mooncake_trace format to transform production trace\ndata before replay. Allows scaling timestamps, token lengths,\nand radix tree structure.","properties":{"speedupRatio":{"default":1.0,"description":"Multiplier for timestamp scaling in synthesized traces. 1.0 = real-time, 2.0 = 2x faster, 0.5 = 2x slower.","title":"Speedupratio","oneOf":[{"default":1.0,"description":"Multiplier for timestamp scaling in synthesized traces. 1.0 = real-time, 2.0 = 2x faster, 0.5 = 2x slower.","exclusiveMinimum":0.0,"title":"Speedupratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"prefixLenMultiplier":{"default":1.0,"description":"Multiplier for core prefix branch lengths in the radix tree. 1.5 means prefix branches are 50%% longer.","title":"Prefixlenmultiplier","oneOf":[{"default":1.0,"description":"Multiplier for core prefix branch lengths in the radix tree. 1.5 means prefix branches are 50%% longer.","exclusiveMinimum":0.0,"title":"Prefixlenmultiplier","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"prefixRootMultiplier":{"default":1,"description":"Number of independent radix trees to distribute traces across. Higher values increase prefix diversity.","title":"Prefixrootmultiplier","oneOf":[{"default":1,"description":"Number of independent radix trees to distribute traces across. Higher values increase prefix diversity.","minimum":1,"title":"Prefixrootmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"promptLenMultiplier":{"default":1.0,"description":"Multiplier for leaf path (unique prompt) lengths. 2.0 means prompts are 2x longer.","title":"Promptlenmultiplier","oneOf":[{"default":1.0,"description":"Multiplier for leaf path (unique prompt) lengths. 2.0 means prompts are 2x longer.","exclusiveMinimum":0.0,"title":"Promptlenmultiplier","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"outputLenMultiplier":{"default":1.0,"description":"Multiplier for output lengths in synthesized traces.","title":"Outputlenmultiplier","oneOf":[{"default":1.0,"description":"Multiplier for output lengths in synthesized traces.","minimum":0.0,"title":"Outputlenmultiplier","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"maxIsl":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum input sequence length filter. Traces with input_length > max_isl are skipped entirely.","title":"Maxisl","x-jinja2-supported":true},"maxOsl":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum output sequence length cap. Traces with output_length > max_osl are capped to this value (not filtered).","title":"Maxosl","x-jinja2-supported":true},"allowDatasetWrap":{"default":false,"description":"Allow the finite weka/agentic trace pool to wrap (reuse distinct roots across concurrency lanes) when concurrency exceeds the number of eligible traces. Defaults to False: over-subscription fails unless this is explicitly enabled or an active cache-bust target already keeps repeated-trace traffic distinct.","title":"Allowdatasetwrap","type":"boolean"}},"title":"SynthesisConfig","type":"object"},"SyntheticDataset":{"additionalProperties":false,"description":"Synthetic dataset configuration.\n\nGenerates prompts programmatically based on token length\nspecifications. Ideal for controlled experiments.","properties":{"systemPrompt":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Verbatim system prompt text, identical across every conversation. Emitted as a system-role message ahead of all turns. When the dataset already carries its own system message, this text is prepended to it rather than replacing it. Tokens are additive: --isl continues to size the generated user prompt only. Mutually exclusive with system_prompt_file and with prefix_prompts.shared_system_length/pool_size/length.","title":"Systemprompt"},"systemPromptFile":{"anyOf":[{"format":"path","type":"string"},{"type":"null"}],"default":null,"description":"Path to a UTF-8 text file holding the verbatim system prompt. Preferred over system_prompt for real production prompts, which are long enough that shell quoting mangles them. Read once at config-validation time so a missing or unreadable file fails at startup. Mutually exclusive with system_prompt.","title":"Systempromptfile"},"name":{"description":"Dataset identifier — used in result file paths. Defaults to 'default' for the singular `dataset:` shorthand.","minLength":1,"title":"Name","type":"string"},"type":{"const":"synthetic","description":"Dataset type discriminator. Must be 'synthetic'.","title":"Type","type":"string"},"entries":{"default":100,"description":"Total number of unique entries to generate for the dataset. Each entry represents a unique prompt with sampled ISL/OSL. Entries are reused across conversations and turns according to the sampling strategy. Higher values provide more diversity.","title":"Entries","oneOf":[{"default":100,"description":"Total number of unique entries to generate for the dataset. Each entry represents a unique prompt with sampled ISL/OSL. Entries are reused across conversations and turns according to the sampling strategy. Higher values provide more diversity.","minimum":1,"title":"Entries","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"randomSeed":{"anyOf":[{"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Random seed for deterministic dataset generation. When set, makes synthetic prompts, sampling, and other random operations reproducible across runs. Essential for A/B testing and debugging. Overrides global random_seed for this dataset.","title":"Randomseed","x-jinja2-supported":true},"sampling":{"$ref":"#/$defs/DatasetSamplingStrategy","default":"sequential","description":"Strategy for selecting entries from dataset during benchmarking. sequential: iterate in order, wrapping to start after end. random: randomly sample with replacement (entries may repeat). shuffle: random permutation without replacement, re-shuffling after exhaustion."},"prompts":{"anyOf":[{"$ref":"#/$defs/PromptConfig"},{"type":"null"}],"default":null,"description":"Prompt/token length configuration specifying ISL, OSL, sequence distributions, and batch processing settings."},"prefixPrompts":{"anyOf":[{"$ref":"#/$defs/PrefixPromptConfig"},{"type":"null"}],"default":null,"description":"Shared prefix configuration for KV cache testing. Generates prefix prompts that are prepended to user prompts, simulating cached context scenarios."},"turns":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Number of request-response turns per conversation. Can be a fixed integer or {mean, stddev} distribution. Each turn consists of a user message and model response. Set to 1 for single-turn interactions. Multi-turn conversations enable testing of context retention and conversation history handling.","title":"Turns"},"turnDelay":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Delay in milliseconds between consecutive turns within a multi-turn conversation. Can be a fixed value or {mean, stddev} distribution. Simulates user think time between receiving a response and sending the next message. Only applies when turns > 1. Set to 0 for back-to-back turns.","title":"Turndelay"},"turnDelayRatio":{"default":1.0,"description":"Multiplier for scaling all turn delays. Applied after mean/stddev calculation: actual_delay = calculated_delay * ratio. Values < 1 speed up conversations, > 1 slow them down. Set to 0 to eliminate delays entirely.","title":"Turndelayratio","oneOf":[{"default":1.0,"description":"Multiplier for scaling all turn delays. Applied after mean/stddev calculation: actual_delay = calculated_delay * ratio. Values < 1 speed up conversations, > 1 slow them down. Set to 0 to eliminate delays entirely.","minimum":0.0,"title":"Turndelayratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"images":{"anyOf":[{"$ref":"#/$defs/ImageConfig"},{"type":"null"}],"default":null,"description":"Synthetic image configuration for multimodal vision-language testing."},"audio":{"anyOf":[{"$ref":"#/$defs/AudioConfig"},{"type":"null"}],"default":null,"description":"Synthetic audio configuration for multimodal speech/audio testing."},"video":{"anyOf":[{"$ref":"#/$defs/VideoConfig"},{"type":"null"}],"default":null,"description":"Synthetic video configuration for multimodal video understanding testing."},"rankings":{"anyOf":[{"$ref":"#/$defs/RankingsConfig"},{"type":"null"}],"default":null,"description":"Rankings/reranking configuration for generating query-passage pairs. Only relevant for rankings endpoint types."},"isl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Shorthand for prompts.isl; hoisted into prompts before validation.","title":"Isl"},"osl":{"anyOf":[{"oneOf":[{"$ref":"#/$defs/FixedDistribution"},{"$ref":"#/$defs/NormalDistribution"},{"$ref":"#/$defs/LogNormalDistribution"},{"$ref":"#/$defs/MultimodalDistribution"},{"$ref":"#/$defs/EmpiricalDistribution"}]},{"type":"null"}],"default":null,"description":"Shorthand for prompts.osl; hoisted into prompts before validation.","title":"Osl"}},"required":["name","type"],"title":"SyntheticDataset","type":"object"},"TcpCommunicationConfig":{"additionalProperties":false,"description":"TCP socket communication configuration.\n\nFor distributed deployments across machines.\nProvides detailed port configuration for all ZMQ proxies.","properties":{"type":{"const":"tcp","description":"Communication type. Must be 'tcp'.","title":"Type","type":"string"},"host":{"default":"127.0.0.1","description":"Host address for TCP communication. Use 0.0.0.0 to listen on all interfaces.","title":"Host","type":"string"},"recordsPort":{"default":5557,"description":"Port for records push/pull communication.","title":"Recordsport","oneOf":[{"default":5557,"description":"Port for records push/pull communication.","maximum":65535,"minimum":1,"title":"Recordsport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"creditRouterPort":{"default":5564,"description":"Port for credit router (ROUTER-DEALER).","title":"Creditrouterport","oneOf":[{"default":5564,"description":"Port for credit router (ROUTER-DEALER).","maximum":65535,"minimum":1,"title":"Creditrouterport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"controlPort":{"default":5667,"description":"Port for control channel (ROUTER-DEALER).","title":"Controlport","oneOf":[{"default":5667,"description":"Port for control channel (ROUTER-DEALER).","maximum":65535,"minimum":1,"title":"Controlport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"eventBusProxy":{"$ref":"#/$defs/TcpProxyConfig","description":"Event bus proxy ports (XPUB/XSUB)."},"datasetManagerProxy":{"$ref":"#/$defs/TcpProxyConfig","description":"Dataset manager proxy ports (DEALER/ROUTER)."},"rawInferenceProxy":{"$ref":"#/$defs/TcpProxyConfig","description":"Raw inference proxy ports (PUSH/PULL)."}},"required":["type"],"title":"TcpCommunicationConfig","type":"object"},"TcpProxyConfig":{"additionalProperties":false,"description":"TCP proxy port configuration for a single ZMQ proxy.","properties":{"frontendPort":{"description":"TCP port for proxy frontend.","title":"Frontendport","oneOf":[{"description":"TCP port for proxy frontend.","maximum":65535,"minimum":1,"title":"Frontendport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"backendPort":{"description":"TCP port for proxy backend.","title":"Backendport","oneOf":[{"description":"TCP port for proxy backend.","maximum":65535,"minimum":1,"title":"Backendport","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"required":["frontendPort","backendPort"],"title":"TcpProxyConfig","type":"object"},"TemplateConfig":{"additionalProperties":false,"description":"Configuration for custom template-based endpoints.\n\nWhen endpoint type is \"template\", this configures how requests\nare formatted and responses are parsed.","properties":{"body":{"description":"Jinja2 template string for request body. Variables: {{prompt}}, {{max_tokens}}, {{model}}, {{messages}}, etc.","title":"Body","type":"string"},"responseField":{"default":"text","description":"JSON path to extract response text from API response. Use dot notation for nested fields: 'choices.0.message.content'.","title":"Responsefield","type":"string"}},"required":["body"],"title":"TemplateConfig","type":"object"},"TimingMode":{"enum":["fixed_schedule","request_rate","adaptive_scale","user_centric_rate","agentic_replay"],"title":"TimingMode","type":"string"},"TokenizerConfig":{"additionalProperties":false,"description":"Tokenizer configuration for token counting and prompt generation.\n\nAIPerf uses a HuggingFace tokenizer for accurate token counting,\nwhich is essential for ISL/OSL enforcement and metrics calculation.","properties":{"name":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"HuggingFace tokenizer identifier, local filesystem path, or `builtin` for a zero-network-access tokenizer backed by tiktoken (o200k_base encoding). Should match the model's tokenizer for accurate token counts. If `--tokenizer` is not set and the model name looks like an obvious placeholder (e.g. `mock-model`, `test-model`, `fake-model`), AIPerf substitutes `builtin` automatically and emits a warning. Example: 'meta-llama/Llama-3.1-8B-Instruct'","title":"Name"},"revision":{"default":"main","description":"Model revision to use: branch name, tag, or commit hash. Use for version pinning to ensure reproducibility.","title":"Revision","type":"string"},"trustRemoteCode":{"default":false,"description":"Allow execution of custom tokenizer code from the repository. Required for some models but poses security risk. Only enable for trusted sources.","title":"Trustremotecode","type":"boolean"},"applyChatTemplate":{"default":false,"description":"Apply the HuggingFace tokenizer's chat template when counting input tokens. When enabled: synthetic ISL is compensated for chat-template wrapping (BOS, role headers, EOT, generation-prompt suffix) and the record processor reports ISL using `apply_chat_template(tokenize=True, add_generation_prompt=True)` for chat-shape payloads. When disabled (default), both paths use bare-text encoding, so reported ISL matches the prompt content the user asked for and ignores template overhead. Requires an HF tokenizer with a chat template configured; no-ops on tiktoken / un-templated models.","title":"Applychattemplate","type":"boolean"}},"title":"TokenizerConfig","type":"object"},"TokenizerOverride":{"additionalProperties":false,"description":"Per-model tokenizer override configuration.\n\nAllows specifying a different tokenizer for a specific model,\nuseful when models require specialized tokenization.","properties":{"name":{"description":"HuggingFace tokenizer identifier or local filesystem path.","title":"Name","type":"string"}},"required":["name"],"title":"TokenizerOverride","type":"object"},"TransportType":{"enum":["http"],"title":"TransportType","type":"string"},"UIType":{"enum":["dashboard","none","simple"],"title":"UIType","type":"string"},"URLSelectionStrategy":{"enum":["round_robin"],"title":"URLSelectionStrategy","type":"string"},"UserCentricPhase":{"additionalProperties":false,"description":"N concurrent users sharing a global request rate.\n\nRequires multi-turn conversations. Each user gets a proportional\nshare of the global ``rate``.","properties":{"adaptiveScale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]},"adaptiveSustainDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds to sustain load near the discovered adaptive scale boundary.","title":"Adaptivesustainduration","x-jinja2-supported":true},"adaptiveAssessmentPeriod":{"anyOf":[{"minimum":1.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Duration in seconds for each adaptive scale SLA assessment window.","title":"Adaptiveassessmentperiod","x-jinja2-supported":true},"adaptiveMinCompletedRequests":{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","title":"Adaptivemincompletedrequests","oneOf":[{"default":1,"description":"Minimum completed requests needed before an adaptive SLA window can make a decision.","minimum":1,"title":"Adaptivemincompletedrequests","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlVariable":{"default":"concurrency","description":"Named adaptive control variable.","enum":["concurrency","prefill_concurrency","request_rate","users"],"title":"Adaptivecontrolvariable","type":"string"},"adaptiveControlMin":{"default":1,"description":"Minimum adaptive control value.","title":"Adaptivecontrolmin","oneOf":[{"default":1,"description":"Minimum adaptive control value.","exclusiveMinimum":0,"title":"Adaptivecontrolmin","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveControlMax":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Maximum adaptive control value. Inferred from the phase target when omitted.","title":"Adaptivecontrolmax","x-jinja2-supported":true},"adaptiveScaleStrategyType":{"const":"ramp_until_fail","default":"ramp_until_fail","description":"Adaptive scale controller strategy. v1 supports ramp_until_fail.","title":"Adaptivescalestrategytype","type":"string"},"adaptiveScaleStepPolicy":{"default":"sla_margin","description":"Adaptive scale increase policy. sla_margin uses normalized SLA margin to choose larger steps when far from the boundary; fixed_percent_step uses a fixed percentage of the current control value.","enum":["sla_margin","fixed_percent_step"],"title":"Adaptivescalesteppolicy","type":"string"},"adaptiveScaleBaseStep":{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","title":"Adaptivescalebasestep","oneOf":[{"default":10,"description":"Minimum adaptive scale step for SLA-margin policy.","minimum":1,"title":"Adaptivescalebasestep","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleMaxStepMultiplier":{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","title":"Adaptivescalemaxstepmultiplier","oneOf":[{"default":4,"description":"Maximum base-step multiplier for SLA-margin policy.","minimum":1,"title":"Adaptivescalemaxstepmultiplier","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptiveScaleStepPercent":{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","title":"Adaptivescalesteppercent","oneOf":[{"default":25.0,"description":"Percent of current concurrency used by fixed-percent adaptive scaling.","exclusiveMinimum":0,"title":"Adaptivescalesteppercent","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"name":{"description":"Unique workflow label for this phase, such as 'baseline_traffic', 'cancellation_stress', or 'recovery_traffic'. This is distinct from phase kind: multiple phases may share kind='profiling' while each keeps a different name. Used in logs, status, sweep targeting, artifact paths, and result file naming. Must be a strict identifier: letters, numbers, underscores, and hyphens; must start with a letter or underscore.","pattern":"^[A-Za-z_][A-Za-z0-9_-]*$","title":"Name","type":"string"},"kind":{"anyOf":[{"enum":["warmup","profiling"],"type":"string"},{"type":"null"}],"default":null,"description":"Semantic runtime role for the phase. Only 'warmup' and 'profiling' are valid kinds because the credit/results pipeline distinguishes those two roles. This field is nullable only as an input compatibility bridge: legacy canonical names ('warmup' and 'profiling') infer kind during normalization, while validated phases always carry a concrete kind.","title":"Kind"},"type":{"const":"user_centric","description":"N users sharing a global request rate.","title":"Type","type":"string"},"excludeFromResults":{"default":false,"description":"Exclude this phase's metrics from final results. Forced by phase kind: kind='warmup' is always excluded, kind='profiling' is always included. Explicitly setting this field to a value inconsistent with the phase kind is rejected.","title":"Excludefromresults","type":"boolean"},"requests":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many requests sent (must be >= 1).","title":"Requests","x-jinja2-supported":true},"duration":{"anyOf":[{"type":"string","pattern":"^(?:\\d+(?:\\.\\d+)?\\s*(?:[sS]|[sS][eE][cC]|[mM]|[mM][iI][nN]|[hH]|[hH][rR]|[hH][oO][uU][rR])?|[iI][nN][fF](?:[iI][nN][iI][tT][yY])?)$","description":"Duration string (e.g., '30s', '5M', '2h', 'inf')."},{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this time elapsed (must be > 0). Supports: 300, '5m', '2h'. Accepts number (seconds) or string like '30s', '5m', '2h'.","gt":0,"title":"Duration","x-jinja2-supported":true},"sessions":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Stop after this many sessions completed (must be >= 1).","title":"Sessions","x-jinja2-supported":true},"concurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent in-flight requests (must be >= 1). For concurrency type: primary control. For rate types: acts as a cap.","title":"Concurrency","x-jinja2-supported":true},"concurrencyRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"prefillConcurrency":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Max concurrent requests in prefill stage (must be >= 1). Limits requests before first token received.","title":"Prefillconcurrency","x-jinja2-supported":true},"prefillRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp prefill_concurrency from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"gracePeriod":{"anyOf":[{"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Seconds to wait for in-flight requests after duration expires (must be >= 0). Requires 'duration' to be set. Supports: 30, '30s', '2m'.","ge":0,"title":"Graceperiod","x-jinja2-supported":true},"cancellation":{"anyOf":[{"$ref":"#/$defs/CancellationConfig"},{"type":"null"}],"default":null,"description":"Request cancellation testing configuration."},"sla":{"description":"SLA filters evaluated by adaptive load controllers. Accepts the canonical list form or compact metric/stat/op mapping.","title":"Sla","anyOf":[{"description":"SLA filters evaluated by adaptive load controllers.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Sla","type":"array"},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"seamless":{"default":false,"description":"Start this phase immediately when previous phase stops, without waiting for in-flight requests to complete. Cannot be True for the first phase.","title":"Seamless","type":"boolean"},"timingMode":{"anyOf":[{"$ref":"#/$defs/TimingMode"},{"type":"null"}],"default":null,"description":"Explicit timing-strategy override for this phase. When set, it WINS over the timing mode derived from ``type`` in ``aiperf.timing.config._phase_timing_mode`` (read via ``_is_agentic_replay`` and ``_build_profiling_config``). This is how a benchmark-scenario invariant-lock stamps AGENTIC_REPLAY onto the profiling phase(s): the scenario validator sets ``phase.timing_mode = TimingMode.AGENTIC_REPLAY`` and the credit pipeline then routes the phase through the agentic-replay strategy. Leave None for normal/dag_jsonl runs so the phase-type mapping applies (REQUEST_RATE / FIXED_SCHEDULE / USER_CENTRIC_RATE). Distinct from the sweep ``scenarios`` strategy."},"failedRequestThreshold":{"anyOf":[{"maximum":1.0,"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Abort the run early when (failed_records / total_records) exceeds this ratio. Default None disables the check. Only PROFILING-phase records count toward the ratio. A grace floor of max(concurrency, 10) records must accumulate before the check is armed, so a single early failure cannot kill the run. When the threshold is exceeded a ProfileCancelCommand is broadcast: in-flight requests drain via the normal cancel path, partial results are still aggregated, and the run exits non-zero. Pairs with the AGENTIC_REPLAY context-overflow drop in record_processor_service so the rate measures real failures only.","title":"Failedrequestthreshold","x-jinja2-supported":true},"trajectoryStartMinRatio":{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","title":"Trajectorystartminratio","oneOf":[{"default":0.25,"description":"AGENTIC_REPLAY only: lower bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). Sampled per trajectory at trajectory-build time; deterministic given --random-seed.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartminratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"trajectoryStartMaxRatio":{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","title":"Trajectorystartmaxratio","oneOf":[{"default":0.75,"description":"AGENTIC_REPLAY only: upper bound (inclusive) on the random start position within each trajectory, expressed as a fraction of the trace's recorded wall-clock duration (timestamped traces) or its total turn count (legacy timestamp-less traces). For the timestamp-less path the effective per-trace ceiling is min(int(max_ratio * n), n - 2) so at least one profile turn remains after warmup.","maximum":1.0,"minimum":0.0,"title":"Trajectorystartmaxratio","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"burstPhaseStarts":{"default":false,"description":"AGENTIC_REPLAY only: collapse the WARMUP-start and PROFILING-start dispatches into synchronized bursts instead of preserving recorded spacing. By default (False), WARMUP requests are aligned globally so every trajectory reaches t* together. PROFILING subtracts one phase-wide minimum from every first request offset: the earliest request starts immediately and all other trajectories retain their recorded spacing. Pass --burst-phase-starts to subtract a separate minimum per lane, making every lane start immediately. Subsequent inter-turn delays are timing-faithful in either mode.","title":"Burstphasestarts","type":"boolean"},"systemIdleGapCapSeconds":{"anyOf":[{"minimum":0.0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: maximum time in seconds the replay may remain globally idle while future requests are scheduled. When no requests are in flight or ready, all pending request timers shift earlier by the same amount so the next request arrives within this limit. Per-trace timing, timer order, and relative spacing are otherwise preserved. None disables the global idle cap.","title":"Systemidlegapcapseconds","x-jinja2-supported":true},"agenticCacheWarmupDuration":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: additional cache-pressure warmup duration in seconds. After the normal snapshot warmup drains, AIPerf continues the live trajectories without recorded idle delays and with one-token outputs for this long, then drains and resumes profiling from the resulting trajectory state. Read off the profiling phase by ``timing.config._build_agentic_warmup_config``. Mutually exclusive with warmup_requests_per_lane. None disables it.","title":"Agenticcachewarmupduration","x-jinja2-supported":true},"warmupRequestsPerLane":{"anyOf":[{"exclusiveMinimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: deterministic cache-pressure warmup request budget per concurrency lane, additional to mandatory snapshot primers. The total warmup request target is the primer count plus this value multiplied by the number of live trajectory lanes. Mutually exclusive with agentic_cache_warmup_duration.","title":"Warmuprequestsperlane","x-jinja2-supported":true},"agenticWarmupGracePeriod":{"anyOf":[{"minimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"AGENTIC_REPLAY only: grace period in seconds the auto-synthesized warmup barrier waits for in-flight priming requests after the warmup burst sends. Read off the profiling phase by ``timing.config._build_agentic_warmup_config`` (the agentic warmup is not a user-declared phase, so it does not inherit ``--warmup-grace-period``, which requires ``--warmup-duration``). None waits indefinitely (the agentic warmup must complete every primed trajectory before profiling starts).","title":"Agenticwarmupgraceperiod","x-jinja2-supported":true},"rate":{"anyOf":[{"exclusiveMinimum":0,"type":"number"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Target request rate in requests per second. Required unless rate_series is set.","title":"Rate","x-jinja2-supported":true},"rateRamp":{"anyOf":[{"$ref":"#/$defs/RampConfig"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Ramp rate from lower value. Can be number (seconds) or {duration, strategy}.","x-jinja2-supported":true},"users":{"description":"Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.","title":"Users","oneOf":[{"description":"Number of simulated concurrent users (must be >= 1). Requests distributed across users to achieve global rate.","minimum":1,"title":"Users","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"adaptive_scale":{"description":"Enable single-run adaptive scale control for this phase. Accepts either a boolean or a nested adaptive-scale configuration block.","title":"Adaptivescale","anyOf":[{"default":false,"description":"Enable single-run adaptive scale control for this phase.","title":"Adaptivescale","type":"boolean"},{"$ref":"#/$defs/AdaptiveScaleNestedBlock"}]}},"required":["name","type","users","rate"],"title":"UserCentricPhase","type":"object","allOf":[{"required":["rate"],"properties":{"rate":{"not":{"type":"null"}}}}]},"UserFile":{"description":"One user-declared output file rendered into the run directory before benchmark start.\n\nPath is relative to the run directory; subdirectories are allowed; absolute\npaths and any segment equal to '..' are rejected. Content is rendered with\njinja2 against a documented context (variables: + system-injected names).","properties":{"path":{"description":"Output path relative to the run directory. Subdirectories allowed. Absolute paths and any segment equal to '..' are rejected.","title":"Path","type":"string"},"format":{"anyOf":[{"enum":["json","yaml","text"],"type":"string"},{"type":"null"}],"default":null,"description":"Serialization format. If omitted: 'text' when content is a string, 'json' otherwise.","title":"Format"},"content":{"description":"Templated value. Structured (dict/list/scalar) for json/yaml; string for text. Jinja2 expressions in any string leaf are rendered with the user_files context.","title":"Content"}},"required":["path","content"],"title":"UserFile","type":"object"},"VideoAudioCodec":{"description":"Audio codecs for embedding audio in synthetic video files.","enum":["aac","libvorbis","libopus"],"title":"VideoAudioCodec","type":"string"},"VideoAudioConfig":{"additionalProperties":false,"description":"Configuration for embedding an audio track in synthetic video files.","properties":{"sampleRate":{"default":44.1,"description":"Audio sample rate in kHz for the embedded audio track. Common values: 8 (telephony), 16 (speech), 44.1 (CD quality), 48 (professional). Higher sample rates increase audio fidelity and file size.","title":"Samplerate","oneOf":[{"default":44.1,"description":"Audio sample rate in kHz for the embedded audio track. Common values: 8 (telephony), 16 (speech), 44.1 (CD quality), 48 (professional). Higher sample rates increase audio fidelity and file size.","maximum":96.0,"minimum":8.0,"title":"Samplerate","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"channels":{"default":0,"description":"Number of audio channels to embed in generated video files. 0 = disabled (no audio track, default), 1 = mono, 2 = stereo. When set to 1 or 2, a Gaussian noise audio track matching the video duration is muxed into each video via FFmpeg.","title":"Channels","oneOf":[{"default":0,"description":"Number of audio channels to embed in generated video files. 0 = disabled (no audio track, default), 1 = mono, 2 = stereo. When set to 1 or 2, a Gaussian noise audio track matching the video duration is muxed into each video via FFmpeg.","maximum":2,"minimum":0,"title":"Channels","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"codec":{"anyOf":[{"$ref":"#/$defs/VideoAudioCodec"},{"type":"null"}],"default":null,"description":"Audio codec for the embedded audio track. If not specified, auto-selects based on video format: libopus for MP4, libvorbis for WebM. Options: libvorbis, libopus, aac. The AIPerf container ships only libvorbis and libopus; aac requires an FFmpeg build that includes an AAC encoder. libopus always encodes at 48 kHz, so any --video-audio-sample-rate is resampled during muxing."},"depth":{"description":"Audio bit depth for the embedded audio track. Supported values: 8, 16, 24, or 32 bits. Higher bit depths provide greater dynamic range but increase file size.","default":16,"oneOf":[{"default":16,"description":"Audio bit depth for the embedded audio track. Supported values: 8, 16, 24, or 32 bits. Higher bit depths provide greater dynamic range but increase file size.","enum":[8,16,24,32],"title":"Depth","type":"integer"},{"type":"string","enum":["8","16","24","32"]},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true}},"title":"VideoAudioConfig","type":"object"},"VideoConfig":{"additionalProperties":false,"description":"Configuration for synthetic video generation in multimodal datasets.\n\nControls the generation of synthetic videos for video understanding\nmodel benchmarking. Requires FFmpeg for video generation.","properties":{"batchSize":{"default":0,"description":"Number of video files to include in each multimodal request. Supported with chat endpoint type for video understanding models. Set to 0 to disable video inputs. Higher batch sizes significantly increase request payload size.","title":"Batchsize","oneOf":[{"default":0,"description":"Number of video files to include in each multimodal request. Supported with chat endpoint type for video understanding models. Set to 0 to disable video inputs. Higher batch sizes significantly increase request payload size.","minimum":0,"title":"Batchsize","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"duration":{"default":1.0,"description":"Duration in seconds for each generated video clip. Combined with fps, determines total frame count (frames = duration * fps). Longer durations increase file size and processing time. Typical values: 1-10 seconds for testing.","title":"Duration","oneOf":[{"default":1.0,"description":"Duration in seconds for each generated video clip. Combined with fps, determines total frame count (frames = duration * fps). Longer durations increase file size and processing time. Typical values: 1-10 seconds for testing.","exclusiveMinimum":0.0,"title":"Duration","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"fps":{"default":4,"description":"Frames per second for generated video. Higher FPS creates smoother video but increases frame count and file size. Common values: 4 (minimal, recommended for Cosmos models), 24 (cinematic), 30 (standard), 60 (high frame rate). Total frames = duration * fps.","title":"Fps","oneOf":[{"default":4,"description":"Frames per second for generated video. Higher FPS creates smoother video but increases frame count and file size. Common values: 4 (minimal, recommended for Cosmos models), 24 (cinematic), 30 (standard), 60 (high frame rate). Total frames = duration * fps.","minimum":1,"title":"Fps","type":"integer"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"width":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Video frame width in pixels. Determines video resolution and file size. Common values: 640 (SD), 1280 (HD), 1920 (Full HD). If not specified, uses codec/format defaults.","title":"Width","x-jinja2-supported":true},"height":{"anyOf":[{"minimum":1,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Video frame height in pixels. Combined with width determines aspect ratio and total pixel count per frame. Common values: 480 (SD), 720 (HD), 1080 (Full HD). If not specified, uses codec/format defaults.","title":"Height","x-jinja2-supported":true},"format":{"$ref":"#/$defs/VideoFormat","default":"webm","description":"Container format for generated video files. webm: VP9 codec, BSD-licensed, recommended for open-source workflows. mp4: widely compatible container, VP9 by default. Format affects compatibility, file size, and encoding options."},"codec":{"default":"libvpx-vp9","description":"Video codec for encoding. Common options: libvpx-vp9 (CPU, BSD-licensed, default), libvpx (CPU, BSD-licensed, VP8). Any codec the local FFmpeg supports can be used, but the AIPerf container ships a minimal FFmpeg build limited to VP8/VP9 video with Vorbis/Opus audio; codecs such as libx264 or h264_nvenc require an FFmpeg build that includes them.","title":"Codec","type":"string"},"synthType":{"$ref":"#/$defs/VideoSynthType","default":"moving_shapes","description":"Algorithm for generating synthetic video content. Different types produce different visual patterns for testing. Content doesn't affect semantic meaning but may impact encoding efficiency and file size."},"audio":{"$ref":"#/$defs/VideoAudioConfig","description":"Audio track configuration for embedding audio in generated videos."}},"title":"VideoConfig","type":"object"},"VideoFormat":{"description":"Video container formats for synthetic video generation.","enum":["mp4","webm"],"title":"VideoFormat","type":"string"},"VideoSynthType":{"enum":["moving_shapes","grid_clock","noise"],"title":"VideoSynthType","type":"string"},"WandbConfig":{"additionalProperties":false,"dependentRequired":{"entity":["project"],"runName":["project"],"tags":["project"]},"description":"Weights & Biases run-upload configuration.","properties":{"project":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Weights & Biases project name. Setting this enables wandb export.","title":"Project"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Weights & Biases entity (team or user). Defaults to the API key's default entity.","title":"Entity"},"runName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Weights & Biases run name.","title":"Runname"},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"default":null,"description":"Additional Weights & Biases run tags to attach on upload.","title":"Tags"}},"title":"WandbConfig","type":"object"},"ZipSweep":{"additionalProperties":false,"description":"Zip sweep — parameters paired element-wise (lockstep).\n\nLike ``GridSweep`` but uses ``zip(strict=True)`` instead of Cartesian\nproduct. All parameter lists must have identical length. Use when you\nwant N runs each setting a coordinated tuple of fields, without the\nNxM blow-up of grid. Canonical use case: paired ISL/OSL.","properties":{"cooldownSeconds":{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","title":"Cooldownseconds","oneOf":[{"default":0.0,"description":"Cooldown between sweep variations (or BO iterations).","minimum":0,"title":"Cooldownseconds","type":"number"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"x-jinja2-supported":true},"slaFilters":{"description":"SLA constraints applied at trial scoring / aggregation time.","items":{"$ref":"#/$defs/SLAFilter"},"title":"Slafilters","type":"array"},"postProcess":{"anyOf":[{"$ref":"#/$defs/PostProcessSpec"},{"type":"null"}],"default":null,"description":"Optional post-aggregation handler emitted by a search recipe."},"recipeName":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Source recipe name (audit trail when expanded via --search-recipe). Used by the live Pareto plot path to look up the recipe's pareto_axes after expansion has stripped the originating --search-recipe flag.","title":"Recipename"},"iterationOrder":{"$ref":"#/$defs/SweepMode","default":"repeated","description":"Trial/variation iteration order. 'repeated': trials outer, variations inner. 'independent': variations outer, trials inner. Inapplicable to adaptive_search."},"sameSeed":{"default":false,"description":"If True, every variation reuses the envelope's random_seed. If False, each variation derives `random_seed + variation.index`.","title":"Sameseed","type":"boolean"},"type":{"const":"zip","default":"zip","description":"Sweep type discriminator.","title":"Type","type":"string"},"parameters":{"additionalProperties":{"items":{},"type":"array"},"description":"Parameters to sweep in lockstep: dot-notation path -> list of values. All lists must have equal length.","minProperties":1,"title":"Parameters","type":"object"}},"required":["parameters"],"title":"ZipSweep","type":"object"},"AdaptiveScaleNestedBlock":{"type":"object","additionalProperties":false,"properties":{"enabled":{"anyOf":[{"type":"boolean"},{"type":"string","enum":["true","false","yes","no","on","off","1","0"]}]},"control":{"type":"object","additionalProperties":false,"properties":{"variable":{"type":"string","enum":["concurrency","prefill_concurrency","request_rate","users"]},"min":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"max":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]}}},"strategy":{"type":"object","additionalProperties":false,"properties":{"type":{"type":"string","enum":["ramp_until_fail"]},"stepPolicy":{"type":"string","enum":["sla_margin","fixed_percent_step"]},"step_policy":{"type":"string","enum":["sla_margin","fixed_percent_step"]},"baseStep":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"base_step":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"maxStepMultiplier":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"max_step_multiplier":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"stepPercent":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"step_percent":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]}}},"sla":{"anyOf":[{"type":"array","items":{"$ref":"#/$defs/SLAFilter"}},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"controlVariable":{"type":"string","enum":["concurrency","prefill_concurrency","request_rate","users"]},"control_variable":{"type":"string","enum":["concurrency","prefill_concurrency","request_rate","users"]},"controlMin":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"control_min":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"controlMax":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"control_max":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"minConcurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"min_concurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"maxConcurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"max_concurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"window":{"anyOf":[{"type":"integer","minimum":1},{"type":"number","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"assessmentPeriod":{"anyOf":[{"type":"integer","minimum":1},{"type":"number","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"assessment_period":{"anyOf":[{"type":"integer","minimum":1},{"type":"number","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"minCompletedRequests":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"min_completed_requests":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"sustainDuration":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"sustain_duration":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]}}},"DisabledAdaptiveScaleNestedBlock":{"type":"object","additionalProperties":false,"properties":{"enabled":{"anyOf":[{"const":false},{"type":"string","enum":["false","no","off","0"]}]},"control":{"type":"object","additionalProperties":false,"properties":{"variable":{"type":"string","enum":["concurrency","prefill_concurrency","request_rate","users"]},"min":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"max":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]}}},"strategy":{"type":"object","additionalProperties":false,"properties":{"type":{"type":"string","enum":["ramp_until_fail"]},"stepPolicy":{"type":"string","enum":["sla_margin","fixed_percent_step"]},"step_policy":{"type":"string","enum":["sla_margin","fixed_percent_step"]},"baseStep":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"base_step":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"maxStepMultiplier":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"max_step_multiplier":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"stepPercent":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"step_percent":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]}}},"sla":{"anyOf":[{"type":"array","items":{"$ref":"#/$defs/SLAFilter"}},{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":["number","string"]}}},"description":"Compact SLA mapping: metric -> stat -> operator -> threshold."}]},"controlVariable":{"type":"string","enum":["concurrency","prefill_concurrency","request_rate","users"]},"control_variable":{"type":"string","enum":["concurrency","prefill_concurrency","request_rate","users"]},"controlMin":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"control_min":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"controlMax":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"control_max":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"minConcurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"min_concurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"maxConcurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"max_concurrency":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"window":{"anyOf":[{"type":"integer","minimum":1},{"type":"number","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"assessmentPeriod":{"anyOf":[{"type":"integer","minimum":1},{"type":"number","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"assessment_period":{"anyOf":[{"type":"integer","minimum":1},{"type":"number","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"minCompletedRequests":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"min_completed_requests":{"oneOf":[{"type":"integer","minimum":1},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"sustainDuration":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]},"sustain_duration":{"anyOf":[{"type":"integer","exclusiveMinimum":0},{"type":"number","exclusiveMinimum":0},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}]}},"required":["enabled"]}},"additionalProperties":false,"description":"JSON Schema for AIPerf YAML configuration files. This schema enables IDE autocompletion and validation for AIPerf benchmark configurations. Schema version: 2.0.0","properties":{"schemaVersion":{"const":"2.0","default":"2.0","description":"AIPerf config schema version. Pinned to '2.0' for the envelope/body shape (envelope keys at top level, swept body under ``benchmark``).","title":"Schemaversion","type":"string"},"benchmark":{"$ref":"#/$defs/BenchmarkConfig","description":"Benchmark workload (the swept body)."},"sweep":{"anyOf":[{"discriminator":{"propertyName":"type","mapping":{"grid":"#/$defs/GridSweep","zip":"#/$defs/ZipSweep","scenarios":"#/$defs/ScenarioSweep","adaptive_search":"#/$defs/AdaptiveSearchSweep","sobol":"#/$defs/SobolSweep","latin_hypercube":"#/$defs/LatinHypercubeSweep"}},"anyOf":[{"$ref":"#/$defs/GridSweep"},{"$ref":"#/$defs/ZipSweep"},{"$ref":"#/$defs/ScenarioSweep"},{"$ref":"#/$defs/AdaptiveSearchSweep"},{"$ref":"#/$defs/SobolSweep"},{"$ref":"#/$defs/LatinHypercubeSweep"}]},{"type":"null"}],"default":null,"description":"Sweep configuration for parameter exploration. Supports grid (Cartesian product), zip (lock-stepped axes), scenarios (hand-picked), sobol / latin_hypercube (QMC sampling), and adaptive_search (Bayesian / monotonic outer-loop) strategies.","title":"Sweep"},"multiRun":{"$ref":"#/$defs/MultiRunConfig","description":"Multi-run benchmarking configuration for statistical reporting. When num_runs > 1, executes multiple runs and computes aggregate statistics."},"plot":{"default":null,"description":"Plot configuration for this run/sweep. When set, ``~/.aiperf/plot_config.yaml`` is ignored. May be a bare-string path to a plot YAML (resolved relative to the AIPerf config file's directory, or absolute), or an inline mapping mirroring ``default_plot_config.yaml``. Presence implies ``artifacts.auto_plot=True`` unless explicitly set False.","title":"Plot"},"variables":{"additionalProperties":true,"description":"User-defined values exposed to Jinja2 in `{{ ... }}` expressions during config load. Cross-variation: scenario `runs[i].variables:` deep-merge over this base. Preserved on the resolved config so run-time renderers can resolve them again.","title":"Variables","type":"object"},"randomSeed":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"},{"type":"string","pattern":".*\\{\\{.*\\}\\}.*","description":"Jinja2 template (e.g., '{{ variable }}')."},{"type":"string","pattern":".*\\$\\{[A-Za-z_][A-Za-z0-9_]*(?::[^}]*)?\\}.*","description":"Environment variable (e.g., '${VAR}' or '${VAR:default}')."}],"default":null,"description":"Global random seed for reproducibility. Base seed for per-variation derivation in sweep mode (variation N gets base + N). Must be non-negative; numpy and scipy reject negative seeds.","title":"Randomseed","x-jinja2-supported":true},"noSweepTable":{"default":false,"description":"Suppress the per-cell streaming sweep table during multi-variation sweeps. Auto-suppressed when stdout is non-interactive, when the dashboard UI is active, or for single-cell sweeps.","title":"Nosweeptable","type":"boolean"}},"required":["benchmark"],"title":"AIPerf Configuration Schema","type":"object","$comment":"Auto-generated by tools/generate_config_schema.py. Do not edit manually.","x-taplo":{"initKeys":["benchmark","sweep","multiRun","variables","randomSeed"]}} From 88c20fbfeadb4dc7aa83d6b7ed8090485e3dfde2 Mon Sep 17 00:00:00 2001 From: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:24:17 -0700 Subject: [PATCH 086/119] fix: preserve terminal benchmark failures across the message bus (#1434) Signed-off-by: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> (cherry picked from commit cbd78cdbcb9e10e5d8f3d70aa7d957a9e2b29c1e) The upstream tests for this change depend on controller and records-manager code from #1325, which is not in 1.0.x. --- src/aiperf/common/models/record_models.py | 2 +- src/aiperf/controller/system_controller.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 41d7e33362..742e6212c8 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -453,7 +453,7 @@ class ProfileResults(AIPerfBaseModel): """The results of a profile run.""" records: list[MetricResult] | None = Field( - ..., description="The records of the profile results" + default=None, description="The records of the profile results" ) warmup_records: list[MetricResult] | None = Field( default=None, diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index 244277694b..1ea08ec83f 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -670,7 +670,7 @@ async def _on_process_records_result_message( self.trace_or_debug( lambda: f"Received profile results message: {message}", lambda: ( - f"Received profile results message: {len(message.results.results.records) if message.results.results else 0} records" + f"Received profile results message: {len(message.results.results.records or []) if message.results.results else 0} records" ), ) if message.results.errors: From 96d94f4d4f51313304867238f832747d58668976 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:41:38 -0500 Subject: [PATCH 087/119] fix(config): drop control-hook routing from the --config resolver 1.0.x does not carry the reset_kv_cache and server_profiler control hooks (#1163), which #1280's resolver change assumes. --- src/aiperf/config/flags/resolver.py | 15 +------ .../config/test_config_telemetry_overrides.py | 45 ------------------- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/src/aiperf/config/flags/resolver.py b/src/aiperf/config/flags/resolver.py index 5d5eadc576..87d36e37f6 100644 --- a/src/aiperf/config/flags/resolver.py +++ b/src/aiperf/config/flags/resolver.py @@ -411,11 +411,7 @@ def _apply_endpoint_overrides(out: dict[str, Any], cli: CLIConfig) -> None: ``--model-names`` lives on the CLIConfig endpoint section but maps to the ``models.items`` block on AIPerfConfig; everything else stays on ``endpoint``. """ - from aiperf.config.flags._converter_endpoint import ( - _ENDPOINT_FIELD_MAP, - _maybe_build_reset_kv_cache, - _maybe_build_server_profiler, - ) + from aiperf.config.flags._converter_endpoint import _ENDPOINT_FIELD_MAP ep_set = cli.model_fields_set & ENDPOINT_FIELDS if not ep_set: @@ -426,15 +422,6 @@ def _apply_endpoint_overrides(out: dict[str, Any], cli: CLIConfig) -> None: for cli_field, aiperf_key in _ENDPOINT_FIELD_MAP.items(): if cli_field in ep_set: endpoint[aiperf_key] = getattr(cli, cli_field) - # The probe sub-blocks have builders the CLI-only path already uses; this - # path simply never called them. They emit only fields the user set, so a - # YAML block keeps whatever the flags do not mention. - for key, built in ( - ("reset_kv_cache", _maybe_build_reset_kv_cache(cli)), - ("server_profiler", _maybe_build_server_profiler(cli)), - ): - if built is not None: - endpoint[key] = built if endpoint: out["endpoint"] = endpoint if "model_names" in ep_set and cli.model_names: diff --git a/tests/unit/config/test_config_telemetry_overrides.py b/tests/unit/config/test_config_telemetry_overrides.py index 438d09c815..0e31f44ca3 100644 --- a/tests/unit/config/test_config_telemetry_overrides.py +++ b/tests/unit/config/test_config_telemetry_overrides.py @@ -204,48 +204,3 @@ def test_otel_secondary_flag_without_any_url_still_errors(base_yaml: Path) -> No # --------------------------------------------------------------------------- # Endpoint probes: reset-kv-cache and server-profiler # --------------------------------------------------------------------------- - - -def test_reset_kv_cache_routes(base_yaml: Path) -> None: - resolved = resolve_config(cli(reset_kv_cache=True), base_yaml) - assert resolved.benchmark.endpoint.reset_kv_cache is not None - - -def test_reset_kv_cache_path_routes(base_yaml: Path) -> None: - resolved = resolve_config( - cli(reset_kv_cache=True, reset_kv_cache_path="/reset"), base_yaml - ) - assert resolved.benchmark.endpoint.reset_kv_cache.path == "/reset" - - -def test_server_profiler_routes(base_yaml: Path) -> None: - resolved = resolve_config( - cli(server_profiler=True, server_profiler_start_path="/start"), base_yaml - ) - assert resolved.benchmark.endpoint.server_profiler.start_path == "/start" - - -def test_probe_flag_overrides_yaml_block(base_yaml: Path, tmp_path: Path) -> None: - """A YAML-supplied probe block must lose to the explicit flag, field-wise.""" - cfg = tmp_path / "with_probe.yaml" - cfg.write_text( - base_yaml.read_text() - .replace( - " dataset:", - " endpoint2_placeholder: 0\n dataset:", - ) - .replace(" endpoint2_placeholder: 0\n", "") - ) - cfg.write_text( - base_yaml.read_text().replace( - " url: http://localhost:8000", - " url: http://localhost:8000\n" - " reset_kv_cache:\n" - " path: /from-yaml\n" - " timeout_seconds: 30", - ) - ) - resolved = resolve_config(cli(reset_kv_cache_path="/from-cli"), cfg) - assert resolved.benchmark.endpoint.reset_kv_cache.path == "/from-cli" - # The field the user did not pass survives. - assert resolved.benchmark.endpoint.reset_kv_cache.timeout_seconds == 30 From e63c15d443fb5395ba2ca0d1ec4dff96db591094 Mon Sep 17 00:00:00 2001 From: Navjot Singh Date: Mon, 21 Sep 2026 13:19:48 -0400 Subject: [PATCH 088/119] feat(dataset): enable live assistant responses for Mooncake messages (#1400) Signed-off-by: Navjot Singh Signed-off-by: navjot10 (cherry picked from commit ab7c8ee87c77b4848f853d7b38dc2c2f0b796157) --- .../benchmark-modes/timing-modes-reference.md | 2 + docs/benchmark-modes/trace-replay.md | 37 +- docs/reference/conversation-context-mode.md | 14 +- docs/tutorials/raw-payload-replay.md | 2 + src/aiperf/common/enums/__init__.py | 4 + src/aiperf/common/enums/dataset_enums.py | 14 + src/aiperf/dataset/loader/models.py | 33 +- src/aiperf/dataset/loader/mooncake_trace.py | 148 ++++- src/aiperf/dataset/loader/raw_payload.py | 9 +- .../dataset/test_mooncake_message_deltas.py | 168 ++++++ .../test_mooncake_message_deltas.py | 306 ++++++++++ .../dataset/loader/test_mooncake_detection.py | 96 ++++ .../loader/test_mooncake_message_deltas.py | 544 ++++++++++++++++++ .../loader/test_mooncake_recorded_warnings.py | 170 ++++++ 14 files changed, 1526 insertions(+), 21 deletions(-) create mode 100644 src/aiperf/common/enums/dataset_enums.py create mode 100644 tests/component_integration/dataset/test_mooncake_message_deltas.py create mode 100644 tests/integration/test_mooncake_message_deltas.py create mode 100644 tests/unit/dataset/loader/test_mooncake_detection.py create mode 100644 tests/unit/dataset/loader/test_mooncake_message_deltas.py create mode 100644 tests/unit/dataset/loader/test_mooncake_recorded_warnings.py diff --git a/docs/benchmark-modes/timing-modes-reference.md b/docs/benchmark-modes/timing-modes-reference.md index 0e90eceec6..7b4dff10d6 100644 --- a/docs/benchmark-modes/timing-modes-reference.md +++ b/docs/benchmark-modes/timing-modes-reference.md @@ -233,6 +233,8 @@ aiperf profile --url localhost:8000 --model llama \ --fixed-schedule-end-offset 120000 ``` +Time window filtering is rejected at load time when the trace contains [`"assistant_responses": "live"`](trace-replay.md#live-responses-for-structured-messages) entries: cropping individual rows would silently truncate the accumulated conversation history. Construct a trace file of complete sessions instead. + ### Using `--user-centric-rate` (KV Cache Benchmarking) Per-user rate limiting for KV cache benchmarking. Each user has a consistent gap between their turns. diff --git a/docs/benchmark-modes/trace-replay.md b/docs/benchmark-modes/trace-replay.md index 647184e22f..598dc0bb43 100644 --- a/docs/benchmark-modes/trace-replay.md +++ b/docs/benchmark-modes/trace-replay.md @@ -130,16 +130,45 @@ or row-order conversion beyond the storage format is required. ## Using Pre-formatted Messages -Instead of synthetic prompts generated from `input_length` and `hash_ids`, you can provide an OpenAI-compatible `messages` array directly per trace entry. This is useful for replaying captured conversations (e.g., coding agent sessions) with exact prompt content. +Instead of synthetic prompts generated from `input_length` and `hash_ids`, you can provide an OpenAI-compatible `messages` array directly per trace entry. This is useful for replaying captured conversations (e.g., coding agent sessions) with exact prompt content. Pass `--custom-dataset-type mooncake_trace` when profiling these files so their structured-message rows are interpreted as Mooncake traces rather than another chat dataset format. -Each entry's `messages` field contains the full conversation history up to that point. In multi-turn sessions, later entries include prior turns so the server receives the complete context: +By default, each entry's `messages` field contains the full conversation history up to that point. In multi-turn sessions, later entries include prior turns so the server receives the complete context: ```json {"session_id": "sess-1", "messages": [{"role": "user", "content": "Hello"}], "output_length": 50, "timestamp": 0} {"session_id": "sess-1", "messages": [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi!"}, {"role": "user", "content": "How are you?"}], "output_length": 30, "timestamp": 2000} ``` -The `messages` field is mutually exclusive with `input_length` and `text_input`. When set, the messages array is sent directly to the API payload, bypassing prompt synthesis entirely. The model's actual response is not carried forward between turns -- each turn uses its pre-defined messages. +The `messages` field is mutually exclusive with `input_length` and `text_input`. When set, the messages array is sent directly to the API payload, bypassing prompt synthesis entirely. By default (`"assistant_responses": "recorded"`), the model's actual response is not carried forward between turns -- each turn uses its pre-defined messages. + +### Live Responses for Structured Messages + +Plain-text multi-turn Mooncake sessions (`text_input` or synthesized `input_length`, including `hash_ids`) already retain live assistant responses by default. Structured `messages` rows default to recorded responses because each row is assumed to contain its complete history. Set `"assistant_responses": "live"` on every `messages` entry in a session to retain the model's generated responses with structured messages too ([`deltas_without_responses` context mode](../reference/conversation-context-mode.md)). + +With `live`, rows must be incremental: the first entry carries the initial history, which may include recorded assistant messages from before the replay begins; each later entry carries only the new messages for that turn. AIPerf inserts the live assistant response between entries, including text and streamed `tool_calls` with reassembled `id`, `function.name`, and `function.arguments` fields. Do not repeat preceding history or recorded substitutes for these newly generated replies in later rows: + +```json +{"session_id": "sess-1", "assistant_responses": "live", "messages": [{"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}], "output_length": 512, "timestamp": 0} +{"session_id": "sess-1", "assistant_responses": "live", "messages": [{"role": "user", "content": "How are you?"}], "output_length": 512, "delay": 2000} +``` + +Request 2 is then `[system, user "Hello", assistant , user "How are you?"]`. + +Both settings send complete message arrays to the chat endpoint. Both can produce growing input lengths: `recorded` uses the histories authored in the trace, while `live` builds the history from replies generated during this run. + +Behavior and constraints: + +- **One setting per session.** Omitting `assistant_responses` (or setting `"recorded"`) keeps the existing verbatim replay for `messages`. A session mixing `"live"` and `"recorded"` entries -- including entries that omit the field and default to `"recorded"` -- is rejected at load time. Repeat `"live"` on every row rather than relying on first-row inheritance. This catches accidental omissions and lets independent sessions in one file use different settings, without a run-wide flag. Values are case-insensitive. +- **Suspicious omissions produce a warning.** For a multi-row `messages` session that omits `assistant_responses` everywhere, AIPerf warns if no row contains an assistant message or a later row does not preserve the preceding row's messages as a prefix. Replay still uses `recorded`. Set `"recorded"` explicitly to acknowledge intentional verbatim replay, or use `"live"` on every row for incremental replay. +- **Pin a compatible runner.** Use an AIPerf build or version that supports `assistant_responses` wherever these traces run. Older readers may silently ignore this field and replay each row as a complete recorded history; validation in newer readers cannot protect runs using those older builds. +- **`messages` only.** Set `assistant_responses` only for `messages` rows. `"live"` is rejected for `payload`, `text_input`, and synthesized (`input_length`) entries; plain-text and synthesized sessions already retain live responses through their existing context mode. +- **Timing is unchanged.** `timestamp`/`delay` values are preserved with millisecond scheduling accuracy, exactly as with recorded responses. +- **No offset cropping.** `--fixed-schedule-start-offset` / `--fixed-schedule-end-offset` drop individual rows before sessions are assembled, which would silently remove the initial history or preceding turns that later rows depend on. If the trace file contains any `"assistant_responses": "live"` entry, configuring either offset fails at load time. Recorded-response and synthesized traces keep the existing offset filtering. To replay a subset, construct a trace file containing only the complete sessions you want instead of cropping with offsets. +- **`output_length` is a ceiling, not a forced length.** It maps to the request's max-completion-tokens cap (e.g. keep your usual 8192 ceiling); the model stops naturally at EOS. Because live replies are threaded into later requests, the actual context length grows with whatever the model generated and can diverge from the recorded transcript's token counts. +- **New tool calls use the live IDs.** Assistant `tool_calls` captured during the run carry the IDs the server actually generated. A later `role: "tool"` delta must reference a `tool_call_id` the live model emitted for the conversation to remain coherent; AIPerf performs no remapping from recorded IDs to live IDs, so recorded tool-result deltas can diverge from what the live model called. AIPerf never executes tools -- tool-result content comes only from your dataset deltas. +- **Failed turns can leave gaps.** If a call fails before producing replayable assistant content, no assistant message is added, but its input messages remain in session history. If replay continues, later tool-result deltas may have no matching assistant tool call. Interrupted streams can leave partial assistant content in history. +- **`extra` cannot override the assembled history.** A per-entry or endpoint-global extra containing `messages` or `input` is rejected for sessions using live responses. These fields carry the assembled conversation for Chat Completions and the Responses API, respectively, and an extra input would overwrite it. +- **Sticky transport is not KV affinity.** Multi-turn sessions reuse a sticky connection/correlation ID so all turns hit the same backend, but that does not guarantee the server keeps the session's KV cache on the same GPU -- prefix-cache hit rates depend on the serving stack. ### Tool Definitions @@ -149,7 +178,7 @@ When replaying conversations that involve tool use (function calling), include t {"messages": [{"role": "user", "content": "What's the weather?"}], "tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}}}], "output_length": 50, "timestamp": 0} ``` -The `tools` field is only valid when `messages` is provided. It is injected directly into the API payload as the `tools` parameter. +The `tools` field is only valid when `messages` is provided. It is injected directly into the API payload as the `tools` parameter. With `"assistant_responses": "live"`, later turns inherit the most recent `tools` declaration (typically the first turn's); a turn that redeclares `tools` replaces them for that turn onward. ## Per-Request Extra Inputs diff --git a/docs/reference/conversation-context-mode.md b/docs/reference/conversation-context-mode.md index f8f5d6a73a..e8fdd93d2f 100644 --- a/docs/reference/conversation-context-mode.md +++ b/docs/reference/conversation-context-mode.md @@ -11,13 +11,13 @@ Conversation context mode controls how prior turns are accumulated when building Two dimensions determine the mode: - **Turn format**: `DELTAS` (incremental per-turn content) vs `MESSAGE_ARRAY` (each turn carries its complete message list) -- **Response inclusion**: `WITH_RESPONSES` (pre-canned assistant turns in dataset) vs `WITHOUT_RESPONSES` (only user content; live responses captured at runtime) +- **Response inclusion**: `WITH_RESPONSES` (assistant replies supplied by the dataset) vs `WITHOUT_RESPONSES` (replies generated during the run are captured and retained) ## Modes ### `deltas_without_responses` -Standard multi-turn chat. Each dataset turn is a user-only delta. AIPerf accumulates turns and threads live inference responses into the history. +Standard multi-turn chat. Each dataset turn supplies new input messages. AIPerf accumulates turns and threads live inference responses into the history. The initial history may include recorded assistant messages from before replay begins; those messages are preserved too. **Dataset:** ``` @@ -42,7 +42,10 @@ Default for: - Synthetic datasets - Multi-turn JSONL - ShareGPT -- Mooncake traces with `hash_ids` +- Mooncake traces with `text_input` or synthesized `input_length` (including `hash_ids`) + +Opt-in for: +- Mooncake traces with `messages` arrays marked `"assistant_responses": "live"` on every row in the session (see [Trace Replay](../benchmark-modes/trace-replay.md#live-responses-for-structured-messages)). The first entry carries the initial history, including any earlier recorded assistant messages; each later entry carries only the new messages for that turn. AIPerf inserts live assistant responses (text and tool calls) between entries. Use `--custom-dataset-type mooncake_trace` to select this format explicitly. ### `deltas_with_responses` @@ -91,10 +94,7 @@ Request 3: sends Turn 3 as-is Each turn is sent exactly as it appears in the dataset. Default for: - -- Mooncake traces with pre-built `messages` arrays -- Bailian traces: a follow-up row's `input_length` and `hash_ids` already cover the history the service retained plus the new text, so each row is sent as its own generated prompt - +- Mooncake traces with pre-built `messages` arrays (the default `"assistant_responses": "recorded"`) ### `message_array_without_responses` diff --git a/docs/tutorials/raw-payload-replay.md b/docs/tutorials/raw-payload-replay.md index 1239d13d97..f03a969b12 100644 --- a/docs/tutorials/raw-payload-replay.md +++ b/docs/tutorials/raw-payload-replay.md @@ -167,6 +167,8 @@ Raw payload conversations use `message_array_with_responses` [context mode](../r This is the correct behavior because raw payloads already contain the complete message history for each turn. In directory mode, each line in a session file should include all prior context needed for that point in the conversation (see the multi-turn examples above). +To retain live model responses (including tool calls) between structured-message turns, use `--custom-dataset-type mooncake_trace` with `"assistant_responses": "live"` on every row in the session. The first row carries the initial history, which may include earlier recorded assistant messages; later rows carry only new messages. See [Live Responses for Structured Messages](../benchmark-modes/trace-replay.md#live-responses-for-structured-messages). + --- ## Tips diff --git a/src/aiperf/common/enums/__init__.py b/src/aiperf/common/enums/__init__.py index bbd8fa3e3e..e359433e3f 100644 --- a/src/aiperf/common/enums/__init__.py +++ b/src/aiperf/common/enums/__init__.py @@ -12,6 +12,9 @@ make_result_producer_capability, parse_result_producer_capability, ) +from aiperf.common.enums.dataset_enums import ( + AssistantResponseMode as AssistantResponseMode, +) from aiperf.common.enums.enums import ( AIPerfLogLevel, AudioFormat, @@ -96,6 +99,7 @@ __all__ = [ "AIPerfLogLevel", + "AssistantResponseMode", "AudioFormat", "BaseMetricUnit", "BaseMetricUnitInfo", diff --git a/src/aiperf/common/enums/dataset_enums.py b/src/aiperf/common/enums/dataset_enums.py new file mode 100644 index 0000000000..fd42461aa1 --- /dev/null +++ b/src/aiperf/common/enums/dataset_enums.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from aiperf.common.enums.base_enums import CaseInsensitiveStrEnum + + +class AssistantResponseMode(CaseInsensitiveStrEnum): + """Source of assistant responses in Mooncake message conversations.""" + + RECORDED = "recorded" + """Replay each row's complete authored history.""" + + LIVE = "live" + """Accumulate message deltas and responses generated during the run.""" diff --git a/src/aiperf/dataset/loader/models.py b/src/aiperf/dataset/loader/models.py index 4399139e5b..e0a5cb2fc5 100644 --- a/src/aiperf/dataset/loader/models.py +++ b/src/aiperf/dataset/loader/models.py @@ -5,6 +5,7 @@ from pydantic import ConfigDict, Field, model_validator +from aiperf.common.enums import AssistantResponseMode from aiperf.common.models import AIPerfBaseModel, Audio, Image, Text, Video from aiperf.plugin.enums import CustomDatasetType @@ -265,6 +266,7 @@ class MooncakeTrace(AIPerfBaseModel): - With messages: {"messages": [{"role": "user", "content": "Hello"}], "output_length": 4} - With payload: {"payload": {"prompt": "Hello", "max_tokens": 50}, "timestamp": 1000} - With timestamp and hash ID: {"timestamp": 1000, "input_length": 10, "hash_ids": [123]} + - Delta messages: {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "Hi"}]} """ type: Literal[CustomDatasetType.MOONCAKE_TRACE] = CustomDatasetType.MOONCAKE_TRACE @@ -290,6 +292,16 @@ class MooncakeTrace(AIPerfBaseModel): "to the transport. Bypasses all endpoint formatting. Cannot be " "combined with other input modes.", ) + assistant_responses: AssistantResponseMode = Field( + default=AssistantResponseMode.RECORDED, + description="Source of assistant responses for 'messages' input. " + "'recorded' (default): replay each row's complete authored history. " + "'live': preserve the first row's initial history (which may include recorded " + "assistant messages), then append subsequent rows' new messages and the " + "responses generated during this run, including tool calls. Live mode " + "requires delta-shaped rows after the first row, not repeated full histories. " + "Declare the same value on every row in a session. Live is only valid with 'messages'.", + ) # Optional fields output_length: int | None = Field( @@ -347,7 +359,12 @@ def validate_payload(self) -> "MooncakeTrace": @model_validator(mode="after") def validate_messages(self) -> "MooncakeTrace": - """Validate the messages and tools field structure.""" + """Validate messages/tools structure and assistant_responses combinations.""" + if "message_mode" in (self.model_extra or {}): + raise ValueError( + "'message_mode' has been renamed to 'assistant_responses'; " + "use 'live' or 'recorded'" + ) if self.tools is not None: if self.messages is None: raise ValueError("'tools' is only allowed when 'messages' is provided") @@ -355,6 +372,20 @@ def validate_messages(self) -> "MooncakeTrace": raise ValueError("'tools' must be a non-empty list") if self.messages is not None: validate_chat_messages(self.messages) + if self.assistant_responses == AssistantResponseMode.LIVE: + if self.messages is None: + raise ValueError( + "assistant_responses='live' is only supported with 'messages'; " + "'payload', 'text_input', and 'input_length' (synthesized) entries cannot be delta turns" + ) + if self.extra is not None and ( + history_keys := {"messages", "input"}.intersection(self.extra) + ): + raise ValueError( + "assistant_responses='live': 'extra' must not contain " + f"conversation input keys {sorted(history_keys)}; " + "it would overwrite the assembled conversation history at dispatch time" + ) return self diff --git a/src/aiperf/dataset/loader/mooncake_trace.py b/src/aiperf/dataset/loader/mooncake_trace.py index fc9925a231..dd3e605c63 100644 --- a/src/aiperf/dataset/loader/mooncake_trace.py +++ b/src/aiperf/dataset/loader/mooncake_trace.py @@ -2,12 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 from collections import defaultdict +from itertools import pairwise from pathlib import Path from typing import Any from pydantic import ValidationError -from aiperf.common.enums import ConversationContextMode +from aiperf.common.enums import AssistantResponseMode, ConversationContextMode from aiperf.common.models import Turn from aiperf.dataset.loader.base_loader import LoaderProbeData from aiperf.dataset.loader.base_trace_loader import BaseTraceDatasetLoader @@ -15,6 +16,31 @@ from aiperf.dataset.loader.speed_bench import is_speed_bench_row +def _looks_like_implicit_message_deltas(traces: list[MooncakeTrace]) -> bool: + """Identify likely incremental input without overriding recorded replay.""" + if len(traces) < 2: + return False + message_rows: list[list[dict[str, Any]]] = [] + for trace in traces: + if trace.messages is None or "assistant_responses" in trace.model_fields_set: + return False + message_rows.append(trace.messages) + + if not any( + message["role"] == "assistant" + for messages in message_rows + for message in messages + ): + return True + return any( + len(current) < len(previous) + or any( + before != after for before, after in zip(previous, current, strict=False) + ) + for previous, current in pairwise(message_rows) + ) + + class MooncakeTraceDatasetLoader(BaseTraceDatasetLoader[MooncakeTrace]): """A dataset loader that loads Mooncake trace data from a file. @@ -35,6 +61,14 @@ class MooncakeTraceDatasetLoader(BaseTraceDatasetLoader[MooncakeTrace]): {"session_id": "abc-123", "input_length": 300, "output_length": 40}, {"session_id": "abc-123", "delay": 2, "input_length": 150, "output_length": 20} ``` + + Message-delta version (opt-in via assistant_responses; the first entry carries the + initial history, later entries carry only the new messages for that turn, + and live assistant responses are threaded into the history between them) + ```json + {"session_id": "abc-123", "assistant_responses": "live", "messages": [{"role": "user", "content": "Hi"}]}, + {"session_id": "abc-123", "assistant_responses": "live", "delay": 2, "messages": [{"role": "user", "content": "More"}]} + ``` """ @classmethod @@ -43,13 +77,21 @@ def can_load( ) -> bool: """Check if this loader can handle the given data format. - For mooncake trace data, simply validate the data against the MooncakeTrace model. - This will handle all of the validation logic for the different input combinations. + Explicit response-mode fields identify Mooncake before validation, so + malformed values cannot fall through to raw-payload replay. Other rows + are probed against the MooncakeTrace schema. """ if data is None: return False if is_speed_bench_row(data): return False + if isinstance(data, dict) and { + "assistant_responses", + "message_mode", + }.intersection(data): + # Explicit Mooncake fields must reach schema validation, even when + # their values are invalid, instead of selecting another loader. + return True try: MooncakeTrace.model_validate(data) @@ -64,6 +106,36 @@ def can_load( def _parse_trace(self, record: dict) -> MooncakeTrace: return MooncakeTrace.model_validate(record) + def _preprocess_trace(self, trace: MooncakeTrace) -> None: + """Fail closed: timestamp-offset cropping is rejected for delta entries. + + ``start_offset``/``end_offset`` drop individual timestamped rows in + :meth:`BaseTraceDatasetLoader.load_dataset` before sessions are + grouped, but each ``assistant_responses='live'`` row depends on every + earlier row in its session (the initial history plus the live + assistant turns threaded between them), so cropping would silently + truncate the assembled conversation. This hook runs before + :meth:`BaseTraceDatasetLoader._filter_and_cap_trace`, so the load + aborts on the first delta row whenever either offset is configured -- + even if offsets already dropped earlier (e.g. history-mode) rows in + the file, nothing partially filtered is ever returned. Both offsets + default to ``None``, so ordinary runs are unaffected; history-mode and + synthesized traces keep the existing offset filtering. + """ + if trace.assistant_responses == AssistantResponseMode.LIVE and ( + self._start_offset is not None or self._end_offset is not None + ): + raise ValueError( + f"mooncake trace: timestamp offset filtering (start_offset=" + f"{self._start_offset}, end_offset={self._end_offset}) cannot " + f"be combined with assistant_responses='live' entries. Offsets drop " + f"individual rows before sessions are assembled, which would " + f"silently remove the initial history or preceding turns that " + f"later delta rows depend on. Remove the offsets and construct " + f"a trace file containing only the complete delta sessions to " + f"replay." + ) + def _group_traces( self, items: list[MooncakeTrace] ) -> dict[str, list[MooncakeTrace]]: @@ -71,6 +143,24 @@ def _group_traces( for trace in items: session_id = trace.session_id or self.session_id_generator.next() data[session_id].append(trace) + # Synthesis serializes defaults, so inspect omission before that round-trip. + suspicious_count = 0 + first_suspicious_id: str | None = None + for session_id, traces in data.items(): + if _looks_like_implicit_message_deltas(traces): + suspicious_count += 1 + if first_suspicious_id is None: + first_suspicious_id = session_id + if suspicious_count: + self.warning( + f"Mooncake trace: {suspicious_count} multi-row messages session(s), " + f"including {first_suspicious_id!r}, omit assistant_responses and " + "may contain incremental rows. Defaulting to recorded replays " + "each row independently; live replies and preceding rows are " + "not accumulated. Set assistant_responses='live' on every row " + "for incremental replay, or explicitly set 'recorded' for " + "intentional self-contained requests." + ) return dict(data) # ------------------------------------------------------------------ @@ -80,12 +170,17 @@ def _group_traces( def _infer_context_mode( self, traces: list[MooncakeTrace] ) -> ConversationContextMode | None: - """Auto-detect MESSAGE_ARRAY_WITH_RESPONSES when all traces are self-contained. + """Auto-detect the context mode for self-contained sessions. Self-contained traces (pre-built `messages` or verbatim `payload`) bypass - endpoint formatting and need to be replayed verbatim. Mixed sessions that - combine self-contained traces with synthesized prompts, or mix `messages` - and `payload` modes, are rejected. + prompt synthesis. All-`messages` sessions resolve by ``assistant_responses``: + the default 'recorded' replays each entry verbatim + (MESSAGE_ARRAY_WITH_RESPONSES) while opt-in 'live' accumulates entries + and threads live assistant responses into the history + (DELTAS_WITHOUT_RESPONSES). Mixed sessions that combine self-contained + traces with synthesized prompts, mix `messages` and `payload` modes, or + mix 'recorded' and 'live' entries (including entries that omit + ``assistant_responses`` and default to 'recorded') are rejected. """ msg_trace_count = sum(1 for trace in traces if trace.messages is not None) payload_trace_count = sum(1 for trace in traces if trace.payload is not None) @@ -99,7 +194,28 @@ def _infer_context_mode( f"the offending sessions or convert all entries to a single " f"self-contained mode." ) + + live_count = sum( + 1 + for trace in traces + if trace.messages is not None + and trace.assistant_responses == AssistantResponseMode.LIVE + ) + if 0 < live_count < msg_trace_count: + raise ValueError( + f"mooncake trace: mixed session contains {live_count} " + f"assistant_responses='live' trace(s) and " + f"{msg_trace_count - live_count} assistant_responses='recorded' " + f"trace(s) (omitted assistant_responses defaults to 'recorded'); each " + f"session must use exactly one assistant_responses value. Mark every " + f"`messages` entry in the session with assistant_responses='live' or " + f"none of them." + ) + if self_contained_count == len(traces) and self_contained_count > 0: + if live_count > 0: + self._validate_endpoint_extra_for_live() + return ConversationContextMode.DELTAS_WITHOUT_RESPONSES return ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES if self_contained_count > 0: raise ValueError( @@ -107,6 +223,24 @@ def _infer_context_mode( ) return None + def _validate_endpoint_extra_for_live(self) -> None: + """Fail closed when an endpoint-global extra input would clobber history. + + Endpoint-level ``extra`` entries (``--extra-inputs`` / + ``endpoint.extra``) are merged into every request body after the + formatter builds ``messages`` (or ``input`` for the Responses API), + so either key could replace the assembled history in every request. + """ + extra = self.run.cfg.endpoint.extra + if history_keys := {"messages", "input"}.intersection(extra): + raise ValueError( + "mooncake trace: assistant_responses='live' cannot be combined with " + f"an endpoint-level extra input named {sorted(history_keys)} " + "(--extra-inputs or endpoint.extra); it " + "would overwrite the assembled conversation history in every " + "request" + ) + def _get_text_input(self, trace: MooncakeTrace) -> str | None: if trace.messages is not None or trace.payload is not None: return "" diff --git a/src/aiperf/dataset/loader/raw_payload.py b/src/aiperf/dataset/loader/raw_payload.py index 59b47342a9..d07bf9baf8 100644 --- a/src/aiperf/dataset/loader/raw_payload.py +++ b/src/aiperf/dataset/loader/raw_payload.py @@ -45,8 +45,9 @@ def can_load( ) -> bool: """Return True when data is a chat API payload or filename is a directory of JSONL files. - Rejects agentic trajectory records (``conversation_id`` present) and - InputsFile structures (``data`` key holding a list). + Rejects agentic trajectory records (``conversation_id`` present), + declared Mooncake response modes, and InputsFile structures (``data`` + key holding a list). """ if data is not None: # Type-dispatch plugins feed arbitrary first-record shapes here; @@ -56,6 +57,10 @@ def can_load( return False if is_speed_bench_row(data): return False + # Mooncake must report invalid modes instead of replaying trace + # metadata as an API payload and discarding session semantics. + if "assistant_responses" in data or "message_mode" in data: + return False if not isinstance(data.get("messages"), list): return False if "conversation_id" in data: diff --git a/tests/component_integration/dataset/test_mooncake_message_deltas.py b/tests/component_integration/dataset/test_mooncake_message_deltas.py new file mode 100644 index 0000000000..7716f165b8 --- /dev/null +++ b/tests/component_integration/dataset/test_mooncake_message_deltas.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Component integration coverage for Mooncake assistant_responses='live' replay. + +Verifies the request payloads actually dispatched to the (fake) transport: +the second request in a session using live responses must contain the initial +history, the LIVE assistant reply captured from the first response, and the +second turn's delta messages - plus per-turn cap/extra semantics and tool +schema inheritance/replacement. +""" + +from pathlib import Path +from typing import Any + +import orjson +import pytest + +from aiperf.common.models import RawRecordInfo, SSEMessage +from tests.component_integration.conftest import ( + ComponentIntegrationTestDefaults as defaults, +) +from tests.harness.utils import AIPerfCLI + +SYSTEM_MSG = {"role": "system", "content": "You are a helpful assistant."} +USER_MSG_1 = {"role": "user", "content": "What is machine learning?"} +USER_MSG_2 = {"role": "user", "content": "Give an example."} +TOOLS_A = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}] +TOOLS_B = [{"type": "function", "function": {"name": "get_forecast", "parameters": {}}}] + + +def _write_trace(path: Path, records: list[dict]) -> Path: + trace_file = path / "trace.jsonl" + with open(trace_file, "wb") as f: + for record in records: + f.write(orjson.dumps(record)) + f.write(b"\n") + return trace_file + + +def _streamed_assistant_text(record: RawRecordInfo) -> str: + """Reassemble the assistant text the (fake) server streamed for a record. + + Mirrors the capture semantics of ``BaseEndpoint.build_assistant_turn``: + concatenated ``delta.content``, falling back to ``delta.reasoning_content`` + when the server streamed everything as reasoning. + """ + content_parts: list[str] = [] + reasoning_parts: list[str] = [] + for response in record.responses: + if not isinstance(response, SSEMessage): + continue + data = response.extract_data_content() + if not data or data.strip() == "[DONE]": + continue + chunk = orjson.loads(data) + choices = chunk.get("choices") or [] + if not choices: + continue + delta = choices[0].get("delta") or {} + if isinstance(delta.get("content"), str): + content_parts.append(delta["content"]) + if isinstance(delta.get("reasoning_content"), str): + reasoning_parts.append(delta["reasoning_content"]) + return "".join(content_parts) or "".join(reasoning_parts) + + +def _token_cap(payload: dict[str, Any]) -> int | None: + return payload.get("max_completion_tokens", payload.get("max_tokens")) + + +def _run_trace_profile( + cli: AIPerfCLI, trace_file: Path, extra_args: str = "" +) -> list[RawRecordInfo]: + result = cli.run_sync( + f""" + aiperf profile \ + --model {defaults.model} \ + --custom-dataset-type mooncake_trace \ + --input-file {trace_file} \ + --streaming \ + --fixed-schedule \ + --workers-max 1 \ + --export-level raw \ + --ui {defaults.ui} {extra_args} + """, + timeout=60.0, + ) + assert result.raw_records is not None + return sorted(result.raw_records, key=lambda r: r.metadata.turn_index) + + +@pytest.mark.component_integration +class TestMooncakeLiveResponsesReplay: + def test_live_second_request_preserves_initial_history_and_new_reply( + self, cli: AIPerfCLI, tmp_path: Path + ) -> None: + """Recorded assistant messages in the initial history survive live replay.""" + initial_history = [ + SYSTEM_MSG, + {"role": "user", "content": "I am learning about computers."}, + {"role": "assistant", "content": "What would you like to learn?"}, + USER_MSG_1, + ] + records = [ + {"session_id": "s1", "assistant_responses": "live", "timestamp": 0, "messages": initial_history, "tools": TOOLS_A, "output_length": 8, "extra": {"temperature": 0.1}}, + {"session_id": "s1", "assistant_responses": "live", "delay": 5, "messages": [USER_MSG_2], "output_length": 6, "extra": {"temperature": 0.5}}, + ] # fmt: skip + trace_file = _write_trace(tmp_path, records) + + raw = _run_trace_profile(cli, trace_file) + assert len(raw) == 2 + first, second = raw + + # First request: exactly the authored initial history. + assert first.payload["messages"] == initial_history + assert first.payload["tools"] == TOOLS_A + assert _token_cap(first.payload) == 8 + assert first.payload["temperature"] == 0.1 + + # Second request: initial history + LIVE assistant reply + delta. + live_text = _streamed_assistant_text(first) + assert live_text, "fake server streamed no assistant text" + messages = second.payload["messages"] + assert messages == [ + *initial_history, + {"role": "assistant", "content": live_text}, + USER_MSG_2, + ] + + # Tool schema inherited from the first turn; per-turn cap/extra + # scoped to the dispatching turn. + assert second.payload["tools"] == TOOLS_A + assert _token_cap(second.payload) == 6 + assert second.payload["temperature"] == 0.5 + assert "messages" not in (records[1].get("extra") or {}) + + def test_live_turn_tools_replacement(self, cli: AIPerfCLI, tmp_path: Path) -> None: + records = [ + {"session_id": "s1", "assistant_responses": "live", "timestamp": 0, "messages": [USER_MSG_1], "tools": TOOLS_A, "output_length": 8}, + {"session_id": "s1", "assistant_responses": "live", "delay": 5, "messages": [USER_MSG_2], "tools": TOOLS_B, "output_length": 6}, + ] # fmt: skip + trace_file = _write_trace(tmp_path, records) + + raw = _run_trace_profile(cli, trace_file) + assert len(raw) == 2 + assert raw[0].payload["tools"] == TOOLS_A + assert raw[1].payload["tools"] == TOOLS_B + + def test_recorded_default_still_replays_verbatim_without_live_responses( + self, cli: AIPerfCLI, tmp_path: Path + ) -> None: + """Backward compatibility: recorded responses remain the default.""" + turn2_history = [ + USER_MSG_1, + {"role": "assistant", "content": "canned"}, + USER_MSG_2, + ] + records = [ + {"session_id": "s1", "timestamp": 0, "messages": [USER_MSG_1], "output_length": 8}, + {"session_id": "s1", "delay": 5, "messages": turn2_history, "output_length": 6}, + ] # fmt: skip + trace_file = _write_trace(tmp_path, records) + + raw = _run_trace_profile(cli, trace_file) + assert len(raw) == 2 + assert raw[0].payload["messages"] == [USER_MSG_1] + # Verbatim replay: the canned assistant turn is used, not the live one. + assert raw[1].payload["messages"] == turn2_history diff --git a/tests/integration/test_mooncake_message_deltas.py b/tests/integration/test_mooncake_message_deltas.py new file mode 100644 index 0000000000..dbd7cc4906 --- /dev/null +++ b/tests/integration/test_mooncake_message_deltas.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End-to-end HTTP integration tests for Mooncake assistant_responses='live' replay. + +Runs the real aiperf CLI (subprocess, real HTTP transport) against an in-test +recording server that streams SSE chat chunks with tool_calls deltas split +across chunks. The test inspects the ACTUAL second HTTP request body received +by the server to prove it contains the newly generated assistant text AND the +reassembled streamed tool_calls (id/name/arguments), followed by the dataset's +tool-result delta whose ``tool_call_id`` matches the live-generated call id. +""" + +from pathlib import Path +from typing import Any + +import orjson +import pytest +from aiohttp import web + +from tests.harness.utils import AIPerfCLI +from tests.integration.conftest import IntegrationTestDefaults as defaults + +LIVE_TOOL_CALL_ID = "call_live_0001" +ASSISTANT_TEXT_TURN_1 = "Checking the weather now." +ASSISTANT_TEXT_TURN_2 = "It is 21C in Paris." + +SYSTEM_MSG = {"role": "system", "content": "You are a weather assistant."} +USER_MSG = {"role": "user", "content": "What's the weather in Paris?"} +TOOL_DEFS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } +] +TOOL_RESULT_MSG = { + "role": "tool", + "tool_call_id": LIVE_TOOL_CALL_ID, + "content": "21C, sunny", +} + + +def _sse(data: dict[str, Any]) -> bytes: + return b"data: " + orjson.dumps(data) + b"\n\n" + + +def _chunk(delta: dict[str, Any], finish_reason: str | None = None) -> bytes: + return _sse( + { + "id": "chatcmpl-recording", + "object": "chat.completion.chunk", + "created": 0, + "model": "recording-model", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + ) + + +class RecordingToolCallServer: + """Localhost SSE chat server that records every request body it receives. + + The first request is answered with assistant text plus a tool call whose + ``id``/``function.name``/``function.arguments`` are deliberately streamed + across separate SSE chunks (matching OpenAI's streaming shape), so the + client must reassemble them. Later requests get a plain text reply. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self._runner: web.AppRunner | None = None + self.port: int = 0 + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + async def start(self) -> None: + app = web.Application() + app.router.add_post("/v1/chat/completions", self._handle_chat) + self._runner = web.AppRunner(app) + await self._runner.setup() + site = web.TCPSite(self._runner, "127.0.0.1", 0) + await site.start() + self.port = self._runner.addresses[0][1] + + async def stop(self) -> None: + if self._runner is not None: + await self._runner.cleanup() + + async def _handle_chat(self, request: web.Request) -> web.StreamResponse: + body = orjson.loads(await request.read()) + self.requests.append(body) + request_index = len(self.requests) + + response = web.StreamResponse( + status=200, headers={"Content-Type": "text/event-stream"} + ) + await response.prepare(request) + + if request_index == 1: + await response.write( + _chunk({"role": "assistant", "content": ASSISTANT_TEXT_TURN_1}) + ) + # Streamed tool_calls deltas: id + name first, then the JSON + # arguments split across two fragments. + await response.write( + _chunk( + { + "tool_calls": [ + { + "index": 0, + "id": LIVE_TOOL_CALL_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + ] + } + ) + ) + await response.write( + _chunk( + { + "tool_calls": [ + {"index": 0, "function": {"arguments": '{"city": '}} + ] + } + ) + ) + await response.write( + _chunk( + { + "tool_calls": [ + {"index": 0, "function": {"arguments": '"Paris"}'}} + ] + }, + finish_reason="tool_calls", + ) + ) + else: + await response.write( + _chunk({"role": "assistant", "content": ASSISTANT_TEXT_TURN_2}) + ) + await response.write(_chunk({}, finish_reason="stop")) + + await response.write(b"data: [DONE]\n\n") + await response.write_eof() + return response + + +def _write_trace(path: Path, records: list[dict[str, Any]]) -> Path: + trace_file = path / "live_trace.jsonl" + with open(trace_file, "wb") as f: + for record in records: + f.write(orjson.dumps(record)) + f.write(b"\n") + return trace_file + + +def _token_cap(payload: dict[str, Any]) -> int | None: + return payload.get("max_completion_tokens", payload.get("max_tokens")) + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestMooncakeLiveResponsesHTTP: + async def test_second_http_request_contains_live_assistant_and_tool_calls( + self, cli: AIPerfCLI, tmp_path: Path + ) -> None: + """The actual second HTTP request must carry the live assistant turn. + + Turn 1 sends the initial history; the server streams text + a tool + call. Turn 2's dataset delta is only the tool-result message, so + everything between the initial history and the tool result in request + 2 must have been captured live from the streamed response - including + the reassembled tool_call id/name/arguments. The tool-result fixture's + ``tool_call_id`` matches the id the live server emits; no ordinal + remapping is performed by AIPerf. + """ + records = [ + {"session_id": "weather-1", "assistant_responses": "live", "timestamp": 0, "messages": [SYSTEM_MSG, USER_MSG], "tools": TOOL_DEFS, "output_length": 64}, + {"session_id": "weather-1", "assistant_responses": "live", "delay": 100, "messages": [TOOL_RESULT_MSG], "output_length": 32, "extra": {"temperature": 0.25}}, + ] # fmt: skip + trace_file = _write_trace(tmp_path, records) + + server = RecordingToolCallServer() + await server.start() + try: + result = await cli.run( + f""" + aiperf profile \ + --model {defaults.model} \ + --url {server.url} \ + --endpoint-type chat \ + --streaming \ + --input-file {trace_file} \ + --custom-dataset-type mooncake_trace \ + --fixed-schedule \ + --workers-max 1 \ + --ui {defaults.ui} + """, + timeout=120.0, + ) + finally: + await server.stop() + + assert result.request_count == 2 + assert len(server.requests) == 2 + first_request, second_request = server.requests + + # Request 1: exactly the authored initial history + tool schemas. + assert first_request["messages"] == [SYSTEM_MSG, USER_MSG] + assert first_request["tools"] == TOOL_DEFS + assert _token_cap(first_request) == 64 + + # Request 2: initial history, then the LIVE assistant message with + # the reassembled streamed tool_call, then the tool-result delta. + messages = second_request["messages"] + assert messages[:2] == [SYSTEM_MSG, USER_MSG] + + assistant_msg = messages[2] + assert assistant_msg["role"] == "assistant" + assert assistant_msg["content"] == ASSISTANT_TEXT_TURN_1 + assert assistant_msg["tool_calls"] == [ + { + "id": LIVE_TOOL_CALL_ID, + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ] + + # The dataset's tool-result delta references the SAME id the live + # server generated - proving id continuity, not positional mapping. + assert messages[3] == TOOL_RESULT_MSG + assert messages[3]["tool_call_id"] == assistant_msg["tool_calls"][0]["id"] + assert messages[3:] == [TOOL_RESULT_MSG] + + # Tool schemas inherited from turn 1; per-turn cap/extra from turn 2. + assert second_request["tools"] == TOOL_DEFS + assert _token_cap(second_request) == 32 + assert second_request["temperature"] == 0.25 + + async def test_third_turn_history_includes_post_tool_assistant_text( + self, cli: AIPerfCLI, tmp_path: Path + ) -> None: + """Turn 3 retains recorded initial history and both live replies.""" + initial_history = [ + SYSTEM_MSG, + {"role": "user", "content": "I am planning a trip."}, + {"role": "assistant", "content": "Where are you going?"}, + USER_MSG, + ] + follow_up = {"role": "user", "content": "Thanks! And in London?"} + records = [ + {"session_id": "weather-1", "assistant_responses": "live", "timestamp": 0, "messages": initial_history, "tools": TOOL_DEFS, "output_length": 64}, + {"session_id": "weather-1", "assistant_responses": "live", "delay": 50, "messages": [TOOL_RESULT_MSG], "output_length": 32}, + {"session_id": "weather-1", "assistant_responses": "live", "delay": 50, "messages": [follow_up], "output_length": 32}, + ] # fmt: skip + trace_file = _write_trace(tmp_path, records) + + server = RecordingToolCallServer() + await server.start() + try: + result = await cli.run( + f""" + aiperf profile \ + --model {defaults.model} \ + --url {server.url} \ + --endpoint-type chat \ + --streaming \ + --input-file {trace_file} \ + --custom-dataset-type mooncake_trace \ + --fixed-schedule \ + --workers-max 1 \ + --ui {defaults.ui} + """, + timeout=120.0, + ) + finally: + await server.stop() + + assert result.request_count == 3 + assert len(server.requests) == 3 + assert server.requests[0]["messages"] == initial_history + third_request = server.requests[2] + messages = third_request["messages"] + + history_length = len(initial_history) + assert messages[:history_length] == initial_history + live_tool_call = messages[history_length] + assert live_tool_call["role"] == "assistant" + assert live_tool_call["tool_calls"][0]["id"] == LIVE_TOOL_CALL_ID + assert messages[history_length + 1 :] == [ + TOOL_RESULT_MSG, + {"role": "assistant", "content": ASSISTANT_TEXT_TURN_2}, + follow_up, + ] diff --git a/tests/unit/dataset/loader/test_mooncake_detection.py b/tests/unit/dataset/loader/test_mooncake_detection.py new file mode 100644 index 0000000000..316976a99d --- /dev/null +++ b/tests/unit/dataset/loader/test_mooncake_detection.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Callable +from pathlib import Path +from typing import Any +from unittest.mock import Mock + +import orjson +import pytest +from pydantic import ValidationError +from pytest import param + +from aiperf.config.dataset.resolver import DatasetResolver +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.dataset.composer.custom import CustomDatasetComposer +from aiperf.plugin import plugins +from aiperf.plugin.enums import CustomDatasetType, PluginType + + +@pytest.mark.parametrize( + "field,value", + [ + param("assistant_responses", "live", id="live"), + param("assistant_responses", "recorded", id="recorded"), + param("assistant_responses", "LIVE", id="uppercase"), + param("assistant_responses", "lve", id="invalid-value"), + param("assistant_responses", None, id="null-value"), + param("assistant_responses", 1, id="non-string-value"), + param("message_mode", "delta", id="obsolete-name"), + ], +) # fmt: skip +def test_declared_response_mode_autodetects_as_mooncake( + tmp_path: Path, + create_cfg_and_composer: Callable[[], tuple[CLIConfig, CustomDatasetComposer]], + field: str, + value: Any, +) -> None: + record = { + "session_id": "s1", + "messages": [{"role": "user", "content": "Hello"}], + field: value, + } + path = tmp_path / "trace.jsonl" + path.write_bytes(orjson.dumps(record) + b"\n") + _, composer = create_cfg_and_composer() + + detected, _ = DatasetResolver._detect_type(str(path)) + + assert detected == CustomDatasetType.MOONCAKE_TRACE + assert composer._infer_dataset_type(str(path)) == CustomDatasetType.MOONCAKE_TRACE + + +@pytest.mark.parametrize( + "fields,error", + [ + param({"assistant_responses": "lve"}, "assistant_responses", id="typo"), + param({"assistant_responses": None}, "assistant_responses", id="null"), + param({"assistant_responses": 1}, "assistant_responses", id="non-string"), + param( + {"message_mode": "delta"}, + "'message_mode' has been renamed to 'assistant_responses'", + id="obsolete-name", + ), + param( + {"assistant_responses": "live", "message_mode": "delta"}, + "'message_mode' has been renamed to 'assistant_responses'", + id="both-names", + ), + ], +) # fmt: skip +def test_invalid_response_mode_rejected_after_autodetection( + tmp_path: Path, + create_cfg_and_composer: Callable[[], tuple[CLIConfig, CustomDatasetComposer]], + mock_prompt_generator: Mock, + fields: dict[str, Any], + error: str, +) -> None: + path = tmp_path / "trace.jsonl" + path.write_bytes( + orjson.dumps( + { + "session_id": "s1", + "messages": [{"role": "user", "content": "Hello"}], + **fields, + } + ) + + b"\n" + ) + _, composer = create_cfg_and_composer() + detected = composer._infer_dataset_type(str(path)) + loader_class = plugins.get_class(PluginType.CUSTOM_DATASET_LOADER, detected) + loader = loader_class(filename=path, prompt_generator=mock_prompt_generator) + + with pytest.raises(ValidationError, match=error): + loader.load_dataset() diff --git a/tests/unit/dataset/loader/test_mooncake_message_deltas.py b/tests/unit/dataset/loader/test_mooncake_message_deltas.py new file mode 100644 index 0000000000..af05feaab4 --- /dev/null +++ b/tests/unit/dataset/loader/test_mooncake_message_deltas.py @@ -0,0 +1,544 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the opt-in MooncakeTrace assistant_responses='live' replay mode.""" + +from pathlib import Path + +import orjson +import pytest +from pydantic import ValidationError +from pytest import param + +from aiperf.common.enums import AssistantResponseMode, ConversationContextMode +from aiperf.config.flags.cli_config import CLIConfig +from aiperf.dataset.loader.models import MooncakeTrace +from aiperf.dataset.loader.mooncake_trace import MooncakeTraceDatasetLoader +from aiperf.plugin.enums import EndpointType + +USER_MSG = [{"role": "user", "content": "Hello"}] +TOOL_DEFS = [ + {"type": "function", "function": {"name": "get_weather", "parameters": {}}} +] + + +def _write_jsonl(path: Path, records: list[dict]) -> Path: + with open(path, "wb") as f: + for record in records: + f.write(orjson.dumps(record)) + f.write(b"\n") + return path + + +def _load_conversations( + tmp_path: Path, records: list[dict], default_cfg, mock_prompt_generator, run=None +): + file = _write_jsonl(tmp_path / "trace.jsonl", records) + loader = MooncakeTraceDatasetLoader( + filename=file, + cfg=default_cfg, + prompt_generator=mock_prompt_generator, + run=run, + ) + return loader.convert_to_conversations(loader.load_dataset()) + + +class TestAssistantResponsesSchema: + """MooncakeTrace model validation for the assistant_responses field.""" + + def test_assistant_responses_defaults_to_history(self): + trace = MooncakeTrace(messages=USER_MSG) + assert trace.assistant_responses == "recorded" + + def test_assistant_responses_history_explicit(self): + trace = MooncakeTrace(messages=USER_MSG, assistant_responses="recorded") + assert trace.assistant_responses == "recorded" + + def test_assistant_responses_delta_with_messages(self): + trace = MooncakeTrace(messages=USER_MSG, assistant_responses="live") + assert trace.assistant_responses == "live" + + def test_assistant_responses_default_preserved_for_non_message_inputs(self): + assert MooncakeTrace(input_length=10).assistant_responses == "recorded" + assert MooncakeTrace(text_input="hi").assistant_responses == "recorded" + assert MooncakeTrace(payload={"prompt": "hi"}).assistant_responses == "recorded" + + @pytest.mark.parametrize( + ("value", "expected"), + [ + param("live", AssistantResponseMode.LIVE, id="live"), + param("LIVE", AssistantResponseMode.LIVE, id="uppercase_live"), + param("Live", AssistantResponseMode.LIVE, id="mixed_case_live"), + param("RECORDED", AssistantResponseMode.RECORDED, id="uppercase_recorded"), + param("Recorded", AssistantResponseMode.RECORDED, id="mixed_case_recorded"), + ], + ) # fmt: skip + def test_assistant_responses_case_insensitive_round_trip( + self, value: str, expected: AssistantResponseMode + ) -> None: + """Hand-authored casing normalizes to the enum's canonical wire value.""" + trace = MooncakeTrace(messages=USER_MSG, assistant_responses=value) + assert trace.assistant_responses is expected + serialized = trace.model_dump(mode="json") + assert serialized["assistant_responses"] == str(expected) + assert MooncakeTrace.model_validate(serialized).assistant_responses is expected + + def test_legacy_message_mode_rejected(self) -> None: + """Stale opt-in traces must not silently become recorded replay.""" + with pytest.raises( + ValidationError, match="has been renamed to 'assistant_responses'" + ): + MooncakeTrace(messages=USER_MSG, message_mode="delta") + + @pytest.mark.parametrize( + "fields", + [ + param({"payload": {"prompt": "hi"}}, id="payload"), + param({"input_length": 100}, id="synthesized_input_length"), + param({"input_length": 100, "hash_ids": [1, 2]}, id="synthesized_hash_ids"), + param({"text_input": "hi"}, id="text_input"), + ], + ) # fmt: skip + def test_delta_without_messages_rejected(self, fields: dict): + with pytest.raises(ValidationError, match="only supported with 'messages'"): + MooncakeTrace(assistant_responses="live", **fields) + + @pytest.mark.parametrize( + "bad_mode", + [ + param("lives", id="plural"), + param("delta", id="old_mode"), + param("message_array", id="context_mode_name"), + param("", id="empty"), + param(1, id="non_string"), + ], + ) # fmt: skip + def test_unsupported_assistant_responses_rejected(self, bad_mode): + with pytest.raises(ValidationError, match="assistant_responses"): + MooncakeTrace(messages=USER_MSG, assistant_responses=bad_mode) + + @pytest.mark.parametrize("key", ["messages", "input"]) + def test_live_with_extra_history_key_rejected(self, key: str) -> None: + """Neither chat nor Responses extras may replace live conversation input.""" + with pytest.raises(ValidationError, match="'extra' must not contain"): + MooncakeTrace( + messages=USER_MSG, + assistant_responses="live", + extra={key: [{"role": "user", "content": "clobber"}]}, + ) + + def test_delta_with_safe_extra_allowed(self): + trace = MooncakeTrace( + messages=USER_MSG, assistant_responses="live", extra={"temperature": 0.5} + ) + assert trace.extra == {"temperature": 0.5} + + def test_history_with_extra_messages_key_still_allowed(self): + """Backward compatibility: history mode does not reject extra['messages'].""" + trace = MooncakeTrace( + messages=USER_MSG, extra={"messages": [{"role": "user", "content": "x"}]} + ) + assert trace.extra is not None + + def test_delta_with_tools_allowed(self): + trace = MooncakeTrace( + messages=USER_MSG, assistant_responses="live", tools=TOOL_DEFS + ) + assert trace.tools == TOOL_DEFS + + def test_delta_serialization_round_trip(self): + trace = MooncakeTrace( + messages=USER_MSG, + assistant_responses="live", + tools=TOOL_DEFS, + output_length=50, + timestamp=1000, + session_id="s1", + ) + round_tripped = MooncakeTrace.model_validate( + orjson.loads(orjson.dumps(trace.model_dump(exclude_none=True))) + ) + assert round_tripped == trace + assert round_tripped.assistant_responses == "live" + + def test_history_default_serialization_round_trip(self): + trace = MooncakeTrace(messages=USER_MSG, output_length=5) + round_tripped = MooncakeTrace.model_validate( + orjson.loads(orjson.dumps(trace.model_dump(exclude_none=True))) + ) + assert round_tripped.assistant_responses == "recorded" + + def test_can_load_accepts_delta_record(self): + assert MooncakeTraceDatasetLoader.can_load( + {"messages": USER_MSG, "assistant_responses": "live"} + ) + + def test_can_load_routes_invalid_live_payload_to_schema_validation(self) -> None: + """An explicit response mode identifies Mooncake even if the row is invalid.""" + assert MooncakeTraceDatasetLoader.can_load( + {"payload": {"prompt": "hi"}, "assistant_responses": "live"} + ) + + +class TestDeltaContextModeInference: + """Loader context-mode inference for delta and history sessions.""" + + def test_all_delta_session_uses_deltas_without_responses( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}], "timestamp": 0}, + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t2"}], "delay": 250}, + ] # fmt: skip + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator + ) + assert len(conversations) == 1 + assert ( + conversations[0].context_mode + == ConversationContextMode.DELTAS_WITHOUT_RESPONSES + ) + + def test_history_session_default_unchanged( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "messages": [{"role": "user", "content": "t1"}]}, + {"session_id": "s1", "messages": [{"role": "user", "content": "t1"}, {"role": "assistant", "content": "a1"}, {"role": "user", "content": "t2"}]}, + ] # fmt: skip + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator + ) + assert ( + conversations[0].context_mode + == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + ) + + def test_synthetic_session_context_mode_unchanged( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "input_length": 10, "output_length": 5}, + {"session_id": "s1", "input_length": 20, "output_length": 5, "delay": 10}, + ] # fmt: skip + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator + ) + assert conversations[0].context_mode is None + + def test_mixed_delta_and_explicit_history_rejected( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}]}, + {"session_id": "s1", "assistant_responses": "recorded", "messages": [{"role": "user", "content": "t2"}]}, + ] # fmt: skip + with pytest.raises(ValueError, match="exactly one assistant_responses"): + _load_conversations(tmp_path, records, default_cfg, mock_prompt_generator) + + def test_mixed_delta_and_omitted_history_default_rejected( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}]}, + {"session_id": "s1", "messages": [{"role": "user", "content": "t2"}]}, + ] # fmt: skip + with pytest.raises(ValueError, match="omitted assistant_responses defaults"): + _load_conversations(tmp_path, records, default_cfg, mock_prompt_generator) + + def test_mixed_delta_messages_and_payload_rejected( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}]}, + {"session_id": "s1", "payload": {"prompt": "t2"}}, + ] # fmt: skip + with pytest.raises(ValueError, match="exactly one mode"): + _load_conversations(tmp_path, records, default_cfg, mock_prompt_generator) + + def test_mixed_delta_messages_and_synthetic_rejected( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}]}, + {"session_id": "s1", "input_length": 10}, + ] # fmt: skip + with pytest.raises(ValueError, match="synthesized prompts are unsupported"): + _load_conversations(tmp_path, records, default_cfg, mock_prompt_generator) + + def test_mixed_history_messages_and_payload_still_rejected( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "messages": [{"role": "user", "content": "t1"}]}, + {"session_id": "s1", "payload": {"prompt": "t2"}}, + ] # fmt: skip + with pytest.raises(ValueError, match="exactly one mode"): + _load_conversations(tmp_path, records, default_cfg, mock_prompt_generator) + + def test_independent_sessions_resolve_modes_independently( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "delta-sess", "assistant_responses": "live", "messages": [{"role": "user", "content": "d1"}]}, + {"session_id": "history-sess", "messages": [{"role": "user", "content": "h1"}]}, + {"session_id": "synthetic-sess", "input_length": 10}, + ] # fmt: skip + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator + ) + modes = {c.session_id: c.context_mode for c in conversations} + assert modes["delta-sess"] == ConversationContextMode.DELTAS_WITHOUT_RESPONSES + assert ( + modes["history-sess"] + == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + ) + assert modes["synthetic-sess"] is None + + +class TestDeltaTurnConstruction: + """Turn-level field preservation for delta traces.""" + + def test_timestamp_delay_and_caps_preserved( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}], "timestamp": 1000.5, "output_length": 64}, + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t2"}], "delay": 250.25, "output_length": 32}, + ] # fmt: skip + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator + ) + turns = conversations[0].turns + assert turns[0].timestamp == 1000.5 + assert turns[0].max_tokens == 64 + assert turns[0].delay is None + assert turns[1].delay == 250.25 + assert turns[1].max_tokens == 32 + assert turns[1].timestamp is None + + def test_delta_turn_carries_raw_messages_tools_and_extra( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "t1"}, + ] + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": messages, "tools": TOOL_DEFS, "extra": {"temperature": 0.1}}, + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t2"}]}, + ] # fmt: skip + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator + ) + turns = conversations[0].turns + assert turns[0].raw_messages == messages + assert turns[0].raw_tools == TOOL_DEFS + assert turns[0].extra_body == {"temperature": 0.1} + assert turns[1].raw_messages == [{"role": "user", "content": "t2"}] + assert turns[1].raw_tools is None + assert turns[1].extra_body is None + + +class TestDeltaEndpointExtraProtection: + """Endpoint-global extra inputs must not clobber assembled delta history.""" + + def _make_run(self, extra_inputs, endpoint_type=EndpointType.CHAT): + from tests.unit.conftest import make_run_from_cli + + return make_run_from_cli( + CLIConfig( + model_names=["test-model"], + extra_inputs=extra_inputs, + endpoint_type=endpoint_type, + ) + ) + + @pytest.mark.parametrize( + ("endpoint_type", "key"), + [ + param(EndpointType.CHAT, "messages", id="chat_messages"), + param(EndpointType.RESPONSES, "input", id="responses_input"), + ], + ) # fmt: skip + def test_endpoint_extra_messages_rejected_for_delta_sessions( + self, tmp_path: Path, default_cfg, mock_prompt_generator, endpoint_type, key + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}]}, + ] # fmt: skip + run = self._make_run( + {key: [{"role": "user", "content": "clobber"}]}, endpoint_type + ) + with pytest.raises(ValueError, match="endpoint-level extra input"): + _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator, run=run + ) + + def test_endpoint_extra_without_messages_allowed_for_delta_sessions( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + records = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}]}, + ] # fmt: skip + run = self._make_run({"temperature": 0.7}) + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator, run=run + ) + assert ( + conversations[0].context_mode + == ConversationContextMode.DELTAS_WITHOUT_RESPONSES + ) + + def test_endpoint_extra_messages_still_allowed_for_history_sessions( + self, tmp_path: Path, default_cfg, mock_prompt_generator + ): + """Narrow fail-closed scope: history-mode replay is unchanged.""" + records = [ + {"session_id": "s1", "messages": [{"role": "user", "content": "t1"}]}, + ] # fmt: skip + run = self._make_run({"messages": [{"role": "user", "content": "override"}]}) + conversations = _load_conversations( + tmp_path, records, default_cfg, mock_prompt_generator, run=run + ) + assert ( + conversations[0].context_mode + == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + ) + + +class TestDeltaOffsetRejection: + """Timestamp offset cropping must fail closed for delta sessions. + + start_offset/end_offset drop individual rows before grouping, which would + silently remove the initial history or preceding turns a later delta row + depends on. Any configured offset combined with any delta row aborts the + load; history-mode and synthesized traces keep the existing filtering. + """ + + DELTA_RECORDS = [ + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t1"}], "timestamp": 1000}, + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t2"}], "timestamp": 2000}, + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "t3"}], "timestamp": 3000}, + ] # fmt: skip + + def _make_offset_run( + self, + file: Path, + start_offset: int | None = None, + end_offset: int | None = None, + ): + from tests.unit.conftest import make_run_from_cli + + input_kwargs: dict = {"input_file": str(file), "fixed_schedule": True} + if start_offset is not None: + input_kwargs["fixed_schedule_start_offset"] = start_offset + if end_offset is not None: + input_kwargs["fixed_schedule_end_offset"] = end_offset + return make_run_from_cli(CLIConfig(model_names=["test-model"], **input_kwargs)) + + def _load_with_offsets( + self, + tmp_path: Path, + records: list[dict], + mock_prompt_generator, + start_offset: int | None = None, + end_offset: int | None = None, + ): + file = _write_jsonl(tmp_path / "trace.jsonl", records) + run = self._make_offset_run(file, start_offset, end_offset) + loader = MooncakeTraceDatasetLoader( + filename=file, prompt_generator=mock_prompt_generator, run=run + ) + return loader.convert_to_conversations(loader.load_dataset()) + + def test_start_offset_prefix_removal_rejected( + self, tmp_path: Path, mock_prompt_generator + ): + """A start offset that would drop the initial history row must abort.""" + with pytest.raises(ValueError, match="assistant_responses='live'"): + self._load_with_offsets( + tmp_path, self.DELTA_RECORDS, mock_prompt_generator, start_offset=1500 + ) + + def test_end_offset_cutoff_rejected(self, tmp_path: Path, mock_prompt_generator): + """An end offset that would drop trailing delta rows must abort.""" + with pytest.raises(ValueError, match="assistant_responses='live'"): + self._load_with_offsets( + tmp_path, self.DELTA_RECORDS, mock_prompt_generator, end_offset=2500 + ) + + def test_both_offsets_rejected(self, tmp_path: Path, mock_prompt_generator): + with pytest.raises(ValueError, match="cannot be combined"): + self._load_with_offsets( + tmp_path, + self.DELTA_RECORDS, + mock_prompt_generator, + start_offset=1500, + end_offset=2500, + ) + + def test_offsets_rejected_even_when_no_delta_row_filtered( + self, tmp_path: Path, mock_prompt_generator + ): + """Fail-closed scope: the offset configuration itself is rejected. + + start_offset=0 would keep every row, but permitting it would make + validity depend on the data's timestamps, so it is rejected outright. + """ + with pytest.raises(ValueError, match="assistant_responses='live'"): + self._load_with_offsets( + tmp_path, self.DELTA_RECORDS, mock_prompt_generator, start_offset=0 + ) + + def test_delta_row_after_filtered_history_rows_rejected( + self, tmp_path: Path, mock_prompt_generator + ): + """History rows filtered earlier in the file cannot mask the abort. + + The two history-session rows fall before the start offset and are + discarded first, but the load still fails when the retained delta row + is parsed -- a partially filtered dataset is never returned. + """ + records = [ + {"session_id": "hist", "messages": [{"role": "user", "content": "h1"}], "timestamp": 100}, + {"session_id": "hist", "messages": [{"role": "user", "content": "h1"}, {"role": "assistant", "content": "a1"}, {"role": "user", "content": "h2"}], "timestamp": 200}, + {"session_id": "s1", "assistant_responses": "live", "messages": [{"role": "user", "content": "d1"}], "timestamp": 2000}, + ] # fmt: skip + with pytest.raises(ValueError, match="assistant_responses='live'"): + self._load_with_offsets( + tmp_path, records, mock_prompt_generator, start_offset=1500 + ) + + def test_no_offset_delta_fixed_schedule_allowed( + self, tmp_path: Path, mock_prompt_generator + ): + """Ordinary fixed-schedule delta replay without offsets is unaffected.""" + conversations = self._load_with_offsets( + tmp_path, self.DELTA_RECORDS, mock_prompt_generator + ) + assert len(conversations) == 1 + assert ( + conversations[0].context_mode + == ConversationContextMode.DELTAS_WITHOUT_RESPONSES + ) + assert len(conversations[0].turns) == 3 + + def test_history_mode_offset_filtering_preserved( + self, tmp_path: Path, mock_prompt_generator + ): + """Backward compatibility: history-mode rows keep offset filtering.""" + records = [ + {"session_id": "a", "messages": [{"role": "user", "content": "m1"}], "timestamp": 1000}, + {"session_id": "b", "messages": [{"role": "user", "content": "m2"}], "timestamp": 2000}, + {"session_id": "c", "messages": [{"role": "user", "content": "m3"}], "timestamp": 2500}, + {"session_id": "d", "messages": [{"role": "user", "content": "m4"}], "timestamp": 3000}, + ] # fmt: skip + conversations = self._load_with_offsets( + tmp_path, records, mock_prompt_generator, start_offset=1500, end_offset=2500 + ) + assert {c.session_id for c in conversations} == {"b", "c"} + assert all( + c.context_mode == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + for c in conversations + ) diff --git a/tests/unit/dataset/loader/test_mooncake_recorded_warnings.py b/tests/unit/dataset/loader/test_mooncake_recorded_warnings.py new file mode 100644 index 0000000000..7a289e6a93 --- /dev/null +++ b/tests/unit/dataset/loader/test_mooncake_recorded_warnings.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Warnings for potentially incremental Mooncake rows using implicit recorded mode.""" + +import logging +from typing import Any +from unittest.mock import Mock + +import pytest +from pytest import param + +from aiperf.common.enums import ConversationContextMode +from aiperf.dataset.loader.mooncake_trace import MooncakeTraceDatasetLoader +from tests.unit.conftest import make_benchmark_run + +USER_1 = {"role": "user", "content": "First question"} +USER_2 = {"role": "user", "content": "Follow-up question"} +ASSISTANT_1 = {"role": "assistant", "content": "Recorded answer"} +SYSTEM = {"role": "system", "content": "Be helpful"} + + +def _omission_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.message + for record in caplog.records + if "omit assistant_responses" in record.message + ] + + +@pytest.mark.parametrize( + "message_rows", + [ + param([[SYSTEM, USER_1], [USER_2]], id="user_only_deltas"), + param([[USER_1], [USER_1, USER_2]], id="no_recorded_assistant"), + param([[USER_1, ASSISTANT_1, USER_2], [USER_2]], id="initial_history_then_delta"), + param([[USER_1, ASSISTANT_1], [SYSTEM, USER_1, ASSISTANT_1, USER_2]], id="non_prefix_history"), + ], +) # fmt: skip +def test_implicit_incremental_rows_warn_without_changing_replay( + message_rows: list[list[dict[str, Any]]], + mock_prompt_generator: Mock, + caplog: pytest.LogCaptureFixture, +) -> None: + """An advisory surfaces ambiguity while preserving existing recorded behavior.""" + caplog.set_level(logging.WARNING) + loader = MooncakeTraceDatasetLoader( + inline_records=[ + {"session_id": "s1", "messages": messages} for messages in message_rows + ], + prompt_generator=mock_prompt_generator, + ) + + conversations = loader.convert_to_conversations(loader.load_dataset()) + + warnings = _omission_warnings(caplog) + assert len(warnings) == 1 + assert "including 's1'" in warnings[0] + assert "each row independently" in warnings[0] + assert "assistant_responses='live' on every row" in warnings[0] + assert "explicitly set 'recorded'" in warnings[0] + assert ( + conversations[0].context_mode + == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + ) + assert [turn.raw_messages for turn in conversations[0].turns] == message_rows + + +@pytest.mark.parametrize( + "rows", + [ + param([{"messages": [USER_1]}, {"messages": [USER_1, ASSISTANT_1, USER_2]}], id="complete_recorded_history"), + param([{"messages": [USER_1, ASSISTANT_1]}, {"messages": [USER_1, ASSISTANT_1]}], id="unchanged_complete_history"), + param([{"messages": [USER_1]}], id="single_row"), + param([{"session_id": "s1", "messages": [USER_1]}, {"session_id": "s2", "messages": [USER_2]}], id="independent_single_rows"), + param([{"messages": [USER_1], "assistant_responses": "recorded"}, {"messages": [USER_2], "assistant_responses": "recorded"}], id="explicit_recorded"), + param([{"messages": [USER_1], "assistant_responses": "Recorded"}, {"messages": [USER_2]}], id="explicit_first_row"), + param([{"messages": [USER_1]}, {"messages": [USER_2], "assistant_responses": "recorded"}], id="explicit_later_row"), + param([{"messages": [USER_1], "assistant_responses": "live"}, {"messages": [USER_2], "assistant_responses": "live"}], id="explicit_live"), + param([{"payload": {"messages": [USER_1]}}, {"payload": {"messages": [USER_2]}}], id="payload"), + param([{"text_input": "First"}, {"text_input": "Second"}], id="text"), + param([{"input_length": 10}, {"input_length": 20}], id="synthetic"), + ], +) # fmt: skip +def test_recorded_warning_skips_explicit_choices_and_other_inputs( + rows: list[dict[str, Any]], + mock_prompt_generator: Mock, + caplog: pytest.LogCaptureFixture, +) -> None: + """Intentional replay choices and unambiguous inputs need no advisory.""" + caplog.set_level(logging.WARNING) + loader = MooncakeTraceDatasetLoader( + inline_records=[{"session_id": "s1", **row} for row in rows], + prompt_generator=mock_prompt_generator, + ) + + data = loader.load_dataset() + + assert sum(len(traces) for traces in data.values()) == len(rows) + assert not _omission_warnings(caplog) + + +def test_recorded_warning_aggregates_once_per_load( + mock_prompt_generator: Mock, + caplog: pytest.LogCaptureFixture, +) -> None: + """Large files emit one warning, and reusing a loader does not suppress it.""" + caplog.set_level(logging.WARNING) + loader = MooncakeTraceDatasetLoader( + inline_records=[ + {"session_id": session_id, "messages": [message]} + for session_id in ("first", "second") + for message in (USER_1, USER_2) + ], + prompt_generator=mock_prompt_generator, + ) + + for _ in range(2): + caplog.clear() + data = loader.load_dataset() + loader.convert_to_conversations(data) + warnings = _omission_warnings(caplog) + assert len(warnings) == 1 + assert "2 multi-row messages session(s)" in warnings[0] + assert "including 'first'" in warnings[0] + + +@pytest.mark.parametrize("explicit_recorded", [False, True]) +def test_recorded_warning_uses_authored_fields_before_synthesis( + explicit_recorded: bool, + mock_prompt_generator: Mock, + caplog: pytest.LogCaptureFixture, +) -> None: + """Synthesis may materialize defaults but must not hide an omitted choice.""" + caplog.set_level(logging.WARNING) + mode = {"assistant_responses": "recorded"} if explicit_recorded else {} + rows = [ + {"session_id": "s1", "timestamp": index * 1000, "messages": [message], **mode} + for index, message in enumerate((USER_1, USER_2)) + ] + run = make_benchmark_run( + extra={ + "datasets": [ + { + "name": "default", + "type": "file", + "format": "mooncake_trace", + "records": rows, + "synthesis": {"speedup_ratio": 2.0}, + } + ] + } + ) + loader = MooncakeTraceDatasetLoader( + inline_records=rows, prompt_generator=mock_prompt_generator, run=run + ) + + data = loader.load_dataset() + conversations = loader.convert_to_conversations(data) + + assert len(_omission_warnings(caplog)) == (0 if explicit_recorded else 1) + assert [trace.timestamp for trace in data["s1"]] == [0, 500] + assert ( + conversations[0].context_mode + == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES + ) + assert [turn.raw_messages for turn in conversations[0].turns] == [ + [USER_1], + [USER_2], + ] From 5d48c602e11f5744778e5404fe3b006d357c87ae Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:48:05 -0500 Subject: [PATCH 089/119] test: drop upstream tests that need skipped Kubernetes helpers The config-envelope cases build specs through aiperf.kubernetes, and the warmup-per-lane flag is routed by the fork's agentic replay converter. --- .../config/flags/_config_flag_routing.py | 1 + tests/unit/config/test_plot_envelope.py | 30 ------------- .../config/test_raw_implies_outputs_json.py | 45 ------------------- 3 files changed, 1 insertion(+), 75 deletions(-) diff --git a/src/aiperf/config/flags/_config_flag_routing.py b/src/aiperf/config/flags/_config_flag_routing.py index f650a82f96..6d237b3166 100644 --- a/src/aiperf/config/flags/_config_flag_routing.py +++ b/src/aiperf/config/flags/_config_flag_routing.py @@ -114,6 +114,7 @@ # agentic phase fields, via _apply_agentic_replay_fields "agentic_cache_warmup_duration", "agentic_warmup_grace_period", + "warmup_requests_per_lane", # publishing targets, via build_mlflow / build_otel / # build_network_latency -- the builders the CLI-only converter has # always used, now called by build_cli_overrides too diff --git a/tests/unit/config/test_plot_envelope.py b/tests/unit/config/test_plot_envelope.py index 58528adc17..a9e3e916cf 100644 --- a/tests/unit/config/test_plot_envelope.py +++ b/tests/unit/config/test_plot_envelope.py @@ -10,8 +10,6 @@ from pytest import param from aiperf.config import AIPerfConfig # noqa: F401 (used by Task 3 tests below) -from aiperf.config.artifacts import ArtifactsConfig -from aiperf.config.loader import load_config_from_mapping from aiperf.config.loader.core import load_config_from_string from aiperf.config.loader.errors import ConfigurationError from aiperf.config.plot import ( @@ -20,7 +18,6 @@ ServerMetricsDownsampling, load_plot_envelope_from_path, ) -from aiperf.kubernetes.spec_converter import build_config_envelope def test_plot_envelope_minimal_inline_form_loads(): @@ -387,30 +384,3 @@ def test_build_benchmark_plan_no_plot_section(): config = load_config_from_string(_BASE_BENCHMARK_YAML) plan = build_benchmark_plan(config) assert plan.plot is None - - -def test_k8s_crd_defaulting_cannot_suppress_implied_auto_plot() -> None: - """A ``plot:`` section must still imply auto_plot under Kubernetes. - - The apiserver materializes CRD ``default:`` values on write, so gating the - implication on ``model_fields_set`` meant every AIPerfJob stored an explicit - ``autoPlot: false`` and silently rendered no plots. ``auto_plot`` is - tri-state so there is no CRD default to materialize. - """ - assert ArtifactsConfig.model_fields["auto_plot"].default is None - - spec = { - "benchmark": { - "model": "test-model", - "endpoint": {"url": "http://localhost:8000"}, - "dataset": {"type": "synthetic"}, - "phases": {"type": "concurrency", "concurrency": 1, "requests": 5}, - }, - "plot": {"visualization": {}}, - } - config = load_config_from_mapping(build_config_envelope(spec)) - assert config.benchmark.artifacts.auto_plot is True - - spec["benchmark"]["artifacts"] = {"autoPlot": False} - config = load_config_from_mapping(build_config_envelope(spec)) - assert config.benchmark.artifacts.auto_plot is False diff --git a/tests/unit/config/test_raw_implies_outputs_json.py b/tests/unit/config/test_raw_implies_outputs_json.py index 058e0d63de..578168f0e7 100644 --- a/tests/unit/config/test_raw_implies_outputs_json.py +++ b/tests/unit/config/test_raw_implies_outputs_json.py @@ -17,8 +17,6 @@ from aiperf.config.flags.cli_config import CLIConfig from aiperf.config.flags.converter import convert_cli_to_aiperf from aiperf.config.flags.resolver import resolve_config -from aiperf.config.loader import load_config_from_mapping -from aiperf.kubernetes.spec_converter import build_config_envelope def _artifacts(**kwargs: Any) -> ArtifactsConfig: @@ -131,46 +129,3 @@ def test_yaml_raw_true_with_cli_opt_out(self, tmp_path: Path) -> None: artifacts = cfg.benchmark.artifacts assert artifacts.export_level == ExportLevel.RAW assert artifacts.export_outputs_json is False - - -class TestRawImpliesOutputsJsonUnderKubernetes: - """The implication must survive Kubernetes CRD structural-schema defaulting. - - A CRD ``default:`` is materialized by the apiserver on write, so a bool - field with a non-null default arrives indistinguishable from a user-authored - value. That is why ``export_outputs_json`` is tri-state rather than gated on - ``model_fields_set``: with a ``False`` default the CRD carried - ``default: false``, every AIPerfJob stored an explicit ``false``, and - ``artifacts.raw: true`` silently produced no outputs.json. - """ - - @staticmethod - def _resolve(artifacts: dict[str, Any]) -> ArtifactsConfig: - spec = { - "benchmark": { - "model": "test-model", - "endpoint": {"url": "http://localhost:8000"}, - "dataset": {"type": "synthetic"}, - "phases": {"type": "concurrency", "concurrency": 1, "requests": 5}, - "artifacts": artifacts, - } - } - return load_config_from_mapping(build_config_envelope(spec)).benchmark.artifacts - - def test_crd_omits_default_for_export_outputs_json(self) -> None: - """No CRD default means the apiserver cannot materialize the field.""" - assert ArtifactsConfig.model_fields["export_outputs_json"].default is None - - def test_k8s_raw_true_implies_outputs_json(self) -> None: - assert self._resolve({"raw": True}).export_outputs_json is True - - def test_k8s_explicit_false_still_opts_out(self) -> None: - artifacts = self._resolve({"raw": True, "exportOutputsJson": False}) - assert artifacts.export_level == ExportLevel.RAW - assert artifacts.export_outputs_json is False - - def test_k8s_non_raw_leaves_outputs_json_off(self) -> None: - assert self._resolve({"raw": False}).export_outputs_json is False - - def test_k8s_flag_alone_works_below_raw(self) -> None: - assert self._resolve({"exportOutputsJson": True}).export_outputs_json is True From 76d2bcb863d7c0a0eb1bffcb4d832eb624b0dcf0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:48:05 -0500 Subject: [PATCH 090/119] ci: drop upstream's uv.lock version-patch tooling Reverts the pick of upstream ea28b2e81; the fork does not track uv.lock. --- .agents/skills/bump-version/SKILL.md | 48 +-- .../update-pyproject-version/action.yml | 42 ++- .github/workflows/nightly.yml | 9 +- .github/workflows/resolve-check.yml | 3 - tests/unit/tools/test_patch_version.py | 304 ------------------ tools/patch_version.py | 184 ----------- 6 files changed, 49 insertions(+), 541 deletions(-) delete mode 100644 tests/unit/tools/test_patch_version.py delete mode 100755 tools/patch_version.py diff --git a/.agents/skills/bump-version/SKILL.md b/.agents/skills/bump-version/SKILL.md index a9b8e9f583..f5b52ebebd 100644 --- a/.agents/skills/bump-version/SKILL.md +++ b/.agents/skills/bump-version/SKILL.md @@ -2,7 +2,7 @@ name: bump-version description: Bump the AIPerf package version (e.g. 0.10.0 -> 0.11.0) across pyproject.toml, the mock server, and the version strings shown in docs, then open a PR to main disable-model-invocation: true -allowed-tools: Bash(git fetch *), Bash(git checkout *), Bash(git branch *), Bash(git status *), Bash(git add *), Bash(git commit *), Bash(git push *), Bash(git log *), Bash(git diff *), Bash(grep *), Bash(uv lock *), Bash(pre-commit run *), Bash(gh pr create *), Read, Edit, AskUserQuestion +allowed-tools: Bash(git fetch *), Bash(git checkout *), Bash(git branch *), Bash(git status *), Bash(git add *), Bash(git commit *), Bash(git push *), Bash(git log *), Bash(grep *), Bash(pre-commit run *), Bash(gh pr create *), Read, Edit, AskUserQuestion --- # Bump AIPerf Version @@ -49,8 +49,7 @@ to stay in sync. ``` Review each hit. Update only strings that mean "the AIPerf version". Do NOT touch unrelated version-like strings (dependency pins, `schema_version`, - sample UUIDs, etc.). `uv.lock` is deliberately outside this grep: other - packages may share the version string, and step 5 regenerates it instead. + sample UUIDs, etc.). The canonical set of files (as of 0.10.0 -> 0.11.0) is: - `pyproject.toml` — `version = "..."` under `[project]` @@ -60,21 +59,14 @@ to stay in sync. - `docs/server-metrics/server-metrics-json-schema.md` — every `"aiperf_version": "..."` JSON example and the `(e.g., "...")` description (3 occurrences) - - `uv.lock` — the root `[[package]] name = "aiperf"` entry, regenerated by - `uv lock` in step 5 (never hand-edited) If the grep surfaces a NEW occurrence not in this list, update it too and add it to this skill's canonical list in the same PR. -5. **Regenerate `uv.lock`:** - ```bash - uv lock - ``` - `uv.lock` is tracked and records the root `aiperf` version, so it must move - with `pyproject.toml`. The PR checks `uv sync --locked` in - `run-unit-tests.yml` and `helm-chart.yml` fail on a stale lock, and after - merge the nightly's `uv lock --check` does too. Tests that need the version - import `aiperf.__version__`; leave those alone. +5. **Do NOT update `uv.lock`:** + `uv.lock` is gitignored and not tracked, so it is not part of the bump. + Tests that need the version import `aiperf.__version__` rather than hard-coding + it; leave those alone. 6. **Verify nothing stale remains:** ```bash @@ -82,16 +74,7 @@ to stay in sync. docs/plugins/creating-your-first-plugin.md \ docs/server-metrics/server-metrics-json-schema.md ``` - This must return no matches. Then confirm `NEW_VERSION` is present in each, - and confirm the lockfile agrees: - ```bash - uv lock --check - git diff uv.lock - ``` - Expect the root `aiperf` `version =` line to change. `uv lock` may also - refresh metadata of already-locked packages (for example add an `sdist =` - line published after the lock was generated); that is fine to commit. - Investigate if any other package's `version =` line changes. + This must return no matches. Then confirm `NEW_VERSION` is present in each. 7. **Run pre-commit on the staged files:** ```bash @@ -103,15 +86,15 @@ to stay in sync. Stage only the version files explicitly (never `git add -A` — the working tree may hold unrelated untracked files): ```bash - git add pyproject.toml uv.lock tests/aiperf_mock_server/pyproject.toml \ + git add pyproject.toml tests/aiperf_mock_server/pyproject.toml \ docs/plugins/creating-your-first-plugin.md \ docs/server-metrics/server-metrics-json-schema.md git commit -s -F - <<'EOF' chore: bump aiperf version to - Updates the package and mock-server pyproject.toml versions, regenerates - uv.lock, and refreshes the version strings shown in the plugin tutorial - example and the server-metrics JSON schema example output. + Updates the package and mock-server pyproject.toml versions and refreshes + the version strings shown in the plugin tutorial example and the + server-metrics JSON schema example output. Co-Authored-By: Claude Opus 4.8 EOF @@ -132,16 +115,14 @@ to stay in sync. --body "$(cat <<'EOF' ## Summary - Bump AIPerf package version from `` to ``. - - Updates `pyproject.toml`, regenerates `uv.lock`, and updates the mock-server - `pyproject.toml` and the version strings in the plugin tutorial and - server-metrics JSON schema docs. + - Updates `pyproject.toml`, the mock-server `pyproject.toml`, and the version + strings in the plugin tutorial and server-metrics JSON schema docs. The runtime version is read dynamically from package metadata, so `pyproject.toml` is the source of truth; the other files are doc/example copies. ## Test plan - `grep` confirms no `` strings remain in the bumped files. - - `uv lock --check` passes. - Pre-commit hooks pass. EOF )" @@ -157,6 +138,5 @@ to stay in sync. - NEVER use `--no-verify` or `git commit -n`. - NEVER `git add -A` / `git add .`; stage the version files by explicit path so unrelated untracked files stay out of the commit. -- NEVER hand-edit `uv.lock` (regenerate it with `uv lock`) or touch - non-AIPerf version-like strings. +- NEVER touch `uv.lock` (gitignored) or non-AIPerf version-like strings. - Branch from an up-to-date `main`; one PR = one concern (just the bump). diff --git a/.github/actions/update-pyproject-version/action.yml b/.github/actions/update-pyproject-version/action.yml index 73a790c3d9..da1f53e4e3 100644 --- a/.github/actions/update-pyproject-version/action.yml +++ b/.github/actions/update-pyproject-version/action.yml @@ -1,13 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: Update pyproject.toml and uv.lock version +name: Update pyproject.toml version description: > - Writes a PEP 440 version to pyproject.toml and to the root package entry in - uv.lock via tools/patch_version.py, so `uv sync --locked` stays satisfied. - Requires a checked-out repository and uv on PATH (astral-sh/setup-uv); the - script runs under a uv-provisioned Python 3.13 so its tomllib verification - always runs, on any runner OS. + Validates a PEP 440 version string and patches the version field + in pyproject.toml using Python regex (safer than sed). inputs: version: @@ -19,8 +16,37 @@ inputs: runs: using: composite steps: - - name: Patch pyproject.toml and uv.lock version + - name: Patch pyproject.toml version shell: bash env: TARGET_VERSION: ${{ inputs.version }} - run: uv run --no-project --python 3.13 python tools/patch_version.py "${TARGET_VERSION}" + run: | + python3 - <<'PYEOF' + import os + import re + + target = os.environ["TARGET_VERSION"] + + # PEP 440: X.Y.Z[(a|b|rc)N][.postN][.devN][+local] + # Local version: [a-zA-Z0-9] segments joined by dots (e.g. "g3e3b3b1"). + if not re.match( + r'^\d+\.\d+\.\d+((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?' + r'(\+[a-zA-Z0-9]+(\.[a-zA-Z0-9]+)*)?$', + target, + ): + raise SystemExit(f"::error::Invalid PEP 440 version '{target}'") + + path = "pyproject.toml" + text = open(path).read() + patched, n = re.subn( + r'^(version\s*=\s*")[^"]*(")', + rf'\g<1>{target}\g<2>', + text, + count=1, + flags=re.MULTILINE, + ) + if n != 1: + raise SystemExit(f"::error::Could not find 'version = \"...\"' in {path}") + open(path, "w").write(patched) + print(f"Patched {path}: version = \"{target}\"") + PYEOF diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2fd1dd7684..d712bf57bc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -379,9 +379,6 @@ jobs: - uses: actions/setup-python@v6 with: python-version: '3.13' - # A bare `uv sync` re-locks silently; fail fast on a stale lockfile instead. - - name: Check uv.lock is up to date - run: uv lock --check - name: Validate dependency resolution against required-environments run: uv sync --active --no-install-project @@ -425,15 +422,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 - - name: Patch version in pyproject.toml and uv.lock + - name: Patch version in pyproject.toml uses: ./.github/actions/update-pyproject-version with: version: ${{ needs.prepare-nightly.outputs.wheel_version }} - # Catch a bad version patch here, not ten minutes into the Docker build. - - name: Verify uv.lock matches the patched pyproject.toml - run: uv lock --check - # Embed the commit SHA two ways for redundant discoverability: # 1. [project.urls] Repository = in pyproject.toml → # lands in the wheel METADATA as `Project-URL: Repository, ...`. diff --git a/.github/workflows/resolve-check.yml b/.github/workflows/resolve-check.yml index 73be182853..74863f991e 100644 --- a/.github/workflows/resolve-check.yml +++ b/.github/workflows/resolve-check.yml @@ -42,9 +42,6 @@ jobs: - uses: actions/setup-python@v6 with: python-version: '3.13' - # A bare `uv sync` re-locks silently; fail fast on a stale lockfile instead. - - name: Check uv.lock is up to date - run: uv lock --check - name: Validate dependency resolution against required-environments # --no-install-project skips installing aiperf itself; only resolves # and installs the dependency graph, which is all we need to diff --git a/tests/unit/tools/test_patch_version.py b/tests/unit/tools/test_patch_version.py deleted file mode 100644 index 615d5db8f9..0000000000 --- a/tests/unit/tools/test_patch_version.py +++ /dev/null @@ -1,304 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for the nightly version patcher used by update-pyproject-version.""" - -import shutil -import tomllib -from collections.abc import Callable -from pathlib import Path - -import pytest -from pytest import param - -import tools.patch_version as patch_version_module -from tools.patch_version import PatchError, main, patch_version - -_REPO_ROOT = Path(__file__).resolve().parents[3] - -# A table with its own `name` precedes [project] so an unscoped regex would -# pick up "pytorch-cpu" instead of the project name. -_PYPROJECT = """\ -[[tool.uv.index]] -name = "pytorch-cpu" -url = "https://download.pytorch.org/whl/cpu" - -[project] -name = "aiperf" -version = "0.13.0" -dependencies = ["aioitertools"] - -[tool.uv] -required-environments = ["sys_platform == 'linux'"] -""" - -# Mirrors uv's layout: a same-versioned dependency before the root and an inline -# `{ name = "aiperf", ... }` requires-dist reference after it, neither of which -# may be touched. -_LOCK = """\ -version = 1 -revision = 3 - -[[package]] -name = "aioitertools" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } - -[[package]] -name = "aiperf" -version = "0.13.0" -source = { editable = "." } -dependencies = [ - { name = "aioitertools" }, -] - -[package.metadata] -requires-dist = [ - { name = "aiperf", extras = ["botorch"], marker = "extra == 'optuna'" }, -] -""" - -_NEW = "0.13.0.dev20260910" - - -def _expected_lock(version: str) -> str: - return _LOCK.replace( - 'name = "aiperf"\nversion = "0.13.0"', f'name = "aiperf"\nversion = "{version}"' - ) - - -@pytest.fixture -def tree(tmp_path: Path) -> tuple[Path, Path]: - """A minimal pyproject.toml and uv.lock pair in a temporary directory.""" - pyproject = tmp_path / "pyproject.toml" - lock = tmp_path / "uv.lock" - pyproject.write_text(_PYPROJECT) - lock.write_text(_LOCK) - return pyproject, lock - - -@pytest.mark.parametrize( - "version", - [ - param("0.13.0.dev20260910", id="scheduled-nightly"), - param("0.13.0.dev20260910+manual.34398837728", id="manual-dispatch"), - param("0.14.0rc1", id="release-candidate"), - ], -) # fmt: skip -def test_patch_version_valid_version_updates_only_project_and_root_entries( - tree: tuple[Path, Path], version: str -) -> None: - pyproject, lock = tree - - assert patch_version(version, pyproject, lock) == "aiperf" - - project = tomllib.loads(pyproject.read_text())["project"] - assert project == { - "name": "aiperf", - "version": version, - "dependencies": ["aioitertools"], - } - assert 'name = "pytorch-cpu"' in pyproject.read_text() - assert lock.read_text() == _expected_lock(version) - - -def test_patch_version_preserves_crlf_line_endings(tmp_path: Path) -> None: - pyproject = tmp_path / "pyproject.toml" - lock = tmp_path / "uv.lock" - pyproject.write_bytes(_PYPROJECT.replace("\n", "\r\n").encode()) - lock.write_bytes(_LOCK.replace("\n", "\r\n").encode()) - - patch_version(_NEW, pyproject, lock) - - assert pyproject.read_bytes() == ( - _PYPROJECT.replace('version = "0.13.0"', f'version = "{_NEW}"') - .replace("\n", "\r\n") - .encode() - ) - assert lock.read_bytes() == _expected_lock(_NEW).replace("\n", "\r\n").encode() - - -@pytest.mark.parametrize( - "version", - [ - param("not-a-version", id="garbage"), - param("1.2", id="two-component"), - param("0.13.0+bad_local", id="underscore-in-local"), - param("0.13.0.dev20260910\n", id="trailing-newline"), - ], -) # fmt: skip -def test_patch_version_invalid_version_raises_before_writing( - tree: tuple[Path, Path], version: str -) -> None: - pyproject, lock = tree - - with pytest.raises(PatchError, match="Invalid PEP 440"): - patch_version(version, pyproject, lock) - - assert pyproject.read_text() == _PYPROJECT - assert lock.read_text() == _LOCK - - -def test_patch_version_normalizes_project_name_for_lock_lookup(tmp_path: Path) -> None: - """uv writes the PEP 503 name into uv.lock even when pyproject spells it differently.""" - pyproject = tmp_path / "pyproject.toml" - lock = tmp_path / "uv.lock" - pyproject.write_text(_PYPROJECT.replace('name = "aiperf"', 'name = "AI_Perf.Tool"')) - lock.write_text(_LOCK.replace('name = "aiperf"', 'name = "ai-perf-tool"')) - - assert patch_version(_NEW, pyproject, lock) == "AI_Perf.Tool" - - assert f'name = "ai-perf-tool"\nversion = "{_NEW}"' in lock.read_text() - - -def test_patch_version_missing_lock_raises_with_remedy(tree: tuple[Path, Path]) -> None: - pyproject, lock = tree - lock.unlink() - - with pytest.raises(PatchError, match=r"uv.lock not found; run `uv lock`"): - patch_version(_NEW, pyproject, lock) - - assert pyproject.read_text() == _PYPROJECT - - -@pytest.mark.parametrize( - ("lock_text", "message"), - [ - param( - _LOCK.replace('source = { editable = "." }', 'source = { editable = "src" }'), - "exactly one editable root", - id="no-root-entry", - ), - param( - _LOCK + '\n[[package]]\nname = "aiperf"\nversion = "0.13.0"\nsource = { editable = "." }\n', - "does not carry", - id="two-root-entries", - ), - ], -) # fmt: skip -def test_patch_version_bad_lock_raises_and_writes_nothing( - tree: tuple[Path, Path], lock_text: str, message: str -) -> None: - pyproject, lock = tree - lock.write_text(lock_text) - - with pytest.raises(PatchError, match=message): - patch_version(_NEW, pyproject, lock) - - assert pyproject.read_text() == _PYPROJECT - assert lock.read_text() == lock_text - - -@pytest.mark.parametrize( - ("pyproject_text", "message"), - [ - param('[tool.poetry]\nname = "aiperf"\nversion = "0.13.0"\n', r"no \[project\] table", id="poetry-only"), - param('[project]\nname = "aiperf"\n', r'no version = "..." line', id="no-version"), - param('[project]\nversion = "0.13.0"\n', "no name", id="no-name"), - param('[project\nname = "aiperf"\n', "not valid TOML", id="invalid-toml"), - ], -) # fmt: skip -def test_patch_version_bad_pyproject_raises_and_writes_nothing( - tree: tuple[Path, Path], pyproject_text: str, message: str -) -> None: - pyproject, lock = tree - pyproject.write_text(pyproject_text) - - with pytest.raises(PatchError, match=message): - patch_version(_NEW, pyproject, lock) - - assert pyproject.read_text() == pyproject_text - assert lock.read_text() == _LOCK - - -def test_patch_version_against_repository_files_changes_exactly_two_lines( - tmp_path: Path, -) -> None: - """Bind the textual patterns to the real pyproject.toml and uv.lock layout.""" - pyproject = shutil.copy(_REPO_ROOT / "pyproject.toml", tmp_path / "pyproject.toml") - lock = shutil.copy(_REPO_ROOT / "uv.lock", tmp_path / "uv.lock") - original_pyproject = (_REPO_ROOT / "pyproject.toml").read_text().splitlines() - original_lock = (_REPO_ROOT / "uv.lock").read_text().splitlines() - current = tomllib.loads((_REPO_ROOT / "pyproject.toml").read_text())["project"][ - "version" - ] - # The nightly runs this suite on an already-patched tree, so derive a target - # that can never equal the current version (dev numbers are dates). - target = f"{current.split('.dev')[0].split('+')[0]}.dev99999999" - - assert patch_version(target, Path(pyproject), Path(lock)) == "aiperf" - - changed_pyproject = [ - (a, b) - for a, b in zip( - original_pyproject, Path(pyproject).read_text().splitlines(), strict=True - ) - if a != b - ] - changed_lock = [ - (a, b) - for a, b in zip(original_lock, Path(lock).read_text().splitlines(), strict=True) - if a != b - ] - assert changed_pyproject == [(f'version = "{current}"', f'version = "{target}"')] - assert changed_lock == [(f'version = "{current}"', f'version = "{target}"')] - assert tomllib.loads(Path(pyproject).read_text())["project"]["version"] == target - - -def test_main_success_prints_both_files_and_returns_zero( - tree: tuple[Path, Path], capsys: pytest.CaptureFixture[str] -) -> None: - pyproject, lock = tree - - rc = main([_NEW, "--pyproject", str(pyproject), "--lock", str(lock)]) - - assert rc == 0 - out = capsys.readouterr().out - assert f"Patched {pyproject}" in out and f"Patched {lock}" in out - - -@pytest.mark.parametrize( - ("break_tree", "message"), - [ - param(lambda pyproject, lock: lock.unlink(), "not found", id="missing-lock"), - param(lambda pyproject, lock: (lock.unlink(), lock.mkdir()), "not a regular file", id="lock-is-a-directory"), - ], -) # fmt: skip -def test_main_failure_emits_github_error_annotation( - tree: tuple[Path, Path], - capsys: pytest.CaptureFixture[str], - break_tree: Callable[[Path, Path], object], - message: str, -) -> None: - pyproject, lock = tree - break_tree(pyproject, lock) - - rc = main([_NEW, "--pyproject", str(pyproject), "--lock", str(lock)]) - - assert rc == 1 - assert message in capsys.readouterr().err - assert pyproject.read_text() == _PYPROJECT - - -def test_main_write_failure_reports_error_and_leaves_both_files_untouched( - tree: tuple[Path, Path], - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - """An OSError while staging the lock must not leave pyproject.toml already patched.""" - pyproject, lock = tree - real_stage = patch_version_module._stage - - def failing_stage(path: Path, text: str) -> Path: - if path == lock: - raise PermissionError(13, "Permission denied", str(path)) - return real_stage(path, text) - - monkeypatch.setattr(patch_version_module, "_stage", failing_stage) - - rc = main([_NEW, "--pyproject", str(pyproject), "--lock", str(lock)]) - - assert rc == 1 - assert capsys.readouterr().err.startswith("::error::") - assert pyproject.read_text() == _PYPROJECT - assert lock.read_text() == _LOCK - assert not (pyproject.parent / "pyproject.toml.tmp").exists() diff --git a/tools/patch_version.py b/tools/patch_version.py deleted file mode 100755 index 4475b5740e..0000000000 --- a/tools/patch_version.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Write a PEP 440 version into pyproject.toml and the root entry of uv.lock. - -The nightly pipeline stamps a date-based version onto a checkout before -building. uv.lock records the root project's version too, so patching only -pyproject.toml leaves the lockfile stale: `uv sync --locked` rejects it and a -plain `uv run` rewrites it. Patching both keeps the tree lock-consistent. - -Both rewrites are textual so each file keeps its formatting and line endings -apart from the one version string. The root lock entry is the `[[package]]` -whose name is the PEP 503 form of `[project].name` and whose source is -`{ editable = "." }` (a build-system-backed root). Both patched documents are -re-parsed with tomllib in memory to confirm the intended entries carry the new -version, then staged to temp files and swapped in last, so a parse or -verification failure never touches either file. - -Requires Python 3.11+ (tomllib). - -Usage: - python3 tools/patch_version.py 0.13.0.dev20260910 [--pyproject P] [--lock L] -""" - -from __future__ import annotations - -import argparse -import os -import re -import sys -import tomllib -from pathlib import Path - -# PEP 440: X.Y.Z[(a|b|rc)N][.postN][.devN][+local]; local segments are [a-zA-Z0-9] joined by dots. -_PEP440 = re.compile( - r"\d+\.\d+\.\d+((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?" - r"(\+[a-zA-Z0-9]+(\.[a-zA-Z0-9]+)*)?" -) -# The [project] table body: from its header to the next table header or EOF. -_PROJECT_TABLE = re.compile( - r"^\[project\][^\r\n]*\r?\n(.*?)(?=^\[|\Z)", re.MULTILINE | re.DOTALL -) -_VERSION_LINE = re.compile(r'^(version\s*=\s*")[^"]*(")', re.MULTILINE) - - -class PatchError(Exception): - """A version patch could not be applied or verified; nothing was written.""" - - -def _canonical(name: str) -> str: - """PEP 503 normalized distribution name, which is what uv writes into uv.lock.""" - return re.sub(r"[-_.]+", "-", name).lower() - - -def _root_package_pattern(name: str) -> re.Pattern[str]: - """Match the uv.lock `[[package]]` block for the editable root project `name`.""" - return re.compile( - r'(\[\[package\]\]\r?\nname = "' - + re.escape(_canonical(name)) - + r'"\r?\nversion = ")[^"]*("\r?\nsource = \{ editable = "\." \})' - ) - - -def _read(path: Path) -> str: - """Read text verbatim; newline="" keeps CRLF so the rewrite preserves it.""" - with path.open("r", encoding="utf-8", newline="") as handle: - return handle.read() - - -def _stage(path: Path, text: str) -> Path: - """Write text verbatim to a same-directory temp file and return its path.""" - tmp = path.with_name(path.name + ".tmp") - with tmp.open("w", encoding="utf-8", newline="") as handle: - handle.write(text) - return tmp - - -def _patch_pyproject(text: str, version: str) -> tuple[str, str]: - """Return the patched pyproject text and the project name it declares.""" - try: - project = tomllib.loads(text)["project"] - except tomllib.TOMLDecodeError as exc: - raise PatchError(f"pyproject.toml is not valid TOML: {exc}") from exc - except KeyError: - raise PatchError("pyproject.toml has no [project] table") from None - name = project.get("name") - if not isinstance(name, str) or not name: - raise PatchError("[project] has no name") - table = _PROJECT_TABLE.search(text) - if table is None: - raise PatchError("Could not locate the [project] table in pyproject.toml") - body, n = _VERSION_LINE.subn(rf"\g<1>{version}\g<2>", table.group(1), count=1) - if n != 1: - raise PatchError('[project] has no version = "..." line') - return text[: table.start(1)] + body + text[table.end(1) :], name - - -def _patch_lock(text: str, name: str, version: str) -> str: - """Return the lock text with the editable root entry for `name` at `version`.""" - patched, n = _root_package_pattern(name).subn( - rf"\g<1>{version}\g<2>", text, count=1 - ) - if n != 1: - raise PatchError( - f"Could not find exactly one editable root package entry for {name!r} in uv.lock" - ) - return patched - - -def _verify(pyproject_text: str, lock_text: str, name: str, version: str) -> None: - """Re-parse both patched documents and confirm the intended entries carry `version`.""" - try: - project_version = tomllib.loads(pyproject_text)["project"].get("version") - packages = tomllib.loads(lock_text).get("package", []) - except tomllib.TOMLDecodeError as exc: - raise PatchError(f"patched document is not valid TOML: {exc}") from exc - if project_version != version: - raise PatchError(f"[project] version is not {version!r} after patching") - roots = [ - p - for p in packages - if p.get("name") == _canonical(name) - and p.get("source", {}).get("editable") == "." - ] - if len(roots) != 1 or roots[0].get("version") != version: - raise PatchError(f"uv.lock root package {name!r} does not carry {version!r}") - - -def patch_version(version: str, pyproject: Path, lock: Path) -> str: - """Write `version` into `pyproject` and the root package entry of `lock`. - - Both documents are patched and verified in memory, staged to temp files, - and only then swapped into place. Returns the project name. Raises - PatchError on an invalid version, a missing file, or an entry that cannot - be located. - """ - if not _PEP440.fullmatch(version): - raise PatchError(f"Invalid PEP 440 version {version!r}") - for path in (pyproject, lock): - if path.exists() and not path.is_file(): - raise PatchError(f"{path} is not a regular file") - if not pyproject.is_file(): - raise PatchError(f"{pyproject} not found") - if not lock.is_file(): - raise PatchError( - f"{lock} not found; run `uv lock` to create it, and check that the " - "repository is checked out before this step" - ) - - pyproject_new, name = _patch_pyproject(_read(pyproject), version) - lock_new = _patch_lock(_read(lock), name, version) - _verify(pyproject_new, lock_new, name, version) - staged: list[tuple[Path, Path]] = [] - try: - staged.append((_stage(pyproject, pyproject_new), pyproject)) - staged.append((_stage(lock, lock_new), lock)) - except OSError: - for tmp, _ in staged: - tmp.unlink(missing_ok=True) - raise - for tmp, target in staged: - os.replace(tmp, target) - return name - - -def main(argv: list[str] | None = None) -> int: - """CLI entry point; failures print a GitHub Actions `::error::` annotation.""" - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument("version", help="PEP 440 version to write") - parser.add_argument("--pyproject", type=Path, default=Path("pyproject.toml")) - parser.add_argument("--lock", type=Path, default=Path("uv.lock")) - args = parser.parse_args(argv) - try: - name = patch_version(args.version, args.pyproject, args.lock) - except (PatchError, OSError) as exc: - print(f"::error::{exc}", file=sys.stderr) - return 1 - print(f'Patched {args.pyproject}: version = "{args.version}"') - print(f'Patched {args.lock}: {name} version = "{args.version}"') - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From cf00948ee4d1155cc44ad20fd9a5ad87b451a211 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:52:33 -0500 Subject: [PATCH 091/119] ci: keep installing unit-test dependencies without uv.lock Reverts the pick of upstream 0968f7e60; the fork does not track uv.lock, so 'uv sync --locked' would fail its unit-test workflow. --- .github/workflows/run-unit-tests.yml | 2 +- CONTRIBUTING.md | 1 - Makefile | 14 +++----------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index ffd2fa7f32..71c6306d2d 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -72,7 +72,7 @@ jobs: PYTHONIOENCODING: utf-8 run: | uv venv .venv --python ${{ matrix.python-version }} - uv sync --locked --extra dev --no-dev + uv pip install -e ".[dev]" uv pip install -e "tests/aiperf_mock_server[dev]" uv run python -c "import aiperf_mock_server" uv run python tools/generate_plugin_artifacts.py --enums diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e86f71dd1e..28bc1599aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,7 +67,6 @@ pre-commit install # Install pre-commit hooks | `make first-time-setup` | Full environment setup (venv + install + hooks) | | `make install` | Install project, mock server, and fake amdsmi bindings in editable mode | | `make install-app` | Install project only | -| `make install-app-locked` | Install project only, at the exact versions in `uv.lock` (what CI's `make ci-install` uses) | | `make install-mock-server` | Install mock server only | | `make install-mock-amdsmi` | Install fake `amdsmi` bindings to exercise the AMD telemetry path on non-AMD hardware (see [Mocking a ROCm Environment](docs/reference/mock-amdsmi.md)) | | `make test` | Unit tests (parallel, excludes integration) | diff --git a/Makefile b/Makefile index a131fa3561..8eeb81b23c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ .PHONY: ruff lint ruff-fix lint-fix format fmt check-format check-fmt \ - test coverage clean install install-app install-app-locked precompile docker docker-run first-time-setup \ + test coverage clean install install-app docker docker-run first-time-setup \ ci-install check-mock-server-install \ test-verbose setup-venv install-mock-server install-mock-amdsmi test-ci test-all \ integration-tests integration-tests-ci integration-tests-verbose integration-tests-ci-macos \ @@ -156,12 +156,6 @@ install: install-app install-mock-server install-mock-amdsmi #? install the proj install-app: #? install the project in editable mode. $(activate_venv) && uv pip install -e ".[dev]" -install-app-locked: #? install the project and its dev extra at the exact versions in uv.lock (CI). - $(activate_venv) && uv sync --locked --active --extra dev --no-dev - -precompile: #? precompile source bytecode so parallel test subprocesses never race to write __pycache__ on first import. - $(activate_venv) && python -m compileall -q -x '/(\.venv|\.ruff_cache|.*\.egg-info)/' src/aiperf tests/aiperf_mock_server tests/aiperf_mock_amdsmi - docker: #? build the docker image. docker build -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) $(args) . @@ -236,10 +230,8 @@ first-time-setup: #? convenience command to setup the environment for the first ci-install: #? CI-only environment setup: venv + project + plugin artifacts. No pre-commit hooks, no redundant mock-server install. $(MAKE) setup-venv --no-print-directory - @# From uv.lock, not a fresh resolve: an unpinned install takes whatever was - @# released minutes ago, so an upstream release could fail every PR at once. - @printf "$(bold)$(green)Installing project from uv.lock (+ mock server)...$(reset)\n" - @PATH="$(UV_PATH):$(PATH)" $(MAKE) --no-print-directory install-app-locked install-mock-server install-mock-amdsmi precompile + @printf "$(bold)$(green)Installing project (+ mock server)...$(reset)\n" + @PATH="$(UV_PATH):$(PATH)" $(MAKE) --no-print-directory install @printf "$(bold)$(green)Verifying mock server install...$(reset)\n" @PATH="$(UV_PATH):$(PATH)" $(MAKE) --no-print-directory check-mock-server-install From 58ddb81377db24d03ef1c5db17d48f60b88031f7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 10:52:49 -0500 Subject: [PATCH 092/119] ci: drop upstream's stale PR and issue auto-close workflow Reverts the pick of upstream 56945bb52; on the fork's default branch it would start closing the fork's own PRs and issues. --- .github/workflows/stale_cleaner.yml | 32 ----------------------------- 1 file changed, 32 deletions(-) delete mode 100644 .github/workflows/stale_cleaner.yml diff --git a/.github/workflows/stale_cleaner.yml b/.github/workflows/stale_cleaner.yml deleted file mode 100644 index 93061c2726..0000000000 --- a/.github/workflows/stale_cleaner.yml +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# https://github.com/actions/stale - -name: 'Close stale issues and PRs' -on: - schedule: - - cron: '30 9 * * *' # 09:30 UTC daily - -permissions: - contents: write # only for delete-branch option - issues: write - pull-requests: write - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 - with: - stale-issue-label: 'stale' - stale-pr-label: 'stale' - stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days. As a reminder, please use backlog label to avoid issues being marked as stale.' - close-issue-message: 'This issue has been closed due to inactivity. If you believe this issue is still relevant, please feel free to reopen it with additional context or information.' - stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' - close-pr-message: 'This PR has been closed due to inactivity. If you believe this PR is still relevant, please feel free to reopen it with additional context or information.' - delete-branch: true - days-before-stale: 30 - days-before-close: 5 - any-of-issue-labels: 'bug' - exempt-issue-labels: 'backlog' From 13da68ca5d23f5f1670369262ece47ffb6d8dbcb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 15 Oct 2026 13:23:47 -0500 Subject: [PATCH 093/119] build: keep aiohttp on 3.13 for 1.0.x measurement parity Upstream #1277 raised the floor to 3.14.3; aiohttp is the client that times streamed chunks, so 1.0.x keeps 1.0.9's 3.13 line. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8624ce7e11..1387f41007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ authors = [ requires-python = ">=3.11,<3.14" dependencies = [ "aiofiles~=24.1.0", - "aiohttp>=3.14.3,<4", + "aiohttp~=3.13.3", "cyclopts>=4,<5", "fastapi>=0.115,<1", "ffmpeg-python~=0.2.0", From 0dd2795411c1e87b4d0e84846940a8d48772f980 Mon Sep 17 00:00:00 2001 From: heihei-push <17313890385@vip.163.com> Date: Tue, 20 Oct 2026 11:29:08 +0800 Subject: [PATCH 094/119] fix(tokenizer): short-circuit local paths in _resolve_local_snapshot (#1560) Signed-off-by: heihei-push <17313890385@vip.163.com> Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Co-authored-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit b0276e19398169eb450f5a79db28509dd4bcb48f) --- src/aiperf/common/tokenizer.py | 46 +++++++---- tests/unit/common/test_tokenizer_cache.py | 96 ++++++++++++++++++++++- 2 files changed, 123 insertions(+), 19 deletions(-) diff --git a/src/aiperf/common/tokenizer.py b/src/aiperf/common/tokenizer.py index 03d4d8f893..e6bb543bdb 100644 --- a/src/aiperf/common/tokenizer.py +++ b/src/aiperf/common/tokenizer.py @@ -291,6 +291,25 @@ def _is_tiktoken_cached(name: str) -> bool: return Path(cache_dir, cache_key).is_file() +def _is_local_path(name: str) -> bool: + """Whether *name* names a local tokenizer path rather than an HF repo id. + + Shared by alias resolution (parent process) and offline snapshot + resolution (worker processes) so both treat the same names as local. + ``path.is_dir()`` mirrors transformers, which loads any existing directory + before consulting the hub, so a bare ``tok`` or ``models/tok`` in the CWD + resolves the same way in both places. + + ``path.anchor`` is used instead of ``path.is_absolute()`` because the + latter requires a drive letter on Windows + (``WindowsPath("/home/user/foo").is_absolute() == False``; pinned by + ``test_resolve_alias_treats_posix_absolute_path_as_local``). Anchor is + truthy for any path with a drive and/or root. + """ + path = Path(name) + return bool(path.anchor) or name.startswith(("./", "../")) or path.is_dir() + + def resolve_alias(name: str) -> AliasResolutionResult: """Resolve a tokenizer name alias to its canonical repository ID. @@ -304,21 +323,7 @@ def resolve_alias(name: str) -> AliasResolutionResult: Returns: AliasResolutionResult with resolved name and any suggestions. """ - # Check if this looks like a local path. Use ``path.anchor`` instead of - # ``path.is_absolute()`` because the latter requires a drive letter on - # Windows (``WindowsPath("/home/user/foo").is_absolute() == False``; - # pinned by ``test_resolve_alias_treats_posix_absolute_path_as_local``). - # Anchor is truthy for any path with a drive AND/OR root, so a POSIX-style - # absolute path passed on Windows is correctly recognized as local. - path = Path(name) - is_local_path = ( - bool(path.anchor) - or name.startswith("./") - or name.startswith("../") - or path.is_dir() - ) - - if is_local_path or _is_offline_mode() or _is_hf_cached(name): + if _is_local_path(name) or _is_offline_mode() or _is_hf_cached(name): return AliasResolutionResult(resolved_name=name) # Lazy import HuggingFace Hub @@ -670,10 +675,21 @@ def _resolve_local_snapshot(cls, name: str, revision: str) -> str: Tries the canonical repo ID first, then alias matches (e.g. ``gpt2`` cached as ``openai-community/gpt2``). Raises ``OSError`` with a clear message if no local snapshot exists. + + A local path is returned as-is: ``snapshot_download`` validates its + argument as a repo id, so an absolute path raises ``HFValidationError`` + and a relative one misses the cache. ``_is_local_path`` is the same + predicate alias resolution uses, so offline workers load the directory + the online parent loaded. """ from huggingface_hub import snapshot_download from huggingface_hub.errors import LocalEntryNotFoundError + if _is_local_path(name): + if Path(name).is_dir(): + return name + raise OSError(f"Local tokenizer path '{name}' is not a directory.") + candidates = [name] if "/" not in name: cached_alias = cls._find_cached_model_for_alias(name) diff --git a/tests/unit/common/test_tokenizer_cache.py b/tests/unit/common/test_tokenizer_cache.py index c379892b55..6c3c3a96bc 100644 --- a/tests/unit/common/test_tokenizer_cache.py +++ b/tests/unit/common/test_tokenizer_cache.py @@ -3,12 +3,14 @@ """Tests for HuggingFace cache detection in the tokenizer module.""" -from pathlib import Path +from pathlib import Path, PureWindowsPath from unittest.mock import patch import pytest +from pytest import param -from aiperf.common.tokenizer import Tokenizer, _is_hf_cached +from aiperf.common.exceptions import TokenizerError +from aiperf.common.tokenizer import Tokenizer, _is_hf_cached, _is_local_path @pytest.fixture @@ -168,7 +170,7 @@ def test_online_cached_load_does_not_force_local_files_only( monkeypatch.delenv("HF_HUB_OFFLINE", raising=False) monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False) - seen_kwargs: dict[str, object] = {} + seen_kwargs: dict[str, str] = {} def fake_from_pretrained(name_or_path, **kwargs): seen_kwargs["name_or_path"] = name_or_path @@ -225,7 +227,7 @@ def fake_snapshot_download(name, revision, local_files_only): monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot_download) - seen: dict[str, object] = {} + seen: dict[str, str] = {} def fake_from_pretrained(name_or_path, **kwargs): seen["name_or_path"] = name_or_path @@ -250,6 +252,92 @@ def fake_from_pretrained(name_or_path, **kwargs): assert "local_files_only" not in seen +class TestOfflineLocalPathLoad: + """Offline workers must load the same local directory the online parent loads. + + Worker pools (``parallel_decode``, ``parallel_convert``) force + ``HF_HUB_OFFLINE=1`` and reload the tokenizer by name. Online, + transformers loads any existing directory before consulting the hub, so + every row below is a local load in the parent. Offline, handing these + names to ``snapshot_download`` raised ``HFValidationError`` (anchored + paths) or missed the cache (relative paths) and broke the process pool, + or silently loaded a different cached tokenizer (shadowed repo id). + """ + + @pytest.fixture + def offline_loader( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> dict[str, str]: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + + def fail_snapshot_download(*args, **kwargs): + raise AssertionError( + f"snapshot_download must not see a local path: {args} {kwargs}" + ) + + monkeypatch.setattr("huggingface_hub.snapshot_download", fail_snapshot_download) + seen: dict[str, str] = {} + + def fake_from_pretrained(name_or_path, **kwargs): + seen["name_or_path"] = name_or_path + return _FakeHfTokenizer() + + monkeypatch.setattr( + "transformers.AutoTokenizer.from_pretrained", fake_from_pretrained + ) + return seen + + @pytest.mark.parametrize( + ("rel_dir", "absolute"), + [ + param("tok", True, id="absolute-path"), + param("./tok", False, id="dot-slash-relative"), + param("models/tok", False, id="relative-no-dot-slash"), + param("tok", False, id="bare-dir-in-cwd"), + param("Org/Model", False, id="cwd-dir-shadows-repo-id"), + ], + ) # fmt: skip + def test_from_pretrained_offline_local_dir_loads_that_dir( + self, + tmp_path: Path, + offline_loader: dict[str, str], + rel_dir: str, + absolute: bool, + ) -> None: + local_dir = tmp_path / rel_dir + local_dir.mkdir(parents=True) + name = str(local_dir) if absolute else rel_dir + + Tokenizer.from_pretrained(name, resolve_alias=False) + + assert Path(offline_loader["name_or_path"]).resolve() == local_dir.resolve() + + def test_from_pretrained_offline_missing_local_path_raises_clear_error( + self, tmp_path: Path, offline_loader: dict[str, str] + ) -> None: + with pytest.raises(TokenizerError, match="is not a directory"): + Tokenizer.from_pretrained(str(tmp_path / "missing"), resolve_alias=False) + + assert "name_or_path" not in offline_loader + + def test_is_local_path_treats_posix_absolute_path_as_local_on_windows( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """``WindowsPath("/data/x").is_absolute()`` is False; the anchor is not.""" + + class _NonexistentWindowsPath(PureWindowsPath): + def is_dir(self) -> bool: + return False + + monkeypatch.setattr("aiperf.common.tokenizer.Path", _NonexistentWindowsPath) + + assert _NonexistentWindowsPath("/data/models/x").is_absolute() is False + assert _is_local_path("/data/models/x") is True + assert _is_local_path("Org/Model") is False + + class _FakeHfTokenizer: """Minimal stub that satisfies ``Tokenizer._apply_kwarg_overrides``.""" From 2bbe0c9e81305e082e872ec073eb252f94ea213a Mon Sep 17 00:00:00 2001 From: Apoorva Garg Date: Thu, 22 Oct 2026 11:45:29 -0400 Subject: [PATCH 095/119] ci: shard integration tests 4-way and cancel superseded PR runs (#1584) Signed-off-by: Apoorva Garg <271577+agarg21@users.noreply.github.com> Co-authored-by: Apoorva Garg <271577+agarg21@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit b1746a84a4665810abebb5ab09a6677d371c9e22) --- .github/workflows/run-integration-tests.yml | 8 +++ tests/harness/ci_shard.py | 52 +++++++++++++++ tests/integration/conftest.py | 27 ++++++++ tests/unit/ci/test_integration_ci_shard.py | 73 +++++++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 tests/harness/ci_shard.py create mode 100644 tests/unit/ci/test_integration_ci_shard.py diff --git a/.github/workflows/run-integration-tests.yml b/.github/workflows/run-integration-tests.yml index 3d61461852..a156af43a4 100644 --- a/.github/workflows/run-integration-tests.yml +++ b/.github/workflows/run-integration-tests.yml @@ -8,6 +8,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: integration-tests: runs-on: ${{ matrix.os }} @@ -16,6 +20,10 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.11", "3.12", "3.13"] + shard: [0, 1, 2, 3] + env: + AIPERF_SHARD_INDEX: ${{ matrix.shard }} + AIPERF_SHARD_TOTAL: 4 steps: - uses: actions/checkout@v6 diff --git a/tests/harness/ci_shard.py b/tests/harness/ci_shard.py new file mode 100644 index 0000000000..bc73d44def --- /dev/null +++ b/tests/harness/ci_shard.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Round-robin CI sharding of the integration suite (AIPERF_SHARD_INDEX / AIPERF_SHARD_TOTAL).""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TypeVar + +import pytest + +T = TypeVar("T") + +SHARD_INDEX_ENV = "AIPERF_SHARD_INDEX" +SHARD_TOTAL_ENV = "AIPERF_SHARD_TOTAL" + + +def ci_shard_from_env(environ: Mapping[str, str]) -> tuple[int, int]: + """Return ``(index, total)``; ``(0, 1)`` (no sharding) when unset. + + Raises ``pytest.UsageError`` for a non-integer value, a total below 1, or an + index outside ``[0, total)``. + """ + try: + total = int(environ.get(SHARD_TOTAL_ENV, "1") or 1) + index = int(environ.get(SHARD_INDEX_ENV, "0") or 0) + except ValueError as e: + raise pytest.UsageError( + f"{SHARD_INDEX_ENV} and {SHARD_TOTAL_ENV} must be integers: {e}" + ) from e + if total < 1: + raise pytest.UsageError(f"{SHARD_TOTAL_ENV}={total} must be >= 1") + if total == 1: + return 0, 1 + if not 0 <= index < total: + raise pytest.UsageError(f"{SHARD_INDEX_ENV}={index} must be in [0, {total})") + return index, total + + +def select_ci_shard( + items: Sequence[T], index: int, total: int +) -> tuple[list[T], list[T]]: + """Split ``items`` into ``(keep, drop)`` for shard ``index`` of ``total``. + + Round-robin over the given order, so every xdist worker (same collection + order) computes the same subset. + """ + keep: list[T] = [] + drop: list[T] = [] + for i, item in enumerate(items): + (keep if i % total == index else drop).append(item) + return keep, drop diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f9229a5841..4640d6aa9c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,6 +23,7 @@ from aiperf_mock_server import serve as aiperf_mock_server_serve from aiperf.common.logging import AIPerfLogger +from tests.harness.ci_shard import ci_shard_from_env, select_ci_shard from tests.harness.subprocess import _new_process_group_kwargs from tests.harness.utils import ( AIPerfCLI, @@ -147,6 +148,31 @@ def setup_integration_tokenizer(): os.environ.pop("TRANSFORMERS_OFFLINE", None) +def pytest_configure(config: pytest.Config) -> None: + """Validate the CI shard settings before any test is collected. + + This runs in the xdist controller before workers start, so a bad + AIPERF_SHARD_INDEX / AIPERF_SHARD_TOTAL fails with a clear usage error + (raised inside a worker it only surfaces as an xdist INTERNALERROR). + """ + ci_shard_from_env(os.environ) + + +def _apply_ci_shard(config: pytest.Config, items: list[pytest.Item]) -> None: + """Keep only this CI shard's tests when AIPERF_SHARD_TOTAL > 1 (no-op when unset). + + Round-robin over the deterministic collection order (before marker + deselection), so every xdist worker computes the same subset. + """ + index, total = ci_shard_from_env(os.environ) + if total <= 1: + return + keep, drop = select_ci_shard(items, index, total) + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = keep + + def pytest_collection_modifyitems(config, items): """Auto-retry integration tests on Windows to absorb parallel-race flakes. @@ -158,6 +184,7 @@ def pytest_collection_modifyitems(config, items): fail only in parallel. Retry up to 2 times to mask that residual contention. Linux runs are unaffected (no marker added there). """ + _apply_ci_shard(config, items) if sys.platform != "win32": return flaky = pytest.mark.flaky(reruns=2, reruns_delay=5) diff --git a/tests/unit/ci/test_integration_ci_shard.py b/tests/unit/ci/test_integration_ci_shard.py new file mode 100644 index 0000000000..26eb3c5a7a --- /dev/null +++ b/tests/unit/ci/test_integration_ci_shard.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The integration-suite CI shards must partition the unsharded selection. + +``tests/integration/conftest.py`` keeps one round-robin shard per CI job +(``AIPERF_SHARD_INDEX`` / ``AIPERF_SHARD_TOTAL``). If the shards ever overlap +or miss an item, tests silently run twice or never. +""" + +from __future__ import annotations + +import pytest +from pytest import param + +from tests.harness.ci_shard import ci_shard_from_env, select_ci_shard + + +@pytest.mark.parametrize( + "n_items,total", + [ + param(0, 4, id="empty"), + param(3, 4, id="fewer-items-than-shards"), + param(352, 4, id="ci-size"), + param(353, 3, id="uneven"), + ], +) # fmt: skip +def test_select_ci_shard_shards_partition_items(n_items: int, total: int) -> None: + items = [f"test_{i}" for i in range(n_items)] + shards = [select_ci_shard(items, index, total)[0] for index in range(total)] + + seen = [item for shard in shards for item in shard] + assert sorted(seen) == sorted(items) + assert len(seen) == len(set(seen)) + assert max(map(len, shards)) - min(map(len, shards)) <= 1 + + +def test_select_ci_shard_keep_and_drop_split_items() -> None: + items = list(range(10)) + keep, drop = select_ci_shard(items, 1, 4) + assert keep == [1, 5, 9] + assert sorted(keep + drop) == items + + +@pytest.mark.parametrize( + "environ,expected", + [ + param({}, (0, 1), id="unset"), + param({"AIPERF_SHARD_TOTAL": "1", "AIPERF_SHARD_INDEX": "3"}, (0, 1), id="total-1"), + param({"AIPERF_SHARD_TOTAL": "4", "AIPERF_SHARD_INDEX": "3"}, (3, 4), id="last-shard"), + param({"AIPERF_SHARD_TOTAL": "4", "AIPERF_SHARD_INDEX": ""}, (0, 4), id="empty-index"), + ], +) # fmt: skip +def test_ci_shard_from_env_valid_values_returns_index_and_total( + environ: dict[str, str], expected: tuple[int, int] +) -> None: + assert ci_shard_from_env(environ) == expected + + +@pytest.mark.parametrize( + "environ,message", + [ + param({"AIPERF_SHARD_TOTAL": "4", "AIPERF_SHARD_INDEX": "4"}, r"must be in \[0, 4\)", id="index-too-high"), + param({"AIPERF_SHARD_TOTAL": "4", "AIPERF_SHARD_INDEX": "-1"}, r"must be in \[0, 4\)", id="negative"), + param({"AIPERF_SHARD_TOTAL": "0"}, r"must be >= 1", id="zero-total"), + param({"AIPERF_SHARD_TOTAL": "-2"}, r"must be >= 1", id="negative-total"), + param({"AIPERF_SHARD_TOTAL": "four"}, "must be integers", id="non-integer"), + ], +) # fmt: skip +def test_ci_shard_from_env_invalid_values_raises_usage_error( + environ: dict[str, str], message: str +) -> None: + with pytest.raises(pytest.UsageError, match=message): + ci_shard_from_env(environ) From 732c7e8ec9ee453ff03b0f3772f337840785404b Mon Sep 17 00:00:00 2001 From: jxj Date: Wed, 28 Oct 2026 04:17:58 +0800 Subject: [PATCH 096/119] docs: make profile export loading examples self-contained (#1499) Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit e65c625800ff8cb7749580a4b605009497fa77dc) --- docs/tutorials/working-with-profile-exports.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials/working-with-profile-exports.md b/docs/tutorials/working-with-profile-exports.md index 4e4beec12e..91ea8aee11 100644 --- a/docs/tutorials/working-with-profile-exports.md +++ b/docs/tutorials/working-with-profile-exports.md @@ -184,6 +184,7 @@ AIPerf output files can be parsed using the native Pydantic models for type-safe ### Synchronous Loading ```python +from pathlib import Path from aiperf.common.models import MetricRecordInfo def load_records(file_path: Path) -> list[MetricRecordInfo]: @@ -202,6 +203,7 @@ def load_records(file_path: Path) -> list[MetricRecordInfo]: For large benchmark runs with thousands of requests, use async file I/O for better performance: ```python +from pathlib import Path import aiofiles from aiperf.common.models import MetricRecordInfo From 3516119316e8394d37ac15a014959ac70ff86f3b Mon Sep 17 00:00:00 2001 From: jxj Date: Wed, 28 Oct 2026 05:52:44 +0800 Subject: [PATCH 097/119] docs: make synthesis API examples runnable (#1497) Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> Signed-off-by: xinjun.jiang Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit be66ff334726a8f27b1eacfff37212181bbe3726) --- docs/api/synthesis.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/api/synthesis.md b/docs/api/synthesis.md index 8b244aa569..cf656c65f2 100644 --- a/docs/api/synthesis.md +++ b/docs/api/synthesis.md @@ -7,6 +7,8 @@ sidebar-title: Prefix Synthesis API Reference Complete API documentation for the prefix synthesis module. +Initialize the global random generator once before constructing an `EmpiricalSampler` or `Synthesizer` directly. The standalone examples below use `rng.init(42)` for reproducible results; when combining examples in one process, initialize it only once. + ## Module: `aiperf.dataset.synthesis` ### Classes @@ -182,8 +184,11 @@ EmpiricalSampler(data: list[int] | list[float]) -> None **Example:** ```python +from aiperf.common import random_generator as rng from aiperf.dataset.synthesis import EmpiricalSampler, EmpiricalSamplerStats +rng.init(42) + # Create sampler from observed data data = [100, 200, 150, 300, 250, 100, 200] sampler = EmpiricalSampler(data) @@ -279,9 +284,12 @@ Synthesizer(params: SynthesisParams | None = None) -> None **Example:** ```python +from aiperf.common import random_generator as rng from aiperf.dataset.synthesis import Synthesizer from aiperf.dataset.synthesis.models import SynthesisParams +rng.init(42) + # Create synthesizer with custom parameters params = SynthesisParams( speedup_ratio=2.0, @@ -295,8 +303,8 @@ synthetic_traces = synthesizer.synthesize_from_file("input.jsonl") # Synthesize from list synthetic_traces = synthesizer.synthesize_traces([ - {"input_length": 512, "output_length": 64, "hash_ids": [1, 2, 3]}, - {"input_length": 768, "output_length": 128, "hash_ids": [1, 2, 4]}, + {"input_length": 1536, "output_length": 64, "hash_ids": [1, 2, 3]}, + {"input_length": 2048, "output_length": 128, "hash_ids": [1, 2, 4]}, ]) # Get statistics @@ -484,7 +492,8 @@ aiperf profile \ **Via SynthesisConfig:** ```python -from aiperf.config.flags._input import SynthesisConfig +from aiperf.config.dataset.trace import SynthesisConfig +from aiperf.dataset.synthesis.models import SynthesisParams config = SynthesisConfig( speedup_ratio=2.0, @@ -493,16 +502,17 @@ config = SynthesisConfig( max_isl=4096, ) -# Check if synthesis would be triggered -if config.should_synthesize(): - print("Synthesis will be applied") +params = SynthesisParams.from_synthesis_config(config) ``` **Direct Instantiation:** ```python +from aiperf.common import random_generator as rng from aiperf.dataset.synthesis import Synthesizer from aiperf.dataset.synthesis.models import SynthesisParams +rng.init(42) + params = SynthesisParams( speedup_ratio=2.0, prefix_len_multiplier=1.5, From 7a5573fb78facd9f79784d4da10933d04a57a86e Mon Sep 17 00:00:00 2001 From: Maze999 <162086986+Maze999@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:29:47 +0530 Subject: [PATCH 098/119] feat(dataset): support self-reported audio_duration_seconds for single_turn datasets (#1481) Signed-off-by: meka-giridhar_hpeprod Signed-off-by: Maze999 <162086986+Maze999@users.noreply.github.com> Co-authored-by: meka-giridhar_hpeprod Co-authored-by: Harshini Komali <157742537+lkomali@users.noreply.github.com> (cherry picked from commit 8bf9af32ddadeec38ddfb7571d957009facb850e) --- docs/metrics-reference.md | 2 +- docs/tutorials/asr.md | 3 + docs/tutorials/custom-dataset.md | 31 +++++- src/aiperf/dataset/loader/models.py | 35 +++++++ src/aiperf/dataset/loader/multi_turn.py | 1 + src/aiperf/dataset/loader/single_turn.py | 1 + src/aiperf/dataset/memory_map_utils.py | 23 ++++- src/aiperf/workers/inference_client.py | 18 +--- src/aiperf/workers/worker.py | 4 +- tests/unit/dataset/loader/test_multi_turn.py | 29 ++++++ tests/unit/dataset/loader/test_single_turn.py | 98 +++++++++++++++++++ tests/unit/dataset/test_payload_mmap.py | 28 ++++-- tests/unit/workers/test_inference_client.py | 37 +++++++ tests/unit/workers/test_worker.py | 8 +- 14 files changed, 289 insertions(+), 29 deletions(-) diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index c8e5c228fc..c5e89c93da 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -762,7 +762,7 @@ video_peak_memory = response.data.peak_memory_mb Per-request input audio duration in seconds. Hidden from the console summary; available in JSON / CSV record exports for characterizing dataset shape and verifying RTFx calculations. **Notes:** -- Only computed when the request carries `audio_duration_seconds` (e.g., ASR datasets such as LibriSpeech). +- Only computed when the request carries `audio_duration_seconds` (e.g., ASR datasets such as LibriSpeech, or a `--custom-dataset-type single_turn` entry with an explicit `audio_duration_seconds` field). - Aggregate stats (avg, p50, p99) are computed automatically. ### Inverse Real-Time Factor (RTFx) diff --git a/docs/tutorials/asr.md b/docs/tutorials/asr.md index 3ac6166611..08e384b2f6 100644 --- a/docs/tutorials/asr.md +++ b/docs/tutorials/asr.md @@ -28,6 +28,9 @@ their full contents. Clips longer than 30 seconds are automatically skipped to stay within typical ASR model context limits. +To benchmark your own audio files instead, see +[Per-Request Audio Duration (RTFx)](custom-dataset.md#per-request-audio-duration-rtfx). + --- ## Start a vLLM Server diff --git a/docs/tutorials/custom-dataset.md b/docs/tutorials/custom-dataset.md index dd07bf0fba..5b9b5b00e0 100644 --- a/docs/tutorials/custom-dataset.md +++ b/docs/tutorials/custom-dataset.md @@ -203,6 +203,35 @@ EOF The `extra` field also works per-turn in multi_turn datasets. +### Per-Request Audio Duration (RTFx) + +For ASR/audio-transcription benchmarks (`--endpoint-type audio_transcription`), AIPerf computes the RTFx (Inverse Real-Time Factor) metric from each request's audio duration. Synthetic audio and HuggingFace ASR datasets (e.g. LibriSpeech) report this automatically, but custom audio files have no built-in way to know their duration. Self-report it with the `audio_duration_seconds` field: + +```bash +cat > audio_with_duration.jsonl << 'EOF' +{"audios": ["/path/to/call_30s.wav"], "audio_duration_seconds": 30.0} +{"audios": ["/path/to/call_120s.wav"], "audio_duration_seconds": 120.0} +EOF + +aiperf profile \ + --model openai/whisper-large-v3 \ + --endpoint-type audio_transcription \ + --input-file audio_with_duration.jsonl \ + --custom-dataset-type single_turn \ + --url localhost:8000 \ + --request-count 2 +``` + +Without `audio_duration_seconds`, RTFx is omitted from the results for these requests. The value is self-reported and unvalidated against the actual audio content; it must be a finite value greater than 0 (`0`, negative, or infinite values are rejected when the dataset is loaded). It must also describe exactly one clip: the transcription endpoint only sends the first clip, so pairing it with more than one clip (several `audios` entries, or one `audios` object with several `contents`) is rejected at load time rather than silently computing RTFx against the wrong duration. Use one JSONL line per clip. + +The field is supported in `single_turn` and `multi_turn` datasets. In a `multi_turn` conversation, set it on each turn that carries audio; every turn is dispatched as its own request and reports RTFx against its own duration: + +```json +{"session_id": "call_1", "turns": [{"audios": ["/path/to/part1_30s.wav"], "audio_duration_seconds": 30.0}, {"audios": ["/path/to/part2_45s.wav"], "audio_duration_seconds": 45.0}]} +``` + +The `random_pool` dataset type does not support `audio_duration_seconds`: the field is ignored there and RTFx is not reported. + --- ## Multi-Turn Datasets @@ -287,7 +316,7 @@ Log File: artifacts/Qwen_Qwen3-0.6B-openai-chat-concurrency2/logs/aiperf.log - Each turn includes full conversation history - Turns execute sequentially within each conversation - Multiple conversations run concurrently (up to `--concurrency`) -- Each turn supports `output_length` and `extra` (same semantics as single_turn — vendor extras shallow-merged into the top of the wire body, latest turn wins for chat-style endpoints) +- Each turn supports `output_length`, `audio_duration_seconds`, and `extra` (same semantics as single_turn — vendor extras shallow-merged into the top of the wire body, latest turn wins for chat-style endpoints; see [Per-Request Audio Duration](#per-request-audio-duration-rtfx)) ### Inline alternative diff --git a/src/aiperf/dataset/loader/models.py b/src/aiperf/dataset/loader/models.py index e0a5cb2fc5..85f99c827b 100644 --- a/src/aiperf/dataset/loader/models.py +++ b/src/aiperf/dataset/loader/models.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, Field, model_validator from aiperf.common.enums import AssistantResponseMode +from aiperf.common.finite import FiniteFloat from aiperf.common.models import AIPerfBaseModel, Audio, Image, Text, Video from aiperf.plugin.enums import CustomDatasetType @@ -114,6 +115,13 @@ class SingleTurn(AIPerfBaseModel): gt=0, description="Maximum number of output tokens to generate for this request. Overrides the global --osl setting when specified.", ) + audio_duration_seconds: FiniteFloat | None = Field( + default=None, + gt=0, + description="Duration of the audio content in seconds, self-reported by the caller. " + "Used to compute RTFx (Inverse Real-Time Factor) for ASR benchmarks when audio " + "duration cannot be derived automatically (e.g. custom/concatenated audio files).", + ) extra: dict[str, Any] | None = Field( default=None, description="Per-turn extra fields shallow-merged into the request body at dispatch time. Keys override formatter defaults on collision.", @@ -164,6 +172,33 @@ def validate_at_least_one_modality(self) -> "SingleTurn": raise ValueError("At least one modality must be provided") return self + @model_validator(mode="after") + def validate_audio_duration_describes_one_audio(self) -> "SingleTurn": + """Ensure audio_duration_seconds unambiguously describes a single clip. + + The transcription endpoint only ever sends the first clip, so a duration + paired with more than one clip would not describe what is actually + transmitted and would silently skew RTFx. Clips are counted across both + the `audios` list and each `Audio` object's `contents` batch. + """ + if self.audio_duration_seconds is None: + return self + clip_count = 1 if self.audio else 0 + for audio in self.audios or []: + clip_count += 1 if isinstance(audio, str) else len(audio.contents) + if clip_count == 0: + raise ValueError( + "audio_duration_seconds requires audio or audios to be set" + ) + if clip_count > 1: + raise ValueError( + f"audio_duration_seconds describes a single audio clip, but " + f"{clip_count} were given. Only the first is sent, so the " + f"duration would not match the request. Split these into one " + f"entry per clip." + ) + return self + class MultiTurn(AIPerfBaseModel): """Defines the schema for multi-turn conversations. diff --git a/src/aiperf/dataset/loader/multi_turn.py b/src/aiperf/dataset/loader/multi_turn.py index f223fead3d..3136231ae4 100644 --- a/src/aiperf/dataset/loader/multi_turn.py +++ b/src/aiperf/dataset/loader/multi_turn.py @@ -279,6 +279,7 @@ def _build_turn(self, single_turn: SingleTurn, media: dict[str, list[Any]]) -> T delay=self._delay_cap_tracker.clamp(single_turn.delay), role=single_turn.role, max_tokens=single_turn.output_length, + audio_duration_seconds=single_turn.audio_duration_seconds, extra_body=single_turn.extra, ) diff --git a/src/aiperf/dataset/loader/single_turn.py b/src/aiperf/dataset/loader/single_turn.py index 6a402ce04f..3d1b59251d 100644 --- a/src/aiperf/dataset/loader/single_turn.py +++ b/src/aiperf/dataset/loader/single_turn.py @@ -140,6 +140,7 @@ def convert_to_conversations( delay=single_turn.delay, role=single_turn.role, max_tokens=single_turn.output_length, + audio_duration_seconds=single_turn.audio_duration_seconds, extra_body=single_turn.extra, ) ) diff --git a/src/aiperf/dataset/memory_map_utils.py b/src/aiperf/dataset/memory_map_utils.py index d9f6181f18..b215ba200e 100644 --- a/src/aiperf/dataset/memory_map_utils.py +++ b/src/aiperf/dataset/memory_map_utils.py @@ -48,6 +48,7 @@ MemoryMapFileOperationError, MemoryMapSerializationError, ) +from aiperf.common.finite import FiniteFloat from aiperf.common.hooks import on_init, on_stop from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.common.models import ( @@ -187,8 +188,8 @@ async def add_conversation( # Pre-encode each turn's raw_payload and write the bytes directly; # workers replay these verbatim with no deserialization. Persist # turn scalars in the index so metric enrichment can restore - # max_tokens / scheduled_send_ms / source_kind without the full - # Conversation. + # max_tokens / scheduled_send_ms / source_kind / audio_duration_seconds + # without the full Conversation. turn_offsets: list[PayloadOffset] = [] for turn in conversation.turns: payload_bytes = orjson.dumps(turn.raw_payload) @@ -199,6 +200,7 @@ async def add_conversation( max_tokens=_resolve_turn_max_tokens(turn), timestamp=turn.timestamp, source_kind=turn.source_kind, + audio_duration_seconds=turn.audio_duration_seconds, ) ) self._current_offset += len(payload_bytes) @@ -524,6 +526,15 @@ class PayloadOffset(AIPerfBaseModel): "PAYLOAD_BYTES path." ), ) + audio_duration_seconds: FiniteFloat | None = Field( + default=None, + gt=0, + description=( + "Audio duration in seconds from Turn.audio_duration_seconds. " + "Restored onto reconstructed Turns so the audio-duration and " + "RTFx metrics stay live on the PAYLOAD_BYTES path." + ), + ) # Wire-body keys that encode the same Turn.max_tokens scalar across endpoints. @@ -542,6 +553,7 @@ class PayloadTurnData: max_tokens: int | None = None timestamp: int | float | None = None source_kind: str | None = None + audio_duration_seconds: float | None = None def max_tokens_from_wire_payload(payload: dict[str, Any] | None) -> int | None: @@ -600,6 +612,7 @@ def turn_from_payload_turn(entry: PayloadTurnData) -> Turn: max_tokens=max_tokens, timestamp=entry.timestamp, source_kind=entry.source_kind, + audio_duration_seconds=entry.audio_duration_seconds, ) @@ -859,8 +872,9 @@ def get_payload_turn( ) -> PayloadTurnData | None: """Get payload bytes plus turn scalars for a specific turn. - Scalars (``max_tokens``, ``timestamp``, ``source_kind``) are restored - from the index when present. When ``max_tokens`` is missing (legacy indexes or turns + Scalars (``max_tokens``, ``timestamp``, ``source_kind``, + ``audio_duration_seconds``) are restored from the index when present. + When ``max_tokens`` is missing (legacy indexes or turns that never set it on the Turn), it is recovered from wire JSON keys ``max_tokens`` / ``max_completion_tokens`` / ``max_output_tokens``. @@ -895,6 +909,7 @@ def get_payload_turn( max_tokens=max_tokens, timestamp=offset_info.timestamp, source_kind=offset_info.source_kind, + audio_duration_seconds=offset_info.audio_duration_seconds, ) def close(self) -> None: diff --git a/src/aiperf/workers/inference_client.py b/src/aiperf/workers/inference_client.py index 0430f2527e..ab11a50e7e 100644 --- a/src/aiperf/workers/inference_client.py +++ b/src/aiperf/workers/inference_client.py @@ -244,21 +244,11 @@ def _finalize_request_record( record: RequestRecord, request_info: RequestInfo, ) -> RequestRecord: - """Enrich a RequestRecord with the original request info. - - The hoisted metric inputs ``max_tokens``, ``audio_duration_seconds``, - and ``scheduled_send_ms`` live only on the originating turn — they are - NOT ``RecordContext`` fields on ``request_info`` and so are not copied - by the downcast in ``_enrich_request_record``. Populate them explicitly - so the record processor (``osl_mismatch`` / ``audio_duration`` / - ``replay_send_schedule_offset`` metrics) reads them directly off the - slim record without the full ``turns`` list on the wire: - ``max_tokens`` and ``scheduled_send_ms`` from the dispatch (last) turn, - ``audio_duration_seconds`` from the first turn (ASR requests are - single-turn; mirrors the pre-hoist ``turns[0]`` read). + """Enrich a RequestRecord with the original request info, hoisting + per-turn values (``max_tokens``, ``audio_duration_seconds``, + ``scheduled_send_ms``) from the dispatched (last) turn. """ last_turn = request_info.turns[-1] if request_info.turns else None - first_turn = request_info.turns[0] if request_info.turns else None turn_model = last_turn.model if last_turn else None record.model_name = turn_model or self.model_endpoint.primary_model_name self._enrich_request_record(record, request_info) @@ -266,7 +256,7 @@ def _finalize_request_record( if record.request_info is not None: record.request_info.max_tokens = last_turn.max_tokens if last_turn else None record.request_info.audio_duration_seconds = ( - first_turn.audio_duration_seconds if first_turn else None + last_turn.audio_duration_seconds if last_turn else None ) record.request_info.scheduled_send_ms = ( float(last_turn.timestamp) diff --git a/src/aiperf/workers/worker.py b/src/aiperf/workers/worker.py index 7a545e1ad3..dcda75f886 100644 --- a/src/aiperf/workers/worker.py +++ b/src/aiperf/workers/worker.py @@ -1150,13 +1150,15 @@ async def _try_payload_bytes_fast_path( payload_bytes=payload_turn.payload_bytes, # Scalars only — payload_bytes carries the wire body; enrichment # reads max_tokens / timestamp off this turn for OSL-mismatch and - # schedule-lag metrics, and source_kind for per-row attribution. + # schedule-lag metrics, source_kind for per-row attribution, and + # audio_duration_seconds for the audio-duration and RTFx metrics. turns=[ Turn( role="user", max_tokens=payload_turn.max_tokens, timestamp=payload_turn.timestamp, source_kind=payload_turn.source_kind, + audio_duration_seconds=payload_turn.audio_duration_seconds, ) ], ) diff --git a/tests/unit/dataset/loader/test_multi_turn.py b/tests/unit/dataset/loader/test_multi_turn.py index 270f237df7..a43b25410d 100644 --- a/tests/unit/dataset/loader/test_multi_turn.py +++ b/tests/unit/dataset/loader/test_multi_turn.py @@ -584,6 +584,35 @@ def test_convert_with_per_turn_output_length(self, default_cfg): assert conversations[0].turns[0].max_tokens == 50 assert conversations[0].turns[1].max_tokens == 500 + def test_convert_with_audio_duration_seconds(self, default_cfg): + """Test converting multi-turn data with audio_duration_seconds sets Turn.audio_duration_seconds.""" + data = { + "session_1": [ + MultiTurn( + session_id="session_1", + turns=[ + SingleTurn( + audio="https://example.com/first.wav", + audio_duration_seconds=10.0, + ), + SingleTurn( + audio="https://example.com/second.wav", + audio_duration_seconds=25.0, + ), + ], + ) + ] + } + + loader = MultiTurnDatasetLoader( + filename="dummy.jsonl", run=make_run_from_cli(default_cfg) + ) + conversations = loader.convert_to_conversations(data) + + assert len(conversations) == 1 + assert conversations[0].turns[0].audio_duration_seconds == 10.0 + assert conversations[0].turns[1].audio_duration_seconds == 25.0 + def test_convert_mixed_output_length(self, default_cfg): """Test converting data where some turns have output_length and others do not.""" data = { diff --git a/tests/unit/dataset/loader/test_single_turn.py b/tests/unit/dataset/loader/test_single_turn.py index 20c99f7718..a75b67979e 100644 --- a/tests/unit/dataset/loader/test_single_turn.py +++ b/tests/unit/dataset/loader/test_single_turn.py @@ -98,6 +98,86 @@ def test_create_with_output_length_negative_raises(self): with pytest.raises(ValueError): SingleTurn(text="Hello", output_length=-1) + def test_create_with_audio_duration_seconds(self): + data = SingleTurn( + audio="https://example.com/audio.wav", audio_duration_seconds=30.0 + ) + + assert data.audio_duration_seconds == 30.0 + + def test_create_with_audio_duration_seconds_default_is_none(self): + data = SingleTurn(text="Hello") + + assert data.audio_duration_seconds is None + + def test_create_with_audio_duration_seconds_negative_raises(self): + with pytest.raises(ValueError): + SingleTurn(text="Hello", audio_duration_seconds=-1.0) + + def test_create_with_audio_duration_seconds_infinite_raises(self): + with pytest.raises(ValueError): + SingleTurn(text="Hello", audio_duration_seconds=float("inf")) + + def test_create_with_audio_duration_seconds_zero_raises(self): + with pytest.raises(ValueError): + SingleTurn(text="Hello", audio_duration_seconds=0.0) + + def test_create_with_audio_duration_seconds_multiple_audios_raises(self): + """audio_duration_seconds is ambiguous when more than one audio is given, + since only the first audio is sent to the transcription endpoint.""" + with pytest.raises(ValueError): + SingleTurn( + audios=["https://example.com/a.wav", "https://example.com/b.wav"], + audio_duration_seconds=45.0, + ) + + def test_create_with_audio_duration_seconds_single_element_audios_allowed(self): + """A single-element audios list still unambiguously describes one clip.""" + data = SingleTurn( + audios=["https://example.com/a.wav"], audio_duration_seconds=30.0 + ) + + assert data.audio_duration_seconds == 30.0 + + def test_create_with_audio_duration_seconds_audio_object_one_content_allowed(self): + data = SingleTurn( + audios=[{"name": "clip", "contents": ["https://example.com/a.wav"]}], + audio_duration_seconds=30.0, + ) + + assert data.audio_duration_seconds == 30.0 + + def test_create_with_audio_duration_seconds_audio_object_batched_contents_raises( + self, + ): + with pytest.raises(ValueError, match="2 were given"): + SingleTurn( + audios=[ + { + "name": "batch", + "contents": [ + "https://example.com/a.wav", + "https://example.com/b.wav", + ], + } + ], + audio_duration_seconds=45.0, + ) + + def test_create_with_audio_duration_seconds_audio_object_empty_contents_raises( + self, + ): + with pytest.raises(ValueError, match="requires audio or audios"): + SingleTurn( + audios=[{"name": "empty", "contents": []}], + audio_duration_seconds=30.0, + ) + + def test_create_with_audio_duration_seconds_without_audio_raises(self): + """audio_duration_seconds requires audio or audios to be set.""" + with pytest.raises(ValueError): + SingleTurn(text="Hello", audio_duration_seconds=30.0) + def test_create_with_delay(self): """Test creating SingleTurn with delay.""" data = SingleTurn(text="Who are you?", delay=1234) @@ -503,6 +583,24 @@ def test_convert_without_output_length_is_none(self, default_cfg): assert len(conversations) == 1 assert conversations[0].turns[0].max_tokens is None + def test_convert_with_audio_duration_seconds(self, default_cfg): + loader = SingleTurnDatasetLoader( + filename="dummy.jsonl", run=make_run_from_cli(default_cfg) + ) + data = { + "session_1": [ + SingleTurn( + audio="https://example.com/audio.wav", + audio_duration_seconds=30.0, + ) + ], + } + + conversations = loader.convert_to_conversations(data) + + assert len(conversations) == 1 + assert conversations[0].turns[0].audio_duration_seconds == 30.0 + def test_convert_multimodal_with_output_length(self, default_cfg): """Test converting multimodal data with output_length sets Turn.max_tokens.""" loader = SingleTurnDatasetLoader( diff --git a/tests/unit/dataset/test_payload_mmap.py b/tests/unit/dataset/test_payload_mmap.py index 65eafecfb4..c2007739bd 100644 --- a/tests/unit/dataset/test_payload_mmap.py +++ b/tests/unit/dataset/test_payload_mmap.py @@ -349,14 +349,16 @@ async def test_adopt_existing_files_compress_only_missing_zst_raises( @pytest.mark.asyncio async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): - """PAYLOAD_BYTES index must round-trip max_tokens, timestamp, and source_kind. + """PAYLOAD_BYTES index must round-trip max_tokens, timestamp, source_kind, + and audio_duration_seconds. Turn scalars live outside the wire body for some loaders (e.g. mooncake - ``output_length`` / ``timestamp``, SPEED-Bench and Weka ``source_kind``). - Persisting them on PayloadOffset keeps OSL-mismatch and schedule-lag metrics - alive on the verbatim path, and keeps each record attributable to the - dataset row it came from -- without it a per-category split collapses to a - single run-level label. + ``output_length`` / ``timestamp``, SPEED-Bench and Weka ``source_kind``, + self-reported ``audio_duration_seconds`` for custom ASR datasets). + Persisting them on PayloadOffset keeps OSL-mismatch, schedule-lag, and + audio-duration/RTFx metrics alive on the verbatim path, and keeps each + record attributable to the dataset row it came from -- without it a + per-category split collapses to a single run-level label. """ from aiperf.dataset.memory_map_utils import ( PayloadOffset, @@ -382,6 +384,7 @@ async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): max_tokens=128, timestamp=42.5, source_kind="coding", + audio_duration_seconds=30.0, ) ], ) @@ -399,12 +402,14 @@ async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): assert entry.max_tokens == 128 assert entry.timestamp == 42.5 assert entry.source_kind == "coding" + assert entry.audio_duration_seconds == 30.0 assert orjson.loads(entry.payload_bytes) == payload turn = turn_from_payload_turn(entry) assert turn.max_tokens == 128 assert turn.timestamp == 42.5 assert turn.source_kind == "coding" + assert turn.audio_duration_seconds == 30.0 assert turn.raw_payload == payload # Wire-JSON fallback recovers max_tokens when index scalars are absent @@ -412,6 +417,7 @@ async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): legacy = PayloadOffset(offset=0, size=0) assert legacy.max_tokens is None assert legacy.source_kind is None + assert legacy.audio_duration_seconds is None assert ( max_tokens_from_wire_payload({"max_completion_tokens": 16, "messages": []}) == 16 @@ -421,3 +427,13 @@ async def test_payload_mmap_persists_turn_scalars(tmp_path, monkeypatch): client.close() await store.stop() + + +@pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) +def test_payload_offset_audio_duration_seconds_rejects_non_positive_or_non_finite( + value, +): + from aiperf.dataset.memory_map_utils import PayloadOffset + + with pytest.raises(ValueError): + PayloadOffset(offset=0, size=0, audio_duration_seconds=value) diff --git a/tests/unit/workers/test_inference_client.py b/tests/unit/workers/test_inference_client.py index 47ebddf09a..bede883a05 100644 --- a/tests/unit/workers/test_inference_client.py +++ b/tests/unit/workers/test_inference_client.py @@ -426,6 +426,43 @@ def test_finalize_request_record_hoists_per_turn_scalars(self, inference_client) assert result.request_info.scheduled_send_ms == 42.5 assert "turns" not in result.request_info.model_dump() + def test_finalize_request_record_uses_last_turn_audio_duration_seconds( + self, inference_client + ): + """audio_duration_seconds must be hoisted from the dispatched (last) turn.""" + first_turn = Turn( + texts=[Text(contents=["first turn"])], + role="user", + audio_duration_seconds=10.0, + ) + second_turn = Turn( + texts=[Text(contents=["second turn"])], + role="user", + audio_duration_seconds=25.0, + ) + request_info = RequestInfo( + model_endpoint=inference_client.model_endpoint, + turns=[first_turn, second_turn], + turn_index=1, + credit_num=0, + credit_phase=CreditPhase.PROFILING, + x_request_id="test-id", + x_correlation_id="test-corr", + conversation_id="test-conv", + ) + record = RequestRecord( + request_info=request_info, + start_perf_ns=1000, + timestamp_ns=1000, + end_perf_ns=2000, + ) + + result = inference_client._finalize_request_record( + record=record, request_info=request_info + ) + + assert result.request_info.audio_duration_seconds == 25.0 + @pytest.mark.parametrize( "strip,expected_payload_bytes", [ diff --git a/tests/unit/workers/test_worker.py b/tests/unit/workers/test_worker.py index 7f462aaa7d..ce5f499c60 100644 --- a/tests/unit/workers/test_worker.py +++ b/tests/unit/workers/test_worker.py @@ -932,7 +932,7 @@ async def test_successful_fast_path_sets_record_emitted( async def test_fast_path_turn_carries_max_tokens_and_timestamp( self, mock_worker, sample_credit_context ): - """PAYLOAD_BYTES fast path must hoist turn scalars (max_tokens, timestamp, source_kind) for metric enrichment, else OSL-mismatch and schedule-lag metrics go silent and every record is attributed to the run-level label.""" + """PAYLOAD_BYTES fast path must hoist turn scalars (max_tokens, timestamp, source_kind, audio_duration_seconds) for metric enrichment, else OSL-mismatch, schedule-lag, and audio-duration/RTFx metrics go silent and every record is attributed to the run-level label.""" mock_worker._is_payload_bytes = True mock_worker._dataset_client = AsyncMock() payload = b'{"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' @@ -941,6 +941,7 @@ async def test_fast_path_turn_carries_max_tokens_and_timestamp( max_tokens=64, timestamp=1234.5, source_kind="coding", + audio_duration_seconds=30.0, ) success_record = RequestRecord( @@ -964,12 +965,13 @@ async def test_fast_path_turn_carries_max_tokens_and_timestamp( assert sent_request_info.turns[0].max_tokens == 64 assert sent_request_info.turns[0].timestamp == 1234.5 assert sent_request_info.turns[0].source_kind == "coding" + assert sent_request_info.turns[0].audio_duration_seconds == 30.0 assert sent_request_info.source_kind == "coding" async def test_retrieve_conversation_for_session_restores_scalars( self, mock_worker, sample_credit_context ): - """Session-path reconstruction must restore max_tokens, timestamp, and source_kind.""" + """Session-path reconstruction must restore max_tokens, timestamp, source_kind, and audio_duration_seconds.""" mock_worker._is_payload_bytes = True mock_worker._dataset_client = AsyncMock() mock_worker._dataset_client.get_payload_turn.return_value = PayloadTurnData( @@ -977,6 +979,7 @@ async def test_retrieve_conversation_for_session_restores_scalars( max_tokens=32, timestamp=99, source_kind="math", + audio_duration_seconds=45.0, ) mock_worker.session_manager = MagicMock() mock_worker.session_manager.default_context_mode = None @@ -989,6 +992,7 @@ async def test_retrieve_conversation_for_session_restores_scalars( assert conversation.turns[0].max_tokens == 32 assert conversation.turns[0].timestamp == 99 assert conversation.turns[0].source_kind == "math" + assert conversation.turns[0].audio_duration_seconds == 45.0 assert conversation.turns[0].raw_payload == { "max_completion_tokens": 32, "messages": [], From 22ab1bb42590fdfd76da49c71de22dc12d36dc7b Mon Sep 17 00:00:00 2001 From: jxj Date: Wed, 28 Oct 2026 08:41:13 +0800 Subject: [PATCH 099/119] fix(dataset): treat image source directories as literal paths (#1496) Signed-off-by: git-jxj <65210887+git-jxj@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 6bc63ef8d076b3e0a66a608f036022a2078340e1) --- docs/tutorials/synthetic-dataset.md | 2 +- src/aiperf/dataset/generator/image.py | 4 +++- .../dataset/generator/test_image_generator.py | 24 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/synthetic-dataset.md b/docs/tutorials/synthetic-dataset.md index 121412c09c..52656aac18 100644 --- a/docs/tutorials/synthetic-dataset.md +++ b/docs/tutorials/synthetic-dataset.md @@ -89,7 +89,7 @@ Available modes: - `noise` (default): A fresh random-noise image is generated at the requested width × height for every request. No filesystem access; pool size is unlimited. - `assets`: Resizes one of the 4 bundled source images in `assets/source_images/` to the requested dimensions. Smaller payload bytes than noise because natural images compress well, but the pool is only 4 images. -- ``: Resizes images from a user-supplied directory (e.g. `--image-source ./my_images`). All readable files in the directory are loaded; non-image files are skipped. +- ``: Resizes images from a user-supplied directory (e.g. `--image-source ./my_images`). The directory path is interpreted literally, including characters such as square brackets. All readable files in the directory are loaded; non-image files are skipped. After source selection, the image is converted to the configured format (PNG, JPEG, or randomly selected) and base64-encoded as a data URI for API requests. diff --git a/src/aiperf/dataset/generator/image.py b/src/aiperf/dataset/generator/image.py index 40b2115a19..4d0731193a 100644 --- a/src/aiperf/dataset/generator/image.py +++ b/src/aiperf/dataset/generator/image.py @@ -83,7 +83,9 @@ def _load_source_image_paths_from_disk(self, source_path: Path) -> list[Path]: supported_extensions = {ext.lower() for ext in Image.registered_extensions()} image_paths = [ Path(path) - for path in sorted(glob.glob(str(source_path / "*"))) + for path in sorted( + glob.glob(str(Path(glob.escape(str(source_path))) / "*")) + ) if Path(path).suffix.lower() in supported_extensions ] if not image_paths: diff --git a/tests/unit/dataset/generator/test_image_generator.py b/tests/unit/dataset/generator/test_image_generator.py index 3980809459..c7acd4fb53 100644 --- a/tests/unit/dataset/generator/test_image_generator.py +++ b/tests/unit/dataset/generator/test_image_generator.py @@ -415,6 +415,30 @@ def test_noise_produces_different_images_per_call(self, noise_config): class TestImageGeneratorCustomDirectory: """Tests for custom directory source mode.""" + @pytest.mark.parametrize("matching_sibling", [False, True]) + def test_custom_directory_treats_brackets_literally( + self, tmp_path: Path, matching_sibling: bool + ) -> None: + source = tmp_path / "images[1]" + source.mkdir() + Image.new("RGB", (5, 5), color="blue").save(source / "source.png") + if matching_sibling: + sibling = tmp_path / "images1" + sibling.mkdir() + Image.new("RGB", (5, 5), color="red").save(sibling / "other.png") + + config = make_image_config( + width_mean=5, + width_stddev=0, + height_mean=5, + height_stddev=0, + source=source, + ) + result = ImageGenerator(config).generate() + image_bytes = base64.b64decode(result.split(",", 1)[1]) + with Image.open(BytesIO(image_bytes)) as image: + assert image.getpixel((0, 0)) == (0, 0, 255) + def test_custom_directory_loads_images(self, tmp_path): img = Image.new("RGB", (5, 5), color="blue") img.save(tmp_path / "test.png") From 0386043b04a0566bdbe235e86256c041f348110e Mon Sep 17 00:00:00 2001 From: Manu Nicholas Jacob <118115267+manunicholasjacob@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:19:06 -0500 Subject: [PATCH 100/119] fix(dashboard): show live GPU telemetry after the pane is toggled (#1473) Signed-off-by: manunicholasjacob Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit ed548a4ca8c44019ce5cc41b2b0189c2262279f3) --- .../dashboard/realtime_telemetry_dashboard.py | 11 +++--- .../ui/test_realtime_telemetry_dashboard.py | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/aiperf/ui/dashboard/realtime_telemetry_dashboard.py b/src/aiperf/ui/dashboard/realtime_telemetry_dashboard.py index 804f59f757..4df144f88d 100644 --- a/src/aiperf/ui/dashboard/realtime_telemetry_dashboard.py +++ b/src/aiperf/ui/dashboard/realtime_telemetry_dashboard.py @@ -306,11 +306,12 @@ def set_status_message(self, message: str) -> None: def on_realtime_telemetry_metrics(self, metrics: list[MetricResult]) -> None: """Handle GPU telemetry metrics updates.""" - - if not self.metrics: - with suppress(Exception): - self.query_one("#telemetry-status").add_class("hidden") - self.all_nodes_view.remove_class("hidden") + # Every batch, not only the first: under --ui dashboard data is already + # flowing when the pane is toggled, and the toggle's status message hides + # the view again. + with suppress(Exception): + self.query_one("#telemetry-status").add_class("hidden") + self.all_nodes_view.remove_class("hidden") self.metrics = metrics self.all_nodes_view.update(metrics) diff --git a/tests/unit/ui/test_realtime_telemetry_dashboard.py b/tests/unit/ui/test_realtime_telemetry_dashboard.py index c550db59bd..6a4bec9b66 100644 --- a/tests/unit/ui/test_realtime_telemetry_dashboard.py +++ b/tests/unit/ui/test_realtime_telemetry_dashboard.py @@ -629,6 +629,40 @@ def test_on_realtime_telemetry_metrics_first_update(self, dashboard): mock_all_nodes.remove_class.assert_called_once_with("hidden") mock_all_nodes.update.assert_called_once_with(metrics) + def test_on_realtime_telemetry_metrics_after_status_message_unhides_view( + self, dashboard: RealtimeTelemetryDashboard + ) -> None: + """Under --ui dashboard, data is already flowing when the pane is toggled, + and the toggle's status message hides the view. The next batch must bring + it back rather than render into a hidden widget.""" + + def widget() -> Mock: + mock = Mock() + mock.classes = set() + mock.add_class.side_effect = mock.classes.add + mock.remove_class.side_effect = mock.classes.discard + return mock + + status, view = widget(), widget() + dashboard.all_nodes_view = view + metrics = [ + MetricResult( + tag="gpu_util_dcgm_http___localhost_9400_metrics_gpu0_GPU-12345678", + header="GPU Utilization | localhost:9400 | GPU 0 | NVIDIA RTX 4090", + unit="%", + avg=75.0, + ) + ] + + with patch.object(dashboard, "query_one", return_value=status): + dashboard.on_realtime_telemetry_metrics(metrics) + dashboard.set_status_message("Enabling live GPU telemetry...") + assert "hidden" in view.classes + dashboard.on_realtime_telemetry_metrics(metrics) + + assert "hidden" not in view.classes + assert "hidden" in status.classes + def test_on_realtime_telemetry_metrics_subsequent_update(self, dashboard): """Test on_realtime_telemetry_metrics on subsequent updates.""" mock_all_nodes = Mock() From 8edf959c94ef6118415e33f26199e9ef2e4dd185 Mon Sep 17 00:00:00 2001 From: Srividhya R Date: Tue, 27 Oct 2026 22:01:36 -0700 Subject: [PATCH 101/119] fix(endpoints): read reasoning_text from non-streaming Responses reasoning items (#1478) Signed-off-by: Srividhya R <195692899+ra-srivid@users.noreply.github.com> Co-authored-by: Srividhya R <195692899+ra-srivid@users.noreply.github.com> (cherry picked from commit 0c4d9dbdf0b184642d194cbbd7da0a6bea617a16) --- docs/tutorials/openai-responses.md | 2 + src/aiperf/endpoints/openai_responses.py | 43 +++++++++++++------ .../unit/endpoints/test_responses_endpoint.py | 33 ++++++++++++++ 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/docs/tutorials/openai-responses.md b/docs/tutorials/openai-responses.md index 7a791ce912..4267387fd9 100644 --- a/docs/tutorials/openai-responses.md +++ b/docs/tutorials/openai-responses.md @@ -413,3 +413,5 @@ For reference, AIPerf processes these Responses API streaming events: This enables accurate measurement of TTFT, ITL, and token throughput metrics when streaming is enabled. +For non-streaming responses, AIPerf reads content from the response's `output[]` items: `reasoning_text` parts of `reasoning` items (matching `response.reasoning_text.delta`), `output_text` parts of `message` items, and `name` and `arguments` of `function_call` items. A `reasoning` item without `reasoning_text` parts contributes its `summary_text` parts instead; the two are never combined, since a summary restates the same reasoning. The first non-empty source in the order reasoning, message, function call populates the parsed response (the same precedence as chat completions): message text is kept alongside reasoning, but function-call arguments are not counted when reasoning is present. + diff --git a/src/aiperf/endpoints/openai_responses.py b/src/aiperf/endpoints/openai_responses.py index f5b8960324..040a6a6740 100644 --- a/src/aiperf/endpoints/openai_responses.py +++ b/src/aiperf/endpoints/openai_responses.py @@ -537,7 +537,8 @@ def _extract_response_content( tokens: - ``message`` items contribute their ``output_text`` parts. - - ``reasoning`` items contribute their ``summary_text`` parts. + - ``reasoning`` items contribute their ``reasoning_text`` parts, or + their ``summary_text`` parts when there are none. - ``function_call`` items contribute ``name`` + ``arguments`` - the model generated those tokens, and the server's ``usage.completion_tokens`` already counts them, so client-side @@ -601,25 +602,39 @@ def _collect_output_item( """ item_type = item.get("type") if item_type == "reasoning": - ResponsesEndpoint._collect_reasoning_summary(item, reasoning_parts) + ResponsesEndpoint._collect_reasoning(item, reasoning_parts) elif item_type == "message": ResponsesEndpoint._collect_message_content(item, text_parts) elif item_type == "function_call": ResponsesEndpoint._collect_function_call(item, tool_call_parts) @staticmethod - def _collect_reasoning_summary( - item: dict[str, Any], reasoning_parts: list[str] - ) -> None: - """Append non-empty ``summary_text`` strings from a reasoning item.""" - summary = item.get("summary") - if not isinstance(summary, list): - return - for part in summary: - if isinstance(part, dict) and part.get("type") == "summary_text": - text = part.get("text") - if text: - reasoning_parts.append(text) + def _collect_reasoning(item: dict[str, Any], reasoning_parts: list[str]) -> None: + """Append a reasoning item's text: its ``reasoning_text`` content parts, + else its ``summary_text`` parts. + + ``reasoning_text`` is the full reasoning, matching the streaming + ``response.reasoning_text.delta`` events; servers such as vLLM serving + gpt-oss return it with an empty ``summary``. A summary condenses the + same reasoning, so taking both would count it twice. + """ + reasoning_parts.extend( + ResponsesEndpoint._part_texts(item.get("content"), "reasoning_text") + or ResponsesEndpoint._part_texts(item.get("summary"), "summary_text") + ) + + @staticmethod + def _part_texts(parts: Any, part_type: str) -> list[str]: + if not isinstance(parts, list): + return [] + return [ + part["text"] + for part in parts + if isinstance(part, dict) + and part.get("type") == part_type + and isinstance(part.get("text"), str) + and part["text"] + ] @staticmethod def _collect_message_content(item: dict[str, Any], text_parts: list[str]) -> None: diff --git a/tests/unit/endpoints/test_responses_endpoint.py b/tests/unit/endpoints/test_responses_endpoint.py index 16a9b8abcd..5b36169098 100644 --- a/tests/unit/endpoints/test_responses_endpoint.py +++ b/tests/unit/endpoints/test_responses_endpoint.py @@ -64,6 +64,16 @@ def _reasoning(text: str) -> dict: return {"type": "reasoning", "summary": [{"type": "summary_text", "text": text}]} +def _reasoning_text(text: str) -> dict: + """Shorthand for a reasoning item carrying ``reasoning_text`` content and an + empty summary, as vLLM returns for gpt-oss.""" + return { + "type": "reasoning", + "summary": [], + "content": [{"type": "reasoning_text", "text": text}], + } + + class TestResponsesEndpoint: """Tests for ResponsesEndpoint.format_payload.""" @@ -821,6 +831,29 @@ def test_parse_response_extracts_text(self, endpoint, json_data, expected_text): "Valid text", id="non_dict_parts_filtered", ), + param( + {"object": "response", "output": [_reasoning_text("Hmm...")]}, + "Hmm...", + None, + id="reasoning_text_only", + ), + param( + { + "object": "response", + "output": [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Short."}], + "content": [ + {"type": "reasoning_text", "text": "Full reasoning."} + ], + }, + ], + }, + "Full reasoning.", + None, + id="reasoning_text_preferred_over_summary", + ), ], ) # fmt: skip def test_parse_response_extracts_reasoning( From 4624766f305c382556c41508a590c4ecdbfa0d42 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev <144015565+v0ropaev@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:39:03 +0300 Subject: [PATCH 102/119] fix(dataset): accept multi-picture JPEGs as JPEG images (#1591) Signed-off-by: Dmitry Voropaev Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit b227ebc3b71bdfb0c0b2605ce3839375bb821830) --- src/aiperf/dataset/dataset_manager.py | 3 +- src/aiperf/dataset/utils.py | 15 ++++++++ tests/unit/dataset/test_dataset_manager.py | 45 ++++++++++++++++++++++ tests/unit/dataset/test_dataset_utils.py | 18 +++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/aiperf/dataset/dataset_manager.py b/src/aiperf/dataset/dataset_manager.py index e059ed7d00..a3706bf3d7 100644 --- a/src/aiperf/dataset/dataset_manager.py +++ b/src/aiperf/dataset/dataset_manager.py @@ -56,7 +56,7 @@ from aiperf.config.dataset import FileDataset, PublicDataset from aiperf.dataset import mmap_cache from aiperf.dataset.memory_map_utils import turn_from_payload_turn -from aiperf.dataset.utils import encode_image +from aiperf.dataset.utils import encode_image, normalize_image_format from aiperf.plugin import plugins from aiperf.plugin.enums import ( ComposerType, @@ -409,6 +409,7 @@ async def _download_and_encode( def _decode_and_encode() -> str: img = PILImage.open(BytesIO(data)) + normalize_image_format(img) if img.format is None: raise RuntimeError( f"Failed to determine image format for URL '{url}'" diff --git a/src/aiperf/dataset/utils.py b/src/aiperf/dataset/utils.py index d8b62cdaa6..de00fab3db 100644 --- a/src/aiperf/dataset/utils.py +++ b/src/aiperf/dataset/utils.py @@ -23,6 +23,20 @@ def check_file_exists(filename: Path) -> None: raise FileNotFoundError(f"The file '{filename}' does not exist.") +def normalize_image_format(img: Image) -> None: + """Reports a multi-picture JPEG as plain JPEG. + + Pillow opens a JPEG whose MPF segment lists more than one image (an Ultra + HDR gain map, a stereo pair) as ``MPO``. The primary frame is a regular + JPEG and ``encode_image`` only re-encodes that frame. + + Args: + img: The PIL Image object to update in place. + """ + if img.format == "MPO": + img.format = "JPEG" + + def open_image(filename: str) -> Image: """Opens an image file. @@ -37,6 +51,7 @@ def open_image(filename: str) -> Image: """ check_file_exists(Path(filename)) img = Image.open(filename) + normalize_image_format(img) if img.format is None: raise RuntimeError(f"Failed to determine image format of '{filename}'.") diff --git a/tests/unit/dataset/test_dataset_manager.py b/tests/unit/dataset/test_dataset_manager.py index 2c2618f81b..7cca54768d 100644 --- a/tests/unit/dataset/test_dataset_manager.py +++ b/tests/unit/dataset/test_dataset_manager.py @@ -1,10 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import base64 +from io import BytesIO from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest +from PIL import Image as PILImage from pydantic import ValidationError from aiperf.common.enums import CacheBustTarget, MemoryMapFormat @@ -741,6 +744,48 @@ async def test_converts_http_urls_to_data_urls(self, dataset_manager): assert len(contents) == 1 assert contents[0].startswith("data:image/png;base64,") + @pytest.mark.asyncio + async def test_converts_multi_picture_jpeg_url(self, dataset_manager): + """A JPEG with an MPF secondary image is inlined as image/jpeg.""" + buffer = BytesIO() + PILImage.new("RGB", (64, 48), color="red").save( + buffer, + format="MPO", + save_all=True, + append_images=[PILImage.new("RGB", (32, 24), color="blue")], + ) + url = "https://example.com/photo.jpg" + dataset_manager.dataset = { + "s1": Conversation( + session_id="s1", + turns=[Turn(images=[Image(contents=[url])], model="test-model")], + ) + } + + with patch( + "aiperf.dataset.dataset_manager.aiohttp.ClientSession" + ) as mock_session_cls: + mock_resp = AsyncMock() + mock_resp.status = 200 + mock_resp.read = AsyncMock(return_value=buffer.getvalue()) + mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) + mock_resp.__aexit__ = AsyncMock(return_value=False) + + mock_session = AsyncMock() + mock_session.get = MagicMock(return_value=mock_resp) + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=False) + mock_session_cls.return_value = mock_session + + await dataset_manager._convert_media_urls_to_inline() + + contents = dataset_manager.dataset["s1"].turns[0].images[0].contents + assert contents[0].startswith("data:image/jpeg;base64,") + encoded = base64.b64decode(contents[0].split(",", 1)[1]) + with PILImage.open(BytesIO(encoded)) as inlined: + assert inlined.format == "JPEG" + assert getattr(inlined, "n_frames", 1) == 1 + @pytest.mark.asyncio async def test_skips_already_encoded_data_urls(self, dataset_manager): """Already-encoded data URLs are left unchanged.""" diff --git a/tests/unit/dataset/test_dataset_utils.py b/tests/unit/dataset/test_dataset_utils.py index daad45ca9d..5667134bda 100644 --- a/tests/unit/dataset/test_dataset_utils.py +++ b/tests/unit/dataset/test_dataset_utils.py @@ -42,6 +42,24 @@ def test_open_valid_image(self): finally: Path(tmp_path).unlink() # Clean up + def test_open_multi_picture_jpeg(self, tmp_path): + """Test that a JPEG carrying an MPF secondary image opens as JPEG.""" + path = tmp_path / "photo.jpg" + primary = Image.new("RGB", (64, 48), color="red") + secondary = Image.new("RGB", (32, 24), color="blue") + primary.save(path, format="MPO", save_all=True, append_images=[secondary]) + with Image.open(path) as raw: + assert raw.format == "MPO" + + opened_img = utils.open_image(str(path)) + assert opened_img.format == "JPEG" + + encoded = utils.encode_image(opened_img, opened_img.format) + decoded_img = Image.open(BytesIO(base64.b64decode(encoded))) + assert decoded_img.format == "JPEG" + assert decoded_img.size == (64, 48) + opened_img.close() + def test_open_non_existing_image(self): """Test that FileNotFoundError is raised for non-existing image.""" with pytest.raises(FileNotFoundError): From d55b962c65cae8612733f0e48de943f62164f5c3 Mon Sep 17 00:00:00 2001 From: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:28:59 -0700 Subject: [PATCH 103/119] perf(export): bound memory usage when generating outputs.json (#1464) Signed-off-by: Jayesh Suyal Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit a8d88a5ce4d48d2cfad110b9c6bcf760ac493506) --- dev/benchmarks/outputs_json_export.py | 99 +++++ docs/reference/json-export-schema.md | 1 + src/aiperf/exporters/outputs_json_exporter.py | 417 +++++++++++++----- .../exporters/test_outputs_json_exporter.py | 311 +++++++++++-- 4 files changed, 675 insertions(+), 153 deletions(-) create mode 100644 dev/benchmarks/outputs_json_export.py diff --git a/dev/benchmarks/outputs_json_export.py b/dev/benchmarks/outputs_json_export.py new file mode 100644 index 0000000000..7d3d30eef0 --- /dev/null +++ b/dev/benchmarks/outputs_json_export.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Measure the real outputs.json exporter in a fresh process per fixture.""" + +import argparse +import asyncio +import math +import resource +import shutil +import time +from pathlib import Path +from unittest.mock import MagicMock + +import orjson + +from aiperf.common.constants import IS_MACOS +from aiperf.config.artifacts import OutputDefaults +from aiperf.exporters.outputs_json_exporter import OutputsJsonExporter + + +def generate(root: Path, records: int, response_bytes: int, shards: int) -> None: + """Write deterministic, unsorted, interleaved fragment files.""" + fragments = root / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + fragments.mkdir(parents=True, exist_ok=True) + handles = [ + (fragments / f"output_fragments_{i:03}.jsonl").open("wb") for i in range(shards) + ] + stride = 7919 + while math.gcd(stride, records) != 1: + stride += 1 + try: + for i in range(records): + index = (i * stride) % records + row = { + "session_num": index // 3, + "turn_index": index % 3, + "conversation_id": f"conversation-{index // 3}", + "x_request_id": f"request-{index}", + "benchmark_phase": "warmup" if index % 13 == 0 else "profiling", + "request_start_ns": index * 1000, + "request_end_ns": index * 1000 + 200, + "metrics": {"output_token_count": 20, "request_latency": 0.2}, + "response_text": "x" * response_bytes, + } + handles[i % shards].write(orjson.dumps(row) + b"\n") + finally: + for handle in handles: + handle.close() + + +async def measure(fixture: Path, work: Path) -> dict[str, int | float | str]: + """Copy inputs before timing and report process peak RSS.""" + shutil.copytree(fixture, work) + cfg = MagicMock() + cfg.cfg.artifacts.export_outputs_json = True + cfg.cfg.artifacts.outputs_json_file = work / "outputs.json" + cfg.cfg.artifacts.artifact_directory = work + exporter = OutputsJsonExporter(cfg) + before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + start = time.perf_counter() + await exporter.export() + elapsed = time.perf_counter() - start + after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # ru_maxrss is bytes on macOS, KiB on Linux. + scale = 1 if IS_MACOS else 1024 + return { + "fixture": fixture.name, + "seconds": elapsed, + "pre_export_peak_rss_bytes": before * scale, + "peak_rss_bytes": after * scale, + "input_bytes": sum(file.stat().st_size for file in fixture.rglob("*.jsonl")), + "output_bytes": (work / "outputs.json").stat().st_size, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + generator = sub.add_parser("generate") + generator.add_argument("root", type=Path) + generator.add_argument("--records", type=int, required=True) + generator.add_argument("--response-bytes", type=int, required=True) + generator.add_argument("--shards", type=int, default=8) + runner = sub.add_parser("run") + runner.add_argument("fixture", type=Path) + runner.add_argument("work", type=Path) + args = parser.parse_args() + if args.command == "generate": + if args.records <= 0 or args.shards <= 0 or args.response_bytes < 0: + parser.error( + "records and shards must be positive; response bytes must be nonnegative" + ) + generate(args.root, args.records, args.response_bytes, args.shards) + else: + print(orjson.dumps(asyncio.run(measure(args.fixture, args.work))).decode()) + + +if __name__ == "__main__": + main() diff --git a/docs/reference/json-export-schema.md b/docs/reference/json-export-schema.md index 8d9c65e619..1a6cae0491 100644 --- a/docs/reference/json-export-schema.md +++ b/docs/reference/json-export-schema.md @@ -186,6 +186,7 @@ It carries its own `schema_version` (`OutputsJsonExporter.SCHEMA_VERSION`, curre - **`response_text`** is the concatenated generated text. For models that emit a separate reasoning channel, reasoning and answer are concatenated. It is `null` when the response carried no content. - **`metrics`** holds an allowlisted subset in display units (`input_sequence_length`, `output_token_count`, `output_sequence_length`, `request_latency`, `time_to_first_token`, `inter_token_latency`). Streaming-only metrics are absent from non-streaming records. - Schema `1.1` added `warmup` and `benchmark_phase`; both are additive over `1.0`. +- The exporter sorts fragments through temporary files beside the final artifact and replaces `outputs.json` only after the complete document is written. Working memory depends on the sorting chunk size, read/write buffers, and merge fan-in. Each merge input can retain a full record, so oversized responses can increase merge memory in proportion to the number of inputs. Failure or cancellation before final replacement preserves the source fragments and any previous `outputs.json`. ### Named phase artifacts diff --git a/src/aiperf/exporters/outputs_json_exporter.py b/src/aiperf/exporters/outputs_json_exporter.py index 2b2909c155..ef553c3b1f 100644 --- a/src/aiperf/exporters/outputs_json_exporter.py +++ b/src/aiperf/exporters/outputs_json_exporter.py @@ -1,9 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Aggregator that concatenates per-processor output fragments into final outputs.json.""" +"""Merge per-processor response fragments into a sorted outputs.json.""" +import asyncio +import heapq +import os +import tempfile +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from contextlib import AsyncExitStack, aclosing, asynccontextmanager from pathlib import Path -from typing import Any +from typing import Any, TypeVar import aiofiles import orjson @@ -16,28 +22,20 @@ from aiperf.exporters.exporter_config import ExporterConfig, FileExportInfo JsonObject = dict[str, Any] +SortRow = tuple[int, int, int, int, JsonObject] +T = TypeVar("T") class OutputsJsonExporter(AIPerfLoggerMixin): - """Aggregates per-processor output fragment files into the final outputs.json. + """Sort output fragments on disk and atomically publish schema 1.1 JSON.""" - Each fragment already carries its allowlisted per-request metrics (captured in - display units by OutputsJsonRecordProcessor), so this exporter performs no - metrics join and does not depend on the records JSONL export being enabled. - - Self-disables unless --export-outputs-json is set. - """ - - # 1.1 added the top-level `warmup` array and the per-entry `benchmark_phase` - # field. Both are additive: `data` is still profiling-only. SCHEMA_VERSION = "1.1" - - # outputs.json is sorted by (session_num, turn_index), so unlike the raw - # record aggregator it cannot stream -- the whole document is materialized - # to sort it, costing roughly 3x the on-disk fragment size at peak. Warn - # past this many records so a large run sees the cost instead of just - # paying it, since --export-level raw now enables this export by default. - LARGE_EXPORT_RECORD_WARNING_THRESHOLD = 50_000 + CHUNK_BYTES = 8 * 1024 * 1024 + CHUNK_RECORDS = 20_000 + MERGE_FAN_IN = 32 + READ_BYTES = 64 * 1024 + WRITE_BYTES = 1024 * 1024 + FRAGMENT_SCAN_BATCH = 64 def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -56,99 +54,174 @@ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: def get_export_info(self) -> FileExportInfo: """Return export metadata for logging.""" - return FileExportInfo( - export_type="Outputs JSON", - file_path=self._file_path, - ) + return FileExportInfo(export_type="Outputs JSON", file_path=self._file_path) async def export(self) -> None: - """Read fragment files (each self-contained) and write the final outputs.json.""" - fragment_files: list[Path] = list( - self._fragments_dir.glob("output_fragments_*.jsonl") + """Publish a sorted document, retaining source fragments on failure.""" + async with aclosing(self._fragment_files()) as fragments: + first = await anext(fragments, None) + if first is None: + self.debug( + "No output fragment files found, skipping outputs.json export" + ) + return + + self._file_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".outputs-json-", dir=self._file_path.parent + ) as scratch_name: + scratch = Path(scratch_name) + consumed = scratch / "consumed.jsonl" + run_count, data_count, warmup_count = await self._make_runs( + self._prepend_first(first, fragments), scratch, consumed + ) + final_run = await self._merge_runs(scratch, run_count) + staged = scratch / "outputs.json" + await self._write_document(staged, final_run) + await asyncio.sleep(0) + # A synchronous rename is the commit point: cancellation cannot land + # between replacing the old artifact and retiring its fragments. + os.replace(staged, self._file_path) + self._cleanup_fragments(consumed) + + self.info( + f"Exported {data_count} records ({warmup_count} warmup) " + f"to {self._file_path}" ) - if not fragment_files: - self.debug("No output fragment files found, skipping outputs.json export") + + @staticmethod + def _next_fragment_batch( + entries: Iterator[os.DirEntry[str]], limit: int + ) -> tuple[list[Path], bool]: + paths: list[Path] = [] + for _ in range(limit): + try: + entry = next(entries) + except StopIteration: + return paths, True + if entry.name.startswith("output_fragments_") and entry.name.endswith( + ".jsonl" + ): + paths.append(Path(entry.path)) + return paths, False + + async def _fragment_files(self) -> AsyncIterator[Path]: + opening = asyncio.ensure_future( + asyncio.to_thread(os.scandir, self._fragments_dir) + ) + try: + entries = await self._settled(opening) + except asyncio.CancelledError: + if ( + opening.done() + and not opening.cancelled() + and opening.exception() is None + ): + await self._settled(asyncio.to_thread(opening.result().close)) + raise + except OSError: return + try: + while True: + paths, finished = await self._settled( + asyncio.to_thread( + self._next_fragment_batch, entries, self.FRAGMENT_SCAN_BATCH + ) + ) + for path in paths: + yield path + if finished: + break + finally: + await self._settled(asyncio.to_thread(entries.close)) - fragments = await self._read_fragments(fragment_files) - - records: list[JsonObject] = [] - warmup: list[JsonObject] = [] - for frag in fragments: - phase = frag.get("benchmark_phase") - entry = { - "session_num": frag["session_num"], - "conversation_id": frag.get("conversation_id"), - "turn_index": frag.get("turn_index"), - "x_request_id": frag.get("x_request_id"), - "benchmark_phase": phase, - "request_start_ns": frag.get("request_start_ns"), - "request_end_ns": frag.get("request_end_ns"), - "metrics": frag.get("metrics") or {}, - "response_text": frag.get("response_text"), - } - # `data` stays profiling-only so consumers that sum over it do not - # silently start counting warmup responses. Test explicitly for - # WARMUP rather than for PROFILING: only a known-warmup record may - # leave `data`, so a missing or newly-added phase surfaces there - # instead of silently vanishing from everyone's denominators. - if phase == CreditPhase.WARMUP: - warmup.append(entry) - else: - records.append(entry) - - def _order(record: JsonObject) -> tuple[int, int]: - return (record["session_num"], record.get("turn_index") or 0) - - records.sort(key=_order) - warmup.sort(key=_order) - - output = { - "schema_version": self.SCHEMA_VERSION, - "data": records, - "warmup": warmup, - } - - total = len(records) + len(warmup) - if total >= self.LARGE_EXPORT_RECORD_WARNING_THRESHOLD: - self.warning( - f"Serializing {total} records to {self._file_path.name}; this " - "export is sorted and therefore held in memory in full, needing " - "roughly 3x the fragment size on disk. Pass " - "--no-export-outputs-json to skip it." - ) + @staticmethod + async def _prepend_first( + first: Path, remaining: AsyncIterator[Path] + ) -> AsyncIterator[Path]: + yield first + async for path in remaining: + yield path - self._file_path.parent.mkdir(parents=True, exist_ok=True) - content = orjson.dumps(scrub_non_finite(output), option=orjson.OPT_INDENT_2) - async with aiofiles.open(self._file_path, "wb") as f: - await f.write(content) + @staticmethod + def _record_consumed(manifest: Path, file: Path) -> None: + with manifest.open("ab") as output: + output.write(orjson.dumps(file.name) + b"\n") - self.info( - f"Exported {len(records)} records ({len(warmup)} warmup) " - f"to {self._file_path}" + @staticmethod + async def _settled(operation: Awaitable[T]) -> T: + """Let one pending file operation finish before cancellation frees scratch.""" + task = asyncio.ensure_future(operation) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + continue + if not task.cancelled(): + task.exception() + raise + + @asynccontextmanager + async def _open(self, path: Path, mode: str) -> AsyncIterator[Any]: + opening = asyncio.ensure_future(aiofiles.open(path, mode)) + try: + handle = await self._settled(opening) + except asyncio.CancelledError: + if ( + opening.done() + and not opening.cancelled() + and opening.exception() is None + ): + await self._settled(opening.result().close()) + raise + try: + yield handle + finally: + await self._settled(handle.close()) + + @staticmethod + def _entry(fragment: JsonObject) -> JsonObject: + phase = fragment.get("benchmark_phase") + return scrub_non_finite( + { + "session_num": fragment["session_num"], + "conversation_id": fragment.get("conversation_id"), + "turn_index": fragment.get("turn_index"), + "x_request_id": fragment.get("x_request_id"), + "benchmark_phase": phase, + "request_start_ns": fragment.get("request_start_ns"), + "request_end_ns": fragment.get("request_end_ns"), + "metrics": fragment.get("metrics") or {}, + "response_text": fragment.get("response_text"), + } ) - self._cleanup_fragments(fragment_files) - - async def _read_fragments(self, fragment_files: list[Path]) -> list[JsonObject]: - """Read all fragment JSONL files, skipping any unparsable lines. - - A crashed processor can leave a half-written trailing line; one bad line - must not sink the whole export, so decode errors and non-object lines are - logged and skipped. - """ - fragments: list[JsonObject] = [] - for file in fragment_files: - async with aiofiles.open(file) as f: - async for line in f: - line = line.strip() - if not line: + async def _lines(self, path: Path) -> AsyncIterator[bytes]: + """Read bounded batches, apart from an individual oversized line.""" + async with self._open(path, "rb") as source: + while batch := await self._settled(source.readlines(self.READ_BYTES)): + for line in batch: + yield line + + async def _make_runs( + self, fragment_files: AsyncIterator[Path], scratch: Path, consumed: Path + ) -> tuple[int, int, int]: + chunk: list[SortRow] = [] + chunk_bytes = 0 + run_count = data_count = warmup_count = ordinal = 0 + async for file in fragment_files: + async with aclosing(self._lines(file)) as lines: + async for line in lines: + if not line.strip(): continue try: fragment = orjson.loads(line) - except orjson.JSONDecodeError as e: + except orjson.JSONDecodeError as exc: self.warning( - f"Skipping unparsable fragment line in {file}: {e}" + f"Skipping unparsable fragment line in {file}: {exc}" ) continue if not isinstance(fragment, dict): @@ -156,13 +229,147 @@ async def _read_fragments(self, fragment_files: list[Path]) -> list[JsonObject]: f"Skipping non-object fragment line in {file}: {line!r}" ) continue - fragments.append(fragment) - return fragments + entry = self._entry(fragment) + phase = int(fragment.get("benchmark_phase") == CreditPhase.WARMUP) + warmup_count += phase + data_count += 1 - phase + chunk.append( + ( + phase, + entry["session_num"], + entry.get("turn_index") or 0, + ordinal, + entry, + ) + ) + ordinal += 1 + chunk_bytes += len(line) + if ( + len(chunk) >= self.CHUNK_RECORDS + or chunk_bytes >= self.CHUNK_BYTES + ): + await self._write_run(scratch / f"run-0-{run_count}", chunk) + run_count += 1 + chunk = [] + chunk_bytes = 0 + await self._settled( + asyncio.to_thread(self._record_consumed, consumed, file) + ) + if chunk: + await self._write_run(scratch / f"run-0-{run_count}", chunk) + run_count += 1 + return run_count, data_count, warmup_count + + async def _write_run(self, path: Path, chunk: list[SortRow]) -> None: + chunk.sort(key=lambda row: row[:4]) + async with self._open(path, "wb") as output: + buffer = bytearray() + for row in chunk: + buffer.extend(orjson.dumps(row) + b"\n") + if len(buffer) >= self.WRITE_BYTES: + await self._settled(output.write(buffer)) + buffer.clear() + if buffer: + await self._settled(output.write(buffer)) + + async def _merge_runs(self, scratch: Path, count: int) -> Path | None: + if count == 0: + return None + pass_index = 0 + while count > 1: + next_count = 0 + for start in range(0, count, self.MERGE_FAN_IN): + source_paths = [ + scratch / f"run-{pass_index}-{i}" + for i in range(start, min(start + self.MERGE_FAN_IN, count)) + ] + target = scratch / f"run-{pass_index + 1}-{next_count}" + if len(source_paths) == 1: + await self._settled( + asyncio.to_thread(os.replace, source_paths[0], target) + ) + else: + await self._merge_group(source_paths, target) + for path in source_paths: + path.unlink() + next_count += 1 + count = next_count + pass_index += 1 + return scratch / f"run-{pass_index}-0" + + async def _merge_group(self, sources: list[Path], target: Path) -> None: + async with AsyncExitStack() as stack: + readers = [self._lines(path) for path in sources] + for reader in readers: + stack.push_async_callback(reader.aclose) + output = await stack.enter_async_context(self._open(target, "wb")) + heap: list[tuple[tuple[int, int, int, int], int, bytes]] = [] + buffer = bytearray() + for index, reader in enumerate(readers): + if line := await anext(reader, None): + row = orjson.loads(line) + heapq.heappush(heap, (tuple(row[:4]), index, line)) + while heap: + _, index, line = heapq.heappop(heap) + buffer.extend(line) + if len(buffer) >= self.WRITE_BYTES: + await self._settled(output.write(buffer)) + buffer.clear() + if next_line := await anext(readers[index], None): + row = orjson.loads(next_line) + heapq.heappush(heap, (tuple(row[:4]), index, next_line)) + if buffer: + await self._settled(output.write(buffer)) + + async def _write_document(self, staged: Path, run: Path | None) -> None: + async with self._open(staged, "wb") as output: + buffer = bytearray() + + async def emit(piece: bytes) -> None: + buffer.extend(piece) + if len(buffer) >= self.WRITE_BYTES: + await self._settled(output.write(buffer)) + buffer.clear() + + await emit(b'{\n "schema_version": "1.1",\n "data": [') + previous_phase = await self._write_entries(run, emit) + if previous_phase == 0: + await emit(b'\n ],\n "warmup": []\n}') + elif previous_phase == 1: + await emit(b"\n ]\n}") + else: + await emit(b'],\n "warmup": []\n}') + if buffer: + await self._settled(output.write(buffer)) + + async def _write_entries( + self, run: Path | None, emit: Callable[[bytes], Awaitable[None]] + ) -> int | None: + if run is None: + return None + previous_phase: int | None = None + async with aclosing(self._lines(run)) as lines: + async for line in lines: + phase, _, _, _, entry = orjson.loads(line) + if previous_phase is None: + if phase == 0: + await emit(b"\n") + else: + await emit(b'],\n "warmup": [\n') + elif phase != previous_phase: + await emit(b'\n ],\n "warmup": [\n') + else: + await emit(b",\n") + encoded = orjson.dumps(entry, option=orjson.OPT_INDENT_2) + await emit(b" " + encoded.replace(b"\n", b"\n ")) + previous_phase = phase + return previous_phase - def _cleanup_fragments(self, fragment_files: list[Path]) -> None: - """Remove fragment files and directory.""" - for file in fragment_files: - file.unlink(missing_ok=True) + def _cleanup_fragments(self, consumed: Path) -> None: + """Remove source fragments only after the final document is published.""" + with consumed.open("rb") as source: + for line in source: + (self._fragments_dir / orjson.loads(line)).unlink(missing_ok=True) try: self._fragments_dir.rmdir() except OSError: diff --git a/tests/unit/exporters/test_outputs_json_exporter.py b/tests/unit/exporters/test_outputs_json_exporter.py index bf1e06c174..50942a0dcf 100644 --- a/tests/unit/exporters/test_outputs_json_exporter.py +++ b/tests/unit/exporters/test_outputs_json_exporter.py @@ -1,14 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio +import os +import threading +from collections.abc import AsyncIterator, Iterator from pathlib import Path from unittest.mock import MagicMock import orjson import pytest +from aiofiles.threadpool.binary import AsyncBufferedIOBase from aiperf.common.exceptions import DataExporterDisabled from aiperf.config.artifacts import OutputDefaults +from aiperf.exporters import outputs_json_exporter from aiperf.exporters.outputs_json_exporter import OutputsJsonExporter @@ -75,6 +81,13 @@ async def test_export_no_fragments_skips(self, tmp_path: Path) -> None: outputs_file = tmp_path / "outputs.json" assert not outputs_file.exists() + @pytest.mark.asyncio + async def test_export_empty_fragments_dir_skips(self, tmp_path: Path) -> None: + (tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER).mkdir() + await _make_exporter(tmp_path).export() + assert not (tmp_path / "outputs.json").exists() + assert not list(tmp_path.glob(".outputs-json-*")) + @pytest.mark.asyncio async def test_export_skips_unparseable_fragment_line(self, tmp_path: Path) -> None: """A corrupt line (e.g. a partial write from a crashed processor) is skipped @@ -379,72 +392,274 @@ async def test_warmup_array_sorted_independently(self, tmp_path: Path) -> None: assert [r["session_num"] for r in data["warmup"]] == [2, 9] -class TestLargeExportWarning: - """The large-export warning fires on the record count, not the file size. +class TestBoundedOutputsExport: + @pytest.mark.asyncio + async def test_many_runs_preserve_order_and_partition(self, tmp_path: Path) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + rows = [] + for index in range(31): + row = _make_fragment( + session_num=index % 4, + turn_index=0 if index % 3 else -1, + response_text=f"original-{index}", + benchmark_phase="warmup" if index % 5 == 0 else "profiling", + ) + if index == 7: + del row["benchmark_phase"] + if index == 8: + row["benchmark_phase"] = "future" + rows.append(row) + _write_jsonl(fragments_dir / "output_fragments_a.jsonl", rows[:15]) + _write_jsonl(fragments_dir / "output_fragments_b.jsonl", rows[15:]) + traversal = [ + orjson.loads(line) + for entry in os.scandir(fragments_dir) + if entry.name.startswith("output_fragments_") + and entry.name.endswith(".jsonl") + for line in Path(entry.path).read_bytes().splitlines() + ] + exporter = _make_exporter(tmp_path) + exporter.CHUNK_RECORDS = 2 + exporter.CHUNK_BYTES = 100 + exporter.MERGE_FAN_IN = 3 + await exporter.export() - outputs.json is sorted, so it cannot stream and holds the whole document in - memory. The warning is the only signal a user gets that a run is paying that - cost, so pin the boundary rather than leaving log-only behavior untested. - """ + document = orjson.loads((tmp_path / "outputs.json").read_bytes()) + assert document["schema_version"] == "1.1" + assert sorted( + row["response_text"] for row in document["data"] + document["warmup"] + ) == sorted(row["response_text"] for row in rows) + for name in ("data", "warmup"): + expected = [ + row["response_text"] + for row in sorted( + ( + r + for r in traversal + if (r.get("benchmark_phase") == "warmup") == (name == "warmup") + ), + key=lambda r: (r["session_num"], r.get("turn_index") or 0), + ) + ] + assert [row["response_text"] for row in document[name]] == expected + assert not list(tmp_path.glob(".outputs-json-*")) - @staticmethod - def _prepare( - tmp_path: Path, record_count: int, threshold: int - ) -> tuple[list[str], OutputsJsonExporter]: - """Build an exporter over ``record_count`` fragments, capturing warnings.""" + @pytest.mark.asyncio + async def test_empty_file_exports_empty_arrays(self, tmp_path: Path) -> None: fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER - fragments_dir.mkdir(parents=True, exist_ok=True) - _write_jsonl( - fragments_dir / "output_fragments_proc1.jsonl", - [_make_fragment(session_num=i) for i in range(record_count)], - ) + fragments_dir.mkdir() + (fragments_dir / "output_fragments_empty.jsonl").touch() + await _make_exporter(tmp_path).export() + assert orjson.loads((tmp_path / "outputs.json").read_bytes()) == { + "schema_version": "1.1", + "data": [], + "warmup": [], + } + assert not (fragments_dir / "output_fragments_empty.jsonl").exists() + @pytest.mark.asyncio + async def test_fragment_scan_consumes_directory_in_bounded_batches( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + for index in range(130): + _write_jsonl( + fragments_dir / f"output_fragments_{index:03}.jsonl", + [_make_fragment(session_num=index)], + ) exporter = _make_exporter(tmp_path) - exporter.LARGE_EXPORT_RECORD_WARNING_THRESHOLD = threshold + actual_scandir = os.scandir + scanned = 0 + scanned_at_first_read: list[int] = [] - warnings: list[str] = [] - exporter.warning = warnings.append # type: ignore[method-assign] - return warnings, exporter + class TrackedEntries: + def __init__(self, entries: Iterator[os.DirEntry[str]]) -> None: + self._entries = entries - @pytest.mark.asyncio - async def test_export_record_count_at_threshold_warns(self, tmp_path: Path) -> None: - warnings, exporter = self._prepare(tmp_path, 3, threshold=2) - await exporter.export() + def __iter__(self) -> "TrackedEntries": + return self + + def __next__(self) -> os.DirEntry[str]: + nonlocal scanned + scanned += 1 + return next(self._entries) + + def close(self) -> None: + self._entries.close() + + def tracked_scandir(path: Path) -> Iterator[os.DirEntry[str]]: + entries = actual_scandir(path) + if path == fragments_dir: + return TrackedEntries(entries) + return entries - assert len(warnings) == 1 - assert "3 records" in warnings[0] - assert "--no-export-outputs-json" in warnings[0] + original_lines = exporter._lines + + async def tracked_lines(path: Path) -> AsyncIterator[bytes]: + if not scanned_at_first_read: + scanned_at_first_read.append(scanned) + async for line in original_lines(path): + yield line + + monkeypatch.setattr(os, "scandir", tracked_scandir) + monkeypatch.setattr(exporter, "_lines", tracked_lines) + await exporter.export() + assert scanned_at_first_read == [exporter.FRAGMENT_SCAN_BATCH] + assert scanned >= 130 + assert ( + len(orjson.loads((tmp_path / "outputs.json").read_bytes())["data"]) == 130 + ) @pytest.mark.asyncio - async def test_export_record_count_below_threshold_does_not_warn( - self, tmp_path: Path + async def test_cleanup_preserves_fragment_created_after_ingestion( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - warnings, exporter = self._prepare(tmp_path, 2, threshold=3) + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + consumed = fragments_dir / "output_fragments_old.jsonl" + late = fragments_dir / "output_fragments_late.jsonl" + _write_jsonl(consumed, [_make_fragment(session_num=1, response_text="old")]) + exporter = _make_exporter(tmp_path) + original_write = exporter._write_document + + async def write_then_add(staged: Path, run: Path | None) -> None: + await original_write(staged, run) + _write_jsonl(late, [_make_fragment(session_num=2, response_text="late")]) + + monkeypatch.setattr(exporter, "_write_document", write_then_add) await exporter.export() + assert not consumed.exists() + assert late.exists() + assert [ + row["response_text"] + for row in orjson.loads((tmp_path / "outputs.json").read_bytes())["data"] + ] == ["old"] - assert warnings == [] + @pytest.mark.asyncio + async def test_large_unicode_record(self, tmp_path: Path) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + row = _make_fragment( + session_num=1, + response_text="雪🎵" * 100_000, + metrics={"request_latency": None, "nested": [None, 3]}, + ) + _write_jsonl(fragments_dir / "output_fragments_a.jsonl", [row]) + exporter = _make_exporter(tmp_path) + exporter.CHUNK_BYTES = 100 + await exporter.export() + actual = orjson.loads((tmp_path / "outputs.json").read_bytes())["data"][0] + assert actual["response_text"] == row["response_text"] + assert actual["metrics"] == {"request_latency": None, "nested": [None, 3]} + + def test_entry_scrubs_non_finite_metrics(self) -> None: + entry = OutputsJsonExporter._entry( + { + "session_num": 1, + "metrics": {"request_latency": float("nan"), "nested": [float("inf")]}, + } + ) + assert entry["metrics"] == {"request_latency": None, "nested": [None]} @pytest.mark.asyncio - async def test_export_warmup_records_count_toward_threshold_warns( - self, tmp_path: Path + async def test_replace_error_retains_sources_and_old_final( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Warmup records are held in memory too, so they count.""" fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER - fragments_dir.mkdir(parents=True) - _write_jsonl( - fragments_dir / "output_fragments_proc1.jsonl", - [ - _make_fragment(session_num=0), - _make_fragment(session_num=1, benchmark_phase="warmup"), - ], - ) + source = fragments_dir / "output_fragments_a.jsonl" + _write_jsonl(source, [_make_fragment(session_num=1)]) + final = tmp_path / "outputs.json" + final.write_bytes(b"old final") + def fail_replace(_source: Path, _target: Path) -> None: + raise OSError("replace denied") + + monkeypatch.setattr(outputs_json_exporter.os, "replace", fail_replace) + with pytest.raises(OSError, match="replace denied"): + await _make_exporter(tmp_path).export() + assert source.exists() + assert final.read_bytes() == b"old final" + assert not list(tmp_path.glob(".outputs-json-*")) + + @pytest.mark.asyncio + async def test_merge_error_retains_sources_and_old_final( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + source = fragments_dir / "output_fragments_a.jsonl" + _write_jsonl(source, [_make_fragment(session_num=i) for i in range(5)]) + final = tmp_path / "outputs.json" + final.write_bytes(b"old final") exporter = _make_exporter(tmp_path) - exporter.LARGE_EXPORT_RECORD_WARNING_THRESHOLD = 2 - warnings: list[str] = [] - exporter.warning = warnings.append # type: ignore[method-assign] + exporter.CHUNK_RECORDS = 1 + original = exporter._merge_group - await exporter.export() + async def fail_after_merge(sources: list[Path], target: Path) -> None: + await original(sources, target) + raise OSError("merge failed") - assert len(warnings) == 1 - assert "2 records" in warnings[0] + monkeypatch.setattr(exporter, "_merge_group", fail_after_merge) + with pytest.raises(OSError, match="merge failed"): + await exporter.export() + assert source.exists() + assert final.read_bytes() == b"old final" + assert not list(tmp_path.glob(".outputs-json-*")) + + @pytest.mark.asyncio + async def test_write_error_closes_file_before_scratch_cleanup( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + source = fragments_dir / "output_fragments_a.jsonl" + _write_jsonl(source, [_make_fragment(session_num=1)]) + final = tmp_path / "outputs.json" + final.write_bytes(b"old final") + opened: list[AsyncBufferedIOBase] = [] + original_write = AsyncBufferedIOBase.write + + async def fail_during_write(handle: AsyncBufferedIOBase, data: bytes) -> int: + opened.append(handle) + await original_write(handle, data[:7]) + raise OSError("write failed") + + monkeypatch.setattr(AsyncBufferedIOBase, "write", fail_during_write) + with pytest.raises(OSError, match="write failed"): + await _make_exporter(tmp_path).export() + assert opened and opened[0].closed + assert source.exists() + assert final.read_bytes() == b"old final" + assert not list(tmp_path.glob(".outputs-json-*")) + + @pytest.mark.asyncio + async def test_cancelled_export_retains_sources_and_old_final( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + fragments_dir = tmp_path / OutputDefaults.OUTPUT_FRAGMENTS_FOLDER + source = fragments_dir / "output_fragments_a.jsonl" + _write_jsonl(source, [_make_fragment(session_num=1)]) + final = tmp_path / "outputs.json" + final.write_bytes(b"old final") + entered = threading.Event() + proceed = threading.Event() + + async def delayed_write(handle: AsyncBufferedIOBase, data: bytes) -> int: + def write_and_wait() -> int: + written = handle._file.write(data) + entered.set() + assert proceed.wait(timeout=2) + return written + + return await asyncio.to_thread(write_and_wait) + + monkeypatch.setattr(AsyncBufferedIOBase, "write", delayed_write) + exporter = _make_exporter(tmp_path) + exporter.CHUNK_RECORDS = 1 + task = asyncio.create_task(exporter.export()) + await asyncio.wait_for(asyncio.to_thread(entered.wait, 2), timeout=3) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + proceed.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=2) + assert source.exists() + assert final.read_bytes() == b"old final" + assert not list(tmp_path.glob(".outputs-json-*")) From b5805079155e8544747baa48527ffbfffbfee0de Mon Sep 17 00:00:00 2001 From: StanleySongNV <49929281+gzydsmz@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:12:27 +0800 Subject: [PATCH 104/119] fix(docs): repair invalid YAML example and broken documentation links (#1585) Signed-off-by: SongMingzhe <65708305+SongMingzhe@users.noreply.github.com> Co-authored-by: SongMingzhe <65708305+SongMingzhe@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 98d8e53b659b509f5e0d1a64526b37559f7acd35) --- ATTRIBUTIONS.md | 10 +++++----- .../reference/gpu-telemetry-metrics-dataflow.md | 6 +++--- docs/reference/validating-isl-distribution.md | 5 +++-- docs/reference/vendor-usage-fields.md | 17 +++++++++++++++-- docs/server-metrics/server-metrics-reference.md | 2 +- docs/tutorials/image-generation.md | 6 +++--- docs/tutorials/plot.md | 2 +- docs/tutorials/sglang-image-edit.md | 2 +- docs/tutorials/sglang-video-generation.md | 6 +++--- tests/aiperf_mock_server/README.md | 2 +- tests/component_integration/sessions/README.md | 2 -- 11 files changed, 36 insertions(+), 24 deletions(-) diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md index 148c58614d..fd1ca19165 100644 --- a/ATTRIBUTIONS.md +++ b/ATTRIBUTIONS.md @@ -12,7 +12,7 @@ This document provides attribution information for public domain assets used in ### Shakespeare Complete Works **Asset Information:** -- **File(s)**: [`aiperf/dataset/generator/assets/shakespeare.txt`](aiperf/dataset/generator/assets/shakespeare.txt) +- **File(s)**: [`src/aiperf/dataset/generator/assets/shakespeare.txt`](src/aiperf/dataset/generator/assets/shakespeare.txt) - **Author**: William Shakespeare (1564-1616) - **License**: Public Domain - **Status**: Works created before copyright law; no restrictions apply @@ -27,10 +27,10 @@ This document provides attribution information for public domain assets used in **Asset Information:** - **Files**: - - [`aiperf/dataset/generator/assets/source_images/0bfd8fdf-457f-43c8-9253-a2346d37d26a_1024.jpg`](aiperf/dataset/generator/assets/source_images/0bfd8fdf-457f-43c8-9253-a2346d37d26a_1024.jpg) - [Object 103QTD](https://www.getty.edu/art/collection/object/103QTD) - - [`aiperf/dataset/generator/assets/source_images/119544eb-9bbf-47d1-8d93-a51de6370295_861.jpg`](aiperf/dataset/generator/assets/source_images/119544eb-9bbf-47d1-8d93-a51de6370295_861.jpg) - [Object 103SNG](https://www.getty.edu/art/collection/object/103SNG) - - [`aiperf/dataset/generator/assets/source_images/1ba066e0-7291-4ef4-8a6e-e8724af7f046_865.jpg`](aiperf/dataset/generator/assets/source_images/1ba066e0-7291-4ef4-8a6e-e8724af7f046_865.jpg) - [Object 10962M](https://www.getty.edu/art/collection/object/10962M) - - [`aiperf/dataset/generator/assets/source_images/c946f826-4a9a-4ca2-b891-9f791d38092b_1024.jpg`](aiperf/dataset/generator/assets/source_images/c946f826-4a9a-4ca2-b891-9f791d38092b_1024.jpg) - [Object 103RQ6](https://www.getty.edu/art/collection/object/103RQ6) + - [`src/aiperf/dataset/generator/assets/source_images/0bfd8fdf-457f-43c8-9253-a2346d37d26a_1024.jpg`](src/aiperf/dataset/generator/assets/source_images/0bfd8fdf-457f-43c8-9253-a2346d37d26a_1024.jpg) - [Object 103QTD](https://www.getty.edu/art/collection/object/103QTD) + - [`src/aiperf/dataset/generator/assets/source_images/119544eb-9bbf-47d1-8d93-a51de6370295_861.jpg`](src/aiperf/dataset/generator/assets/source_images/119544eb-9bbf-47d1-8d93-a51de6370295_861.jpg) - [Object 103SNG](https://www.getty.edu/art/collection/object/103SNG) + - [`src/aiperf/dataset/generator/assets/source_images/1ba066e0-7291-4ef4-8a6e-e8724af7f046_865.jpg`](src/aiperf/dataset/generator/assets/source_images/1ba066e0-7291-4ef4-8a6e-e8724af7f046_865.jpg) - [Object 10962M](https://www.getty.edu/art/collection/object/10962M) + - [`src/aiperf/dataset/generator/assets/source_images/c946f826-4a9a-4ca2-b891-9f791d38092b_1024.jpg`](src/aiperf/dataset/generator/assets/source_images/c946f826-4a9a-4ca2-b891-9f791d38092b_1024.jpg) - [Object 103RQ6](https://www.getty.edu/art/collection/object/103RQ6) - **License**: CC0 1.0 Universal (Public Domain Dedication) - **Attribution** (recommended): Digital image courtesy of Getty's Open Content Program diff --git a/docs/reference/gpu-telemetry-metrics-dataflow.md b/docs/reference/gpu-telemetry-metrics-dataflow.md index f17b6b48fe..abfca7205e 100644 --- a/docs/reference/gpu-telemetry-metrics-dataflow.md +++ b/docs/reference/gpu-telemetry-metrics-dataflow.md @@ -12,9 +12,9 @@ so each vendor's totals sum only its own GPUs. A mixed NVIDIA+AMD run emits both lanes' sections; a vendor with no reporting GPU is omitted entirely. > [!NOTE] -> A self-contained, richly-styled HTML version of this diagram lives alongside -> this page at [`gpu-telemetry-metrics-dataflow.html`](gpu-telemetry-metrics-dataflow.html) -> (open it directly in a browser). +> A self-contained, richly-styled HTML version of this diagram lives in the +> repository at [`docs/reference/gpu-telemetry-metrics-dataflow.html`](https://github.com/ai-dynamo/aiperf/blob/main/docs/reference/gpu-telemetry-metrics-dataflow.html) +> (download it and open it directly in a browser). ## Dataflow diff --git a/docs/reference/validating-isl-distribution.md b/docs/reference/validating-isl-distribution.md index c00eff0324..d548441277 100644 --- a/docs/reference/validating-isl-distribution.md +++ b/docs/reference/validating-isl-distribution.md @@ -18,8 +18,9 @@ parameters. Blue = aiperf, red = vLLM Docker; overlapping bars appear purple. ![Llama 3.1 8B Instruct ISL distribution — aiperf vs vLLM Docker](isl-example-llama.svg) -An interactive version with additional tokenizer comparisons is available in -[isl-distribution-examples.html](isl-distribution-examples.html). +An interactive version with additional tokenizer comparisons is available in the +repository at [`docs/reference/isl-distribution-examples.html`](https://github.com/ai-dynamo/aiperf/blob/main/docs/reference/isl-distribution-examples.html) +(download it and open it directly in a browser). ## Quick start diff --git a/docs/reference/vendor-usage-fields.md b/docs/reference/vendor-usage-fields.md index 405a659575..03eeac2ed7 100644 --- a/docs/reference/vendor-usage-fields.md +++ b/docs/reference/vendor-usage-fields.md @@ -81,7 +81,7 @@ All field names match AIPerf's modelled synonyms. `cached_tokens` is read-only o ### vLLM -**Verified against:** [`vllm` / `vllm/entrypoints/openai/engine/protocol.py`](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/engine/protocol.py). +**Verified against:** [`vllm` / `vllm/entrypoints/serve/engine/protocol.py`](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/serve/engine/protocol.py). ```python class UsageInfo(OpenAIBaseModel): @@ -89,12 +89,25 @@ class UsageInfo(OpenAIBaseModel): total_tokens: int = 0 completion_tokens: int | None = 0 prompt_tokens_details: PromptTokenUsageInfo | None = None + completion_tokens_details: CompletionTokenUsageInfo | None = None class PromptTokenUsageInfo(OpenAIBaseModel): cached_tokens: int | None = None + created_cache_tokens: int | None = None + multimodal_tokens: dict[str, int] | None = None + +class CompletionTokenUsageInfo(OpenAIBaseModel): + reasoning_tokens: int = 0 ``` -vLLM is OpenAI-compatible. Its `prompt_tokens_details` is narrower than OpenAI's (only `cached_tokens`, no `audio_tokens`). vLLM may emit `prompt_tokens_details: null` and `completion_tokens_details: null` explicitly; AIPerf's nested-field walk handles that case (the `isinstance(details, dict)` guard returns False, and the property returns None). +vLLM is OpenAI-compatible, but its `*_tokens_details` sub-objects are neither a strict subset nor a superset of OpenAI's: + +- `prompt_tokens_details` has no `audio_tokens`, but adds `created_cache_tokens` (cache *writes*, which OpenAI does not surface at all) and `multimodal_tokens` — a per-modality breakdown (`image`, `audio`, `video`) of placeholder tokens already counted in `prompt_tokens`, `None` when the request has no multimodal input. +- `completion_tokens_details` carries only `reasoning_tokens`, with none of OpenAI's `audio_tokens` / `accepted_prediction_tokens` / `rejected_prediction_tokens`. + +AIPerf models `cached_tokens` and `reasoning_tokens`; `created_cache_tokens` and `multimodal_tokens` are not currently mapped to metrics, so vLLM cache-write and per-modality counts are available on the wire but unreported. + +vLLM may emit `prompt_tokens_details: null` and `completion_tokens_details: null` explicitly; AIPerf's nested-field walk handles that case (the `isinstance(details, dict)` guard returns False, and the property returns None). **Enablement (opt-in):** vLLM populates `cached_tokens` only when the server is launched with `--enable-prompt-tokens-details` (default off, per [vllm-project/vllm#10174](https://github.com/vllm-project/vllm/pull/10174) — "guarded by a flag ... OFF by default"). Without the flag, `prompt_tokens_details` is `null` and AIPerf's cache-read metrics are all-None even when prefix caching is active. The flag is independent of `--enable-prefix-caching` (the cache must still be on for `cached_tokens` to be non-zero). diff --git a/docs/server-metrics/server-metrics-reference.md b/docs/server-metrics/server-metrics-reference.md index 9556e58482..8472bae6c3 100644 --- a/docs/server-metrics/server-metrics-reference.md +++ b/docs/server-metrics/server-metrics-reference.md @@ -193,7 +193,7 @@ Key equivalent metrics across backends: ### Understanding Percentiles -Histogram percentiles are *estimated* from bucket boundaries, not exact values. Accuracy depends on bucket granularity. See [Histogram Buckets](#histogram-buckets) for bucket definitions. +Histogram percentiles are *estimated* from bucket boundaries, not exact values. Accuracy depends on bucket granularity. Per-backend bucket definitions appear under the **Histogram buckets** notes in [Detailed Metric Definitions](#detailed-metric-definitions). ### Multiple Endpoints diff --git a/docs/tutorials/image-generation.md b/docs/tutorials/image-generation.md index 9972c9d116..92192e771f 100644 --- a/docs/tutorials/image-generation.md +++ b/docs/tutorials/image-generation.md @@ -16,8 +16,8 @@ This guide shows how to benchmark image generation APIs using a Docker-based ser ## References For the most up-to-date information, please refer to the following resources: - [OpenAI Image Generation API](https://platform.openai.com/docs/api-reference/images/create) -- [SGLang Diffusion Installation Guide](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/installation.md) -- [SGLang Diffusion CLI Reference](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/api/cli.md) +- [SGLang Diffusion Installation Guide](https://docs.sglang.ai/docs/sglang-diffusion/installation) +- [SGLang Diffusion CLI Reference](https://docs.sglang.ai/docs/sglang-diffusion/api/cli) ## Setting up the server @@ -53,7 +53,7 @@ pip install yunchang remote_pdb imageio diffusers diffusion > [!WARNING] > > The following arguments will setup the server to use the FLUX.1-dev model on a single GPU, on port 30000. > > You can modify these arguments to use a different model, different number of GPUs, different port, etc. -> > See the [SGLang Diffusion CLI Reference](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/api/cli.md) for more details. +> > See the [SGLang Diffusion CLI Reference](https://docs.sglang.ai/docs/sglang-diffusion/api/cli) for more details. ```bash SERVER_ARGS=( --model-path black-forest-labs/FLUX.1-dev --text-encoder-cpu-offload --pin-cpu-memory --num-gpus 1 --port 30000 --host 0.0.0.0 ) ``` diff --git a/docs/tutorials/plot.md b/docs/tutorials/plot.md index 773b658a62..fcf64ad0f1 100644 --- a/docs/tutorials/plot.md +++ b/docs/tutorials/plot.md @@ -331,7 +331,7 @@ artifacts/ **Advanced**: Use `group_extraction_pattern` to aggregate variants: ```yaml -group_extraction_pattern: "^(treatment_\d+)" # Groups treatment_1_varA + treatment_1_varB → "treatment_1" +group_extraction_pattern: '^(treatment_\d+)' # Groups treatment_1_varA + treatment_1_varB → "treatment_1" ``` > [!TIP] diff --git a/docs/tutorials/sglang-image-edit.md b/docs/tutorials/sglang-image-edit.md index 5a2a15008d..1f9ae11709 100644 --- a/docs/tutorials/sglang-image-edit.md +++ b/docs/tutorials/sglang-image-edit.md @@ -50,7 +50,7 @@ docker run --gpus all \ > [!WARNING] > These arguments set up FLUX.2-Klein-4B on a single GPU at port 30000. > Adjust the model path, GPU count, or port to match your environment. -> The flags below come from upstream SGLang diffusion and may change over time — treat the [SGLang Diffusion CLI Reference](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/api/cli.md) as the source of truth if any flag here is rejected. +> The flags below come from upstream SGLang diffusion and may change over time — treat the [SGLang Diffusion CLI Reference](https://docs.sglang.ai/docs/sglang-diffusion/api/cli) as the source of truth if any flag here is rejected. ```bash SERVER_ARGS=( --model-path black-forest-labs/FLUX.2-klein-4B --num-gpus 1 --port 30000 --host 0.0.0.0 --warmup --enable-torch-compile ) ``` diff --git a/docs/tutorials/sglang-video-generation.md b/docs/tutorials/sglang-video-generation.md index 2bfee5654e..3eb08aaca5 100644 --- a/docs/tutorials/sglang-video-generation.md +++ b/docs/tutorials/sglang-video-generation.md @@ -20,9 +20,9 @@ AIPerf handles this polling workflow automatically. ## References For the most up-to-date information, please refer to the following resources: -- [SGLang Diffusion OpenAI API Reference](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/api/openai_api.md) -- [SGLang Diffusion Installation Guide](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/installation.md) -- [SGLang Diffusion CLI Reference](https://github.com/sgl-project/sglang/blob/main/docs/diffusion/api/cli.md) +- [SGLang Diffusion OpenAI API Reference](https://docs.sglang.ai/docs/sglang-diffusion/api/openai_api) +- [SGLang Diffusion Installation Guide](https://docs.sglang.ai/docs/sglang-diffusion/installation) +- [SGLang Diffusion CLI Reference](https://docs.sglang.ai/docs/sglang-diffusion/api/cli) - [OpenAI Videos API](https://platform.openai.com/docs/api-reference/videos) ## Supported Models diff --git a/tests/aiperf_mock_server/README.md b/tests/aiperf_mock_server/README.md index 9ee848ddc1..8d02045f69 100644 --- a/tests/aiperf_mock_server/README.md +++ b/tests/aiperf_mock_server/README.md @@ -19,7 +19,7 @@ A mock server for integration testing and performance benchmarking of LLM applic - [**Error Injection**](#error-injection): Reproducible fault testing with configurable error rates - [**Deterministic Responses**](#token-generation): Hash-based generation for identical outputs - [**Fast Mode**](#quick-start): Zero-latency mode (`--fast`) for integration testing -- [**Corpus**](#corpus): Pre-tokenized corpus loaded from aiperf's shakespeare.txt for deterministic output +- [**Corpus**](#tokenizer-options): Pre-tokenized corpus loaded from aiperf's shakespeare.txt for deterministic output - [**Request Recording**](#request-recording): Per-request ISL + requested OSL capture with JSONL output and a shutdown summary ### Supported Endpoints diff --git a/tests/component_integration/sessions/README.md b/tests/component_integration/sessions/README.md index 0eb1f15750..2fc9ba4a6d 100644 --- a/tests/component_integration/sessions/README.md +++ b/tests/component_integration/sessions/README.md @@ -312,6 +312,4 @@ AssertionError: Cancelled request req_456 has wrong error code: ## See Also -- [HELPER_USAGE.md](./HELPER_USAGE.md) - Comprehensive usage examples -- [/SESSION_HELPERS_SUMMARY.md](../../../SESSION_HELPERS_SUMMARY.md) - Implementation summary - [/tests/component_integration/conftest.py](../conftest.py) - Source code From 3e7d6db22cb5783c8495e011f7c1d7b96280bfd9 Mon Sep 17 00:00:00 2001 From: Harshini Komali <157742537+lkomali@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:22:50 -0700 Subject: [PATCH 105/119] ci(windows): use -n logical so xdist does not run the suite in one worker (#1623) Signed-off-by: lkomali (cherry picked from commit 6e096ab758d8582a781f8aa985373ca1c27e27b1) --- .github/workflows/run-unit-tests.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 71c6306d2d..23523c145d 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -126,8 +126,13 @@ jobs: # --continue-on-collection-errors preserves the previous two-step # behaviour where a unit collection error did not block the zmq suite. if: runner.os == 'Windows' + # -n logical, not -n auto: xdist's ``auto`` asks psutil for the PHYSICAL + # core count, and the GitHub x86 Windows VM reports 1 physical core for + # its 4 vCPUs, so ``auto`` ran the whole suite in a single worker (~3x + # slower than windows-11-arm, which happens to report 3). ``logical`` + # uses the logical count on both and tracks the runner size if it grows. run: | - uv run pytest tests/unit tests/zmq -n auto --dist=worksteal --tb=short --continue-on-collection-errors -m 'not performance and not stress and not slow' + uv run pytest tests/unit tests/zmq -n logical --dist=worksteal --tb=short --continue-on-collection-errors -m 'not performance and not stress and not slow' - name: Run component integration tests (Windows) # ``!cancelled()`` overrides the implicit ``success()`` so component # integration still runs even if the unit-test step failed — matching @@ -135,8 +140,9 @@ jobs: # Cancellation is still respected so we don't burn runner minutes # finishing after a manual stop. if: ${{ !cancelled() && runner.os == 'Windows' }} + # -n logical for the reason given on the unit-test step above. run: | - uv run pytest tests/component_integration -n auto --dist=worksteal --tb=short -v -m 'not performance and not stress and not slow' + uv run pytest tests/component_integration -n logical --dist=worksteal --tb=short -v -m 'not performance and not stress and not slow' - name: Upload results to Codecov if: matrix.os == 'linux' && matrix.python-version == '3.12' uses: codecov/codecov-action@v5 From 9a5272fabc2ed4cee0ede298779d943b7bb60d2d Mon Sep 17 00:00:00 2001 From: Nate Mailhot Date: Sun, 25 Oct 2026 13:54:04 -0700 Subject: [PATCH 106/119] =?UTF-8?q?fix(security):=20bump=20pillow,=20mlflo?= =?UTF-8?q?w,=20gitpython,=20cryptography=20for=20known=E2=80=A6=20(#1450)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 2602253970a44f0100d93a96d6ed888216f0d678) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1387f41007..29e557ee48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "optuna>=3.6", "orjson~=3.11.7", "pandas~=2.3.3", - "pillow~=12.2.0", + "pillow~=12.3.0", "plotly~=6.4.0", "dash~=3.1.0", "dash-bootstrap-components~=2.0.0", From 0aed7209896d6476597b9fc8eac71f3a545813b2 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:43:03 +0800 Subject: [PATCH 107/119] fix(metrics): skip ragged values of records ingested after the timeslice snapshot Root cause: export_results runs in a worker thread while ingestion can continue. _build_slice_ragged_index indexed record_bin, sized at the snapshot, with every current record index, so a ragged value appended for a later record raised IndexError and aborted the export. Fix: map record indices beyond the snapshot to no slice, as get_values_for_mask does, keeping value positions unchanged. Drop the redundant _compute_timeslice_bins docstring. Validation: new test appends ragged values for a record past the snapshot while the index is built; the timeslices match the export without them. Signed-off-by: weireweire <20922698+weireweire@users.noreply.github.com> (cherry picked from commit 0b166c33f9d13df9726d83740646351812068585) --- src/aiperf/metrics/accumulator.py | 11 +++++-- .../test_metrics_accumulator.py | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index 428797bdaa..8c0c9c2460 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -920,9 +920,15 @@ def _build_slice_ragged_index( backend = self._column_store.ragged(tag) if not getattr(backend, "SUPPORTS_PER_RECORD_REPLAY", False): continue - if len(backend.record_indices) == 0: + rec_idx = backend.record_indices + if len(rec_idx) == 0: continue - value_bin = record_bin[backend.record_indices] + # Records ingested after ``record_bin`` was sized fall in no slice, + # as in get_values_for_mask. + in_bounds = rec_idx < len(record_bin) + value_bin = np.where( + in_bounds, record_bin[np.where(in_bounds, rec_idx, 0)], -1 + ) order = np.argsort(value_bin, kind="stable") sorted_bin = value_bin[order] index[tag] = ( @@ -943,7 +949,6 @@ def _compute_timeslice_bins( max_ts: float, adjusted_arrays: dict[str, FloatArray] | None, ) -> list[TimesliceResult]: - """Per-slice results for ``_compute_timeslices`` (see its docstring).""" timeslices: list[TimesliceResult] = [] for bin_idx in range(len(edges) - 1): bin_mask_local = bins == bin_idx diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index a5f846d07e..eadbe8ed91 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -1194,6 +1194,35 @@ async def test_indexed_timeslices_match_per_slice_mask_path( ) assert self._dump(indexed) == self._dump(reference) + @pytest.mark.asyncio + async def test_ragged_values_ingested_after_snapshot_are_excluded( + self, mock_run + ) -> None: + mock_run.cfg.artifacts.slice_duration = 1.0 + accumulator = MetricsAccumulator(mock_run) + await self._ingest(accumulator) + reference = await accumulator.export_results(ExportContext()) + + backend = accumulator._column_store.ragged(InterChunkLatencyMetric.tag) + build_index = MetricsAccumulator._build_slice_ragged_index + + def build_index_after_late_ingest(self, record_bin, n_bins): + backend.extend(len(record_bin) + 3, [1e9, 2e9]) + return build_index(self, record_bin, n_bins) + + with patch.object( + MetricsAccumulator, + "_build_slice_ragged_index", + autospec=True, + side_effect=build_index_after_late_ingest, + ): + indexed = await accumulator.export_results(ExportContext()) + + assert indexed.timeslices + assert [t.model_dump(mode="json") for t in indexed.timeslices] == [ + t.model_dump(mode="json") for t in reference.timeslices or [] + ] + class TestMetricsSummary: def test_to_json(self) -> None: From c16820956ee40226cac56c349241c210ea69dbe8 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:43:19 +0800 Subject: [PATCH 108/119] refactor(server-metrics): align the warmup baseline tagging with upstream Root cause: #49 fixed the agentic warmup-start scrape in two places: it set phase_kind="warmup" on the synthesized agentic warmup, and it added a PhaseBaselineRequestMessage.phase fallback for boundaries without a kind. Upstream aiperf (ai-dynamo/aiperf#1325) fixes it with phase_kind="warmup" alone. The fallback cannot trigger: validated PhaseConfigs always carry a kind, and the agentic warmup now sets one. Fix: keep phase_kind="warmup" and its test assertion; drop the phase fallback, the message field, the publisher wiring, their tests, and the added comments, so this code matches upstream. Validation: tests/unit/server_metrics and tests/unit/timing. Signed-off-by: weireweire <20922698+weireweire@users.noreply.github.com> (cherry picked from commit 70cfe62ed486f7b4dd74de6440dde14052f539bc) --- .../common/messages/baseline_messages.py | 6 +-- src/aiperf/server_metrics/manager.py | 14 ++---- src/aiperf/timing/config.py | 2 - src/aiperf/timing/phase/publisher.py | 1 - .../test_server_metrics_manager.py | 49 +------------------ .../test_phase_config_agentic_replay.py | 2 - 6 files changed, 7 insertions(+), 67 deletions(-) diff --git a/src/aiperf/common/messages/baseline_messages.py b/src/aiperf/common/messages/baseline_messages.py index bf564669af..ffb1a91b57 100644 --- a/src/aiperf/common/messages/baseline_messages.py +++ b/src/aiperf/common/messages/baseline_messages.py @@ -6,7 +6,7 @@ from pydantic import Field -from aiperf.common.enums import BaselineKind, CreditPhase, MessageType +from aiperf.common.enums import BaselineKind, MessageType from aiperf.common.messages.base_messages import Message from aiperf.common.types import MessageTypeT, PhaseKind @@ -34,10 +34,6 @@ class PhaseBaselineRequestMessage(Message): phase_kind: PhaseKind | None = Field( default=None, description="Semantic phase kind: warmup or profiling." ) - phase: CreditPhase | None = Field( - default=None, - description="Credit phase of the boundary; used when phase_kind is unset.", - ) kind: BaselineKind = Field( ..., description="START before credits are issued; END after returns drain." ) diff --git a/src/aiperf/server_metrics/manager.py b/src/aiperf/server_metrics/manager.py index 5beb990609..583e441366 100644 --- a/src/aiperf/server_metrics/manager.py +++ b/src/aiperf/server_metrics/manager.py @@ -216,15 +216,11 @@ async def collect_baseline(self, message: PhaseBaselineRequestMessage) -> None: """Capture a one-shot server-metrics scrape for a phase boundary.""" if self._server_metrics_disabled or not self._collectors: return - # Tag the scrape with its own phase. Falling back to PROFILING for an - # unset kind put a warmup-start scrape into the profiling series, so - # counters that already existed then counted the whole warmup. - if message.phase_kind == "warmup": - boundary_phase = CreditPhase.WARMUP - elif message.phase_kind == "profiling": - boundary_phase = CreditPhase.PROFILING - else: - boundary_phase = message.phase or CreditPhase.PROFILING + boundary_phase = ( + CreditPhase.WARMUP + if message.phase_kind == "warmup" + else CreditPhase.PROFILING + ) errors: list[str] = [] for endpoint_url, collector in list(self._collectors.items()): try: diff --git a/src/aiperf/timing/config.py b/src/aiperf/timing/config.py index bc6983fe79..789b10d5b3 100644 --- a/src/aiperf/timing/config.py +++ b/src/aiperf/timing/config.py @@ -643,8 +643,6 @@ def _build_agentic_warmup_config(phase: PhaseConfig) -> CreditPhaseConfig | None total_expected_requests = concurrency return CreditPhaseConfig( phase=CreditPhase.WARMUP, - # Synthesized phases have no PhaseConfig.kind; consumers that key on - # phase_kind (server-metrics phase-boundary baselines) need it set. phase_kind="warmup", timing_mode=TimingMode.AGENTIC_REPLAY, # Duration mode is strategy-terminated by its timer. Count mode uses diff --git a/src/aiperf/timing/phase/publisher.py b/src/aiperf/timing/phase/publisher.py index eb8a615b94..5064d0197b 100644 --- a/src/aiperf/timing/phase/publisher.py +++ b/src/aiperf/timing/phase/publisher.py @@ -74,7 +74,6 @@ async def publish_phase_baseline_request( phase_name=config.phase_name or f"{config.phase.value}_{config.phase_index}", phase_kind=config.phase_kind, - phase=config.phase, kind=kind, ) await self._pub_client.publish(msg) diff --git a/tests/unit/server_metrics/test_server_metrics_manager.py b/tests/unit/server_metrics/test_server_metrics_manager.py index 3a7c8616d7..2bec3384e1 100644 --- a/tests/unit/server_metrics/test_server_metrics_manager.py +++ b/tests/unit/server_metrics/test_server_metrics_manager.py @@ -21,10 +21,7 @@ CreditPhaseStartMessage, ) from aiperf.plugin.enums import EndpointType, TimingMode -from aiperf.server_metrics.manager import ( - _SERVER_METRICS_SCRAPE_PHASE, - ServerMetricsManager, -) +from aiperf.server_metrics.manager import ServerMetricsManager from aiperf.timing.config import CreditPhaseConfig from tests.unit.conftest import make_run_from_cli @@ -670,50 +667,6 @@ async def slow_scrape(): assert manager._active_phase == CreditPhase.WARMUP - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("phase_kind", "phase", "expected"), - [ - ("warmup", None, CreditPhase.WARMUP), - ("profiling", None, CreditPhase.PROFILING), - # Synthesized phases (agentic auto-warmup) may carry no kind; the - # credit phase must decide, not a PROFILING default. - (None, CreditPhase.WARMUP, CreditPhase.WARMUP), - (None, CreditPhase.PROFILING, CreditPhase.PROFILING), - ], - ) - async def test_baseline_scrape_is_tagged_with_boundary_phase( - self, - cfg_with_endpoint: CLIConfig, - phase_kind, - phase, - expected, - ): - manager = ServerMetricsManager( - run=make_run_from_cli(cfg_with_endpoint), - ) - seen: list[CreditPhase | None] = [] - - async def scrape(): - seen.append(_SERVER_METRICS_SCRAPE_PHASE.get()) - - mock_collector = MagicMock() - mock_collector.collect_and_process_metrics = AsyncMock(side_effect=scrape) - manager._collectors = {"http://localhost:8000/metrics": mock_collector} - - await manager.collect_baseline( - PhaseBaselineRequestMessage( - service_id="timing-manager", - phase_id="phase-0", - phase_name="warmup_0", - phase_kind=phase_kind, - phase=phase, - kind=BaselineKind.START, - ) - ) - - assert seen == [expected] - class TestDisabledServerMetrics: """Test server metrics disabled scenarios.""" diff --git a/tests/unit/timing/test_phase_config_agentic_replay.py b/tests/unit/timing/test_phase_config_agentic_replay.py index f97f2e93f4..086401c6ac 100644 --- a/tests/unit/timing/test_phase_config_agentic_replay.py +++ b/tests/unit/timing/test_phase_config_agentic_replay.py @@ -35,8 +35,6 @@ def test_warmup_config_uses_agentic_replay_when_top_level_is_agentic_replay() -> assert warmup is not None assert warmup.timing_mode == TimingMode.AGENTIC_REPLAY assert warmup.phase == CreditPhase.WARMUP - # Server-metrics phase-boundary baselines key on phase_kind; an unset kind - # tagged the warmup-start scrape as profiling. assert warmup.phase_kind == "warmup" From 947f740df3244a6abe571e4ea924af7c8147c929 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:33:22 +0800 Subject: [PATCH 109/119] fix(metrics): pass timeslice ragged positions through the call instead of shared state Root cause: the slice index and the active slice were stored on the accumulator while the export's slice loop ran. export_results runs in a worker thread, and the realtime summarize runs on the event loop with no lock, so a summary that overlapped the slice loop gathered ragged values from the export's current slice instead of its own mask. Fix: build the index as a local in _compute_timeslices and pass each slice's value positions to _compute_results as an argument; the accumulator holds no per-computation state. Also trim the index: int32 slice ids and value positions, and slice bounds from bincount + cumsum instead of a sorted copy and two searchsorted passes. Validation: new test runs a phase-scoped summary in the middle of the slice loop; both the export and the summary match their standalone results. Existing index tests still match the per-slice mask path. Signed-off-by: weireweire <20922698+weireweire@users.noreply.github.com> (cherry picked from commit 8847786a61f29523da3e3ce6a4870fe220b3bfbf) --- src/aiperf/metrics/accumulator.py | 141 +++++++----------- .../test_metrics_accumulator.py | 43 ++++++ 2 files changed, 100 insertions(+), 84 deletions(-) diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index 8c0c9c2460..6e30b4614a 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -65,8 +65,8 @@ FloatArray: TypeAlias = NDArray[np.float64] BoolArray: TypeAlias = NDArray[np.bool_] -# (value order, per-slice start, per-slice end) of one ragged column's values sorted by slice -SliceIndex: TypeAlias = tuple[NDArray[np.int64], NDArray[np.int64], NDArray[np.int64]] +# (value positions sorted by slice, slice b at order[bounds[b] : bounds[b + 1]]) +SliceIndex: TypeAlias = tuple[NDArray[np.integer], NDArray[np.int64]] _AGGREGATE_FUNCS: dict[AggregationKind, Callable[[np.ndarray], float]] = { @@ -175,9 +175,6 @@ def __init__( self._slice_duration_ns: int | None = ( int(slice_dur * NANOS_PER_SECOND) if slice_dur else None ) - # Set only while _compute_timeslices runs. - self._slice_ragged_index: dict[MetricTagT, SliceIndex] | None = None - self._active_slice_bin: int | None = None @property def column_store(self) -> ColumnStore: @@ -418,6 +415,7 @@ def _compute_results( window_start_ns: int | None = None, window_end_ns: int | None = None, is_timeslice: bool = False, + ragged_positions: dict[MetricTagT, NDArray[np.integer]] | None = None, ) -> dict[MetricTagT, MetricResult]: """Phases: collect scalars/arrays, resolve derived, build MetricResults. @@ -427,6 +425,9 @@ def _compute_results( ``is_timeslice`` skips derived metrics anchored to a run-global reference (``timeslice_derivable = False``); see ``_non_timeslice_derived_tags``. + + ``ragged_positions`` gives, per ragged tag, the positions of the values + selected by ``mask``, so they are gathered without a scan of every value. """ scalar_dict: MetricResultsDict = MetricResultsDict() scalar_dict.window_start_ns = window_start_ns @@ -435,7 +436,11 @@ def _compute_results( sketch_results: dict[MetricTagT, MetricResult] = {} self._collect_scalars_and_arrays( - mask, scalar_dict, record_arrays, sketch_results + mask, + scalar_dict, + record_arrays, + sketch_results, + ragged_positions=ragged_positions, ) self._resolve_derived_metrics(scalar_dict, is_timeslice=is_timeslice) @@ -488,6 +493,8 @@ def _collect_scalars_and_arrays( scalar_dict: MetricResultsDict, record_arrays: dict[MetricTagT, tuple[FloatArray, float]], sketch_results: dict[MetricTagT, MetricResult], + *, + ragged_positions: dict[MetricTagT, NDArray[np.integer]] | None = None, ) -> None: """Iterate columns, populating scalar_dict and record_arrays in-place.""" store = self._column_store @@ -520,6 +527,9 @@ def _collect_scalars_and_arrays( scalar_dict=scalar_dict, record_arrays=record_arrays, sketch_results=sketch_results, + positions=( + ragged_positions.get(tag) if ragged_positions is not None else None + ), ) def _collect_one_list_column( @@ -531,6 +541,7 @@ def _collect_one_list_column( scalar_dict: MetricResultsDict, record_arrays: dict[MetricTagT, tuple[FloatArray, float]], sketch_results: dict[MetricTagT, MetricResult], + positions: NDArray[np.integer] | None = None, ) -> None: """Forks on the backend's ``SUPPORTS_PER_RECORD_REPLAY`` flag. @@ -545,21 +556,10 @@ def _collect_one_list_column( # a view into the ragged buffer that compute_sweep_curves reads later # (against unsorted offsets/record_indices), so the full-dataset branch # must copy. get_values_for_mask already returns a fresh masked copy. - slice_index = ( - self._slice_ragged_index.get(tag) - if self._slice_ragged_index is not None - and self._active_slice_bin is not None - else None - ) if full_dataset: filtered = backend.values.copy() - elif slice_index is not None: - # The same values in the same (storage) order as - # get_values_for_mask(mask) for this slice's mask, without a - # pass over every stored value. - order, starts, ends = slice_index - b = self._active_slice_bin - filtered = backend.values[order[starts[b] : ends[b]]] + elif positions is not None: + filtered = backend.values[positions] else: filtered = backend.get_values_for_mask(mask) if len(filtered) == 0: @@ -884,72 +884,12 @@ def _compute_timeslices( # Assign each record to a bin — O(n) total via digitize bins = np.digitize(filled_ts, edges) - 1 + timeslices: list[TimesliceResult] = [] filled_indices = np.where(filled)[0] - - n_bins = len(edges) - 1 - record_bin = np.full(n, -1, dtype=np.int64) + record_bin = np.full(n, -1, dtype=np.int32) record_bin[filled_indices] = bins - self._slice_ragged_index = self._build_slice_ragged_index(record_bin, n_bins) - try: - return self._compute_timeslice_bins( - sweeps, - edges=edges, - bins=bins, - filled_indices=filled_indices, - n=n, - max_ts=max_ts, - adjusted_arrays=adjusted_arrays, - ) - finally: - self._slice_ragged_index = None - self._active_slice_bin = None - - def _build_slice_ragged_index( - self, record_bin: NDArray[np.int64], n_bins: int - ) -> dict[MetricTagT, SliceIndex]: - """Bucket every replay-capable ragged value by its record's slice, once. - - ``record_bin`` maps each record to its slice (-1 for none). A stable sort - keeps storage order inside each slice, so ``values[order[starts[b]:ends[b]]]`` - equals ``get_values_for_mask`` for slice ``b``'s mask element for element, - at O(values in the slice) per slice instead of O(all values). - """ - bin_ids = np.arange(n_bins, dtype=np.int64) - index: dict[MetricTagT, SliceIndex] = {} - for tag in self._column_store.ragged_tags(): - backend = self._column_store.ragged(tag) - if not getattr(backend, "SUPPORTS_PER_RECORD_REPLAY", False): - continue - rec_idx = backend.record_indices - if len(rec_idx) == 0: - continue - # Records ingested after ``record_bin`` was sized fall in no slice, - # as in get_values_for_mask. - in_bounds = rec_idx < len(record_bin) - value_bin = np.where( - in_bounds, record_bin[np.where(in_bounds, rec_idx, 0)], -1 - ) - order = np.argsort(value_bin, kind="stable") - sorted_bin = value_bin[order] - index[tag] = ( - order, - np.searchsorted(sorted_bin, bin_ids, side="left"), - np.searchsorted(sorted_bin, bin_ids, side="right"), - ) - return index + ragged_index = self._build_slice_ragged_index(record_bin, len(edges) - 1) - def _compute_timeslice_bins( - self, - sweeps: Any, - *, - edges: NDArray[np.float64], - bins: NDArray[np.intp], - filled_indices: NDArray[np.intp], - n: int, - max_ts: float, - adjusted_arrays: dict[str, FloatArray] | None, - ) -> list[TimesliceResult]: - timeslices: list[TimesliceResult] = [] for bin_idx in range(len(edges) - 1): bin_mask_local = bins == bin_idx if not bin_mask_local.any(): @@ -966,14 +906,16 @@ def _compute_timeslice_bins( is_complete = raw_window_end <= max_ts window_end = raw_window_end if is_complete else max_ts - self._active_slice_bin = bin_idx results = self._compute_results( full_mask, window_start_ns=int(window_start), window_end_ns=int(window_end), is_timeslice=True, + ragged_positions={ + tag: order[bounds[bin_idx] : bounds[bin_idx + 1]] + for tag, (order, bounds) in ragged_index.items() + }, ) - self._active_slice_bin = None if len(results) == 0: continue results.update(sweeps.compute_metrics(window_start, window_end)) @@ -999,3 +941,34 @@ def _compute_timeslice_bins( ) return timeslices + + def _build_slice_ragged_index( + self, record_bin: NDArray[np.int32], n_bins: int + ) -> dict[MetricTagT, SliceIndex]: + """Bucket every replay-capable ragged value by its record's slice, once. + + ``record_bin`` maps each record to its slice (-1 for none). A stable sort + keeps storage order inside each slice, so ``values[order[bounds[b]:bounds[b + 1]]]`` + equals ``get_values_for_mask`` for slice ``b``'s mask element for element, + at O(values in the slice) per slice instead of O(all values). + """ + index: dict[MetricTagT, SliceIndex] = {} + for tag in self._column_store.ragged_tags(): + backend = self._column_store.ragged(tag) + if not getattr(backend, "SUPPORTS_PER_RECORD_REPLAY", False): + continue + rec_idx = backend.record_indices + if len(rec_idx) == 0: + continue + # Records ingested after ``record_bin`` was sized fall in no slice, + # as in get_values_for_mask. + in_bounds = rec_idx < len(record_bin) + value_bin = np.full(len(rec_idx), -1, dtype=np.int32) + value_bin[in_bounds] = record_bin[rec_idx[in_bounds]] + order = np.argsort(value_bin, kind="stable") + if len(order) <= np.iinfo(np.int32).max: + order = order.astype(np.int32) + # Unassigned values (-1) sort first; bounds[b] is where slice b starts. + bounds = np.cumsum(np.bincount(value_bin + 1, minlength=n_bins + 1)) + index[tag] = (order, bounds) + return index diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index eadbe8ed91..2f96e63d12 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -1223,6 +1223,49 @@ def build_index_after_late_ingest(self, record_bin, n_bins): t.model_dump(mode="json") for t in reference.timeslices or [] ] + @pytest.mark.asyncio + async def test_summary_overlapping_timeslice_export_keeps_its_own_values( + self, mock_run + ) -> None: + mock_run.cfg.artifacts.slice_duration = 1.0 + accumulator = MetricsAccumulator(mock_run) + await self._ingest(accumulator) + warmup = ExportContext(phase=CreditPhase.WARMUP) + profiling = ExportContext(phase=CreditPhase.PROFILING) + expected_warmup = self._dump(await accumulator.export_results(warmup)) + expected_profiling = self._dump( + accumulator._summarize_for_export_context( + profiling, include_timeslices=False + ) + ) + + compute_results = MetricsAccumulator._compute_results + pending = [profiling] + overlapped: list[AccumulatorMetricsSummary] = [] + + def summarize_mid_slice(self, *args, **kwargs): + # The realtime summary runs on the event loop while the export's + # slice loop runs in a worker thread. + if kwargs.get("is_timeslice") and pending: + overlapped.append( + self._summarize_for_export_context( + pending.pop(), include_timeslices=False + ) + ) + return compute_results(self, *args, **kwargs) + + with patch.object( + MetricsAccumulator, + "_compute_results", + autospec=True, + side_effect=summarize_mid_slice, + ): + exported = await accumulator.export_results(warmup) + + assert overlapped + assert self._dump(exported) == expected_warmup + assert self._dump(overlapped[0]) == expected_profiling + class TestMetricsSummary: def test_to_json(self) -> None: From d4b1563e3a1483aa58905551fc06a707a25b311a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 29 Oct 2026 19:07:05 -0500 Subject: [PATCH 110/119] chore: fix pre-commit after the upstream cherry-picks Allow the "unparseable" spelling in codespell, as upstream does, and apply ruff format to the inter-token latency test. --- pyproject.toml | 3 ++- tests/unit/metrics/test_inter_token_latency_metric.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 29e557ee48..6c0b0daab2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -281,4 +281,5 @@ filterwarnings = [ [tool.codespell] skip = "*.pyc,*build*,tests/unit/transports/test_aiohttp_sse.py,tests/integration/assets/canary_reference_inputs.json,src/aiperf/server_metrics/units.py,src/aiperf/api/static/dashboard.html,src/aiperf/config/schema/aiperf-config.schema.json" -ignore-words-list = "timeslice,timeslices,optiona,disjointness,concurency,strat,clen,anull" +# "unparseable" is an accepted English variant, as upstream allows. +ignore-words-list = "timeslice,timeslices,optiona,disjointness,concurency,strat,clen,anull,unparseable" diff --git a/tests/unit/metrics/test_inter_token_latency_metric.py b/tests/unit/metrics/test_inter_token_latency_metric.py index 5be7acec6c..d03354d9ce 100644 --- a/tests/unit/metrics/test_inter_token_latency_metric.py +++ b/tests/unit/metrics/test_inter_token_latency_metric.py @@ -141,6 +141,7 @@ def test_requires_dependencies(self) -> None: FullResponseInterTokenLatencyMetric().parse_record( create_record(), MetricRecordDict() ) + def test_inter_token_latency_subtracts_bundled_first_chunk(self): """When the server reports the first content chunk carried N tokens, ITL divides the decode window by (OSL - N), not (OSL - 1), so a server that From da20f4e6fea06783af80fef096a79036d180c15f Mon Sep 17 00:00:00 2001 From: jxj Date: Fri, 16 Oct 2026 02:54:13 +0800 Subject: [PATCH 111/119] fix(transport): preserve Unicode separators in SSE data (#1535) Signed-off-by: xinjun.jiang (cherry picked from commit 34335eb2fdf2a3663e23b216f0e172b9af5f9e6f) --- src/aiperf/common/models/record_models.py | 5 +- tests/unit/transports/test_sse_utils.py | 63 +++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 742e6212c8..461235974b 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -746,7 +746,10 @@ def parse(cls, raw_message: AnyStr, perf_ns: int) -> SSEMessage: ) message = cls(perf_ns=perf_ns) - for line in raw_message.splitlines(): + # SSE fields end at CR/LF; Unicode line separators belong to the data. + if "\r" in raw_message: + raw_message = raw_message.replace("\r\n", "\n").replace("\r", "\n") + for line in raw_message.split("\n"): if not (line := line.strip()): continue diff --git a/tests/unit/transports/test_sse_utils.py b/tests/unit/transports/test_sse_utils.py index 5ec7863e47..ba462ce90b 100644 --- a/tests/unit/transports/test_sse_utils.py +++ b/tests/unit/transports/test_sse_utils.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import time +from collections.abc import AsyncIterator import orjson import pytest @@ -470,6 +471,51 @@ def test_differently_cased_data_field_is_not_extracted( assert result.get_json() is None +class TestParseSSEMessageUnicode: + """Unicode separators in data are not SSE field delimiters.""" + + @pytest.mark.parametrize( + "separator,line_ending", + [ + param("\u0085", "\n", id="next-line-lf"), + param("\u2028", "\n", id="line-separator-lf"), + param("\u2029", "\n", id="paragraph-separator-lf"), + param("\u2028", "\r\n", id="line-separator-crlf"), + param("\u2028", "\r", id="line-separator-cr"), + ], + ) # fmt: skip + def test_parse_named_event_preserves_unicode_json_content( + self, separator: str, line_ending: str, base_perf_ns: int + ) -> None: + payload = {"delta": {"type": "text_delta", "text": f"before{separator}after"}} + data = orjson.dumps(payload).decode() + raw_message = f"event: content_block_delta{line_ending}data: {data}" + result = SSEMessage.parse(raw_message, base_perf_ns) + + assert result.get_json() == payload + assert result.extract_data_content() == data + assert [packet.name for packet in result.packets] == ["event", "data"] + + def test_parse_named_event_preserves_unicode_bytes(self, base_perf_ns: int) -> None: + payload = {"delta": {"type": "text_delta", "text": "a\u0085b\u2028c\u2029d"}} + data = orjson.dumps(payload) + result = SSEMessage.parse( + b"event: content_block_delta\ndata: " + data, base_perf_ns + ) + + assert result.get_json() == payload + assert [packet.name for packet in result.packets] == ["event", "data"] + + def test_parse_named_event_preserves_unicode_plain_text( + self, base_perf_ns: int + ) -> None: + text = "before\u2028after" + result = SSEMessage.parse(f"event: message\ndata: {text}", base_perf_ns) + + assert result.get_text() == text + assert [packet.name for packet in result.packets] == ["event", "data"] + + class TestParseSSEMessageIncompleteJSON: """Tests for SSE messages where a literal newline in a JSON value splits a data field.""" @@ -721,6 +767,23 @@ async def mock_async_iter(): return _factory +@pytest.mark.asyncio +async def test_sse_reader_preserves_unicode_in_named_event() -> None: + payload = {"type": "response.output_text.delta", "delta": "a\u0085b\u2028c\u2029d"} + body = ( + f"event: response.output_text.delta\ndata: {orjson.dumps(payload).decode()}\n\n" + ).encode() + + async def chunks() -> AsyncIterator[bytes]: + for byte in body: + yield bytes([byte]) + + messages = await AsyncSSEStreamReader(chunks()).read_complete_stream() + + assert len(messages) == 1 + assert messages[0].get_json() == payload + + @pytest.mark.asyncio class TestAsyncSSEStreamReaderParsing: """Tests for SSE stream reader buffer management and delimiter handling.""" From aff2e3a1b182c0f1513ec18fdaf16397a603c69d Mon Sep 17 00:00:00 2001 From: "Yanze (David) Wu" <133224895+David-Wu1119@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:46:01 -0400 Subject: [PATCH 112/119] fix(metrics): end a phase's timeslice grid at that phase's last record (#1494) Signed-off-by: David-Wu1119 <133224895+David-Wu1119@users.noreply.github.com> Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit b359fbaf9c02b19fd6c23dd1ff1fc2b50815eef2) --- src/aiperf/metrics/accumulator.py | 2 +- .../test_accumulator_timeslice_phase_mask.py | 64 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tests/unit/metrics/test_accumulator_timeslice_phase_mask.py diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index 6e30b4614a..fb2871aec5 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -871,7 +871,7 @@ def _compute_timeslices( # bucket every record. Falls back to max(start_ns) if no end_ns is # recorded. max_start_ts = float(np.nanmax(filled_ts)) - filled_end = ~np.isnan(end_ns) + filled_end = filled & ~np.isnan(end_ns) if filled_end.any(): max_ts = max(max_start_ts, float(np.nanmax(end_ns[filled_end]))) else: diff --git a/tests/unit/metrics/test_accumulator_timeslice_phase_mask.py b/tests/unit/metrics/test_accumulator_timeslice_phase_mask.py new file mode 100644 index 0000000000..3dc0f68b04 --- /dev/null +++ b/tests/unit/metrics/test_accumulator_timeslice_phase_mask.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import asyncio + +from aiperf.common.accumulator_protocols import ExportContext +from aiperf.common.enums import CreditPhase +from aiperf.common.messages import MetricRecordsData +from aiperf.common.models import MetricRecordMetadata +from aiperf.metrics.accumulator import MetricsAccumulator +from tests.unit.conftest import make_benchmark_run + +NS_PER_S = 1_000_000_000 +T0 = 1_780_000_000 * NS_PER_S + + +def _record(index: int, second: int, phase: CreditPhase) -> MetricRecordsData: + start, end = T0 + second * NS_PER_S, T0 + (second + 1) * NS_PER_S + return MetricRecordsData( + metadata=MetricRecordMetadata( + session_num=index, + turn_index=0, + request_start_ns=start, + request_end_ns=end, + worker_id="worker", + record_processor_id="rp", + benchmark_phase=phase, + ), + metrics={ + "request_latency": NS_PER_S, + "request_count": 1, + "output_sequence_length": 100, + "input_sequence_length": 50, + "min_request_timestamp": start, + "max_response_timestamp": end, + }, + error=None, + ) + + +def test_last_slice_ends_with_the_exported_phase() -> None: + asyncio.run(_run_last_slice_ends_with_the_exported_phase()) + + +async def _run_last_slice_ends_with_the_exported_phase() -> None: + acc = MetricsAccumulator( + make_benchmark_run(extra={"artifacts": {"slice_duration": 10}}) + ) + for second in range(15): + await acc.process_record(_record(second, second, CreditPhase.PROFILING)) + for second in range(15, 40): + await acc.process_record(_record(second, second, CreditPhase.WARMUP)) + + summary = await acc.export_results( + ExportContext( + start_ns=T0, end_ns=T0 + 15 * NS_PER_S, phase=CreditPhase.PROFILING + ) + ) + + last = summary.timeslices[-1] + assert (last.start_ns, last.end_ns) == (T0 + 10 * NS_PER_S, T0 + 15 * NS_PER_S) + assert last.is_complete is False + assert last.metric_results["request_throughput"].avg == 1.0 From ab25b74abfa45ccc42be9f0e821044dcd047e6a7 Mon Sep 17 00:00:00 2001 From: Henry Richter <101796272+henrichter@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:56:02 +0100 Subject: [PATCH 113/119] fix(synthesis): clamp prefix_len to input_len for partial blocks (#1385) Signed-off-by: Richter Signed-off-by: henrichter Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 4f0deb69f16843fe9238bc2511f97b89f258c95f) --- src/aiperf/dataset/synthesis/synthesizer.py | 16 ++++--- .../dataset/synthesis/test_synthesizer.py | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/aiperf/dataset/synthesis/synthesizer.py b/src/aiperf/dataset/synthesis/synthesizer.py index be2abe76da..322634ec29 100644 --- a/src/aiperf/dataset/synthesis/synthesizer.py +++ b/src/aiperf/dataset/synthesis/synthesizer.py @@ -213,14 +213,18 @@ def _apply_multipliers( else: break - # Compute prefix_len and prompt_len + # Clamp prefix_len to input_len only when the overshoot is smaller + # than one block (a final partial block). Larger mismatches mean + # the trace claims fewer tokens than its full shared-prefix blocks. prefix_len = len(prefix_ids) * block_size + if prefix_len > input_len: + if prefix_len - input_len >= block_size: + raise ValueError( + f"input_len ({input_len}) < prefix_len ({prefix_len}): " + f"trace has fewer tokens than its shared prefix blocks" + ) + prefix_len = input_len prompt_len = input_len - prefix_len - if prompt_len < 0: - raise ValueError( - f"input_len ({input_len}) < prefix_len ({prefix_len}): " - f"trace has fewer tokens than its shared prefix blocks" - ) # Step 3: Apply prefix_len_mult - stretch or squeeze prefix if prefix_mult > 1.0: diff --git a/tests/unit/dataset/synthesis/test_synthesizer.py b/tests/unit/dataset/synthesis/test_synthesizer.py index 57d556d3b0..146fe373c1 100644 --- a/tests/unit/dataset/synthesis/test_synthesizer.py +++ b/tests/unit/dataset/synthesis/test_synthesizer.py @@ -165,6 +165,50 @@ def test_speedup_ratio_variations( assert synthetic[0].get("timestamp") == expected_ts + def test_speedup_ratio_with_partial_final_block(self) -> None: + """Traces whose shared prefix ends in a partial block should not crash.""" + # block_size=16; 756 tokens = 47 full blocks + a 4-token remainder. + # All hash_ids are shared, so the prefix consumes the partial final block. + hash_ids = list(range(1, 49)) + traces = [ + { + "input_length": 756, + "output_length": 10, + "timestamp": 1000, + "hash_ids": hash_ids, + }, + { + "input_length": 756, + "output_length": 10, + "timestamp": 2000, + "hash_ids": hash_ids, + }, + ] + params = SynthesisParams(speedup_ratio=2.0, block_size=16) + synthesizer = Synthesizer(params=params) + synthetic = synthesizer.synthesize_traces(traces) + + assert synthetic[0].get("timestamp") == 500 + assert synthetic[1].get("timestamp") == 1000 + assert synthetic[0]["input_length"] == 756 + assert synthetic[1]["input_length"] == 756 + assert len(synthetic[0]["hash_ids"]) == 48 + assert len(synthetic[1]["hash_ids"]) == 48 + + def test_shared_prefix_overshoot_rejected(self) -> None: + """Reject traces whose shared prefix overshoots input_length by a full block.""" + # 3 shared blocks * 16 = 48 tokens, but input_length claims only 32. + # The overshoot (16) is exactly one block_size, so this pins >= vs >. + traces = [ + {"input_length": 32, "output_length": 10, "hash_ids": [1, 2, 3]}, + {"input_length": 32, "output_length": 10, "hash_ids": [1, 2, 3]}, + ] + params = SynthesisParams(speedup_ratio=2.0, block_size=16) + synthesizer = Synthesizer(params=params) + + with pytest.raises(ValueError, match="trace has fewer tokens than"): + synthesizer.synthesize_traces(traces) + # ============================================================================ # Prefix Multiplier Tests # ============================================================================ From f98995f15ade5ea6ae55995a9347c792cadd5aed Mon Sep 17 00:00:00 2001 From: Manu Nicholas Jacob <118115267+manunicholasjacob@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:15:10 -0500 Subject: [PATCH 114/119] fix(telemetry): read the DCGM hostname label in either case (#1438) Signed-off-by: manunicholasjacob Co-authored-by: Frank <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit d4b286191f3ef11eb37f4855522a0d2cb78bfac6) --- src/aiperf/gpu_telemetry/dcgm_collector.py | 4 +- .../test_dcgm_collector_labels.py | 80 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tests/unit/gpu_telemetry/test_dcgm_collector_labels.py diff --git a/src/aiperf/gpu_telemetry/dcgm_collector.py b/src/aiperf/gpu_telemetry/dcgm_collector.py index d6f9fe394d..b187255956 100644 --- a/src/aiperf/gpu_telemetry/dcgm_collector.py +++ b/src/aiperf/gpu_telemetry/dcgm_collector.py @@ -154,7 +154,9 @@ def _parse_metrics_to_records(self, metrics_data: str) -> list[TelemetryRecord]: gpu_uuid=labels.get("UUID"), pci_bus_id=labels.get("pci_bus_id"), device=labels.get("device"), - hostname=labels.get("Hostname"), + # dcgm-exporter#655 lowercased this label; the + # legacy form is kept for older exporters. + hostname=labels.get("hostname") or labels.get("Hostname"), namespace=labels.get("namespace"), pod_name=labels.get("pod"), platform=NVIDIA_GPU_TELEMETRY_PLATFORM, diff --git a/tests/unit/gpu_telemetry/test_dcgm_collector_labels.py b/tests/unit/gpu_telemetry/test_dcgm_collector_labels.py new file mode 100644 index 0000000000..c45ddeaf1e --- /dev/null +++ b/tests/unit/gpu_telemetry/test_dcgm_collector_labels.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Label handling in the DCGM collector's Prometheus parser. + +dcgm-exporter#655 lowercased ``Hostname`` to ``hostname``. Both forms are in +the wild, so the parser has to read either one. +""" + +from __future__ import annotations + +import pytest + +from aiperf.gpu_telemetry.dcgm_collector import DCGMTelemetryCollector + +METRIC = ( + "# HELP DCGM_FI_DEV_GPU_TEMP GPU temperature\n" + "# TYPE DCGM_FI_DEV_GPU_TEMP gauge\n" + 'DCGM_FI_DEV_GPU_TEMP{{gpu="0",UUID="GPU-0",device="nvidia0",' + 'modelName="GPU",{label}="node-1"}} 42\n' +) + + +@pytest.fixture +def collector() -> DCGMTelemetryCollector: + return DCGMTelemetryCollector("http://localhost:9400/metrics") + + +def test_lowercase_hostname_label_is_read(collector): + records = collector._parse_metrics_to_records(METRIC.format(label="hostname")) + + assert records + assert records[0].hostname == "node-1" + + +def test_legacy_uppercase_only_still_read(collector): + records = collector._parse_metrics_to_records(METRIC.format(label="Hostname")) + + assert records + assert records[0].hostname == "node-1" + + +def test_lowercase_wins_when_both_labels_are_present(collector): + metric = ( + "# HELP DCGM_FI_DEV_GPU_TEMP GPU temperature\n" + "# TYPE DCGM_FI_DEV_GPU_TEMP gauge\n" + 'DCGM_FI_DEV_GPU_TEMP{gpu="0",UUID="GPU-0",device="nvidia0",' + 'modelName="GPU",hostname="new",Hostname="legacy"} 42\n' + ) + records = collector._parse_metrics_to_records(metric) + + assert records + assert records[0].hostname == "new" + + +def test_an_empty_lowercase_label_falls_back_to_the_legacy_one(collector): + """An empty label names no host, so a populated legacy label is the better + value rather than an empty string.""" + metric = ( + "# HELP DCGM_FI_DEV_GPU_TEMP GPU temperature\n" + "# TYPE DCGM_FI_DEV_GPU_TEMP gauge\n" + 'DCGM_FI_DEV_GPU_TEMP{gpu="0",UUID="GPU-0",device="nvidia0",' + 'modelName="GPU",hostname="",Hostname="legacy"} 42\n' + ) + records = collector._parse_metrics_to_records(metric) + + assert records + assert records[0].hostname == "legacy" + + +def test_hostname_absent_stays_none(collector): + metric = ( + "# HELP DCGM_FI_DEV_GPU_TEMP GPU temperature\n" + "# TYPE DCGM_FI_DEV_GPU_TEMP gauge\n" + 'DCGM_FI_DEV_GPU_TEMP{gpu="0",UUID="GPU-0",device="nvidia0",' + 'modelName="GPU"} 42\n' + ) + records = collector._parse_metrics_to_records(metric) + + assert records + assert records[0].hostname is None From 93d9e9be034b7ed18331ad99465a0c521157886b Mon Sep 17 00:00:00 2001 From: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:28:21 -0700 Subject: [PATCH 115/119] fix(endpoints): replay only the answer from streamed reasoning replies (#1628) (cherry picked from commit fa07384e19f330ebe18e6e91e573af57620be092) --- docs/benchmark-modes/dag.md | 2 +- src/aiperf/cli_commands/chat.py | 4 +- src/aiperf/endpoints/base_endpoint.py | 39 ++++++------ .../test_base_endpoint_assistant_turn.py | 35 +++++++++++ .../test_openai_chat_tool_call_reassembly.py | 62 +++++++++++++++++++ 5 files changed, 119 insertions(+), 23 deletions(-) diff --git a/docs/benchmark-modes/dag.md b/docs/benchmark-modes/dag.md index d0c5dd0934..f31e12c6e6 100644 --- a/docs/benchmark-modes/dag.md +++ b/docs/benchmark-modes/dag.md @@ -154,7 +154,7 @@ By default a bare-string `forks: ["c"]` entry is **terminal**: the parent has no The seed history a FORK child inherits is the parent's `messages` plus the captured assistant reply, with two intentional simplifications: -- **`reasoning` may be dropped from the captured assistant turn on chat/completions.** The base chat endpoint's `build_assistant_turn` keeps `content` (and tool/function calls when present) but discards `reasoning_content`/`reasoning` because most chat templates do not round-trip reasoning back to the model on a follow-up. Only the `content` field of a `ReasoningResponseData` survives into the child's seed; if the parent emitted reasoning *only* (empty `content`), the reasoning text is used as a fallback so the child still sees something. **Anthropic Messages** is different: it reassembles `thinking` (+ signature) → `text` → `tool_use` for FORK replay — see [Anthropic Messages endpoint](../tutorials/anthropic-messages-endpoint.md). For workloads where chain-of-thought continuity across turns matters on chat endpoints, prefer SPAWN mode — its children start fresh with the same `system` prompt rather than inheriting a stripped CoT. +- **`reasoning` may be dropped from the captured assistant turn on chat/completions.** The base chat endpoint's `build_assistant_turn` keeps `content` (and tool/function calls when present) but discards `reasoning_content`/`reasoning` because most chat templates do not round-trip reasoning back to the model on a follow-up. Only the `content` field of a `ReasoningResponseData` survives into the child's seed; if the parent's whole reply carried reasoning *only* (no `content` in any chunk, e.g. a turn truncated mid-thinking), the reasoning text is used as a fallback so the child still sees something. The rule is applied to the whole reply, so `--streaming` and non-streaming runs replay the same assistant turn. **Anthropic Messages** is different: it reassembles `thinking` (+ signature) → `text` → `tool_use` for FORK replay — see [Anthropic Messages endpoint](../tutorials/anthropic-messages-endpoint.md). For workloads where chain-of-thought continuity across turns matters on chat endpoints, prefer SPAWN mode — its children start fresh with the same `system` prompt rather than inheriting a stripped CoT. - **Responses-API output items that are server-built tool outputs are filtered.** When the parent runs against `endpoint=responses` and the model emitted `web_search_call`, `file_search_call`, `image_generation_call`, `code_interpreter_call`, `computer_call`, or `reasoning` items, those are stripped from the seed before splicing into the child's `input` array — the Responses API rejects them as input unless paired with the corresponding tool config, which the child does not redeclare. `message` and `function_call` items round-trip cleanly and remain. ### FORK mode with `background: true` (fork-and-continue) diff --git a/src/aiperf/cli_commands/chat.py b/src/aiperf/cli_commands/chat.py index a1bb61871f..81af35ab24 100644 --- a/src/aiperf/cli_commands/chat.py +++ b/src/aiperf/cli_commands/chat.py @@ -189,9 +189,7 @@ async def _run_turn( print(format_stats(compute_record_metrics(record), reasoning_tokens)) # Fall back to the reasoning text only when the reply had no visible content, # so a reasoning-only turn isn't recorded as an empty assistant message. - # Intentionally content-only when content exists: unlike profile's per-chunk - # build_assistant_turn (which concatenates reasoning + content), we don't - # resend reasoning, matching how chat APIs replay assistant turns. + # Same answer-else-reasoning rule as BaseEndpoint.build_assistant_turn. return "".join(output_parts) or "".join(reasoning_parts) diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index 9dfb2a9b81..bd4331712a 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -166,16 +166,16 @@ def build_assistant_turn(self, record: RequestRecord) -> Turn | None: that inherit the parent's history - see what the model actually said. The default implementation captures plain text plus the ``content`` - field of any ``ReasoningResponseData``. When a response carries only - ``reasoning`` (some servers and the mock return everything in that - field with empty ``content`` - e.g. Qwen3-style outputs), the - reasoning text is used as a fallback so FORK-mode DAG children - still inherit a non-empty parent context. Endpoints whose responses - carry structured fields beyond plain text (e.g. chat ``tool_calls`` - / ``function_call``) should override this to preserve those fields - verbatim by returning a Turn whose ``raw_messages`` re-renders as - the same assistant message - that way ``build_messages`` extends - them back into the wire body unchanged. + field of any ``ReasoningResponseData``, dropping ``reasoning`` the + way chat templates drop prior-turn thinking. When the whole reply + carries only ``reasoning`` (a turn truncated mid-thinking, or the + mock server's Qwen3-style output), the reasoning text is used as a + fallback so FORK-mode DAG children still inherit a non-empty parent + context. Endpoints whose responses carry structured fields beyond + plain text (e.g. chat ``tool_calls`` / ``function_call``) should + override this to preserve those fields verbatim by returning a Turn + whose ``raw_messages`` re-renders as the same assistant message - that + way ``build_messages`` extends them back into the wire body unchanged. Returns ``None`` when the record has no replayable assistant content (error response, empty body, etc.). @@ -189,21 +189,22 @@ def _build_assistant_turn_from_parsed( parsed_responses: list[ParsedResponse], ) -> Turn | None: """Build the default text-only assistant turn from parsed responses.""" - output_texts: list[str] = [] + answer_texts: list[str] = [] + reasoning_texts: list[str] = [] for response in parsed_responses: if not response.data: continue if isinstance(response.data, ReasoningResponseData): if response.data.content: - output_texts.append(response.data.content) - elif response.data.reasoning: - # Reasoning-only fallback: without this, FORK children - # of a Qwen3-style/mock-server response would inherit - # a parent context with no captured assistant turn. - output_texts.append(response.data.reasoning) + answer_texts.append(response.data.content) + if response.data.reasoning: + reasoning_texts.append(response.data.reasoning) else: - output_texts.append(response.data.get_text()) - resp_text = "".join(output_texts) + answer_texts.append(response.data.get_text()) + # Decided once per reply, not per parsed response: a streamed reply + # splits reasoning and answer across chunks, and replaying both glues + # the thinking into the answer. + resp_text = "".join(answer_texts) or "".join(reasoning_texts) if not resp_text: return None return Turn(role="assistant", texts=[Text(contents=[resp_text])]) diff --git a/tests/unit/endpoints/test_base_endpoint_assistant_turn.py b/tests/unit/endpoints/test_base_endpoint_assistant_turn.py index 1cacaaf891..e437183141 100644 --- a/tests/unit/endpoints/test_base_endpoint_assistant_turn.py +++ b/tests/unit/endpoints/test_base_endpoint_assistant_turn.py @@ -144,6 +144,41 @@ def test_mixed_reasoning_and_text_combines_content(self): assert turn is not None assert turn.texts[0].contents == ["HelloWorld"] + def test_streaming_reasoning_chunks_replay_answer_only(self): + # Streaming splits a reasoning reply across chunks that each carry + # only ``reasoning`` or only ``content``; the answer-else-reasoning + # rule must be decided once per reply, not per chunk. + canned = { + 10: ParsedResponse(perf_ns=10, data=ReasoningResponseData(reasoning="a")), + 20: ParsedResponse(perf_ns=20, data=ReasoningResponseData(reasoning="b")), + 30: ParsedResponse(perf_ns=30, data=ReasoningResponseData(content="X")), + 40: ParsedResponse(perf_ns=40, data=TextResponseData(text="Y")), + } + ep = _StubEndpoint(canned) + record = RequestRecord( + responses=[_text_response(ns) for ns in canned], + start_perf_ns=0, + end_perf_ns=40, + ) + turn = ep.build_assistant_turn(record) + assert turn is not None + assert turn.texts[0].contents == ["XY"] + + def test_streaming_reasoning_only_chunks_fall_back_to_joined_reasoning(self): + canned = { + 10: ParsedResponse(perf_ns=10, data=ReasoningResponseData(reasoning="a")), + 20: ParsedResponse(perf_ns=20, data=ReasoningResponseData(reasoning="b")), + } + ep = _StubEndpoint(canned) + record = RequestRecord( + responses=[_text_response(ns) for ns in canned], + start_perf_ns=0, + end_perf_ns=20, + ) + turn = ep.build_assistant_turn(record) + assert turn is not None + assert turn.texts[0].contents == ["ab"] + def test_empty_record_returns_none(self): ep = _StubEndpoint() record = RequestRecord( diff --git a/tests/unit/endpoints/test_openai_chat_tool_call_reassembly.py b/tests/unit/endpoints/test_openai_chat_tool_call_reassembly.py index 246022a59b..ed9822f71c 100644 --- a/tests/unit/endpoints/test_openai_chat_tool_call_reassembly.py +++ b/tests/unit/endpoints/test_openai_chat_tool_call_reassembly.py @@ -181,3 +181,65 @@ def test_streaming_tool_calls_index_keyed_concat(self, chat_endpoint): assert tc["id"] == "call_1" assert tc["function"]["name"] == "get_weather" assert tc["function"]["arguments"] == '{"city":"SF"}' + + +def _reasoning_reply_chunks() -> list[dict]: + """Streaming SSE chunks for one reasoning-model reply (vLLM/SGLang shape).""" + deltas = [ + {"role": "assistant", "content": ""}, + {"reasoning_content": "I should walk "}, + {"reasoning_content": "the list."}, + {"content": "def rev(head): "}, + {"content": "..."}, + ] + return [ + {"object": "chat.completion.chunk", "choices": [{"delta": d}]} for d in deltas + ] + + +def _record_from(json_objs: list[dict]) -> RequestRecord: + return RequestRecord( + responses=[ + TextResponse(perf_ns=i, text=orjson.dumps(o).decode()) + for i, o in enumerate(json_objs) + ] + ) + + +class TestBuildAssistantTurnReasoningReplay: + def test_streaming_reasoning_reply_replays_answer_only(self, chat_endpoint): + turn = chat_endpoint.build_assistant_turn( + _record_from(_reasoning_reply_chunks()) + ) + assert turn is not None + assert turn.texts[0].contents == ["def rev(head): ..."] + + def test_streaming_reasoning_without_answer_replays_reasoning(self, chat_endpoint): + chunks = [ + c + for c in _reasoning_reply_chunks() + if "content" not in c["choices"][0]["delta"] + ] + turn = chat_endpoint.build_assistant_turn(_record_from(chunks)) + assert turn is not None + assert turn.texts[0].contents == ["I should walk the list."] + + def test_streaming_and_non_streaming_replay_identical_turn(self, chat_endpoint): + non_streaming = { + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "reasoning_content": "I should walk the list.", + "content": "def rev(head): ...", + } + } + ], + } + streamed = chat_endpoint.build_assistant_turn( + _record_from(_reasoning_reply_chunks()) + ) + whole = chat_endpoint.build_assistant_turn(_record_from([non_streaming])) + assert streamed is not None and whole is not None + assert streamed.model_dump() == whole.model_dump() From 4f9a7942c1f606e674fd2e6596de67e5aea05203 Mon Sep 17 00:00:00 2001 From: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:03:20 -0700 Subject: [PATCH 116/119] fix: count completed requests from phase records (#1448) Signed-off-by: Jayesh Suyal <93232720+jayeshsuyal@users.noreply.github.com> (cherry picked from commit afab9d1ca1078c0c521bd72be2e34da5f99c093e) --- docs/architecture.md | 5 ++ src/aiperf/records/records_manager.py | 6 +- tests/unit/records/test_records_manager.py | 67 ++++++++++++++++------ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index e7af8ef5e0..4b7a22bf70 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -114,6 +114,11 @@ The Records Manager handles the collection, organization, and storage of benchma - Supporting the generation of reports and artifacts for performance evaluation - Managing the final export of aggregated performance summaries and per-request details +`ProfileResults.completed` counts requests processed for the result's phase +(successful plus failed), independently of how many metric summaries accumulators +or analyzers produce. Profiling counts exclude warmup requests, and cancelled +summaries count the requests processed so far. + #### Record-Type Channels The Records Manager does not hard-wire one handler per producer. Instead, every typed record carries a `record_type` class attribute and is fanned out through a metadata-driven routing table: each accumulator and stream exporter declares the record types it consumes via `record_types` in `plugins.yaml`, and `_dispatch_record` routes each record by `getattr(record, "record_type")` to all registered handlers. This makes each record type a dedicated channel. diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 2d948e8e6e..03fb7af0e1 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -2280,10 +2280,6 @@ async def _process_results_impl( metric_duration_coverage, fatal_errors = ( self._validate_profile_metric_duration_coverage(phase, cancelled) ) - # Snapshot count BEFORE extending with derived aggregates (efficiency, - # analyzers) — `completed` reports request-derived records only. - records_completed = len(records_results) - # Cross-accumulator analyzer plugins (e.g. energy efficiency) run after # all accumulators have summarized, reading peers via the SummaryContext. records_results.extend(await self._run_analyzers(summary_ctx)) @@ -2293,7 +2289,7 @@ async def _process_results_impl( records=records_results, warmup_records=warmup_records_results, timeslices=timeslices or None, - completed=records_completed, + completed=phase_stats.total_records, start_ns=phase_stats.start_ns or time.time_ns(), end_ns=phase_stats.requests_end_ns or time.time_ns(), error_summary=self._error_tracker.get_error_summary_for_phase(phase), diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 5c7f8e3f2e..a045264932 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from pytest import param from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.enums import CreditPhase, ProfileCancelReason @@ -1418,11 +1419,20 @@ async def test_on_records_defers_finalization_for_multi_profiling_until_credits_ class TestRecordsManagerAnalyzerMetrics: - """Pin the invariant that `completed` counts request-derived records only, - and that analyzer-injected metrics are merged after the snapshot.""" - + @pytest.mark.parametrize( + ("success_count", "error_count", "cancelled"), + [ + param(3, 0, False, id="successful"), + param(3, 1, False, id="mixed"), + param(0, 3, False, id="errors-only"), + param(1, 2, True, id="cancelled-partial"), + param(0, 0, True, id="cancelled-empty"), + ], + ) # fmt: skip @pytest.mark.asyncio - async def test_completed_excludes_analyzer_metrics(self) -> None: + async def test_completed_counts_phase_requests_not_metric_summaries( + self, success_count: int, error_count: int, cancelled: bool + ) -> None: manager = RecordsManager.__new__(RecordsManager) manager.debug = MagicMock() @@ -1441,15 +1451,16 @@ async def test_completed_excludes_analyzer_metrics(self) -> None: manager.run.cfg.gpu_telemetry_disabled = True manager.run.cfg.server_metrics_disabled = True manager.run.cfg.network_latency.enabled = False + manager.run.cfg.scenario = None - request_records = [ + summary_metrics = [ MetricResult(tag="request_latency", header="h", unit="ms", avg=1.0), MetricResult(tag="output_token_count", header="h", unit="tokens", avg=2.0), ] metric_accumulator = MagicMock() metric_accumulator.summarize = AsyncMock( return_value=AccumulatorMetricsSummary( - results={r.tag: r for r in request_records}, + results={r.tag: r for r in summary_metrics}, ) ) manager._accumulators = {AccumulatorType.METRIC_RESULTS: metric_accumulator} @@ -1458,8 +1469,6 @@ async def test_completed_excludes_analyzer_metrics(self) -> None: manager._gpu_telemetry_accumulator = None manager._server_metrics_accumulator = None - # An analyzer contributes derived aggregates that must NOT inflate - # `completed` (which counts request-derived records only). analyzer_metrics = [ MetricResult(tag="total_gpu_power", header="h", unit="W", avg=200.0), MetricResult(tag="total_gpu_energy", header="h", unit="J", avg=1000.0), @@ -1478,23 +1487,47 @@ async def test_completed_excludes_analyzer_metrics(self) -> None: ] manager._run_analyzers = RecordsManager._run_analyzers.__get__(manager) - manager._records_tracker = MagicMock() - manager._records_tracker.create_stats_for_phase.return_value = MagicMock( - start_ns=1_000_000_000, - requests_end_ns=2_000_000_000, - success_records=2, - error_records=0, + manager._records_tracker = RecordsTracker() + manager._records_tracker.update_phase_info( + CreditPhaseStats( + phase=CreditPhase.PROFILING, + start_ns=1_000_000_000, + requests_end_ns=2_000_000_000, + ) ) + for phase, successes, errors in ( + (CreditPhase.WARMUP, 5, 1), + (CreditPhase.PROFILING, success_count, error_count), + ): + for index in range(successes + errors): + manager._records_tracker.update_from_request( + MetricRecordMetadata( + session_num=index, + request_start_ns=1_000_000_000, + request_end_ns=2_000_000_000, + worker_id="worker", + record_processor_id="processor", + benchmark_phase=phase, + ), + None + if index < successes + else ErrorDetails(code=500, type="ServerError", message="failed"), + ) manager._error_tracker = MagicMock() manager._error_tracker.get_error_summary_for_phase.return_value = [] manager._process_results_lock = asyncio.Lock() manager._processed_results = {} - result = await manager._process_results(CreditPhase.PROFILING, cancelled=False) + result = await manager._process_results( + CreditPhase.PROFILING, cancelled=cancelled + ) - assert result.results.completed == len(request_records) - assert len(result.results.records) == len(request_records) + len( + assert result.results.completed == success_count + error_count + assert result.results.successful_request_count == success_count + assert result.results.error_request_count == error_count + assert result.results.was_cancelled is cancelled + assert len(result.results.records) == len(summary_metrics) + len( analyzer_metrics ) assert {r.tag for r in result.results.records} == { From 3077492f70c30f2ce64cc8112ba6a2c9330515bf Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:12:18 -0700 Subject: [PATCH 117/119] fix(records): preserve binary responses across JSON serialization (#1546) Signed-off-by: Elias Bermudez (cherry picked from commit e2c149ee29d3b64798d8ba2176e9f42e0aa89280) --- .../tutorials/working-with-profile-exports.md | 50 +++++++ src/aiperf/common/models/record_models.py | 14 +- .../test_binary_response_accounting.py | 125 ++++++++++++++++++ tests/unit/common/messages/test_messages.py | 76 ++++++++++- .../unit/common/models/test_record_models.py | 51 ++++++- .../test_raw_record_writer_processor.py | 52 +++++++- 6 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_binary_response_accounting.py diff --git a/docs/tutorials/working-with-profile-exports.md b/docs/tutorials/working-with-profile-exports.md index 91ea8aee11..2425151bea 100644 --- a/docs/tutorials/working-with-profile-exports.md +++ b/docs/tutorials/working-with-profile-exports.md @@ -14,6 +14,7 @@ AIPerf generates multiple output formats after each benchmark run, each optimize - [**`inputs.json`**](#input-dataset-json) - Complete input dataset with formatted payloads for each request - [**`profile_export.jsonl`**](#per-request-records-jsonl) - Per-request metric records in JSON Lines format with one record per line +- [**`profile_export_raw.jsonl`**](#raw-request-and-response-records) - Full retained request/response records, written with `--export-level raw` - [**`profile_export_aiperf.json`**](#aggregated-statistics-json) - Aggregated statistics and user configuration as a single JSON object - [**`profile_export_aiperf.csv`**](#aggregated-statistics-csv) - Aggregated statistics in CSV format - **`outputs.json`** - Generated response text per request, with a small set of per-request metrics. Written when `--export-outputs-json` is set, which `--export-level raw` implies. See [`outputs.json` schema](../reference/json-export-schema.md#outputsjson-schema). @@ -216,6 +217,55 @@ async def process_streaming_records_async(file_path: Path) -> None: # ... Process the streaming records here ... ``` +### Raw Request and Response Records + +Use `--export-level raw` to write `profile_export_raw.jsonl`. Binary responses store +`raw_bytes` as a padded URL-safe base64 string in JSON. For example, a response +containing the bytes `b"\xff\xfe"` appears as: + +```json +{"perf_ns":123,"raw_bytes":"__4=","content_type":"image/png"} +``` + +All binary bodies use this encoding, including bodies that happen to be valid +UTF-8. Empty bytes serialize as `""`. The in-memory `BinaryResponse.raw_bytes` +and `get_raw()` values remain bytes. Updated `RawRecordInfo` readers decode the +base64 string automatically: + +```python +from pathlib import Path +from aiperf.common.models import BinaryResponse +from aiperf.common.models.record_models import RawRecordInfo + +with Path("artifacts/my-run/profile_export_raw.jsonl").open(encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + record = RawRecordInfo.model_validate_json(line) + for response in record.responses: + if isinstance(response, BinaryResponse): + body = response.raw_bytes +``` + +If you read JSON dictionaries directly, decode binary response fields explicitly: + +```python +import base64 +import orjson + +record = orjson.loads(line) +for response in record["responses"]: + if "raw_bytes" in response: + body = base64.urlsafe_b64decode(response["raw_bytes"]) +``` + +Legacy binary-response exports produced before this encoding change used UTF-8 +text for serializable `raw_bytes` values. Choose a legacy migration using the +producer's version; do not infer the encoding from the string. For example, +`"abcd"` can be either legacy text or base64 for different bytes. Updated model +readers interpret strings as base64 and cannot automatically recover untagged +legacy text. Use matching application versions for workers and record processors. + ### Working with Input Datasets Load and analyze the `inputs.json` file to understand what data was sent during the benchmark: diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 461235974b..f9c11ab0a6 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -22,6 +22,7 @@ field_validator, model_serializer, ) +from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.functional_validators import AfterValidator from aiperf.common.aiperf_logger import AIPerfLogger @@ -664,19 +665,20 @@ def get_json(self) -> JsonObject | None: return None -@dataclass(slots=True) +# Nested stdlib dataclasses ignore byte encoding settings; use Pydantic here. +# Reject extra fields so response unions cannot select the wrong dataclass. +@pydantic_dataclass( + slots=True, + config=ConfigDict(extra="forbid", ser_json_bytes="base64", val_json_bytes="base64"), +) class BinaryResponse: """Raw binary response from an inference client for non-text content types.""" - # Reject extra fields so Pydantic's union discrimination (e.g. in - # RequestRecord.responses) doesn't match the wrong dataclass type. - __pydantic_config__ = ConfigDict(extra="forbid") - perf_ns: int """The performance timestamp of the response in nanoseconds (perf_counter_ns).""" raw_bytes: bytes - """The raw binary body of the response.""" + """The original response bytes; JSON encodes them as URL-safe base64.""" content_type: str | None = None """The content type of the response. e.g. 'video/mp4', 'application/octet-stream'.""" diff --git a/tests/integration/test_binary_response_accounting.py b/tests/integration/test_binary_response_accounting.py new file mode 100644 index 0000000000..03891f8db7 --- /dev/null +++ b/tests/integration/test_binary_response_accounting.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Binary bodies must cross the results bus and reach request accounting.""" + +from collections.abc import AsyncIterator + +import orjson +import pytest +from aiohttp import web +from pytest import param + +from aiperf.common.models import BinaryResponse +from tests.harness.utils import AIPerfCLI +from tests.integration.conftest import IntegrationTestDefaults as defaults + +BINARY_BODY = bytes(range(256)) +REQUEST_COUNT = 6 + + +@pytest.fixture +async def binary_response_server() -> AsyncIterator[tuple[str, list[str]]]: + """Serve one valid chat response followed by arbitrary binary bodies.""" + request_ids: list[str] = [] + + async def handle(request: web.Request) -> web.Response: + """Track request identities and select the control or binary response.""" + await request.read() + request_ids.append(request.headers["X-Request-ID"]) + # All-error runs skip aggregate export; retain a successful control response. + if len(request_ids) == 1: + return web.Response( + body=orjson.dumps( + { + "object": "chat.completion", + "choices": [ + {"message": {"role": "assistant", "content": "ok"}} + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 1, + "total_tokens": 9, + }, + } + ), + content_type="application/json", + ) + return web.Response(body=BINARY_BODY, content_type="application/octet-stream") + + app = web.Application() + app.router.add_post("/v1/chat/completions", handle) + runner = web.AppRunner(app) + await runner.setup() + try: + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + yield f"http://127.0.0.1:{runner.addresses[0][1]}", request_ids + finally: + await runner.cleanup() + + +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "export_level", [param("records", id="records"), param("raw", id="raw")] +) # fmt: skip +async def test_binary_http_response_counted_and_run_completes( + cli: AIPerfCLI, + binary_response_server: tuple[str, list[str]], + export_level: str, +) -> None: + """Binary results cross the bus, reach accounting, and survive RAW export.""" + url, request_ids = binary_response_server + result = await cli.run( + f""" + aiperf profile + --model {defaults.model} + --url {url} + --endpoint-type chat + --request-count {REQUEST_COUNT} + --concurrency 2 + --workers-max 1 + --isl 8 --isl-stddev 0 --osl 1 + --export-level {export_level} + --ui none + """, + timeout=60, + ) + + assert result.exit_code == 0 + assert len(request_ids) == REQUEST_COUNT + assert len(set(request_ids)) == REQUEST_COUNT + assert result.json is not None + successes = result.json.request_count.avg if result.json.request_count else 0 + errors = ( + result.json.error_request_count.avg if result.json.error_request_count else 0 + ) + assert successes + errors == REQUEST_COUNT + assert successes == 1 + # The chat parser cannot extract content from this legitimate binary HTTP body. + assert errors == REQUEST_COUNT - 1 + assert result.jsonl is not None + assert len(result.jsonl) == REQUEST_COUNT + assert {record.metadata.x_request_id for record in result.jsonl} == set(request_ids) + for record in result.jsonl: + if record.metadata.x_request_id == request_ids[0]: + assert record.error is None + else: + assert record.error is not None + assert record.error.type == "InvalidInferenceResultError" + + if export_level == "raw": + assert result.raw_records is not None + assert len(result.raw_records) == REQUEST_COUNT + assert {record.metadata.x_request_id for record in result.raw_records} == set( + request_ids + ) + for record in result.raw_records: + assert record.status == 200 + assert len(record.responses) == 1 + if record.metadata.x_request_id == request_ids[0]: + continue + response = record.responses[0] + assert isinstance(response, BinaryResponse) + assert response.raw_bytes == BINARY_BODY + assert response.content_type == "application/octet-stream" diff --git a/tests/unit/common/messages/test_messages.py b/tests/unit/common/messages/test_messages.py index 2e1194a4d6..c0bfa92b43 100644 --- a/tests/unit/common/messages/test_messages.py +++ b/tests/unit/common/messages/test_messages.py @@ -1,19 +1,29 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import base64 import json import time import orjson import pytest +from pytest import param from aiperf.common.enums import LifecycleState, MessageType from aiperf.common.messages import ( ErrorMessage, HeartbeatMessage, + InferenceResultsMessage, + Message, ShutdownCommand, StatusMessage, ) -from aiperf.common.models import ErrorDetails +from aiperf.common.models import ( + BinaryResponse, + ErrorDetails, + RequestRecord, + SSEMessage, + TextResponse, +) from aiperf.plugin.enums import ServiceType @@ -56,6 +66,70 @@ def test_status_message(): ) +class TestBinaryInferenceResultsMessage: + """Binary response bodies survive the worker results-message boundary.""" + + @pytest.mark.parametrize( + "body", + [ + param(bytes(range(256)), id="all-bytes"), + param(b"a\x00b", id="embedded-nul"), + param(b"", id="empty"), + param(b"hello", id="ascii"), + param("café".encode(), id="utf8"), + ], + ) # fmt: skip + def test_binary_response_message_roundtrip_preserves_bytes( + self, body: bytes + ) -> None: + """Each message-reader route restores the exact base64-encoded body.""" + original = InferenceResultsMessage( + service_id="worker-1", + record=RequestRecord( + responses=[BinaryResponse(123, body, "image/png")], + ), + ) + + wire = original.to_json_bytes() + dumped = orjson.loads(wire) + assert dumped["record"]["responses"][0]["raw_bytes"] == ( + base64.urlsafe_b64encode(body).decode("ascii") + ) + for restored in ( + Message.from_json(wire), + Message.from_json(dumped), + InferenceResultsMessage.model_validate_json(wire), + ): + assert isinstance(restored, InferenceResultsMessage) + response = restored.record.responses[0] + assert isinstance(response, BinaryResponse) + assert response.raw_bytes == body + assert response.perf_ns == 123 + assert response.content_type == "image/png" + assert original.to_json_bytes() == wire + assert original.record.responses[0].raw_bytes == body + + def test_mixed_response_message_roundtrip_preserves_types_and_contents( + self, + ) -> None: + """Binary encoding preserves sibling response types and their contents.""" + responses = [ + BinaryResponse(1, b"\xff\xfe", "image/png"), + TextResponse(2, "café", "text/plain"), + SSEMessage.parse('data: {"delta":"token"}\n\n', perf_ns=3), + ] + original = InferenceResultsMessage( + service_id="worker-1", record=RequestRecord(responses=responses) + ) + + restored = Message.from_json(original.to_json_bytes()) + + assert [type(response) for response in restored.record.responses] == [ + type(response) for response in responses + ] + assert restored.record.responses == responses + + class TestBaseStatusMessageTimestamp: """Tests for BaseStatusMessage default_factory timestamp behavior.""" diff --git a/tests/unit/common/models/test_record_models.py b/tests/unit/common/models/test_record_models.py index 448a567506..c685a08fea 100644 --- a/tests/unit/common/models/test_record_models.py +++ b/tests/unit/common/models/test_record_models.py @@ -1,11 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import orjson import pytest -from pydantic import BaseModel, Field, SerializeAsAny +from pydantic import BaseModel, Field, SerializeAsAny, ValidationError +from pytest import param from aiperf.common.messages import InferenceResultsMessage from aiperf.common.models import ( + BinaryResponse, MetricResult, ProfileResults, RequestRecord, @@ -16,6 +19,52 @@ from aiperf.common.models.export_models import JsonMetricResult +class TestBinaryResponse: + """Binary response construction and record validation preserve the contract.""" + + def test_python_construction_and_dump_preserve_raw_bytes(self) -> None: + """Python construction and dumps retain bytes and the slots layout.""" + body = bytes(range(256)) + positional = BinaryResponse(123, body, "application/octet-stream") + keyword = BinaryResponse( + perf_ns=123, raw_bytes=body, content_type="application/octet-stream" + ) + record = RequestRecord(responses=[positional]) + + assert positional == keyword + assert not hasattr(positional, "__dict__") + assert positional.get_raw() == body + assert positional.get_text() is None + assert positional.get_json() is None + assert record.model_dump()["responses"][0]["raw_bytes"] == body + assert positional.raw_bytes == body + + @pytest.mark.parametrize( + "encoded", + [param("!", id="invalid-character"), param("a", id="invalid-length")], + ) # fmt: skip + @pytest.mark.parametrize("json_mode", [False, True]) + def test_malformed_base64_response_rejected( + self, encoded: str, json_mode: bool + ) -> None: + """Both record validation routes reject malformed base64 bodies.""" + data = {"responses": [{"perf_ns": 123, "raw_bytes": encoded}]} + with pytest.raises(ValidationError): + if json_mode: + RequestRecord.model_validate_json(orjson.dumps(data)) + else: + RequestRecord.model_validate(data) + + def test_valid_base64_response_decoded_without_legacy_text_detection(self) -> None: + """An ambiguous legacy text value is always interpreted as base64.""" + record = RequestRecord.model_validate( + {"responses": [{"perf_ns": 123, "raw_bytes": "abcd"}]} + ) + + assert isinstance(record.responses[0], BinaryResponse) + assert record.responses[0].raw_bytes == b"i\xb7\x1d" + + class TestProfileResults: """Test cases for ProfileResults model.""" diff --git a/tests/unit/post_processors/test_raw_record_writer_processor.py b/tests/unit/post_processors/test_raw_record_writer_processor.py index 4cda4f7acb..89ba0f62cf 100644 --- a/tests/unit/post_processors/test_raw_record_writer_processor.py +++ b/tests/unit/post_processors/test_raw_record_writer_processor.py @@ -1,14 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import base64 + import orjson import pytest +from pytest import param from aiperf.common.enums import CreditPhase -from aiperf.common.models import ParsedResponseRecord +from aiperf.common.models import BinaryResponse, ParsedResponseRecord from aiperf.common.models.record_models import RawRecordInfo from aiperf.config.artifacts import OutputDefaults from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.resolution.plan import BenchmarkRun from aiperf.post_processors.raw_record_writer_processor import ( RawRecordAggregator, RawRecordWriterProcessor, @@ -184,6 +188,52 @@ async def test_process_multiple_records( class TestRawRecordWriterProcessorFileFormat: """Test RawRecordWriterProcessor file format.""" + @pytest.mark.asyncio + @pytest.mark.parametrize( + "payload_bytes", + [param(b'{"prompt":"test"}', id="fragment"), param(None, id="fallback")], + ) # fmt: skip + async def test_binary_response_jsonl_roundtrip_preserves_bytes( + self, + run_raw: BenchmarkRun, + sample_parsed_record: ParsedResponseRecord, + payload_bytes: bytes | None, + ) -> None: + """Both raw writer paths preserve binary bodies through JSONL readers.""" + body = bytes(range(256)) + sample_parsed_record.request.responses = [ + BinaryResponse(123, body, "application/octet-stream") + ] + sample_parsed_record.request.request_info.payload_bytes = payload_bytes + async with raw_record_processor("binary-worker", run_raw) as processor: + await processor.observe( + RecordObserverContext( + record=sample_parsed_record, + metadata=create_metric_metadata(), + produced={}, + ) + ) + + assert processor.dropped_record_count == 0 + lines = processor.output_file.read_bytes().splitlines() + assert len(lines) == 1 + dumped = orjson.loads(lines[0]) + assert dumped["responses"][0]["raw_bytes"] == ( + base64.urlsafe_b64encode(body).decode("ascii") + ) + assert dumped.get("payload") == ( + orjson.loads(payload_bytes) if payload_bytes is not None else None + ) + for restored in ( + RawRecordInfo.model_validate_json(lines[0]), + RawRecordInfo.model_validate(dumped), + ): + response = restored.responses[0] + assert isinstance(response, BinaryResponse) + assert response.raw_bytes == body + assert response.perf_ns == 123 + assert response.content_type == "application/octet-stream" + @pytest.mark.asyncio async def test_output_is_valid_jsonl_and_record_structure( self, From 0b00388f5b7cb006ceb8b52c09229857a16a9981 Mon Sep 17 00:00:00 2001 From: Frank <3429989+FrankD412@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:27:44 -0700 Subject: [PATCH 118/119] fix: address post-merge review feedback on #1268 (verbatim system prompts) (#1328) Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> (cherry picked from commit 3fc7d4c54248080336750e4f901c6c969acfd8d3) --- docs/tutorials/anthropic-messages-endpoint.md | 9 +- docs/tutorials/custom-dataset.md | 9 +- docs/tutorials/openai-responses.md | 10 +- src/aiperf/common/models/record_models.py | 6 + src/aiperf/dataset/dataset_manager.py | 64 +++ src/aiperf/dataset/loader/multi_turn.py | 17 +- src/aiperf/endpoints/base_endpoint.py | 43 ++ src/aiperf/endpoints/openai_chat.py | 21 +- src/aiperf/endpoints/openai_responses.py | 148 ++++++- src/aiperf/workers/session_manager.py | 23 +- src/aiperf/workers/worker.py | 66 ++- tests/unit/conftest.py | 21 + .../test_dataset_manager_payload_paths.py | 143 +++++++ tests/unit/endpoints/conftest.py | 2 + .../endpoints/test_message_merging_fixes.py | 82 ++++ .../unit/endpoints/test_responses_endpoint.py | 393 ++++++++++++++++++ tests/unit/workers/test_session_manager.py | 38 +- tests/unit/workers/test_worker.py | 173 ++++++++ .../test_worker_cache_bust_injection.py | 62 ++- 19 files changed, 1250 insertions(+), 80 deletions(-) diff --git a/docs/tutorials/anthropic-messages-endpoint.md b/docs/tutorials/anthropic-messages-endpoint.md index 2eeea16b9e..19fd363d1f 100644 --- a/docs/tutorials/anthropic-messages-endpoint.md +++ b/docs/tutorials/anthropic-messages-endpoint.md @@ -117,7 +117,7 @@ Key formatting rules: - **Stream**: Set from `--streaming` flag (default: `false`) - **System**: Placed as a top-level string field when a system message is provided (never in the messages array) - **Tools**: Included at the top level when tool definitions are present in the conversation -- **System (raw)**: When a turn carries `raw_system` (a list of content blocks), it is placed verbatim in the payload's top-level `system` field, overriding the conversation-level `system_message` string. See [Per-block system prompts (`raw_system`)](#per-block-system-prompts-raw_system) +- **System (raw)**: When a turn carries `raw_system` (a list of content blocks), it is placed verbatim in the payload's top-level `system` field. A conversation-level `system_message` is not discarded — it is prepended as a leading text block. See [Per-block system prompts (`raw_system`)](#per-block-system-prompts-raw_system) - **Extra inputs**: Merged into the payload via `--extra-inputs` (e.g., `--extra-inputs temperature:0.7`) ### Simple Text Content @@ -346,7 +346,7 @@ The usage object may include `cache_creation_input_tokens` and `cache_read_input The conversation-level `system_message` string (set, for example, by `--shared-system-prompt-length`) is rendered as a plain top-level `system: "..."` string. That works for most cases, but cannot express Anthropic's per-block extensions — most importantly `cache_control: {"type": "ephemeral"}` for prompt caching. -To opt in, set `raw_system` on a `Turn` to a list of content blocks. The endpoint resolves it the same way as `raw_tools`: walking the turn list from the end and taking the first non-`None` value (`base_endpoint.py::_latest_turn_attr`). When set, it wins over `system_message`: +To opt in, set `raw_system` on a `Turn` to a list of content blocks. The endpoint resolves it the same way as `raw_tools`: walking the turn list from the end and taking the first non-`None` value (`base_endpoint.py::_latest_turn_attr`). When set, it replaces the plain-string rendering — but it does not displace a conversation-level `system_message`, which is prepended as a leading text block ahead of the authored ones (see below): ```json { @@ -370,8 +370,9 @@ Notes: - Latest-non-`None` turn wins — a FORK-mode DAG child that does not redeclare `raw_system` still inherits the parent's value. - The field is Anthropic-specific. Other endpoints (`chat`, `responses`, ...) ignore it. - Block contents flow into ISL accounting via the endpoint's system-prompt walk; see [Input token accounting (ISL)](#input-token-accounting-isl) below. -- `raw_system` is currently populated by trace-replay loaders that ingest Anthropic-shaped traces; programmatic callers building `Turn` objects directly can set it as well. -- `--cache-bust system-prefix` / `system-suffix` targets `raw_system` when it is set, since that is what actually ships. The marker is appended or prepended as its own `{"type": "text", ...}` block rather than being spliced into an existing block's text, so a neighbouring `cache_control` breakpoint stays attached to the bytes it was authored against. +- Built-in dataset loaders do not currently populate `raw_system`; programmatic callers building `Turn` objects directly can set it. +- A conversation-level `system_message` (from `--system-prompt`/`--system-prompt-file` or `--shared-system-prompt-length`) is **prepended as block 0**, ahead of the authored blocks, which are passed through untouched along with their `cache_control`. The two coexist rather than one winning. +- `--cache-bust system-prefix` marks `system_message` when both are present, because that is block 0 — marking `raw_system` would leave a constant leading block the server could still prefix-hit on. `--cache-bust system-suffix` does the opposite: the tail of the system section is the last `raw_system` block, so the marker goes there and `system_message` is left unchanged; `system_message` receives the suffix marker only when no `raw_system` follows it. On the `raw_system` path the marker is appended or prepended as its own `{"type": "text", ...}` block rather than being spliced into an existing block's text, so a neighbouring `cache_control` breakpoint stays attached to the bytes it was authored against. ### Audio and video are unsupported diff --git a/docs/tutorials/custom-dataset.md b/docs/tutorials/custom-dataset.md index 5b9b5b00e0..d478167bc5 100644 --- a/docs/tutorials/custom-dataset.md +++ b/docs/tutorials/custom-dataset.md @@ -544,7 +544,8 @@ conversation. Unlike `--shared-system-prompt-length`, which generates synthetic target token length, these take the **exact text** — so prefix-cache hit rates and TTFT reflect the system prompt your deployment actually sends. -They work with every dataset kind: synthetic, file-based, and public. +They work with synthetic, public, and structured file-based datasets. The one exclusion is +datasets that author their own request payloads — see the note at the end of this section. ```bash # Inline, for short prompts @@ -580,6 +581,12 @@ Behavior worth knowing: - **Supported on `chat`, `responses`, `messages`, and `chat_embeddings`.** Endpoints with no system role (`completions`, `embeddings`, the rankings endpoints) reject the option at startup rather than silently dropping it. +- **Not supported on datasets that author their own request payloads** — `raw_payload`, + `inputs_json`, and `mooncake_trace` in payload mode. Those payloads are sent to the server + byte-for-byte, bypassing the formatting step that puts the system message on the wire, so + the prompt would be silently dropped. AIPerf rejects the combination at startup. Either + author the system message directly into the trace payloads, or use a dataset type that + produces structured turns (`single_turn`, `multi_turn`, `dag_jsonl`). - **Mutually exclusive** with `--shared-system-prompt-length` and with `--num-prefix-prompts`/`--prefix-prompt-length`, all of which fill the same slot. Setting two of them is rejected at startup rather than one taking precedence, so a misconfigured diff --git a/docs/tutorials/openai-responses.md b/docs/tutorials/openai-responses.md index 4267387fd9..6ea91f6a9e 100644 --- a/docs/tutorials/openai-responses.md +++ b/docs/tutorials/openai-responses.md @@ -19,7 +19,7 @@ AIPerf's `responses` endpoint type handles the key differences between the Respo | Chat Completions | Responses API | |---|---| | `messages` array | `input` array | -| `system` role message | Top-level `instructions` field | +| `system` role message | Top-level `instructions` field (see [System Instructions](#system-instructions) for the authored-system-item case) | | `max_completion_tokens` | `max_output_tokens` | | `{"type": "text", ...}` content | `{"type": "input_text", ...}` content | | `{"type": "image_url", ...}` content | `{"type": "input_image", ...}` content | @@ -139,6 +139,14 @@ aiperf profile \ This generates a synthetic system prompt of approximately 50 tokens and places it in the `"instructions"` field of the Responses API payload, rather than adding a system message to the input array. The same prompt is shared across all requests in the session. +### When the dataset authors its own system item + +A custom dataset whose turns carry authored messages that go through the Responses formatter (for example `mooncake_trace` or `dag_jsonl` rows with a `messages` array) may already carry a leading `{"role": "system", ...}` input item. (Datasets that author complete request payloads — `raw_payload`, `inputs_json` — bypass the formatter and are rejected outright when combined with a system prompt; see [Adding a System Prompt](custom-dataset.md#adding-a-system-prompt).) Shipping that alongside `instructions` would give the server two system prompts, so AIPerf merges instead: the `--system-prompt` / `--system-prompt-file` text is prepended to the authored item's content (joined with a blank line), the merged item stays at index 0 of `input`, and `instructions` is omitted. This mirrors how the `chat` endpoint handles the same collision and keeps the system prompt as the leading wire bytes, which `--cache-bust system-prefix` relies on. + +`instructions` is only dropped in that collision case. With no authored system item, the prompt goes to `instructions` as described above. + +With stateful chaining (`--extra-inputs store:true`), the server replays the merged item from stored history on every later request in the chain. Top-level `instructions` is not carried across `previous_response_id`, so AIPerf also omits `instructions` from those chained requests to avoid sending the system prompt twice. AIPerf tracks this per stored response, alongside the response ID it chains onto: a request that fails never enters stored history, so it does not count, and the next request chains onto the last successful response and resends `instructions` if that response's history does not carry the prompt. A `reset_context` turn starts a new chain and clears the state; a forked conversation inherits its parent's. An explicit `previous_response_id` in `extra` (endpoint or per-turn) wins: chaining onto a different response resends the prompt, and `null` breaks the chain, so the request ships the full conversation as a stateless request would. + --- ## Vision (Image Inputs) diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index f9c11ab0a6..b96cdf44bc 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -1090,6 +1090,12 @@ class RequestInfo(RecordContext): description="Response ID from the previous turn (e.g. 'resp_') " "used for stateful chaining in the Responses API.", ) + system_prompt_in_stored_history: bool = Field( + default=False, + description="Whether the stored server-side history behind " + "previous_response_id already contains the system prompt as an input " + "item. Only meaningful when previous_response_id is set.", + ) class RequestRecord(AIPerfBaseModel): diff --git a/src/aiperf/dataset/dataset_manager.py b/src/aiperf/dataset/dataset_manager.py index a3706bf3d7..d59bc2d59f 100644 --- a/src/aiperf/dataset/dataset_manager.py +++ b/src/aiperf/dataset/dataset_manager.py @@ -714,6 +714,53 @@ def _reject_body_mutators_for_payload_bytes( "dataset cache directory." ) + def _reject_system_prompt_for_raw_payload(self, has_raw_payload: bool) -> None: + """Refuse a verbatim system prompt when the dataset authors its own payloads. + + ``inference_client`` ships ``Turn.raw_payload`` verbatim and never calls + ``format_payload``, which is the only place a conversation-level + ``system_message`` becomes wire bytes. The prompt would therefore be + composed, stored, and silently never sent -- measuring prefix-cache hit + rate and TTFT against a system prompt the server never saw. + + Two call sites compute ``has_raw_payload`` from what they have: + + - The cache-miss path passes ``any(turn.raw_payload)`` over the loaded + conversations. ``any`` rather than ``all``: a dataset where only some + conversations carry ``raw_payload`` falls back to the CONVERSATION + mmap format and never reaches ``_select_mmap_format``'s + PAYLOAD_BYTES guard, but its raw turns are still dispatched verbatim. + Must run BEFORE ``_preformat_payloads`` -- payloads synthesized there + go through ``format_payload`` with ``system_message`` attached, so + checking afterwards cannot tell an authored ``raw_payload`` from a + pre-formatted one and would reject a run that works. + - The cache-hit path passes ``manifest.all_turns_source_loaded_payloads``. + Conversations are never loaded on a HIT, so this is the only signal + available; it is ``all`` rather than ``any``, but a mixed dataset + cannot be populated with a system prompt in the first place because + the miss path rejects it before ``mmap_cache.populate``. + + ``get_system_prompt()`` (--system-prompt / --system-prompt-file) is the + only carrier checked because it is the only one that can coexist with + authored payloads. The other source of ``conversation.system_message``, + ``prefix_prompts.shared_system_length`` (--shared-system-prompt-length), + is a ``SyntheticDataset``-only field: every loader that emits + ``raw_payload`` is file-backed, and the CLI converter rejects the flag + on file/public datasets (``_FILE_DATASET_INCOMPATIBLE_TRIGGERS``). + """ + if self.run.cfg.get_system_prompt() is None or not has_raw_payload: + return + raise ValueError( + "--system-prompt/--system-prompt-file is incompatible with a dataset " + "whose turns carry their own raw_payload (raw_payload, inputs_json, " + "and mooncake_trace in payload mode). Those payloads are sent to the " + "server byte-for-byte, bypassing the payload formatting that puts the " + "system message on the wire, so the prompt would be silently dropped. " + "Either author the system message directly into the trace payloads, or " + "use a dataset type that produces structured turns (e.g. single_turn / " + "multi_turn / dag_jsonl)." + ) + def _select_mmap_format(self, conversations: list[Conversation]) -> MemoryMapFormat: """Pick the dataset mmap format and refuse PAYLOAD_BYTES for body-mutators. @@ -813,6 +860,12 @@ async def _configure_from_cache_hit(self, hit: mmap_cache.CacheHit) -> None: manifest, falls back to a MISS (``_cache_hit_used`` stays False). """ self._reject_body_mutators_for_payload_bytes(hit.manifest.mmap_format) + # Conversations are never loaded on a HIT, so the composition-path + # guard in _configure_dataset never runs; re-check from the manifest + # before adopting the cached bytes. + self._reject_system_prompt_for_raw_payload( + hit.manifest.all_turns_source_loaded_payloads + ) run_data_path, run_index_path = self._run_mmap_paths() await asyncio.to_thread( @@ -952,6 +1005,17 @@ async def _configure_dataset(self) -> None: for turn in conv.turns ) + # Same pre-preformat window as the capture above, for the same reason: + # once _preformat_payloads runs, a synthesized raw_payload (which does + # carry the system message) is indistinguishable from an authored one. + self._reject_system_prompt_for_raw_payload( + any( + turn.raw_payload is not None + for conv in conversations + for turn in conv.turns + ) + ) + endpoint_meta: EndpointMetadata = plugins.get_endpoint_metadata( self.run.cfg.endpoint.type ) diff --git a/src/aiperf/dataset/loader/multi_turn.py b/src/aiperf/dataset/loader/multi_turn.py index 3136231ae4..26d95dce55 100644 --- a/src/aiperf/dataset/loader/multi_turn.py +++ b/src/aiperf/dataset/loader/multi_turn.py @@ -114,11 +114,10 @@ class MultiTurnDatasetLoader(BaseFileLoader, MediaConversionMixin): chat_embeddings); on others the system turn is left in place rather than silently dropped. A conversation that leads with two or more consecutive ``role: "system"`` turns is - un-hoisted: the endpoint only merges ``system_message`` into a rendered - leading system message during warmup, so a hoisted prompt sitting in front - of another leading system turn would never reach the wire in profiling - while still counting toward ISL. Both system turns are dispatched normally - instead, matching pre-hoist behavior. + un-hoisted: hoisting would fold the first system turn into the second on + the wire, collapsing two authored messages the trace deliberately kept + apart. Both system turns are dispatched normally instead, matching + pre-hoist behavior. """ _hoist_leading_system_message: ClassVar[bool] = True @@ -228,13 +227,7 @@ def convert_to_conversations( conversation.turns.append(self._build_turn(*hoisted)) hoisted = None - # A second consecutive leading system turn fails the hoist guard and - # stays as turn 0 (role="system"). The endpoint only merges - # system_message into a rendered leading system message during - # warmup, so in profiling the hoisted prompt never reaches the wire - # while ISL still counts it. Un-hoist: restore the hoisted turn as - # turn 0 so both system turns dispatch normally, matching pre-hoist - # behavior for this ambiguous input. + # Two leading system turns: un-hoist (see class docstring). elif hoisted is not None and conversation.turns[0].role == "system": conversation.system_message = None conversation.turns.insert(0, self._build_turn(*hoisted)) diff --git a/src/aiperf/endpoints/base_endpoint.py b/src/aiperf/endpoints/base_endpoint.py index bd4331712a..f62f09e573 100644 --- a/src/aiperf/endpoints/base_endpoint.py +++ b/src/aiperf/endpoints/base_endpoint.py @@ -7,6 +7,7 @@ from collections.abc import Callable from typing import Any, ClassVar +from aiperf.common.constants import SYSTEM_PROMPT_JOIN_SEP from aiperf.common.enums import MediaType from aiperf.common.environment import Environment from aiperf.common.mixins import AIPerfLoggerMixin @@ -382,6 +383,48 @@ def _extend_parts( continue parts.append(render_fn(content)) + def _prepend_system_text(self, prefix: str, content: Any) -> Any: + """Return ``content`` with ``prefix`` prepended, preserving its shape. + + System-message content is normally a plain string, but the chat and + Responses schemas also permit a list of content parts; a raw-payload + dataset may author either. Both branches join with + ``SYSTEM_PROMPT_JOIN_SEP`` so the two shapes tokenize identically -- + servers that concatenate content parts would otherwise see the verbatim + prompt run into the dataset's own text with no separator. + + The list branch renders through ``_render_text_part`` so each caller + contributes its own part shape (``text`` for chat, ``input_text`` for + Responses) without a shape table here. A single dict is treated as a + one-part list. ``MessagesEndpoint`` does not use this helper: Anthropic's + ``system`` is already a block list, so it prepends the prompt as its + own block with no separator. + + Content of any other shape (a number, a nested structure the schema + does not allow) cannot be preserved; it is dropped with a warning so a + malformed dataset is visible rather than silently rewritten. Empty + string content carries nothing to preserve and is not warned about. + + Returns a new object in both cases -- callers pass content that aliases + reusable turn state. + """ + if isinstance(content, dict): + content = [content] + if isinstance(content, list): + return [ + self._render_text_part(prefix + SYSTEM_PROMPT_JOIN_SEP), + *content, + ] + if isinstance(content, str): + return f"{prefix}{SYSTEM_PROMPT_JOIN_SEP}{content}" if content else prefix + if content is not None: + self.warning( + f"Dropping authored system content of unsupported type " + f"{type(content).__name__} while merging the verbatim system " + "prompt; expected str, dict, or list of content parts." + ) + return prefix + # --- Content-part hooks: override per endpoint to change type names ------ def _render_text_part(self, text: str) -> dict[str, Any]: diff --git a/src/aiperf/endpoints/openai_chat.py b/src/aiperf/endpoints/openai_chat.py index 10a9399d71..16ee3839a0 100644 --- a/src/aiperf/endpoints/openai_chat.py +++ b/src/aiperf/endpoints/openai_chat.py @@ -5,7 +5,6 @@ from typing import Any -from aiperf.common.constants import SYSTEM_PROMPT_JOIN_SEP from aiperf.common.models import ( BaseResponseData, Image, @@ -21,21 +20,6 @@ from aiperf.endpoints.base_endpoint import BaseEndpoint -def _prepend_system_text(prefix: str, content: Any) -> Any: - """Return ``content`` with ``prefix`` prepended, preserving its shape. - - System-message content is normally a plain string, but the OpenAI schema - also permits a list of content parts; a raw-payload dataset may author - either. Returns a new object in both cases -- callers pass content that - aliases reusable turn state. - """ - if isinstance(content, list): - return [{"type": "text", "text": prefix}, *content] - if isinstance(content, str) and content: - return f"{prefix}{SYSTEM_PROMPT_JOIN_SEP}{content}" - return prefix - - class ChatEndpoint(BaseEndpoint): """OpenAI Chat Completions endpoint. @@ -106,9 +90,8 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: self.trace(lambda: f"Formatted payload: {payload}") return payload - @staticmethod def _format_messages( - request_info: RequestInfo, rendered: list[dict[str, Any]] + self, request_info: RequestInfo, rendered: list[dict[str, Any]] ) -> list[dict[str, Any]]: """Build chat messages with RequestInfo-level prompts applied. @@ -130,7 +113,7 @@ def _format_messages( # raw_messages, which are reused across credits in a session, so # an in-place edit would restack the prefix on every replay. merged = dict(rendered[0]) - merged["content"] = _prepend_system_text( + merged["content"] = self._prepend_system_text( request_info.system_message, merged.get("content") ) messages.append(merged) diff --git a/src/aiperf/endpoints/openai_responses.py b/src/aiperf/endpoints/openai_responses.py index 040a6a6740..4733acd447 100644 --- a/src/aiperf/endpoints/openai_responses.py +++ b/src/aiperf/endpoints/openai_responses.py @@ -180,30 +180,80 @@ def _render(turn: Turn) -> dict[str, Any]: def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: """Format OpenAI Responses API request payload from RequestInfo.""" + payload, _ = self._build_payload(request_info) + self.trace(lambda: f"Formatted payload: {payload}") + return payload + + def _build_payload(self, request_info: RequestInfo) -> tuple[dict[str, Any], bool]: + """Build the wire payload and report whether the system prompt was + merged into an input item that actually ships. + + The flag is read off the final payload, after endpoint ``extra`` and + per-turn ``extra_body`` are applied, because either may replace + ``input`` and discard the merged item. + """ if not request_info.turns: raise ValueError("Responses endpoint requires at least one turn.") turns = request_info.turns model_endpoint = request_info.model_endpoint - # Responses API doesn't nest the system prompt into ``input``; it - # lives in top-level ``instructions``. The per-conversation - # ``user_context_message`` is prepended as a leading user item. - input_items: list[dict[str, Any]] = [] - if request_info.previous_response_id: + # Responses API carries the system prompt in top-level ``instructions`` + # rather than in ``input``. The per-conversation ``user_context_message`` + # is a leading user item. Branch on the outgoing id, not the session's: + # an override to ``None`` breaks the chain, so the server has no history + # to replay and the full conversation must ship as in a stateless run. + if self._outgoing_previous_response_id(request_info): # Stateful chaining: previous_response_id points to server-side history. self._warn_chaining_isl_once() - input_items.extend(self.build_messages([turns[-1]])) + rendered = self.build_messages([turns[-1]]) + context_items: list[dict[str, Any]] = [] else: - if request_info.user_context_message: - input_items.append( + rendered = self.build_messages(turns) + context_items = ( + [ { "type": "message", "role": self.DEFAULT_TURN_ROLE, "content": request_info.user_context_message, } - ) - input_items.extend(self.build_messages(turns)) + ] + if request_info.user_context_message + else [] + ) + instructions, merge_prefix = self._system_prompt_carrier(request_info, rendered) + + # A dataset that authored its own leading ``role: system`` input item + # collides with ``instructions``: both ship, and the server sees two + # system prompts. Merge into the authored item and drop + # ``instructions``, matching ``ChatEndpoint._format_messages`` -- the + # de-dup invariant is endpoint-wide, and repeated system roles are + # mishandled by many OpenAI-compatible servers. + # + # The merged item is placed AHEAD of ``user_context_message`` so the + # system prompt stays the leading wire bytes, as it does for chat and + # as ``SYSTEM_PREFIX`` cache-bust assumes. The alternative -- folding + # the authored item's text into ``instructions`` and dropping it from + # ``input`` -- would keep the "system lives in instructions" contract + # unconditional, but flattens list-part content to text and discards + # any extra fields on the authored item; the dataset's explicit wire + # shape wins here. + leading_system: list[dict[str, Any]] = [] + if merge_prefix is not None: + # Copy rather than mutate: ``rendered`` aliases the turn's + # raw_messages, reused across credits in a session. + merged = dict(rendered[0]) + merged["content"] = self._prepend_system_text( + merge_prefix, merged.get("content") + ) + leading_system = [merged] + rendered = rendered[1:] + + input_items: list[dict[str, Any]] = [ + *leading_system, + *context_items, + *rendered, + ] # Conversation-level fields walk turns from the end so FORK-mode # children whose final turn lacks model/tools still inherit the parent's @@ -221,7 +271,7 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: payload["previous_response_id"] = request_info.previous_response_id for key, value in ( - ("instructions", request_info.system_message or None), + ("instructions", instructions), ("max_output_tokens", max_tokens), ("tools", self._latest_turn_attr(turns, "raw_tools")), ): @@ -235,8 +285,80 @@ def format_payload(self, request_info: RequestInfo) -> dict[str, Any]: self._maybe_enable_usage_stream_options(payload, model_endpoint) - self.trace(lambda: f"Formatted payload: {payload}") - return payload + merged_shipped = merge_prefix is not None and payload["input"] is input_items + return payload, merged_shipped + + @staticmethod + def _outgoing_previous_response_id(request_info: RequestInfo) -> str | None: + """The ``previous_response_id`` the payload will actually carry. + + Endpoint ``extra`` and the dispatching turn's ``extra_body`` are applied + after the payload is built and win over the session's value, including + an explicit ``None`` that breaks the chain. + """ + outgoing = request_info.previous_response_id + for overrides in ( + dict(request_info.model_endpoint.endpoint.extra or {}), + request_info.turns[-1].extra_body or {}, + ): + if "previous_response_id" in overrides: + outgoing = overrides["previous_response_id"] + return outgoing + + def _system_prompt_already_stored(self, request_info: RequestInfo) -> bool: + """Whether the chain this request continues already holds the prompt. + + The session flag describes the session's last stored response, so it + only applies when the outgoing ``previous_response_id`` is still that + response; an override to another response or to ``None`` does not + inherit it. + """ + return bool( + request_info.previous_response_id + and request_info.system_prompt_in_stored_history + and self._outgoing_previous_response_id(request_info) + == request_info.previous_response_id + ) + + def _system_prompt_carrier( + self, request_info: RequestInfo, rendered: list[dict[str, Any]] + ) -> tuple[str | None, str | None]: + """Decide how this request carries ``system_message``. + + Returns ``(instructions, merge_prefix)``: at most one is set, and + ``merge_prefix`` means "prepend to the authored leading system item". + When the chained response's stored history already holds the prompt + as an input item, neither is set: top-level ``instructions`` is not + carried across ``previous_response_id``, but stored input items are, + so sending it again would double the prompt. + """ + system_message = request_info.system_message or None + if system_message is None or self._system_prompt_already_stored(request_info): + return None, None + if ( + rendered + and isinstance(rendered[0], dict) + and rendered[0].get("role") == "system" + ): + return None, system_message + return system_message, None + + def stored_history_has_system_prompt(self, request_info: RequestInfo) -> bool: + """Whether the stored history of the response to ``request_info`` + contains the system prompt. + + The worker records this next to the response ID on success only, so + the flag always describes the response being chained onto. It holds + when the request still chains onto a history that carried the prompt, + or when the merged input item survived onto the wire; prompts sent as + ``instructions`` are not stored. Both are read from the built payload + so explicit overrides of ``input`` or ``previous_response_id`` are + reflected rather than inferred from the turns. + """ + if not request_info.system_message: + return False + _, merged_shipped = self._build_payload(request_info) + return self._system_prompt_already_stored(request_info) or merged_shipped _warned_chaining_isl: bool = False diff --git a/src/aiperf/workers/session_manager.py b/src/aiperf/workers/session_manager.py index 2f08d83780..db8f71d891 100644 --- a/src/aiperf/workers/session_manager.py +++ b/src/aiperf/workers/session_manager.py @@ -100,6 +100,12 @@ class UserSession(AIPerfBaseModel): description="Response ID from the previous turn (e.g. 'resp_') " "used for stateful chaining in the Responses API.", ) + system_prompt_in_stored_history: bool = Field( + default=False, + description="Whether the stored history behind previous_response_id " + "contains the system prompt as an input item. Set together with " + "previous_response_id so the two always describe the same response.", + ) def advance_turn(self, turn_index: int) -> Turn: """Append the next turn onto ``turn_list`` and return it. @@ -134,6 +140,7 @@ def advance_turn(self, turn_index: int) -> Turn: # Clearing is the safe direction (falls back to sending full history). if turn.reset_context: self.previous_response_id = None + self.system_prompt_in_stored_history = False if self.context_mode == ConversationContextMode.MESSAGE_ARRAY_WITH_RESPONSES: self.turn_list = [turn] @@ -173,9 +180,20 @@ def store_response(self, response_turn: Turn) -> None: """ self.turn_list.append(response_turn) - def store_response_id(self, response_id: str | None) -> None: - """Store the response ID from the server for stateful chaining.""" + def store_response_id( + self, response_id: str | None, *, system_prompt_in_history: bool = False + ) -> None: + """Store the response ID from the server for stateful chaining. + + ``system_prompt_in_history`` records whether that response's stored + history carries the system prompt as an input item. Only successful + responses are stored, so a failed request leaves both values on the + last good response. + """ self.previous_response_id = response_id + self.system_prompt_in_stored_history = ( + response_id is not None and system_prompt_in_history + ) class UserSessionManager: @@ -359,6 +377,7 @@ def seed_from_parent( return child.turn_list = list(parent.turn_list) child.previous_response_id = parent.previous_response_id + child.system_prompt_in_stored_history = parent.system_prompt_in_stored_history def release_fork_child(self, x_correlation_id: str) -> None: """Decrement the FORK-pin refcount on the session, floored at 0. diff --git a/src/aiperf/workers/worker.py b/src/aiperf/workers/worker.py index dcda75f886..f4f53fa17d 100644 --- a/src/aiperf/workers/worker.py +++ b/src/aiperf/workers/worker.py @@ -257,10 +257,14 @@ def _find_effective_raw_system(turn_list: list[Turn]) -> list[dict] | None: Mirrors ``base_endpoint._latest_turn_attr``: walks ``turn_list`` from the end and returns the first non-None ``raw_system``. Only ``MessagesEndpoint`` - reads this field, and there it OUTRANKS both the conversation-level - ``system_message`` string and any system-role ``raw_messages`` entry — so a - ``SYSTEM_*`` marker written to either of those would be silently dropped - from the wire whenever a turn carries ``raw_system``. + reads this field, and there it outranks any system-role ``raw_messages`` + entry — so a ``SYSTEM_*`` marker written to ``raw_messages`` would be + silently dropped from the wire whenever a turn carries ``raw_system``. + + It does NOT outrank a conversation-level ``system_message``: that is + prepended as ``system`` block 0, ahead of these blocks. Callers marking + the front of the system section must therefore prefer ``system_message`` + when both are present; callers marking the tail must prefer these blocks. """ for turn in reversed(turn_list): if turn.raw_system is not None: @@ -476,14 +480,28 @@ def _apply_system_target_cache_bust( :func:`_effective_prefix_turns`). Four sub-paths, ordered to match how the endpoint resolves the system field on the wire — marking a lower-precedence carrier would leave the bytes that actually ship unchanged: - 1. ``raw_system`` present on any turn: it outranks both ``system_message`` - and system-role ``raw_messages`` in ``MessagesEndpoint``, so the marker - goes there. Resolved over ``all_turns`` (not the prefix slice) because - ``_latest_turn_attr`` ignores ``reset_context``. - 2. Conversation-level ``system_message`` present: marker applied every + 1. Conversation-level ``system_message`` present: marker applied every turn (string mutation re-applied per credit). Unaffected by ``reset_context`` — the ``system_message`` rides on ``RequestInfo`` and is re-emitted every turn independent of ``build_messages``' reset. + Checked first for prefix targets because every system-aware endpoint + places it at or ahead of the other carriers: ``MessagesEndpoint`` + makes it ``system`` block 0 in front of ``raw_system``, and + ``ChatEndpoint`` merges it into the front of a leading system + ``raw_messages`` entry. Marking a later carrier would leave a constant + leading block the server can still prefix-hit on, which is exactly + what ``SYSTEM_PREFIX`` must prevent. + For ``SYSTEM_SUFFIX`` the same wire order means ``system_message`` is + the tail only when no ``raw_system`` follows it: ``MessagesEndpoint`` + ships ``[system_message, *raw_system]``, so a marker appended to + ``system_message`` would sit ahead of constant trailing blocks and + the shared bytes the suffix contract keeps cacheable would land after + it. Suffix therefore defers to step 2 when ``raw_system`` is present. + 2. ``raw_system`` present on any turn (and either no ``system_message`` + or a suffix target): it outranks system-role ``raw_messages`` in + ``MessagesEndpoint``, so the marker goes there. Resolved over + ``all_turns`` (not the prefix slice) because ``_latest_turn_attr`` + ignores ``reset_context``. 3. ``raw_messages`` first dict has ``role=="system"``: marker injected into the first system message of the prefix slice. 4. No system anywhere -> first-user-turn fallback: marker injected @@ -494,11 +512,11 @@ def _apply_system_target_cache_bust( Returns the (possibly modified) ``system_message``. """ raw_system_blocks = _find_effective_raw_system(all_turns) + if system_message is not None and (is_prefix or raw_system_blocks is None): + return _apply_cache_bust_to_system_message(system_message, marker, target) if raw_system_blocks is not None: _inject_marker_into_raw_system(raw_system_blocks, marker, is_prefix=is_prefix) return system_message - if system_message is not None: - return _apply_cache_bust_to_system_message(system_message, marker, target) raw_system = _find_first_system_message(prefix_turns) if raw_system is not None: _inject_marker_into_raw_messages(raw_system, marker, is_prefix=is_prefix) @@ -1105,7 +1123,7 @@ async def _process_credit_with_session( credit_context, request_info, first_token_callback ) - self._finalize_session_response(session, credit_context, record) + self._finalize_session_response(session, credit_context, record, request_info) async def _try_payload_bytes_fast_path( self, @@ -1204,9 +1222,14 @@ def _finalize_session_response( session: UserSession, credit_context: CreditContext, record: RequestRecord, + request_info: RequestInfo | None = None, ) -> None: """Store the assistant turn (when retained) and populate metrics from a - single response-processing pass, shared with the payload-bytes path.""" + single response-processing pass, shared with the payload-bytes path. + + ``request_info`` is the full request that produced ``record``; the + record itself only carries the slim ``RecordContext``. + """ parsed_responses, assistant_turn = self._process_responses_for_record( record, capture_assistant_turn=session.should_store_response(), @@ -1229,7 +1252,19 @@ def _finalize_session_response( ) ) ): - session.store_response_id(extract_response_id(record)) + compute_stored_flag = getattr( + self.inference_client.endpoint, + "stored_history_has_system_prompt", + None, + ) + session.store_response_id( + extract_response_id(record), + system_prompt_in_history=bool( + compute_stored_flag + and request_info is not None + and compute_stored_flag(request_info) + ), + ) self._populate_response_metrics(credit_context, record, parsed_responses) def _populate_response_metrics( @@ -1529,6 +1564,9 @@ def _create_request_info( source_kind=source_turn.source_kind if source_turn else None, turns=turns, previous_response_id=session.previous_response_id if session else None, + system_prompt_in_stored_history=session.system_prompt_in_stored_history + if session + else False, drop_perf_ns=credit_context.drop_perf_ns, credit_issued_ns=credit.issued_at_ns, system_message=system_message, diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 0627e55633..2b71d26443 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -358,6 +358,27 @@ class is initialized" occurs when a singleton instance from a previous test SingletonMeta._instances.clear() +@pytest.fixture(autouse=True) +def isolate_fake_communication_bus(monkeypatch: pytest.MonkeyPatch) -> None: + """Give every test its own ``FakeCommunication`` bus. + + ``FakeCommunication.shared_bus`` is class-level, and every instance + registers on it. A service built by one test (e.g. a ``Worker`` subscribed + to ``DatasetConfigured``) stays subscribed for the rest of the xdist + process, so a later test that publishes on the bus delivers to it. That + surfaced as an order-dependent failure: a dataset-manager test's + placeholder mmap index was opened by a ``Worker`` left over from + ``test_worker.py``. ``monkeypatch`` restores the attribute, so tests that + install their own bus via ``set_shared_bus`` still unwind cleanly. + """ + from tests.harness.fake_communication import ( + FakeCommunication, + FakeCommunicationBus, + ) + + monkeypatch.setattr(FakeCommunication, "shared_bus", FakeCommunicationBus()) + + @pytest.fixture def temporary_registry() -> Generator[PluginRegistry, None, None]: """Fixture for isolated plugin registry testing. diff --git a/tests/unit/dataset/test_dataset_manager_payload_paths.py b/tests/unit/dataset/test_dataset_manager_payload_paths.py index 752693d23d..8f497cd45c 100644 --- a/tests/unit/dataset/test_dataset_manager_payload_paths.py +++ b/tests/unit/dataset/test_dataset_manager_payload_paths.py @@ -34,6 +34,7 @@ from aiperf.common.models import Conversation, ModelEndpointInfo, Text, Turn from aiperf.config.flags.cli_config import CLIConfig from aiperf.config.resolution.plan import BenchmarkRun +from aiperf.dataset import mmap_cache from aiperf.dataset.dataset_manager import DatasetManager from aiperf.plugin.enums import CustomDatasetType from tests.unit.conftest import make_run_from_cli @@ -205,6 +206,148 @@ def _raw_conversation(session_id: str = "raw1", num_turns: int = 1) -> Conversat ) +def _make_system_prompt_run(prompt: str = "VERBATIM-SYS") -> BenchmarkRun: + return make_run_from_cli( + CLIConfig(model_names=["test-model"], system_prompt=prompt) + ) + + +def _has_raw(conversations: list[Conversation]) -> bool: + """The predicate the cache-miss call site feeds to the guard.""" + return any( + turn.raw_payload is not None for conv in conversations for turn in conv.turns + ) + + +def _make_cache_hit( + tmp_path: Path, *, all_turns_source_loaded_payloads: bool +) -> mmap_cache.CacheHit: + entry_dir = tmp_path / "entry" + entry_dir.mkdir() + (entry_dir / "dataset.dat").write_bytes(b"DATA") + (entry_dir / "index.dat").write_bytes(b"IDX") + return mmap_cache.CacheHit( + entry_dir=entry_dir, + data_path=entry_dir / "dataset.dat", + index_path=entry_dir / "index.dat", + manifest=mmap_cache.CacheManifest( + cache_key="k", + created_at=0.0, + num_conversations=1, + total_size_bytes=4, + compressed=False, + compressed_size_bytes=0, + mmap_format=str(MemoryMapFormat.PAYLOAD_BYTES), + all_turns_source_loaded_payloads=all_turns_source_loaded_payloads, + dataset_metadata_json='{"conversations": [], "sampling_strategy": "random"}', + ), + ) + + +class TestRejectSystemPromptForRawPayload: + """A verbatim system prompt cannot ride on datasets that author payloads. + + ``inference_client`` ships ``Turn.raw_payload`` byte-for-byte and never + calls ``format_payload``, the only place ``system_message`` becomes wire + bytes -- so without this gate the prompt is composed, cached, and silently + never sent. + """ + + def _manager(self, run: BenchmarkRun) -> DatasetManager: + return DatasetManager(run=run, service_id="dm-test") + + def test_no_system_prompt_allows_raw_payload(self) -> None: + dm = self._manager(_make_synthetic_run()) + dm._reject_system_prompt_for_raw_payload(_has_raw([_raw_conversation()])) + + def test_system_prompt_without_raw_payload_is_allowed(self) -> None: + dm = self._manager(_make_system_prompt_run()) + dm._reject_system_prompt_for_raw_payload( + _has_raw([_single_turn_conversation()]) + ) + + def test_system_prompt_with_raw_payload_raises(self) -> None: + dm = self._manager(_make_system_prompt_run()) + with pytest.raises(ValueError, match="raw_payload"): + dm._reject_system_prompt_for_raw_payload(_has_raw([_raw_conversation()])) + + def test_error_names_the_flag_and_the_way_out(self) -> None: + dm = self._manager(_make_system_prompt_run()) + with pytest.raises(ValueError) as excinfo: + dm._reject_system_prompt_for_raw_payload(_has_raw([_raw_conversation()])) + + message = str(excinfo.value) + assert "--system-prompt/--system-prompt-file" in message + assert "inputs_json" in message + assert "multi_turn" in message + + def test_partially_raw_dataset_raises(self) -> None: + """The mixed shape falls back to the CONVERSATION mmap format and never + reaches ``_select_mmap_format``'s PAYLOAD_BYTES guard, but its raw turns + are still dispatched verbatim -- so ``any``, not ``all``.""" + dm = self._manager(_make_system_prompt_run()) + conversations = [_single_turn_conversation(), _raw_conversation()] + + with pytest.raises(ValueError, match="raw_payload"): + dm._reject_system_prompt_for_raw_payload(_has_raw(conversations)) + + def test_preformatted_payloads_are_not_rejected( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Payloads synthesized by ``_preformat_payloads`` go through + ``format_payload`` with ``system_message`` attached, so they carry the + prompt correctly. Ordering the gate after pre-formatting would reject + this working run -- pin the ordering, not just the predicate. + """ + monkeypatch.setattr(Environment.DATASET, "PREFORMAT_PAYLOADS", True) + dm = self._manager(_make_system_prompt_run()) + conversations = [_single_turn_conversation()] + conversations[0].system_message = "VERBATIM-SYS" + + dm._reject_system_prompt_for_raw_payload(_has_raw(conversations)) + dm._preformat_payloads(conversations) + + # The synthesized payload really does carry the system message, which + # is why this run must be allowed to proceed. + payload = conversations[0].turns[0].raw_payload + assert payload is not None + assert payload["messages"][0] == { + "role": "system", + "content": "VERBATIM-SYS", + } + + # ...and running the same gate afterwards would now reject it. That is + # the whole reason the call site sits before _preformat_payloads; if it + # ever moves, this assertion documents what breaks. + with pytest.raises(ValueError, match="raw_payload"): + dm._reject_system_prompt_for_raw_payload(_has_raw(conversations)) + + @pytest.mark.asyncio + async def test_cache_hit_with_raw_payload_manifest_raises( + self, tmp_path: Path + ) -> None: + """A warm HIT never loads conversations, so the composition-path guard + never runs. The manifest's ``all_turns_source_loaded_payloads`` is the + only signal left; the same rejection must fire before the cached bytes + are adopted.""" + dm = self._manager(_make_system_prompt_run()) + hit = _make_cache_hit(tmp_path, all_turns_source_loaded_payloads=True) + + with pytest.raises(ValueError, match="raw_payload"): + await dm._configure_from_cache_hit(hit) + assert dm._cache_hit_used is False + + @pytest.mark.asyncio + async def test_cache_hit_without_raw_payload_manifest_passes_guard( + self, tmp_path: Path + ) -> None: + dm = self._manager(_make_system_prompt_run()) + hit = _make_cache_hit(tmp_path, all_turns_source_loaded_payloads=False) + + await dm._configure_from_cache_hit(hit) + assert dm._cache_hit_used is True + + class TestPreformatPayloads: """Opt-in pre-encoding of structured conversations to raw_payload.""" diff --git a/tests/unit/endpoints/conftest.py b/tests/unit/endpoints/conftest.py index ecda605d1d..96ff198f48 100644 --- a/tests/unit/endpoints/conftest.py +++ b/tests/unit/endpoints/conftest.py @@ -64,6 +64,7 @@ def create_request_info( user_context_message: str | None = None, cache_bust_target: CacheBustTarget | None = None, previous_response_id: str | None = None, + system_prompt_in_stored_history: bool = False, **turn_kwargs, ) -> RequestInfo: """Helper to create RequestInfo with all required fields. @@ -97,6 +98,7 @@ def create_request_info( user_context_message=user_context_message, cache_bust_target=cache_bust_target, previous_response_id=previous_response_id, + system_prompt_in_stored_history=system_prompt_in_stored_history, ) diff --git a/tests/unit/endpoints/test_message_merging_fixes.py b/tests/unit/endpoints/test_message_merging_fixes.py index a6eae84166..13a5b5795f 100644 --- a/tests/unit/endpoints/test_message_merging_fixes.py +++ b/tests/unit/endpoints/test_message_merging_fixes.py @@ -8,8 +8,11 @@ from __future__ import annotations +import logging + import orjson import pytest +from pytest import param from aiperf.common.models import ( RequestRecord, @@ -377,6 +380,38 @@ def test_authored_leading_system_untouched_without_request_info( assert len(systems) == 1 assert systems[0]["content"] == "authored system" + def test_authored_leading_system_list_content_keeps_separator(self, chat_endpoint): + """The list branch must join with the same separator as the string one. + + Servers that concatenate content parts would otherwise see + ``request_info systemauthored system`` -- a different prefix + tokenization for a semantically identical dataset. + """ + authored = [ + { + "role": "system", + "content": [{"type": "text", "text": "authored system"}], + }, + {"role": "user", "content": "hi"}, + ] + turn = Turn(role="user", raw_messages=authored) + request_info = create_request_info( + model_endpoint=chat_endpoint.model_endpoint, + turns=[turn], + system_message="request_info system", + ) + payload = chat_endpoint.format_payload(request_info) + systems = [m for m in payload["messages"] if m["role"] == "system"] + assert len(systems) == 1 + assert systems[0]["content"] == [ + {"type": "text", "text": "request_info system\n\n"}, + {"type": "text", "text": "authored system"}, + ] + # Concatenating the parts must match what the string branch produces. + assert "".join(p["text"] for p in systems[0]["content"]) == ( + "request_info system\n\nauthored system" + ) + def test_authored_leading_system_not_restacked_across_calls(self, chat_endpoint): """Formatting twice must not stack the prefix onto shared turn state. @@ -400,6 +435,53 @@ def test_authored_leading_system_not_restacked_across_calls(self, chat_endpoint) assert authored[0]["content"] == "authored system" +class TestPrependSystemTextShapes: + """Every shape the helper accepts must either be preserved or warned about.""" + + def test_dict_content_is_treated_as_a_one_part_list(self, chat_endpoint) -> None: + part = {"type": "text", "text": "authored"} + + out = chat_endpoint._prepend_system_text("prefix", part) + + assert out == [{"type": "text", "text": "prefix\n\n"}, part] + + def test_empty_string_content_yields_bare_prefix_without_warning( + self, chat_endpoint, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + out = chat_endpoint._prepend_system_text("prefix", "") + + assert out == "prefix" + assert "Dropping authored system content" not in caplog.text + + def test_none_content_yields_bare_prefix_without_warning( + self, chat_endpoint, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING): + out = chat_endpoint._prepend_system_text("prefix", None) + + assert out == "prefix" + assert "Dropping authored system content" not in caplog.text + + @pytest.mark.parametrize( + "content", + [ + param(42, id="int"), + param(1.5, id="float"), + param(True, id="bool"), + ], + ) # fmt: skip + def test_unsupported_content_is_dropped_with_warning( + self, chat_endpoint, caplog: pytest.LogCaptureFixture, content: object + ) -> None: + with caplog.at_level(logging.WARNING): + out = chat_endpoint._prepend_system_text("prefix", content) + + assert out == "prefix" + assert "Dropping authored system content" in caplog.text + assert type(content).__name__ in caplog.text + + # --------------------------------------------------------------------------- # raw_messages=[] is an explicit zero-message delta, not a synthesis request # --------------------------------------------------------------------------- diff --git a/tests/unit/endpoints/test_responses_endpoint.py b/tests/unit/endpoints/test_responses_endpoint.py index 5b36169098..4b67c04034 100644 --- a/tests/unit/endpoints/test_responses_endpoint.py +++ b/tests/unit/endpoints/test_responses_endpoint.py @@ -178,6 +178,117 @@ def test_format_payload_no_system_message_no_instructions( assert "instructions" not in payload + def test_format_payload_authored_system_item_merges_not_duplicates( + self, endpoint, model_endpoint + ): + """An authored ``role: system`` input item collides with instructions. + + Both used to ship, so the server saw two system prompts. The verbatim + prompt is now folded into the authored item and ``instructions`` is + dropped, matching ChatEndpoint._format_messages. + """ + turn = Turn( + raw_messages=[ + {"role": "system", "content": "SYS-A", "type": "message"}, + {"role": "user", "content": "hi", "type": "message"}, + ], + model="test-model", + ) + request_info = create_request_info( + model_endpoint=model_endpoint, + turns=[turn], + system_message="VERBATIM-SYS", + ) + + payload = endpoint.format_payload(request_info) + + assert "instructions" not in payload + system_items = [i for i in payload["input"] if i.get("role") == "system"] + assert len(system_items) == 1 + assert system_items[0]["content"] == "VERBATIM-SYS\n\nSYS-A" + + def test_format_payload_authored_system_item_list_content_merges( + self, endpoint, model_endpoint + ): + """List content gets the Responses part shape, with the separator.""" + turn = Turn( + raw_messages=[ + { + "role": "system", + "content": [{"type": "input_text", "text": "SYS-A"}], + "type": "message", + }, + ], + model="test-model", + ) + request_info = create_request_info( + model_endpoint=model_endpoint, + turns=[turn], + system_message="VERBATIM-SYS", + ) + + payload = endpoint.format_payload(request_info) + + assert "instructions" not in payload + assert payload["input"][0]["content"] == [ + {"type": "input_text", "text": "VERBATIM-SYS\n\n"}, + {"type": "input_text", "text": "SYS-A"}, + ] + + def test_format_payload_merge_does_not_mutate_turn_state( + self, endpoint, model_endpoint + ): + """``raw_messages`` is reused across credits; a replay must not restack.""" + raw_messages = [ + {"role": "system", "content": "SYS-A", "type": "message"}, + ] + turn = Turn(raw_messages=raw_messages, model="test-model") + request_info = create_request_info( + model_endpoint=model_endpoint, + turns=[turn], + system_message="VERBATIM-SYS", + ) + + first = endpoint.format_payload(request_info) + second = endpoint.format_payload(request_info) + + assert first["input"][0]["content"] == "VERBATIM-SYS\n\nSYS-A" + assert second["input"][0]["content"] == "VERBATIM-SYS\n\nSYS-A" + assert raw_messages[0]["content"] == "SYS-A" + + def test_format_payload_merged_system_item_precedes_user_context( + self, endpoint, model_endpoint + ) -> None: + """The merged system item must be the leading input item. + + ``user_context_message`` used to be appended before the rendered turns + were extended, so the merged system item landed at index 1 behind a + user message -- user-controlled context ahead of the system policy, + diverging from ChatEndpoint and defeating SYSTEM_PREFIX cache-bust. + """ + turn = Turn( + raw_messages=[ + {"role": "system", "content": "DATASET POLICY", "type": "message"}, + {"role": "user", "content": "QUESTION", "type": "message"}, + ], + model="test-model", + ) + request_info = create_request_info( + model_endpoint=model_endpoint, + turns=[turn], + system_message="CLI POLICY", + user_context_message="USER CONTEXT", + ) + + payload = endpoint.format_payload(request_info) + + assert "instructions" not in payload + assert [(i["role"], i["content"]) for i in payload["input"]] == [ + ("system", "CLI POLICY\n\nDATASET POLICY"), + ("user", "USER CONTEXT"), + ("user", "QUESTION"), + ] + def test_format_payload_user_context_message(self, endpoint, model_endpoint): """user_context_message is prepended as a user input item.""" turn = Turn(texts=[Text(contents=["Hello"])], model="test-model") @@ -1236,3 +1347,285 @@ def test_format_payload_turn1_with_previous_response_id_sends_only_latest( assert payload["previous_response_id"] == "resp_b6c65395f4fb8c7d" assert len(payload["input"]) == 1 assert payload["input"][0]["content"] == "Second message" + + @staticmethod + def _system_user_turn(system: str, user: str) -> Turn: + return Turn( + raw_messages=[ + {"role": "system", "content": system, "type": "message"}, + {"role": "user", "content": user, "type": "message"}, + ], + ) + + @staticmethod + def _user_turn(user: str) -> Turn: + return Turn(raw_messages=[{"role": "user", "content": user, "type": "message"}]) + + def _request( + self, + endpoint: ResponsesEndpoint, + turns: list[Turn], + *, + previous_response_id: str | None = None, + in_history: bool = False, + ): + return create_request_info( + model_endpoint=endpoint.model_endpoint, + turns=turns, + system_message="CLI POLICY", + previous_response_id=previous_response_id, + system_prompt_in_stored_history=in_history, + ) + + def test_merged_request_reports_system_prompt_stored_and_next_omits_instructions( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """Request 1 merges into an input item; request 2 must not resend it.""" + turn0 = self._system_user_turn("DATASET POLICY", "FIRST") + first_info = self._request(store_endpoint, [turn0]) + first = store_endpoint.format_payload(first_info) + stored = store_endpoint.stored_history_has_system_prompt(first_info) + + second = store_endpoint.format_payload( + self._request( + store_endpoint, + [turn0, self._user_turn("SECOND")], + previous_response_id="resp_first", + in_history=stored, + ) + ) + + assert "instructions" not in first + assert first["input"][0]["content"] == "CLI POLICY\n\nDATASET POLICY" + assert stored is True + assert "instructions" not in second + assert [i["content"] for i in second["input"]] == ["SECOND"] + + def test_chain_after_failed_system_bearing_request_keeps_instructions( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """Request 2 merged the prompt but failed, so ``resp_1`` never stored it. + + The session still chains onto ``resp_1`` with the flag request 1 left, + and the attempted turn's system item must not suppress ``instructions``. + """ + first_info = self._request(store_endpoint, [self._user_turn("FIRST")]) + assert store_endpoint.stored_history_has_system_prompt(first_info) is False + + third = store_endpoint.format_payload( + self._request( + store_endpoint, + [ + self._user_turn("FIRST"), + self._system_user_turn("DATASET POLICY", "SECOND"), + self._user_turn("THIRD"), + ], + previous_response_id="resp_1", + in_history=False, + ) + ) + + assert third["instructions"] == "CLI POLICY" + assert [i["content"] for i in third["input"]] == ["THIRD"] + + def test_stateless_request_with_mid_history_system_item_reports_not_stored( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """After request 1 fails, request 2 goes stateless with the system item + mid-array; the prompt rides ``instructions``, which is not stored.""" + turns = [ + self._user_turn("FIRST"), + self._system_user_turn("DATASET POLICY", "SECOND"), + ] + info = self._request(store_endpoint, turns) + payload = store_endpoint.format_payload(info) + + assert payload["instructions"] == "CLI POLICY" + assert [i["content"] for i in payload["input"]] == [ + "FIRST", + "DATASET POLICY", + "SECOND", + ] + assert store_endpoint.stored_history_has_system_prompt(info) is False + + def test_chain_without_stored_system_prompt_keeps_instructions( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """``instructions`` does not carry across the chain, so it is resent.""" + info = self._request( + store_endpoint, + [self._user_turn("FIRST"), self._user_turn("SECOND")], + previous_response_id="resp_first", + ) + assert store_endpoint.format_payload(info)["instructions"] == "CLI POLICY" + assert store_endpoint.stored_history_has_system_prompt(info) is False + + def test_chained_turn_with_own_system_item_does_not_remerge( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """A later authored system item ships as authored once history has the prompt.""" + info = self._request( + store_endpoint, + [ + self._system_user_turn("DATASET POLICY", "FIRST"), + self._system_user_turn("TURN POLICY", "SECOND"), + ], + previous_response_id="resp_first", + in_history=True, + ) + payload = store_endpoint.format_payload(info) + + assert "instructions" not in payload + assert payload["input"][0]["content"] == "TURN POLICY" + assert store_endpoint.stored_history_has_system_prompt(info) is True + + @pytest.mark.parametrize("override_source", ["turn_extra", "endpoint_extra"]) + def test_input_override_drops_merged_prompt_and_next_resends_instructions( + self, store_endpoint: ResponsesEndpoint, override_source: str + ) -> None: + """An explicit ``input`` override wins on the wire, so the merged prompt + never reaches stored history and the next chained request must resend it.""" + override = [{"role": "user", "content": "OVERRIDE"}] + turn0 = self._system_user_turn("DATASET POLICY", "FIRST") + if override_source == "turn_extra": + turn0.extra_body = {"input": override} + else: + store_endpoint.model_endpoint.endpoint.extra = [("input", override)] + first_info = self._request(store_endpoint, [turn0]) + + first = store_endpoint.format_payload(first_info) + stored = store_endpoint.stored_history_has_system_prompt(first_info) + store_endpoint.model_endpoint.endpoint.extra = [] + second = store_endpoint.format_payload( + self._request( + store_endpoint, + [turn0, self._user_turn("SECOND")], + previous_response_id="resp_first", + in_history=stored, + ) + ) + + assert first["input"] == override + assert "instructions" not in first + assert stored is False + assert second["instructions"] == "CLI POLICY" + + @pytest.mark.parametrize( + "source", + [param("turn_extra", id="turn_extra"), param("endpoint_extra", id="endpoint_extra")], + ) # fmt: skip + def test_previous_response_id_override_sends_instructions_and_drops_flag( + self, store_endpoint: ResponsesEndpoint, source: str + ) -> None: + """The session's stored-prompt fact describes only the session's chain. + + Chaining onto another response leaves the request without that + history, so the current request must carry the prompt and the next + flag must not inherit the session's. (A ``None`` override instead ships + the full history; see the test below.) + """ + outgoing_id = "resp_elsewhere" + turn = self._user_turn("SECOND") + if source == "turn_extra": + turn.extra_body = {"previous_response_id": outgoing_id} + else: + store_endpoint.model_endpoint.endpoint.extra = [ + ("store", True), + ("previous_response_id", outgoing_id), + ] + info = self._request( + store_endpoint, + [self._system_user_turn("DATASET POLICY", "FIRST"), turn], + previous_response_id="resp_first", + in_history=True, + ) + + payload = store_endpoint.format_payload(info) + + assert payload["previous_response_id"] == outgoing_id + assert payload["instructions"] == "CLI POLICY" + assert store_endpoint.stored_history_has_system_prompt(info) is False + + @pytest.mark.parametrize( + "source", + [param("turn_extra", id="turn_extra"), param("endpoint_extra", id="endpoint_extra")], + ) # fmt: skip + def test_null_previous_response_id_override_ships_full_history( + self, store_endpoint: ResponsesEndpoint, source: str + ) -> None: + """A chain broken by ``None`` has no server-side history to replay, so + the request renders the whole conversation like a stateless one.""" + turn = self._user_turn("SECOND") + if source == "turn_extra": + turn.extra_body = {"previous_response_id": None} + else: + store_endpoint.model_endpoint.endpoint.extra = [ + ("store", True), + ("previous_response_id", None), + ] + info = create_request_info( + model_endpoint=store_endpoint.model_endpoint, + turns=[self._system_user_turn("DATASET POLICY", "FIRST"), turn], + system_message="CLI POLICY", + user_context_message="USER CONTEXT", + previous_response_id="resp_first", + system_prompt_in_stored_history=True, + ) + + payload = store_endpoint.format_payload(info) + + assert payload["previous_response_id"] is None + # Without the full render the policy was not on the wire at all. + assert orjson.dumps(payload).count(b"CLI POLICY") == 1 + assert "instructions" not in payload + assert [(i["role"], i["content"]) for i in payload["input"]] == [ + ("system", "CLI POLICY\n\nDATASET POLICY"), + ("user", "USER CONTEXT"), + ("user", "FIRST"), + ("user", "SECOND"), + ] + assert store_endpoint.stored_history_has_system_prompt(info) is True + + def test_other_previous_response_id_override_ships_latest_turn_only( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """Chaining onto another stored response still relies on its history.""" + turn = self._user_turn("SECOND") + turn.extra_body = {"previous_response_id": "resp_elsewhere"} + info = self._request( + store_endpoint, + [self._system_user_turn("DATASET POLICY", "FIRST"), turn], + previous_response_id="resp_first", + in_history=True, + ) + + payload = store_endpoint.format_payload(info) + + assert [i["content"] for i in payload["input"]] == ["SECOND"] + + def test_previous_response_id_override_to_session_id_keeps_suppression( + self, store_endpoint: ResponsesEndpoint + ) -> None: + """Restating the session's own ID is still the stored chain.""" + turn = self._user_turn("SECOND") + turn.extra_body = {"previous_response_id": "resp_first"} + info = self._request( + store_endpoint, + [self._system_user_turn("DATASET POLICY", "FIRST"), turn], + previous_response_id="resp_first", + in_history=True, + ) + + payload = store_endpoint.format_payload(info) + + assert "instructions" not in payload + assert store_endpoint.stored_history_has_system_prompt(info) is True + + def test_stored_history_has_system_prompt_false_without_system_message( + self, store_endpoint: ResponsesEndpoint + ) -> None: + info = create_request_info( + model_endpoint=store_endpoint.model_endpoint, + turns=[self._system_user_turn("DATASET POLICY", "FIRST")], + ) + assert store_endpoint.stored_history_has_system_prompt(info) is False diff --git a/tests/unit/workers/test_session_manager.py b/tests/unit/workers/test_session_manager.py index 55f3764276..c2047681a4 100644 --- a/tests/unit/workers/test_session_manager.py +++ b/tests/unit/workers/test_session_manager.py @@ -456,13 +456,14 @@ def test_advance_turn_clears_previous_response_id_on_reset_context( context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, ) session.advance_turn(0) - session.store_response_id("resp_turn0") + session.store_response_id("resp_turn0", system_prompt_in_history=True) assert session.previous_response_id == "resp_turn0" # A context reset always breaks the wire chain, regardless of whether the # turn carries raw_messages: chaining would retain discarded history. session.advance_turn(1) assert session.previous_response_id is None + assert session.system_prompt_in_stored_history is False def test_advance_turn_preserves_previous_response_id_without_reset_context( self, @@ -481,11 +482,12 @@ def test_advance_turn_preserves_previous_response_id_without_reset_context( context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, ) session.advance_turn(0) - session.store_response_id("resp_turn0") + session.store_response_id("resp_turn0", system_prompt_in_history=True) assert session.previous_response_id == "resp_turn0" session.advance_turn(1) assert session.previous_response_id == "resp_turn0" + assert session.system_prompt_in_stored_history is True def test_seed_from_parent_copies_previous_response_id(self) -> None: manager = UserSessionManager() @@ -494,7 +496,7 @@ def test_seed_from_parent_copies_previous_response_id(self) -> None: turns=[Turn(messages=[{"role": "user", "content": "Q1"}])], ) parent = manager.create_and_store("parent-corr", conv, num_turns=1) - parent.store_response_id("resp_parent_last") + parent.store_response_id("resp_parent_last", system_prompt_in_history=True) child = manager.create_and_store( "child-corr", conv, @@ -503,7 +505,37 @@ def test_seed_from_parent_copies_previous_response_id(self) -> None: branch_mode=ConversationBranchMode.FORK, ) assert child.previous_response_id is None + assert child.system_prompt_in_stored_history is False manager.seed_from_parent("child-corr", "parent-corr") assert child.previous_response_id == "resp_parent_last" + assert child.system_prompt_in_stored_history is True + + @pytest.mark.parametrize( + "response_id, in_history, expected", + [ + param("resp_1", True, True, id="stored_with_prompt"), + param("resp_1", False, False, id="stored_without_prompt"), + param(None, True, False, id="no_id_never_claims_prompt"), + ], + ) # fmt: skip + def test_store_response_id_sets_system_prompt_flag_with_id( + self, response_id: str | None, in_history: bool, expected: bool + ) -> None: + conv = Conversation( + conversation_id="test-conv-flag", + turns=[Turn(messages=[{"role": "user", "content": "Q1"}])], + ) + session = UserSession( + x_correlation_id="test-corr", + num_turns=1, + conversation=conv, + context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, + ) + session.store_response_id("resp_0", system_prompt_in_history=not expected) + + session.store_response_id(response_id, system_prompt_in_history=in_history) + + assert session.previous_response_id == response_id + assert session.system_prompt_in_stored_history is expected diff --git a/tests/unit/workers/test_worker.py b/tests/unit/workers/test_worker.py index ce5f499c60..7a3133b79b 100644 --- a/tests/unit/workers/test_worker.py +++ b/tests/unit/workers/test_worker.py @@ -12,14 +12,18 @@ Conversation, ErrorDetails, ParsedResponse, + RequestInfo, RequestRecord, SSEMessage, + Text, TextResponseData, Turn, ) from aiperf.config.phases import ConcurrencyPhase from aiperf.credit.structs import Credit, CreditContext from aiperf.dataset.memory_map_utils import PayloadTurnData +from aiperf.endpoints.openai_responses import ResponsesEndpoint +from aiperf.plugin.enums import EndpointType from aiperf.workers.session_manager import UserSession from aiperf.workers.worker import ( Worker, @@ -30,6 +34,7 @@ from tests.harness.fake_service_manager import FakeServiceManager as FakeServiceManager from tests.harness.fake_tokenizer import FakeTokenizer from tests.harness.fake_transport import FakeTransport as FakeTransport +from tests.unit.endpoints.conftest import create_model_endpoint @pytest.mark.parametrize( @@ -1062,6 +1067,174 @@ async def test_finalize_session_response_success_stores_new_id( assert session.previous_response_id == "resp_new" + async def test_finalize_session_response_success_records_system_prompt_flag( + self, + mock_worker: Worker, + sample_credit_context: CreditContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The flag is computed from the full RequestInfo that was sent, not the + slim RecordContext on the record.""" + monkeypatch.setattr( + mock_worker, "_process_responses_for_record", Mock(return_value=([], None)) + ) + monkeypatch.setattr(mock_worker, "_populate_response_metrics", Mock()) + endpoint = mock_worker.inference_client.endpoint + monkeypatch.setattr( + endpoint, "extract_response_id", Mock(return_value="resp_new") + ) + stored = Mock(return_value=True) + monkeypatch.setattr( + endpoint, "stored_history_has_system_prompt", stored, raising=False + ) + session = self._chaining_session() + request_info = Mock(spec=RequestInfo) + record = RequestRecord(timestamp_ns=1, start_perf_ns=1, end_perf_ns=2) + + mock_worker._finalize_session_response( + session, sample_credit_context, record, request_info + ) + + stored.assert_called_once_with(request_info) + assert session.previous_response_id == "resp_new" + assert session.system_prompt_in_stored_history is True + + async def test_finalize_real_responses_endpoint_other_id_override_records_false( + self, + mock_worker: Worker, + sample_credit_context: CreditContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Real ``RequestInfo`` + real ``ResponsesEndpoint`` flag computation. + + Storing the assistant turn appends to ``session.turn_list``. The sent + request must keep ending at the dispatch turn (``RequestInfo`` copies + the list), and an override chaining onto another response must record + the prompt as not stored, even though the session's own chain held it. + """ + endpoint = ResponsesEndpoint( + model_endpoint=create_model_endpoint( + EndpointType.RESPONSES, extra=[("store", True)] + ) + ) + monkeypatch.setattr(mock_worker.inference_client, "endpoint", endpoint) + monkeypatch.setattr( + endpoint, "extract_response_id", Mock(return_value="resp_new") + ) + assistant = Turn(role="assistant", texts=[Text(contents=["REPLY"])]) + monkeypatch.setattr( + mock_worker, + "_process_responses_for_record", + Mock(return_value=([], assistant)), + ) + monkeypatch.setattr(mock_worker, "_populate_response_metrics", Mock()) + + dispatch = Turn( + raw_messages=[{"role": "user", "content": "SECOND", "type": "message"}], + extra_body={"previous_response_id": "resp_elsewhere"}, + ) + conv = Conversation( + conversation_id="test-conv-override", + turns=[ + Turn( + raw_messages=[ + {"role": "system", "content": "DATASET POLICY"}, + {"role": "user", "content": "FIRST"}, + ] + ), + dispatch, + ], + ) + session = UserSession( + x_correlation_id="test-corr", + num_turns=2, + conversation=conv, + context_mode=ConversationContextMode.DELTAS_WITHOUT_RESPONSES, + ) + session.advance_turn(0) + session.store_response_id("resp_first", system_prompt_in_history=True) + session.advance_turn(1) + request_info = mock_worker._create_request_info( + session=session, + credit_context=sample_credit_context, + x_request_id="x-req", + system_message="CLI POLICY", + user_context_message=None, + ) + record = RequestRecord(timestamp_ns=1, start_perf_ns=1, end_perf_ns=2) + + mock_worker._finalize_session_response( + session, sample_credit_context, record, request_info + ) + + assert session.turn_list[-1] is assistant + assert request_info.turns[-1] == dispatch + assert session.previous_response_id == "resp_new" + assert session.system_prompt_in_stored_history is False + + @pytest.mark.parametrize("last_good_flag", [param(True, id="flag_true"), param(False, id="flag_false")]) # fmt: skip + async def test_finalize_session_response_error_keeps_last_good_system_prompt_flag( + self, + mock_worker: Worker, + sample_credit_context: CreditContext, + monkeypatch: pytest.MonkeyPatch, + last_good_flag: bool, + ) -> None: + """A failed request never reached stored history, so neither the id nor + the flag may move off the last successful response.""" + monkeypatch.setattr( + mock_worker, "_process_responses_for_record", Mock(return_value=([], None)) + ) + monkeypatch.setattr(mock_worker, "_populate_response_metrics", Mock()) + endpoint = mock_worker.inference_client.endpoint + monkeypatch.setattr( + endpoint, "extract_response_id", Mock(return_value="resp_partial") + ) + stored = Mock(return_value=not last_good_flag) + monkeypatch.setattr( + endpoint, "stored_history_has_system_prompt", stored, raising=False + ) + session = self._chaining_session() + session.store_response_id("resp_good", system_prompt_in_history=last_good_flag) + record = RequestRecord(error=ErrorDetails(message="boom", type="TestError")) + + mock_worker._finalize_session_response( + session, sample_credit_context, record, Mock(spec=RequestInfo) + ) + + stored.assert_not_called() + assert session.previous_response_id == "resp_good" + assert session.system_prompt_in_stored_history is last_good_flag + + async def test_process_credit_with_session_finalizes_with_sent_request_info( + self, + mock_worker: Worker, + sample_credit_context: CreditContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The record only carries a slim RecordContext, so the full RequestInfo + that was sent must reach the finalizer for the flag to be computed.""" + session = self._chaining_session() + monkeypatch.setattr( + mock_worker.session_manager, "get", Mock(return_value=session) + ) + sent = Mock(spec=RequestInfo) + monkeypatch.setattr( + mock_worker, "_create_request_info", Mock(return_value=sent) + ) + record = RequestRecord(timestamp_ns=1, start_perf_ns=1, end_perf_ns=2) + monkeypatch.setattr( + mock_worker, "_execute_request", AsyncMock(return_value=record) + ) + finalize = Mock() + monkeypatch.setattr(mock_worker, "_finalize_session_response", finalize) + + await mock_worker._process_credit_with_session( + sample_credit_context, "x-req", "test-corr", None + ) + + finalize.assert_called_once_with(session, sample_credit_context, record, sent) + @pytest.mark.asyncio class TestMakeFirstTokenCallback: diff --git a/tests/unit/workers/test_worker_cache_bust_injection.py b/tests/unit/workers/test_worker_cache_bust_injection.py index a8bdf34583..0cdf2e8972 100644 --- a/tests/unit/workers/test_worker_cache_bust_injection.py +++ b/tests/unit/workers/test_worker_cache_bust_injection.py @@ -252,7 +252,7 @@ def test_system_prefix_uses_existing_raw_system_role_when_no_conversation_system def _make_raw_system_session( - raw_system: list[dict], + raw_system: list[dict] | None, *, system_message: str | None = None, raw_messages: list[dict] | None = None, @@ -303,23 +303,63 @@ def test_system_target_injects_into_raw_system_blocks(target, marker, expected): assert session.turn_list[-1].raw_system == expected -def test_raw_system_outranks_conversation_system_message(): - """``raw_system`` wins over ``system_message`` on the wire, so marking the - string instead would ship unchanged system bytes.""" +def test_conversation_system_message_outranks_raw_system(): + """``system_message`` becomes ``system`` block 0, ahead of ``raw_system``. + + MessagesEndpoint prepends it as a leading text block, so it -- not + ``raw_system[0]`` -- is the first system content on the wire. Marking + ``raw_system`` instead would leave a constant leading block that the server + can still prefix-hit on, defeating SYSTEM_PREFIX. + """ session = _make_raw_system_session( - [{"type": "text", "text": "sys"}], system_message="ignored-on-wire" + [{"type": "text", "text": "sys"}], system_message="on-wire-first" ) credit = _make_credit( target=CacheBustTarget.SYSTEM_PREFIX, marker=_PREFIX_MARKER, turn_index=0 ) - out = _apply_cache_bust(session, credit, system_message="ignored-on-wire") + out = _apply_cache_bust(session, credit, system_message="on-wire-first") - assert out == "ignored-on-wire" - assert session.turn_list[-1].raw_system[0] == { - "type": "text", - "text": _PREFIX_MARKER.strip(), - } + assert out == _PREFIX_MARKER + "on-wire-first" + # raw_system is the lower-precedence carrier and must be left alone. + assert session.turn_list[-1].raw_system == [{"type": "text", "text": "sys"}] + + +def test_suffix_with_system_message_and_raw_system_marks_raw_system_tail(): + """``SYSTEM_SUFFIX`` must land at the end of the whole system section. + + MessagesEndpoint ships ``[system_message, *raw_system]``, so with both + carriers present the tail is the last ``raw_system`` block. Appending to + ``system_message`` instead would leave constant blocks trailing the marker, + inverting the suffix contract (shared bytes cacheable, unique tail). + """ + session = _make_raw_system_session( + [{"type": "text", "text": "sys"}], system_message="on-wire-first" + ) + credit = _make_credit( + target=CacheBustTarget.SYSTEM_SUFFIX, marker=_SUFFIX_MARKER, turn_index=0 + ) + + out = _apply_cache_bust(session, credit, system_message="on-wire-first") + + assert out == "on-wire-first" + assert session.turn_list[-1].raw_system == [ + {"type": "text", "text": "sys"}, + {"type": "text", "text": _SUFFIX_MARKER.strip()}, + ] + + +def test_suffix_with_system_message_and_no_raw_system_marks_system_message(): + """Without ``raw_system``, ``system_message`` is the entire system section.""" + session = _make_raw_system_session(None, system_message="only-carrier") + credit = _make_credit( + target=CacheBustTarget.SYSTEM_SUFFIX, marker=_SUFFIX_MARKER, turn_index=0 + ) + + out = _apply_cache_bust(session, credit, system_message="only-carrier") + + assert out == "only-carrier" + _SUFFIX_MARKER + assert session.turn_list[-1].raw_system is None def test_raw_system_outranks_system_role_raw_messages(): From dac6e191afa5c4c64a85adb7569a89a711b953de Mon Sep 17 00:00:00 2001 From: Elias Bermudez <6505145+debermudez@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:31:51 -0700 Subject: [PATCH 119/119] fix(transport): keep Unicode separators at SSE and eventstream value edges (#1571) Signed-off-by: Elias Bermudez (cherry picked from commit eecf78c1031093325b41d60ed1ce01234a2b4490) --- src/aiperf/common/models/record_models.py | 18 ++++-- src/aiperf/transports/sse_utils.py | 6 +- tests/unit/transports/test_sse_utils.py | 73 +++++++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index b96cdf44bc..a213105a14 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -52,6 +52,9 @@ _SSE_COMMENT_FIELD_NAME = "comment" _SSE_DATA_FIELD_NAME = "data" _SSE_DATA_PREFIX = "data:" +# ASCII padding only: str.strip() would also eat U+0085/U+2028/U+2029 and +# \x0b-\x0c/\x1c-\x1f, which are data, not SSE whitespace. +_SSE_PADDING = " \t" class MetricResult(JsonMetricResult): @@ -742,7 +745,7 @@ def parse(cls, raw_message: AnyStr, perf_ns: int) -> SSEMessage: packets=[ SSEField( name=_SSE_DATA_FIELD_NAME, - value=raw_message[len(_SSE_DATA_PREFIX) :].strip(), + value=raw_message[len(_SSE_DATA_PREFIX) :].strip(_SSE_PADDING), ) ], ) @@ -752,7 +755,7 @@ def parse(cls, raw_message: AnyStr, perf_ns: int) -> SSEMessage: if "\r" in raw_message: raw_message = raw_message.replace("\r\n", "\n").replace("\r", "\n") for line in raw_message.split("\n"): - if not (line := line.strip()): + if not (line := line.strip(_SSE_PADDING)): continue prev_value = message.packets[-1].value if message.packets else None @@ -775,7 +778,9 @@ def parse(cls, raw_message: AnyStr, perf_ns: int) -> SSEMessage: parts = line.split(":", 1) if len(parts) < 2: # Fields without a colon have no value, so the whole line is the field name - message.packets.append(SSEField(name=parts[0].strip(), value=None)) + message.packets.append( + SSEField(name=parts[0].strip(_SSE_PADDING), value=None) + ) continue field_name, value = parts @@ -784,11 +789,14 @@ def parse(cls, raw_message: AnyStr, perf_ns: int) -> SSEMessage: # Field name is empty, so this is a comment field_name = _SSE_COMMENT_FIELD_NAME - # Spec says strip only one leading space; we strip() all whitespace + # Spec says strip only one leading space; we strip all ASCII padding # to normalize inconsistent servers for downstream exact comparisons # (e.g. "[DONE]", "error"). message.packets.append( - SSEField(name=field_name.strip(), value=value.strip()) + SSEField( + name=field_name.strip(_SSE_PADDING), + value=value.strip(_SSE_PADDING), + ) ) return message diff --git a/src/aiperf/transports/sse_utils.py b/src/aiperf/transports/sse_utils.py index 03c41567ad..d8438eaccc 100644 --- a/src/aiperf/transports/sse_utils.py +++ b/src/aiperf/transports/sse_utils.py @@ -14,6 +14,8 @@ _SSE_DATA_FIELD_NAME = "data" _SSE_ERROR_EVENT_VALUE = "error" _SSE_EVENT_FIELD_NAME = "event" +# Bytes-level, so non-ASCII separators and \x0b-\x0c survive into the parser. +_SSE_MESSAGE_PADDING = b" \t\r\n" class AsyncSSEStreamReader: @@ -164,8 +166,8 @@ async def __aiter__(self) -> AsyncIterator[SSEMessage]: raw_message = ( buffer[consumed:delimiter_index] + .strip(_SSE_MESSAGE_PADDING) .decode("utf-8", errors="replace") - .strip() ) consumed = delimiter_index + delimiter_length search_offset = consumed @@ -190,7 +192,7 @@ async def __aiter__(self) -> AsyncIterator[SSEMessage]: # Handle any remaining data in buffer after stream ends # Some servers don't send final delimiter - remaining = buffer[consumed:].strip() + remaining = buffer[consumed:].strip(_SSE_MESSAGE_PADDING) if remaining: final_perf_ns = time.perf_counter_ns() raw_message = remaining.decode("utf-8", errors="replace") diff --git a/tests/unit/transports/test_sse_utils.py b/tests/unit/transports/test_sse_utils.py index ba462ce90b..4d38a22152 100644 --- a/tests/unit/transports/test_sse_utils.py +++ b/tests/unit/transports/test_sse_utils.py @@ -516,6 +516,50 @@ def test_parse_named_event_preserves_unicode_plain_text( assert [packet.name for packet in result.packets] == ["event", "data"] +class TestParseSSEMessageEdgeWhitespace: + """Only ASCII space and tab are stripped from field edges.""" + + @pytest.mark.parametrize( + "prefix,value", + [ + param("", "\u2028", id="fast-path-line-separator-only"), + param("", "tok\u2029", id="fast-path-trailing-paragraph-separator"), + param("", "\x1ftok", id="fast-path-leading-unit-separator"), + param("event: message\n", "\u0085tok", id="named-event-leading-next-line"), + param("event: message\n", "tok\x0c", id="named-event-trailing-form-feed"), + ], + ) # fmt: skip + def test_parse_preserves_non_padding_edges( + self, prefix: str, value: str, base_perf_ns: int + ) -> None: + result = SSEMessage.parse(f"{prefix}data: {value}", base_perf_ns) + + assert result.get_text() == value + + @pytest.mark.parametrize( + "prefix", + [ + param("", id="fast-path"), + param("event: message\n", id="named-event"), + ], + ) # fmt: skip + def test_parse_strips_ascii_padding_around_done( + self, prefix: str, base_perf_ns: int + ) -> None: + result = SSEMessage.parse(f"{prefix}data: \t [DONE] \t", base_perf_ns) + + assert result.extract_data_content() == "[DONE]" + assert result.get_json() is None + + def test_parse_continuation_line_preserves_leading_separator( + self, base_perf_ns: int + ) -> None: + result = SSEMessage.parse('event: x\ndata: {"a":"b\n\u2028c"}', base_perf_ns) + + assert result.extract_data_content() == '{"a":"b\\n\u2028c"}' + assert result.get_json() == {"a": "b\n\u2028c"} + + class TestParseSSEMessageIncompleteJSON: """Tests for SSE messages where a literal newline in a JSON value splits a data field.""" @@ -784,6 +828,35 @@ async def chunks() -> AsyncIterator[bytes]: assert messages[0].get_json() == payload +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [ + param("event: message\ndata: tok\u2028\n\n", id="delimited"), + param("event: message\ndata: tok\u2028", id="no-final-delimiter"), + ], +) # fmt: skip +async def test_sse_reader_preserves_trailing_separator(body: str) -> None: + async def chunks() -> AsyncIterator[bytes]: + yield body.encode() + + messages = await AsyncSSEStreamReader(chunks()).read_complete_stream() + + assert [message.get_text() for message in messages] == ["tok\u2028"] + + +@pytest.mark.asyncio +async def test_sse_reader_preserves_trailing_form_feed_without_final_delimiter() -> ( + None +): + async def chunks() -> AsyncIterator[bytes]: + yield b"data: tok\x0c" + + messages = await AsyncSSEStreamReader(chunks()).read_complete_stream() + + assert [message.get_text() for message in messages] == ["tok\x0c"] + + @pytest.mark.asyncio class TestAsyncSSEStreamReaderParsing: """Tests for SSE stream reader buffer management and delimiter handling."""