diff --git a/mkdocs.yml b/mkdocs.yml index 84ac0f3b4..efde929f5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -103,6 +103,8 @@ plugins: "docs/tasks.md": "docs/concepts/tasks.md" "docs/services.md": "docs/concepts/services.md" "docs/fleets.md": "docs/concepts/fleets.md" + "docs/concepts/endpoints.md": "docs/concepts/presets.md" + "docs/reference/cli/dstack/endpoint.md": "docs/reference/cli/dstack/preset.md" "docs/examples/llms/llama31.md": "docs/examples/inference/vllm.md" "docs/examples/llms/llama32.md": "docs/examples/inference/vllm.md" "docs/examples/llms/qwen36.md": "docs/examples/models/qwen36.md" diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index 1ea18073f..a10f5a40c 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -5,7 +5,7 @@ description: Creating and reusing optimized model inference configurations # Presets -A preset configuration lets you use an agent to create a preset: a validated and optimized model inference configuration. Once created, the preset can be reused to deploy model inference on validated hardware without an agent. +A preset configuration lets you use an agent to create a preset: a verified and optimized model inference configuration. Once created, the preset can be reused to deploy model inference on verified hardware without an agent. The value of presets comes from combining two fundamental features: agent-driven model inference optimization and the `dstack` [service](services.md) primitive, which can deploy model inference to any cloud, Kubernetes, or on-prem cluster. @@ -25,14 +25,26 @@ The filename must end with `.dstack.yml` (e.g. `.dstack.yml` or `preset.dstack.y ```yaml type: preset -name: qwen25-7b +name: dsv4-flash # The agent picks a compatible variant of the base model -base: Qwen/Qwen2.5-7B-Instruct +base: deepseek-ai/DeepSeek-V4-Flash # The number of benchmarked trials -max_trials: 3 +trials: 5 +# The requirements the preset must meet (time to first token is in milliseconds) +min_context_length: 1048576 +max_ttft: 675 + +# The number of simultaneous requests every benchmark uses +concurrency: 1 + +# The request shape every benchmark uses (defaults to 1024 and 1024) +input_tokens: 10000 +output_tokens: 1500 + +# The environment variables the agent may pass to runs env: - HF_TOKEN ``` @@ -44,19 +56,23 @@ To create the preset, pass the configuration to the `dstack preset create` comma
```shell -$ dstack preset create -f preset.dstack.yml -Create the preset qwen25-7b? [y/n]: y -[2026-07-15 11:32:01] Starting preset creation for Qwen/Qwen2.5-7B-Instruct. Allowed fleets: gpu-fleet. -[2026-07-15 11:41:06] Prototype task qwen25-7b-a1b2c3-2 verified vLLM on an L4:24GB. -[2026-07-15 11:52:06] Final service qwen25-7b-a1b2c3-3 verified with context length 32768. -[2026-07-15 11:52:18] Benchmark via guidellm 0.7.1: 32/32 requests succeeded. +$ dstack preset create -f preset.dstack.yml --fleet b200-fleet +Create the preset dsv4-flash? [y/n]: y +[2026-08-04 11:38:34] Starting preset creation for deepseek-ai/DeepSeek-V4-Flash. Allowed fleets: b200-fleet. +[2026-08-04 12:31:19] Trial 3 switched from vLLM to SGLang: 319 tok/s per user, 2.2x the baseline. +[2026-08-04 13:04:52] Final service dsv4-flash-c83375b4-4 verified with context length 1048576. +[2026-08-04 13:12:07] Benchmark via sglang.bench_serving: 32/32 requests succeeded. ```
-The command executes entirely locally and uses the locally installed `claude` CLI along with `dstack`'s bundled skills. The agent uses a `dstack` task to find the best serving configuration for the available fleet offers, then submits it as a `dstack` service for a final benchmark. The validated preset is saved locally under `~/.dstack/presets`. +> It's highly recommended to specify the exact hardware you want the preset to use, so that the +> optimization is done against that hardware. Point `dstack preset create` to a fleet configured +> correspondingly, via `fleets` inside the preset configuration or via `--fleet` in the CLI. + +The command executes entirely locally and uses the locally installed `claude` CLI along with `dstack`'s bundled skills. The agent uses a `dstack` task to find the best serving configuration for the available fleet offers, then submits it as a `dstack` service for a final benchmark. -You can stop watching with Ctrl+C at any time. The agent keeps running, and `dstack preset logs -f` follows it again. Resume an interrupted creation with `dstack preset create --resume`: +You can stop watching with `Ctrl`+`C` at any time. The agent keeps running, and `dstack preset logs -f` follows it again. Resume an interrupted creation with `dstack preset create --resume`:
@@ -66,26 +82,40 @@ $ dstack preset create -f preset.dstack.yml --resume a1b2c3d4
+When resuming, the constraints are read from the original session, not from the configuration file. Editing them and resuming has no effect. To change any of them, create a new preset. + To stop a creation and its runs, use `dstack preset stop`. -!!! info "Claude configuration" +??? info "Claude configuration" By default, preset creation uses the existing `claude` login. To use an Anthropic API key instead, set: ```shell export DSTACK_AGENT_ANTHROPIC_API_KEY=... ``` - By default, the agent uses `claude-opus-4-8` and the default `claude` CLI effort. To override them, set: + By default, the agent uses `claude-opus-4-8`. It doesn't set an effort level, so the `claude` CLI default applies. To override them, set: ```shell - export DSTACK_AGENT_ANTHROPIC_MODEL=claude-fable-5 - export DSTACK_AGENT_CLAUDE_EFFORT=high + export DSTACK_AGENT_ANTHROPIC_MODEL=claude-opus-5 + export DSTACK_AGENT_CLAUDE_EFFORT=max ``` Supported effort levels are `low`, `medium`, `high`, `xhigh`, and `max`. +??? info "Presets directory" + The verified presets are saved locally under `~/.dstack/presets`, and `dstack preset` reads them from there. Presets aren't stored on the server. + ## Configuration options +### Fleets + +Set `fleets` to restrict creation and reuse to specific [fleets](fleets.md). It's highly recommended to specify a fleet with exactly the hardware that you'd like the preset to use. + +Alternatively, pass `--fleet` to `dstack preset create` or `dstack preset apply`. + +> Profile settings such as `spot_policy`, `max_price`, and `backends` are ignored during preset +> creation. Configure them on the fleet instead. + ### Model === "Base" @@ -104,21 +134,27 @@ To stop a creation and its runs, use `dstack preset stop`. repo: Qwen/Qwen2.5-7B-Instruct ``` -### Trials +### Shared prefix -`max_trials` is required and sets how many benchmarked trials the agent runs before promoting the best one. Set `concurrency` to control the benchmark concurrency. +By default every request is unique, so the cache hit rate is near zero. Set `shared_prefix_tokens` to control how much of each request the serving framework can serve from its prefix cache. -### Context length +
-Set `context_length` to require a minimum supported context length. +```yaml +input_tokens: 8192 +output_tokens: 1024 -### Fleets +# Roughly 90% of prompt tokens can be served from cache +shared_prefix_tokens: 7360 +``` -Set `fleets` to restrict creation and reuse to specific [fleets](fleets.md). Placement properties such as `backends`, `max_price`, and `spot_policy` constrain both creation and reuse too. +
+ +The `shared_prefix_tokens` value is the part of `input_tokens` that is identical across requests, such as a system prompt or conversation history, and must be less than `input_tokens`. ### Prompt -Set `prompt` to guide the agent with custom objectives, target metrics, or an experimentation approach. It accepts inline text or a file `path`. +The `prompt` property is optional. Set it to guide the agent with custom objectives, target metrics, or an experimentation approach. It accepts inline text or a file `path`.
@@ -129,6 +165,10 @@ prompt: |
+### Baseline + +Set `baseline: true` to make the first trial a baseline: the agent serves the model the way the chosen serving framework recommends, without tuning it for performance. Later trials are optimization attempts. + !!! info "Reference" The `preset` configuration supports many more options. See the [`.dstack.yml` reference](../reference/dstack.yml/preset.md). @@ -139,20 +179,23 @@ To deploy a preset as a service, pass the preset configuration and the preset ID
```shell -$ dstack preset apply -f preset.dstack.yml --id 532f3f4b +$ dstack preset apply -f preset.dstack.yml --id c83375b4 Project main User admin Type service - Resources cpu=2.. mem=8GB.. disk=100GB.. gpu=RTXPRO4500:32GB:1.. + Resources cpu=8.. mem=64GB.. disk=500GB gpu=B200:180GB:2 Spot policy on-demand Max price off - Model Qwen/Qwen3.5-27B (base) - Preset 532f3f4b (ctx=8K con=8 387 tok/s TTFT 582ms) + Retry policy off + Idle duration 5m + Max duration off + Model deepseek-ai/DeepSeek-V4-Flash (base) + Preset c83375b4 (io=10000/1500 conc=1 tok/s/user=309 tok/s=296 ttft=213ms ctx=1M) - # BACKEND RESOURCES INSTANCE TYPE PRICE - 1 runpod (EU-RO-1) cpu=12 mem=54GB disk=100GB gpu=RTXPRO4500:32GB:1 NVIDIA RTX PRO 4500 Blackwell $0.74 + # BACKEND RESOURCES INSTANCE TYPE PRICE + 1 runpod (US-CA-2) cpu=48 mem=502GB disk=500GB gpu=B200:180GB:2 NVIDIA B200 $11.78 -Submit the run qwen35-27b? [y/n]: y +Submit the run dsv4-flash? [y/n]: y ```
@@ -167,20 +210,32 @@ Use `dstack preset` to list presets: ```shell $ dstack preset list - BASE ID GPU BENCHMARK STATUS SUBMITTED NAME - Qwen/Qwen2.5-0.5B - bc592b38 clauding (0/3) 23 sec ago qwen05 - Qwen/Qwen3-32B - f91d6b60 RTX5090:32GB:1 con=8 576 tok/s TTFT 368ms verified (10/10) 2 days ago qwen3-32b - Qwen/Qwen3.5-27B - 3c4d5e6f verifying (3/3) 2 min ago qwen35-27b-2 - 532f3f4b RTXPRO4500:32GB:1.. con=8 387 tok/s TTFT 582ms verified (4/4) yesterday qwen35-27b - d1c2e12b RTX5090:32GB:1 con=8 266 tok/s TTFT 2.15s verified (7/10) yesterday + ID BASE GPU CONSTRAINTS BENCHMARK STATUS SUBMITTED + c83375b4 deepseek-ai/DeepSeek-V4-Flash B200:180GB:2 io=10000/1500 conc=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago ``` -Presets are grouped by base model. In-progress creations appear too, with a live status like `clauding` or `verifying`. Pass `-w` to watch in realtime. Pass `-v` to include validation resources and all benchmark metrics, or `--json` for complete preset objects. Filter with `--base` or `--repo`. +By default, `dstack preset` shows creations that are still running, or the most recent one if none are. Pass `-a` to show every preset, or `-n` to show the last N: + +
+ +```shell +$ dstack preset list -a + ID BASE GPU CONSTRAINTS BENCHMARK STATUS SUBMITTED + c83375b4 deepseek-ai/DeepSeek-V4-Flash B200:180GB:2 io=10000/1500 conc=1 tok/s/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + 092c792b Qwen/Qwen3.5-397B-A17B RTXPRO6000:4 io=8K/1K conc=64 tok/s/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago + 9ab0fa65 Qwen/Qwen3.6-27B RTXPRO4500:1 io=1K/1K conc=8 tok/s/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago + f91d6b60 Qwen/Qwen3-32B RTX5090:32GB:1 io=1K/512 conc=8 tok/s/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago +``` + +
+ +The `CONSTRAINTS` column is what the creation was asked for, and `BENCHMARK` is the best trial so far. `tok/s/user` is the steady decode rate, measured as one second divided by the median time per output token, so it excludes the time to the first token. + +The glyphs after the benchmark are one per trial: height is throughput, a yellow bar is a trial whose benchmark broke a constraint, and a red `·` is one that produced no benchmark at all. The shape shows whether a run converged or wandered. + +Pass `-w` to watch in realtime, `-v` for more detail, or `--json` for complete preset objects. Filter with `--base` or `--repo`. ### Delete presets @@ -189,14 +244,22 @@ Delete a preset by ID or name, or all presets for a base model with `--base`:
```shell -$ dstack preset delete 8f3a12c4 +$ dstack preset delete c83375b4 ```
For command options and agent settings, see the [`dstack preset` CLI reference](../reference/cli/dstack/preset.md). -> Presets are experimental, and we’d love your feedback. Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). +!!! info "Roadmap and feedback" + Here's what is coming soon: + + * Allow the agent to change the source code, compile binaries, etc. + * Support for PD disaggregation + * Allow passing multiple `--previous ` to `dstack preset create` to reuse the insights from previous sessions + * Allow passing ranges to `concurrency` + + Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). !!! info "What's next?" 1. Learn how dstack [services](services.md) work diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index ee0e2690b..562b3f31b 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -104,10 +104,10 @@ def _register(self) -> None: help="Leave the verified service running", ) create_parser.add_argument( - "--max-trials", + "--trials", type=int, metavar="N", - help="The maximum number of benchmarked trials before the best one is promoted", + help="The number of benchmarked trials before the best one is promoted", ) create_parser.add_argument( "--debug", @@ -235,12 +235,26 @@ def _list(self, args: argparse.Namespace) -> None: verbose = args.verbose if not getattr(args, "watch", False): presets, sessions = self._list_presets_and_sessions(base=base, repo=repo) - print_presets(presets, sessions=sessions, verbose=verbose) + print_presets( + presets, + sessions=sessions, + verbose=verbose, + all_presets=args.all_presets, + limit=args.limit, + ) return with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live: while True: presets, sessions = self._list_presets_and_sessions(base=base, repo=repo) - live.update(get_presets_table(presets, sessions=sessions, verbose=verbose)) + live.update( + get_presets_table( + presets, + sessions=sessions, + verbose=verbose, + all_presets=args.all_presets, + limit=args.limit, + ) + ) time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS) def _list_presets_and_sessions( @@ -267,18 +281,21 @@ def _create(self, args: argparse.Namespace) -> None: resume_session = None if getattr(args, "resume", None): resume_session = load_resumable_agent_session(args.resume) - if getattr(args, "max_trials", None) is not None: + if getattr(args, "trials", None) is not None: console.print( - "[warning]--max-trials is ignored when resuming: " + "[warning]--trials is ignored when resuming: " "the constraints are fixed at creation[/]" ) api = Client.from_config(project_name=args.project) allowed_fleets = None if resume_session is None: - if configuration.max_trials is None: + if configuration.trials is None: raise ConfigurationError( - "max_trials is required. Set it in the configuration or pass --max-trials" + "trials is required. Set it in the configuration or pass --trials" ) + for field in ("max_ttft", "min_context_length", "concurrency"): + if getattr(configuration, field) is None: + raise ConfigurationError(f"{field} is required") allowed_fleets = plan_preset(api=api, configuration=configuration) if not _confirm_preset_creation(store, configuration.name, assume_yes=args.yes): console.print("\nExiting...") @@ -408,6 +425,21 @@ def _add_list_args(parser: argparse.ArgumentParser) -> None: action="store_true", help="Output in JSON format", ) + parser.add_argument( + "-a", + "--all", + action="store_true", + dest="all_presets", + help="Show all presets. By default, it only shows unfinished creations or the last one.", + ) + parser.add_argument( + "-n", + "--last", + metavar="COUNT", + type=int, + dest="limit", + help="Show only the last N presets. Implies --all", + ) model_filter = parser.add_mutually_exclusive_group() model_filter.add_argument( "--base", @@ -490,8 +522,8 @@ def _get_effective_configuration( require_name: bool = True, ) -> PresetConfiguration: _apply_name(configuration, args.name, required=require_name) - if getattr(args, "max_trials", None) is not None: - configuration.max_trials = args.max_trials + if getattr(args, "trials", None) is not None: + configuration.trials = args.trials profile = load_profile_from_args(args=args, repo_dir=Path.cwd()) for field in ProfileParams.model_fields: if getattr(configuration, field) is None: diff --git a/src/dstack/_internal/cli/models/configurations.py b/src/dstack/_internal/cli/models/configurations.py index f7a355424..e1bb40a7c 100644 --- a/src/dstack/_internal/cli/models/configurations.py +++ b/src/dstack/_internal/cli/models/configurations.py @@ -6,6 +6,7 @@ field_validator, model_validator, ) +from typing_extensions import Self from dstack._internal.core.models.common import ( CoreModel, @@ -14,7 +15,9 @@ from dstack._internal.core.models.envs import Env from dstack._internal.core.models.profiles import ProfileParams -DEFAULT_CONCURRENCY = 8 +DEFAULT_INPUT_TOKENS = 1024 +DEFAULT_OUTPUT_TOKENS = 1024 +DEFAULT_BASELINE = False class PresetModelRepo(CoreModel): @@ -130,14 +133,23 @@ class PresetConfiguration( ) ), ] = None - context_length: Annotated[ + min_context_length: Annotated[ Optional[PositiveInt], Field(description="The minimum required context length") ] = None - max_trials: Annotated[ + max_ttft: Annotated[ Optional[PositiveInt], Field( description=( - "The maximum number of benchmarked trials during preset creation" + "The maximum p50 time to first token, in milliseconds, that any benchmark" + " may report" + ) + ), + ] = None + trials: Annotated[ + Optional[PositiveInt], + Field( + description=( + "The number of benchmarked trials during preset creation" " before the best one is promoted" ) ), @@ -146,8 +158,45 @@ class PresetConfiguration( Optional[PositiveInt], Field( description=( - "The number of simultaneous requests used for benchmarks during" - f" preset creation. Defaults to `{DEFAULT_CONCURRENCY}`" + "The number of simultaneous requests used for benchmarks during preset creation" + ) + ), + ] = None + input_tokens: Annotated[ + Optional[PositiveInt], + Field( + description=( + "The number of input tokens per request used for benchmarks during" + f" preset creation. Defaults to `{DEFAULT_INPUT_TOKENS}`" + ) + ), + ] = None + output_tokens: Annotated[ + Optional[PositiveInt], + Field( + description=( + "The number of output tokens per request used for benchmarks during" + f" preset creation. Defaults to `{DEFAULT_OUTPUT_TOKENS}`" + ) + ), + ] = None + shared_prefix_tokens: Annotated[ + Optional[PositiveInt], + Field( + description=( + "How many of `input_tokens` are a prefix identical in every benchmark request," + " as a repeated system prompt or conversation history would be. Defaults to `0`," + " meaning every request is fully unique" + ) + ), + ] = None + baseline: Annotated[ + Optional[bool], + Field( + description=( + "Whether the first trial must be a baseline that serves the model with the" + " serving framework's recommended defaults instead of an optimization attempt." + " Defaults to `false`" ) ), ] = None @@ -167,8 +216,29 @@ class PresetConfiguration( ) @property - def effective_concurrency(self) -> int: - return self.concurrency if self.concurrency is not None else DEFAULT_CONCURRENCY + def effective_input_tokens(self) -> int: + return self.input_tokens if self.input_tokens is not None else DEFAULT_INPUT_TOKENS + + @property + def effective_output_tokens(self) -> int: + return self.output_tokens if self.output_tokens is not None else DEFAULT_OUTPUT_TOKENS + + @property + def effective_baseline(self) -> bool: + return self.baseline if self.baseline is not None else DEFAULT_BASELINE + + @model_validator(mode="after") + def validate_shared_prefix_tokens(self) -> Self: + # The prefix is carved out of the request, so something has to be left + # to differ between requests. + if self.shared_prefix_tokens is None: + return self + input_tokens = self.input_tokens or DEFAULT_INPUT_TOKENS + if self.shared_prefix_tokens >= input_tokens: + raise ValueError( + f"shared_prefix_tokens must be less than input_tokens ({input_tokens})" + ) + return self @model_validator(mode="before") @classmethod @@ -211,9 +281,14 @@ class PresetConstraints(CoreModel): run_name_prefix: str model: PresetModelSpec - context_length: Optional[PositiveInt] = None - max_trials: PositiveInt + min_context_length: PositiveInt + max_ttft: PositiveInt + trials_num: PositiveInt concurrency: PositiveInt + input_tokens: PositiveInt + output_tokens: PositiveInt + shared_prefix_tokens: int = 0 + baseline: bool = False fleets: list[str] = Field(min_length=1) env: list[str] = [] diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 8c1710d76..25c740083 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -35,6 +35,7 @@ "input_tokens": {"type": "integer", "minimum": 1}, "output_tokens": {"type": "integer", "minimum": 2}, "concurrency": {"type": "integer", "minimum": 1}, + "shared_prefix_tokens": {"type": "integer", "minimum": 0}, }, "required": [ "api", @@ -42,6 +43,7 @@ "input_tokens", "output_tokens", "concurrency", + "shared_prefix_tokens", ], "additionalProperties": False, }, @@ -53,6 +55,8 @@ "duration_seconds": {"type": "number", "exclusiveMinimum": 0}, "total_input_tokens": {"type": "integer", "minimum": 0}, "total_output_tokens": {"type": "integer", "minimum": 0}, + "output_tok_per_s": {"type": "number", "exclusiveMinimum": 0}, + "per_user_tok_per_s": {"type": "number", "exclusiveMinimum": 0}, "ttft_ms": _LATENCY_JSON_SCHEMA, "tpot_ms": _LATENCY_JSON_SCHEMA, }, @@ -62,6 +66,8 @@ "duration_seconds", "total_input_tokens", "total_output_tokens", + "output_tok_per_s", + "per_user_tok_per_s", "ttft_ms", "tpot_ms", ], diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index 500bcf42b..a61f9ccdf 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -23,6 +23,9 @@ class PresetBenchmarkWorkload(CoreModel): input_tokens: PositiveInt output_tokens: Annotated[int, Field(ge=2)] concurrency: PositiveInt + # Defaulted rather than required: presets saved before this field existed + # must still load, and for them the benchmark was fully unique. + shared_prefix_tokens: Annotated[int, Field(ge=0)] = 0 class PresetBenchmarkLatency(CoreModel): @@ -37,6 +40,10 @@ class PresetBenchmarkMetrics(CoreModel): duration_seconds: PositiveFloat total_input_tokens: Annotated[int, Field(ge=0)] total_output_tokens: Annotated[int, Field(ge=0)] + # Defaulted rather than required: presets saved before these fields existed + # must still load. The `effective_*` properties derive them when absent. + output_tok_per_s: Optional[PositiveFloat] = None + per_user_tok_per_s: Optional[PositiveFloat] = None ttft_ms: PresetBenchmarkLatency tpot_ms: PresetBenchmarkLatency @@ -58,6 +65,19 @@ class PresetBenchmark(CoreModel): target: Optional[PresetBenchmarkTarget] = None client: Optional[PresetBenchmarkClient] = None + @property + def effective_output_tok_per_s(self) -> float: + """Performance as defined in the agent prompt's `## Performance`. Derived + rather than read, so a miscomputed field cannot become the displayed truth.""" + return self.metrics.total_output_tokens / self.metrics.duration_seconds + + @property + def effective_per_user_tok_per_s(self) -> float: + """Per-user output speed as the serving literature defines it: the steady + decode rate, `1/TPOT`, which excludes time to first token. Dividing the + aggregate by concurrency instead folds TTFT and the ramp into it.""" + return 1000 / self.metrics.tpot_ms.p50 + @field_validator("tool", "tool_version", "command") @classmethod def validate_non_empty(cls, value: str) -> str: diff --git a/src/dstack/_internal/cli/services/presets/agent.py b/src/dstack/_internal/cli/services/presets/agent.py index 201c2ed35..6dd1d8bbb 100644 --- a/src/dstack/_internal/cli/services/presets/agent.py +++ b/src/dstack/_internal/cli/services/presets/agent.py @@ -423,6 +423,14 @@ async def _session_tailers( offset_key="trials", echo=agent_session.echo, ), + _RecordMirror( + source=workspace.verifications_path, + target=agent_session.verifications_path, + redacted_values=redacted_values, + offset_store=offset_store, + offset_key="verifications", + echo=agent_session.echo, + ), ] tailer_tasks = [ asyncio.create_task(tailer.run()) for tailer in [progress_tailer, *record_mirrors] diff --git a/src/dstack/_internal/cli/services/presets/apply.py b/src/dstack/_internal/cli/services/presets/apply.py index fe6eb837a..f5a15d22f 100644 --- a/src/dstack/_internal/cli/services/presets/apply.py +++ b/src/dstack/_internal/cli/services/presets/apply.py @@ -8,6 +8,7 @@ from dstack._internal.cli.services.configurators.run import ServiceConfigurator from dstack._internal.cli.services.presets.output import ( format_preset_benchmark, + format_preset_objective, ) from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.core.errors import CLIError @@ -58,11 +59,11 @@ def _validate_preset_matches(preset: Preset, *, configuration: PresetConfigurati service_model = preset.service.model if service_model is None or service_model.name.lower() != model_name.lower(): raise CLIError(f"Preset {preset.id} does not serve {model_name}") - if configuration.context_length is not None: - if preset.context_length < configuration.context_length: + if configuration.min_context_length is not None: + if preset.context_length < configuration.min_context_length: raise CLIError( f"Preset {preset.id} does not support context length" - f" {configuration.context_length}" + f" {configuration.min_context_length}" ) if configuration.model.allows_variant_selection: if preset.base.lower() != model_name.lower(): @@ -94,5 +95,7 @@ def _format_requested_model(configuration: PresetConfiguration) -> str: def _format_selected_preset(preset: Preset) -> str: - details = format_preset_benchmark(preset, verbose=True) - return f"{escape(preset.id)} ([secondary]{details}[/])" + # The formatter dims its own keys; wrapping it again would flatten that. + # One line, so the objective and the result are joined rather than columned. + details = f"{format_preset_objective(preset)} {format_preset_benchmark(preset, verbose=True)}" + return f"{escape(preset.id)} ({details})" diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index 6c84c0c3c..4ca4d3665 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -554,7 +554,10 @@ async def _create_preset( workspace=setup.workspace, token=token, ) - prompt = get_preset_agent_system_prompt(user_prompt=setup.user_prompt) + prompt = get_preset_agent_system_prompt( + user_prompt=setup.user_prompt, + baseline=configuration.effective_baseline, + ) if setup.write_constraints: if setup.user_prompt: agent_session.write_user_prompt(setup.user_prompt) @@ -849,9 +852,14 @@ def _build_constraints( { "run_name_prefix": build_name, "model": json.loads(configuration.model.model_dump_json(exclude_none=True)), - "context_length": configuration.context_length, - "max_trials": configuration.max_trials, - "concurrency": configuration.effective_concurrency, + "min_context_length": configuration.min_context_length, + "max_ttft": configuration.max_ttft, + "trials_num": configuration.trials, + "concurrency": configuration.concurrency, + "input_tokens": configuration.effective_input_tokens, + "output_tokens": configuration.effective_output_tokens, + "shared_prefix_tokens": configuration.shared_prefix_tokens or 0, + "baseline": configuration.effective_baseline, "fleets": list(allowed_fleets), "env": list(configuration.env), } diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 35da0994f..758131c23 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -12,7 +12,7 @@ _STATUS_DISPLAY = { "ready": ("verified", "grey"), - "running": ("clauding", "bold sea_green3"), + "running": ("trialing", "bold sea_green3"), "verifying": ("verifying", "bold deep_sky_blue1"), "interrupted": ("interrupted", "bold gold1"), "failed": ("failed", "indian_red1"), @@ -24,15 +24,55 @@ def _format_status(status: str) -> str: return f"[{style}]{text}[/]" if style else text -def _trials_exhausted(session: dict[str, Any]) -> bool: +def _verifying(session: dict[str, Any]) -> bool: + """Whether the agent has moved on to the final service. Read from the session's + verification records rather than inferred from a spent trial budget, which + misses every run that stopped early. An attempt that failed still counts: the + agent is picking the next trial to verify, not trialing again.""" + return isinstance(session.get("verification"), dict) + + +_SPARK_BLOCKS = "▁▂▃▄▅▆▇█" + + +def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: + """One glyph per trial, scaled within the run: the shape of the search. + A red `·` marks a trial that produced no benchmark at all; a yellow bar marks + one that measured but broke a constraint, since its number is real.""" + if not isinstance(session, dict): + return "" trials = session.get("trials") - max_trials = session.get("max_trials") - return ( - isinstance(trials, dict) - and isinstance(max_trials, int) - and isinstance(trials.get("count"), int) - and trials["count"] >= max_trials - ) + series = trials.get("series") if isinstance(trials, dict) else None + if not isinstance(series, list) or not series: + return "" + failed = trials.get("failed") + if not isinstance(failed, list) or len(failed) != len(series): + failed = [False] * len(series) + values = [v for v in series if isinstance(v, (int, float))] + if not values: + return "·" * len(series) + low, high = min(values), max(values) + span = high - low + out = [] + for value, is_failed in zip(series, failed): + if not isinstance(value, (int, float)): + out.append("[indian_red1]·[/]") + continue + glyph = ( + _SPARK_BLOCKS[-1] + if span <= 0 + else _SPARK_BLOCKS[round((value - low) / span * (len(_SPARK_BLOCKS) - 1))] + ) + # The best trial is the answer the run found; everything else is context. + # Yellow, not red: the trial measured, its number is real, and only the + # constraint breach makes it unusable. Red is reserved for `·`, where + # nothing came back at all. + if is_failed: + style = "gold1" + else: + style = "bold sea_green3" if value >= high else "secondary" + out.append(f"[{style}]{glyph}[/]") + return "".join(out) def _format_trial_progress(session: Optional[dict[str, Any]]) -> str: @@ -41,12 +81,12 @@ def _format_trial_progress(session: Optional[dict[str, Any]]) -> str: if not isinstance(session, dict): return "" trials = session.get("trials") - max_trials = session.get("max_trials") - if not isinstance(trials, dict) or not (trials.get("count") or isinstance(max_trials, int)): + trials_num = session.get("trials_num") + if not isinstance(trials, dict) or not (trials.get("count") or isinstance(trials_num, int)): return "" progress = str(trials.get("count") or 0) - if isinstance(max_trials, int): - progress += f"/{max_trials}" + if isinstance(trials_num, int): + progress += f"/{trials_num}" return f" [secondary]({progress})[/]" @@ -54,8 +94,14 @@ def print_presets( presets: list[Preset], sessions: Optional[list[dict[str, Any]]] = None, verbose: bool = False, + all_presets: bool = False, + limit: Optional[int] = None, ) -> None: - console.print(get_presets_table(presets, sessions=sessions, verbose=verbose)) + console.print( + get_presets_table( + presets, sessions=sessions, verbose=verbose, all_presets=all_presets, limit=limit + ) + ) console.print() @@ -63,15 +109,22 @@ def get_presets_table( presets: list[Preset], sessions: Optional[list[dict[str, Any]]] = None, verbose: bool = False, + all_presets: bool = False, + limit: Optional[int] = None, ) -> Table: table = Table(box=None) - table.add_column("BASE", no_wrap=True) table.add_column("ID", no_wrap=True) + table.add_column("BASE", no_wrap=True, style="secondary") table.add_column("RESOURCES" if verbose else "GPU", style="secondary") - table.add_column("BENCHMARK", min_width=len("con=1"), overflow="fold") + # CONSTRAINTS is the test that was asked for; BENCHMARK is the best trial under it. + table.add_column("CONSTRAINTS", no_wrap=True) + table.add_column("BENCHMARK", min_width=len("tps=1"), overflow="fold") + # The search shape, one glyph per trial. Unlabelled: it reads on sight. + table.add_column("", no_wrap=True) table.add_column("STATUS", no_wrap=True) table.add_column("SUBMITTED", no_wrap=True, style="secondary") - table.add_column("NAME", no_wrap=True, style="secondary") + if verbose: + table.add_column("NAME", no_wrap=True, style="secondary") presets_by_base: dict[str, list[Preset]] = defaultdict(list) repo_to_base: dict[str, str] = {} for preset in presets: @@ -87,23 +140,32 @@ def get_presets_table( model = str(session.get("model") or "unknown") sessions_by_model[repo_to_base.get(model, model)].append(session) - for base in sorted({*presets_by_base, *sessions_by_model}, key=str.lower): - add_row_from_dict(table, {"BASE": base}) - # Newest first within a group, as in `dstack ps`. - for preset in sorted( - presets_by_base.get(base, []), key=lambda p: p.created_at, reverse=True - ): - _add_preset(table, preset, verbose=verbose, creation=creations_by_id.get(preset.id)) - for session in sorted( - sessions_by_model.get(base, []), - key=lambda s: str(s.get("created_at") or ""), - reverse=True, - ): - _add_session(table, session) + # One flat list, newest first, as in `dstack ps`. The base is a column, so + # runs of different models still sort together by when they were submitted. + # Same contract as `dstack ps`: only active by default, or the single most + # recent row when nothing is active. `-a` and `-n` show everything. + rows: list[tuple[str, Any, bool]] = [] + for preset_list in presets_by_base.values(): + rows += [(preset.created_at.isoformat(), preset, True) for preset in preset_list] + for session_list in sessions_by_model.values(): + rows += [ + (str(session.get("created_at") or ""), session, False) for session in session_list + ] + rows.sort(key=lambda r: r[0], reverse=True) + only_active = not all_presets and limit is None + if only_active: + active = [r for r in rows if not r[2] and str(r[1].get("status")) == "running"] + rows = active or rows[:1] + + for _, item, is_preset in rows[:limit] if limit is not None else rows: + if is_preset: + _add_preset(table, item, verbose=verbose, creation=creations_by_id.get(item.id)) + else: + _add_session(table, item, verbose=verbose) return table -def _add_session(table: Table, session: dict[str, Any]) -> None: +def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False) -> None: created = "" created_at = session.get("created_at") if isinstance(created_at, str): @@ -116,28 +178,82 @@ def _add_session(table: Table, session: dict[str, Any]) -> None: benchmark = "" gpu = "" status_key = str(session.get("status", "")) - if status_key == "running" and _trials_exhausted(session): - # The trial budget is spent, so the agent is deploying and verifying - # the final service. + if status_key == "running" and _verifying(session): status_key = "verifying" status = _format_status(status_key) + _format_trial_progress(session) trials = session.get("trials") - if isinstance(trials, dict): - best = trials.get("best") - if isinstance(best, dict): - parts = ["best trial:"] - if best.get("concurrency"): - parts.append(f"con={best['concurrency']}") - parts.append(f"{_format_number(best['tok_s'])} tok/s") + best = trials.get("best") if isinstance(trials, dict) else None + # Nothing passed: fall back to the fastest attempt that did not. + breached = not isinstance(best, dict) + if breached and isinstance(trials, dict): + best = trials.get("best_failed") + if isinstance(best, dict): + gpu = best.get("gpu") or "" + if not gpu and isinstance(trials, dict): + # Trials can record hardware without ever producing a benchmark. + gpu = trials.get("gpu") or "" + constraints = session.get("constraints") or {} + parts = [] + objective = [] + if constraints.get("input_tokens") and constraints.get("output_tokens"): + objective.append( + f"io={_format_token_count(constraints['input_tokens'])}" + f"/{_format_token_count(constraints['output_tokens'])}" + ) + # Shown even at `0%`: rows are only comparable when it matches. + input_tokens = constraints.get("input_tokens") + if input_tokens: + shared_prefix_tokens = constraints.get("shared_prefix_tokens") or 0 + share = round(100 * shared_prefix_tokens / input_tokens) + objective.append(f"prefix={share}%") + concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") + if concurrency: + objective.append(f"conc={concurrency}") + min_context_length = constraints.get("min_context_length") + if verbose and isinstance(min_context_length, int): + objective.append(f"ctx>={_format_token_count(min_context_length)}") + max_ttft = constraints.get("max_ttft") + if verbose and isinstance(max_ttft, (int, float)): + objective.append(f"ttft<={_format_duration_ms(max_ttft)}") + # Stays empty until a trial has produced a benchmark: a run that has measured + # nothing yet has no best, and `n/a` is noise in a column of numbers. + if isinstance(best, dict): + tps = _format_number(best["tok_s"]) + if objective: + # Per-user leads: aggregate rises with concurrency, so it makes rows at + # different concurrencies look better or worse than they serve. + # Same definition as `effective_per_user_tok_per_s`: the steady decode + # rate, not the aggregate divided by concurrency. + tpot_ms = best.get("tpot_ms") + if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: + parts.append(f"tok/s/user={_format_number(1000 / tpot_ms)}") + if verbose: + parts.append(f"tok/s={tps}") + ttft_ms = best.get("ttft_ms") + if isinstance(ttft_ms, (int, float)): + parts.append(f"ttft={_format_duration_ms(ttft_ms)}") + context_length = best.get("context_length") + if isinstance(context_length, int): + parts.append(f"ctx={_format_token_count(context_length)}") benchmark = " ".join(parts) - gpu = best.get("gpu") or "" + else: + benchmark = f"conc={best.get('concurrency')} tps={tps}" + benchmark = benchmark.strip() + if benchmark and breached: + # Marked, not only dimmed: colour alone is not a signal. + benchmark = f"[secondary]*{benchmark}[/]" add_row_from_dict( table, { "ID": str(session.get("id", "")), + "BASE": str(session.get("model") or ""), "NAME": str(session.get("name") or ""), "GPU": gpu, "RESOURCES": gpu, + "": _format_trial_spark(session), + # The constraints are context for the number, so the whole cell recedes; + # the benchmark beside it is what the reader came for and stays bright. + "CONSTRAINTS": f"[secondary]{' '.join(objective)}[/]" if objective else "", "BENCHMARK": benchmark, "STATUS": status, "SUBMITTED": created, @@ -158,7 +274,15 @@ def _add_preset( "ID": preset.id, "NAME": preset.name or "", column: _format_resources(groups[0].resources, verbose=verbose), + "BASE": preset.base, "STATUS": _format_status("ready") + _format_trial_progress(creation), + "": _format_trial_spark(creation), + "CONSTRAINTS": format_preset_objective( + preset, + min_context_length=(creation or {}).get("constraints", {}).get("min_context_length"), + max_ttft=(creation or {}).get("constraints", {}).get("max_ttft"), + verbose=verbose, + ), "BENCHMARK": format_preset_benchmark(preset, verbose=verbose), "SUBMITTED": pretty_date(preset.created_at), } @@ -177,21 +301,61 @@ def _add_preset( ) +def format_preset_objective( + preset: Preset, + *, + min_context_length: Optional[int] = None, + max_ttft: Optional[float] = None, + verbose: bool = False, +) -> str: + """What was asked for. The context the configuration actually reached is a + result and sits next to the numbers; the context that was *required* is shown + here under `-v`. Two runs can share every constraint and still serve different + context lengths, so both are worth seeing.""" + workload = preset.validations[0].benchmark.workload + parts = [ + f"io={_format_token_count(workload.input_tokens)}" + f"/{_format_token_count(workload.output_tokens)}", + ] + share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) + parts.append(f"prefix={share}%") + parts.append(f"conc={workload.concurrency}") + # Absent for presets saved before the creation record was consulted. + if verbose and min_context_length is not None: + parts.append(f"ctx>={_format_token_count(min_context_length)}") + # The latency ceiling only explains a number that is near it, so it waits for `-v`. + if verbose and max_ttft is not None: + parts.append(f"ttft<={_format_duration_ms(max_ttft)}") + # Context for the number, so the whole cell recedes. + return f"[secondary]{' '.join(parts)}[/]" + + def format_preset_benchmark(preset: Preset, *, verbose: bool = False) -> str: benchmark = preset.validations[0].benchmark - workload = benchmark.workload metrics = benchmark.metrics - output_tokens_per_second = metrics.total_output_tokens / metrics.duration_seconds + # The workload and context define the number, so they are always shown next + # to it: two presets are comparable only when all three match. parts = [ - f"con={workload.concurrency}", - f"{_format_number(output_tokens_per_second)} tok/s", - f"TTFT {_format_latency(metrics.ttft_ms.p50)}", + f"tok/s/user={_format_number(benchmark.effective_per_user_tok_per_s)}", ] if verbose: - parts.insert(0, f"ctx={_format_token_count(preset.context_length)}") + parts.append(f"tok/s={_format_number(benchmark.effective_output_tok_per_s)}") + parts += [ + f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", + f"ctx={_format_token_count(preset.context_length)}", + ] return " ".join(parts) +def _format_duration_ms(value: float) -> str: + """Milliseconds below a second, seconds above it. A bare `4152` reads as small + until you notice the unit; `4.15s` does not.""" + # 999.6 rounds to 1000, which must read as 1s rather than 1000ms. + if value < 999.5: + return f"{_format_number(value)}ms" + return f"{_format_number(value / 1000)}s" + + def _format_token_count(value: int) -> str: for divisor, suffix in ((1024 * 1024, "M"), (1024, "K")): if value >= divisor and value % divisor == 0: diff --git a/src/dstack/_internal/cli/services/presets/prompt.py b/src/dstack/_internal/cli/services/presets/prompt.py index 0d0052c32..e565fc2fd 100644 --- a/src/dstack/_internal/cli/services/presets/prompt.py +++ b/src/dstack/_internal/cli/services/presets/prompt.py @@ -11,26 +11,40 @@ # the document; this module only applies the rule. _DIRECTIVE_PATTERN = re.compile(r"", re.DOTALL) +# `` is a note for maintainers and is dropped before the agent sees +# the document. Any other comment is left alone, so that a plain `` or a +# malformed directive stays visible instead of disappearing silently. +_NOTE_PATTERN = re.compile(r"\n?", re.DOTALL) + # TODO: reintroduce a `# Resume` section in system_prompt.md once session resume # (seeded from `runs.jsonl` and `trials.jsonl`) is designed. -def get_preset_agent_system_prompt(user_prompt: Optional[str] = None) -> str: +def get_preset_agent_system_prompt( + user_prompt: Optional[str] = None, + baseline: bool = False, +) -> str: text = _SYSTEM_PROMPT_PATH.read_text(encoding="utf-8").strip() - variables = {"prompt": user_prompt.strip() if user_prompt else None} - applied = 0 + variables = { + "prompt": user_prompt.strip() if user_prompt else None, + # Rendered for its presence only; the directive body must not interpolate it. + "baseline": "on" if baseline else None, + } + applied: set[str] = set() def substitute(match: re.Match) -> str: - nonlocal applied name, content = match.group(1), match.group(2) if name not in variables: raise CLIError(f"Unknown variable {name!r} in the agent system prompt") value = variables[name] if not value: return "" - applied += 1 + applied.add(name) return content.replace("{" + name + "}", value) - rendered = _DIRECTIVE_PATTERN.sub(substitute, text) - if variables["prompt"] and not applied: + rendered = _NOTE_PATTERN.sub("", _DIRECTIVE_PATTERN.sub(substitute, text)) + if variables["prompt"] and "prompt" not in applied: raise CLIError("The agent system prompt has no place for the user prompt") + for name, value in variables.items(): + if value and name not in applied: + raise CLIError(f"The agent system prompt has no place for {name!r}") return re.sub(r"\n{3,}", "\n\n", rendered) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 238709970..d18187cb0 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -1,9 +1,19 @@ # Objective -Your goal is to find a model serving configuration with the best -performance through sequential experimental trials. Once the best-performing -candidate is found, it is deployed as a `dstack` service for the final -benchmark and saved as a reusable preset. +Your goal is to get the best serving performance for a given model on given +hardware, through sequential experimental trials. + +## Performance + +Performance is `output_tok_per_s`, the output token throughput of a benchmark +run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark +tools report several throughput numbers; performance is the output one. + +Significantly better performance goes beyond what conventional configurations +and published benchmarks reach; they are not the ceiling. Estimate the +practical limits of the chosen hardware (memory bandwidth for decode, compute +for prefill) and treat the gap between measured performance and those limits as +headroom: while the gap is large, assume a better configuration exists. # Constraints @@ -20,11 +30,21 @@ Field semantics: different precision or quantization, or another trusted compatible repo. The client-facing model name of the final service is `model.name` when set, otherwise `model.repo` or `model.base`. -- `context_length`: the minimum context length the selected repo/path and - the final service must support. `null` means no minimum is required. -- `max_trials`: the maximum number of trials in this session. +- `min_context_length`: the minimum context length the selected repo/path and + the final service must support. +- `max_ttft`: the maximum p50 time to first token, in milliseconds, that any + benchmark in this session may report. +- `trials_num`: the number of trials in this session. - `concurrency`: the number of simultaneous requests for every benchmark in this session. It is fixed so that benchmark results are comparable. + +- `input_tokens`, `output_tokens`: the request shape for every benchmark in + this session. They are fixed for the same reason. +- `shared_prefix_tokens`: how many of `input_tokens` are identical in every + request. `0` means every request is fully unique. +- `baseline`: whether the first trial must be a baseline rather than an + optimization attempt; see `# Trials`. - `fleets`: use these existing `dstack` fleets only. Do not create, delete, apply, or edit fleets. - `env`: the environment variable names available to runs; the values are @@ -35,8 +55,12 @@ pick the hardware (the best available within the allowed `dstack` fleets), the model variant (only if `model` has `base`), the serving framework, the Docker image and dependencies, the serving framework parameters, and anything else within these constraints — except generating custom kernels, -patching drivers, or patching serving framework source code. + - -Trial ideas must not rely only on what you already know. Research how to -get the best performance for the chosen model, serving framework, and -hardware in trustworthy sources. Start with these: +how to get better performance than the previous trials. Sometimes it is worth +continuing to improve a previous trial's idea, but when that risks settling +into a local optimum, search for a substantially different approach rather than +tweaking parameters further. + + + +Trial ideas must not rely only on what you already know. Research what limits +performance and how to improve it for the chosen model, serving framework, and +hardware. Actively seek credible and recent sources: benchmarks, newly +published optimizations, release notes, papers, and issue threads. Start with +these: - vLLM recipes: `https://recipes.vllm.ai/` (model index: `https://recipes.vllm.ai/models.json`) @@ -152,10 +190,12 @@ hardware in trustworthy sources. Start with these: - Performance-loop methodology (profiling, benchmark contracts): `https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development` -Go beyond this list proactively — official docs, repo issues, and reputable -benchmarks — whenever that can help the trial. Research before the first +Go beyond this list whenever it can help the trial. Research before the first trial and whenever a benchmark exposes a bottleneck. +Don't skip profiling the serving engine, especially if it could help you find +an idea for significantly improving the current numbers. + For each trial, use `dstack` tasks (see `# Task Usage`). During a trial, run commands interactively inside the task (over SSH) and measure the performance when needed, following `## Benchmark` below. @@ -172,12 +212,14 @@ the final performance into a complete `dstack` task configuration with exact commands, and log it together with the corresponding benchmark results (see `## Benchmark` for the structure) to `trials.jsonl`. The benchmark may be skipped in one case only: you failed to make the configuration run at -all — a failed trial. +all — a failed trial. A trial is also failed when its benchmark does not meet +the constraints (see `# Constraints`). When a trial that changed several things +fails, be mindful of which specific change was the root cause. -Each `trials.jsonl` record is one JSON line with exactly three fields: +Each `trials.jsonl` record is one JSON line with exactly four fields: ``` -{"task": {...}, "resources": {...}, "benchmark": {...}} +{"task": {...}, "resources": {...}, "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} ``` - `task`: the compiled `dstack` task configuration described above, as JSON. @@ -193,16 +235,17 @@ Each `trials.jsonl` record is one JSON line with exactly three fields: `dstack run get --json`, converting MiB values to GB and the `gpus` list into one `gpu` object with the GPU `name`, per-GPU `memory`, and `count`. +- `context_length`: the largest context the trial's configuration handles, + found as described in `## Benchmark`; `null` only when the benchmark couldn't + be done at all. - `benchmark`: the trial benchmark (see `## Benchmark` for the structure); - `null` only for a failed trial. + `null` only when the benchmark couldn't be done at all. +- `learned`: the major things this trial taught you that you did not know + before it ran. Required for every trial, including a failed one. +- `failed`: `true` if the benchmark broke a constraint such as `max_ttft` or + `min_context_length`, absent otherwise. -Continue the trials until `max_trials` from `constraints.json` is reached or -you are entirely out of ideas on how to reach better performance. Before -stopping early, step back and think once more about what could still improve -performance within the constraints (see `# Constraints`). An early stop must -be justified in -`progress.jsonl` (see `# Progress`): report what you considered and why none -of it is worth a trial. +You're expected to do exactly `trials_num` trials (see `# Constraints`). Once the trials are over, pick the best trial and deploy it as a `dstack` service to verify that it works and benchmark it finally (see @@ -216,14 +259,35 @@ no trials remain. In that case, log the failure to `final_report.json` (see ## Benchmark During trials, run benchmarks via SSH inside the task, directly against the -serving engine: use `concurrency` from `constraints.json` and measure all -trials the same way so that their results are comparable with each other. -In trial benchmarks too, all measured requests must succeed. - -Before any benchmark — a trial one or the final one — warm the engine up by -verifying that the model works as expected: send real requests and check -the responses, including reasoning output when the model supports it. These -verification requests are never part of the measured metrics. +serving engine: use `concurrency`, `input_tokens`, `output_tokens`, and +`shared_prefix_tokens` from `constraints.json` and measure all trials the same +way so that their results are comparable with each other. +Before any benchmark, ensure it uses a different seed than the previous +benchmark. Otherwise the benchmark will depend on what has been cached by the +previous benchmark. + +When `shared_prefix_tokens` is not 0, every measured request must begin with the +same `shared_prefix_tokens` tokens, and the rest of each request must differ +from every other request's. Choose the benchmark tool's dataset and options that +do this, and confirm from the tool's own documentation, for the version you run, +that they do. Never produce the per-request difference by varying request +lengths. For example, the shared-prefix options are: + +| tool | shared-prefix options | +| --- | --- | +| `vllm bench serve` | `--dataset-name random --random-prefix-len --random-input-len ` | +| `sglang.benchmark.serving` | `--dataset-name generated-shared-prefix --gsp-num-groups 1 --gsp-system-prompt-len --gsp-question-len --gsp-prompts-per-group ` | + +The table is an example and not a full command: the remaining options still come +from `concurrency` and `output_tokens`, option names and defaults differ between +versions, and any other tool needs its own equivalent. + +Before any benchmark — a trial one or the final one — verify that the model +works as expected: send real requests and check the responses, including +reasoning output when the model supports it. These verification requests are +never part of the measured metrics. + +All verification and benchmark requests must succeed. Record every benchmark using the following structure and field names — trial benchmarks in `trials.jsonl`, the final benchmark as @@ -234,24 +298,35 @@ trial benchmarks in `trials.jsonl`, the final benchmark as "tool": "vllm bench serve", "tool_version": "0.11.0", "command": "vllm bench serve ...", - "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8}, + "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "shared_prefix_tokens": 768}, "metrics": { "successful_requests": 16, "failed_requests": 0, "duration_seconds": 4.0, "total_input_tokens": 16384, "total_output_tokens": 2048, + "output_tok_per_s": 512.0, "per_user_tok_per_s": 64.0, "ttft_ms": {"mean": 110.9, "p50": 108.2, "p99": 121.6}, "tpot_ms": {"mean": 7.5, "p50": 7.4, "p99": 8.1} } } ``` +Compute `output_tok_per_s` as `total_output_tokens / duration_seconds` and +`per_user_tok_per_s` as `output_tok_per_s / workload.concurrency`. These are +the numbers used to compare trials (see `## Performance`). + Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free invocation. For the final benchmark, run it with streaming responses, set `workload.concurrency` to `concurrency` from `constraints.json`, produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and -warmup requests; all measured requests must succeed. Never invent missing -values. +warmup requests. Never invent missing values. + +After each benchmark, find the largest context the configuration handles by +sending real requests, and record it: for a trial, as the `context_length` +field of its `trials.jsonl` record; for the final benchmark, as +`final_report.json.context_length`. Stopping at the required minimum is not +enough. + # Task Usage @@ -274,7 +349,9 @@ instances/instance volumes: later runs reuse the instance and cached model weights, while container-based backends start clean on every re-submission. If the backend allows, use instance volumes to mount cache and model weights -between runs. +between runs. Even if the backend doesn't allow instance volumes (like most +container-based backends), still use them the same way, just mark them +`optional: true`. When submitting a `dstack` task or service, pass exact `fleets`, `backends`, and an intentional `resources` range based on the choice made from offers, so @@ -302,17 +379,34 @@ the trial. Set the service `model` name to the client-facing model name from serve requests. If the service never passes the probe, treat that as a real failure of the configuration, not something to work around by removing `model`. +Record every attempt in `verifications.jsonl`, append-only: one line +immediately after submitting the service, one when the attempt ends (values +are illustrative): + +```json +{"trial": 3, "run_name": "qwen-preset-2", "status": "verifying"} +{"trial": 3, "run_name": "qwen-preset-2", "status": "failed", "reason": "..."} +{"trial": 2, "run_name": "qwen-preset-3", "status": "verifying"} +{"trial": 2, "run_name": "qwen-preset-3", "status": "verified"} +``` + +`trial` is the 1-based line number of that trial in `trials.jsonl`. Keep +`reason` to one sentence. + Before the final benchmark, verify the model through the service: send real requests using the client-facing model name and check that the model works -as it should, including reasoning output when the model supports it. This -verification also warms the service up. Only then run the final benchmark -(see `## Benchmark`). When verifying or benchmarking the service, use its -`service.url` reported by `dstack run get --json`, along with +as it should, including reasoning output when the model supports it. When +verifying the service, use its `service.url` reported by +`dstack run get --json`, along with `DSTACK_TOKEN` as the bearer token. If `service.url` is a relative path, prepend `DSTACK_SERVER_URL` to build the absolute URL. -During the service verification, test the context length the service -actually supports and report it as `final_report.json.context_length`. +Only then run the final benchmark (see `## Benchmark`). Unlike the +verification above, do not run it through `service.url`: run it via SSH inside +the service replica, directly against the serving engine, the same way as the +trial benchmarks so that the results are comparable with each other. Attach to +the service with `dstack attach `, which enables `ssh ` +into the replica. If the service or its benchmark cannot be completed, stop that service, pick the next-best trial, and repeat, until a service is verified or there @@ -330,7 +424,8 @@ print, copy, or summarize tokens, secrets, or environment variable values. Do not expose the value of `DSTACK_TOKEN` or the value of any environment variable listed under `env` in `constraints.json`. -Do not put secret values in `final_report.json` or print them. Use env +Do not put secret values in `final_report.json`, or print +them. Use env references in `final_report.json.service_yaml`; use environment variable names or redacted values in `final_report.json.benchmark.command`. @@ -348,7 +443,8 @@ On success, include exactly: - `service_yaml`: the full YAML of the verified final service - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `context_length`: the context length verified for the final service +- `context_length`: the largest context verified for the final service, as + described in `## Benchmark` - `benchmark`: the final service benchmark described in `## Benchmark` Set `final_report.json.base` as follows: @@ -372,9 +468,10 @@ through `StructuredOutput`. Verify that `final_report.json` is correct and matches the required schema. -Stop only after `final_report.json` is written and submitted: either one -final `dstack` service was verified and benchmarked, or the trials and -unverified candidates were exhausted (see `# Trials` and `# Final Service`). +Stop only after `final_report.json` is written, and the +report submitted: either one final `dstack` service was verified and +benchmarked, or the trials and unverified candidates were exhausted (see +`# Trials` and `# Final Service`). Ending your turn stops the session even while background commands are still running. Wait for long-running work — weight downloads, engine startup, diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 1f32bea6d..eb66f8112 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -30,6 +30,7 @@ _PROGRESS_FILENAME = "progress.jsonl" _RUNS_FILENAME = "runs.jsonl" _TRIALS_FILENAME = "trials.jsonl" +_VERIFICATIONS_FILENAME = "verifications.jsonl" _CONSTRAINTS_FILENAME = "constraints.json" _FINAL_REPORT_FILENAME = "final_report.json" _SESSION_FILENAME = "session.json" @@ -68,6 +69,10 @@ def runs_path(self) -> Path: def trials_path(self) -> Path: return self.path / _TRIALS_FILENAME + @property + def verifications_path(self) -> Path: + return self.path / _VERIFICATIONS_FILENAME + def write_prompt(self, prompt: str) -> None: _write_private_text(self.path / "prompt.md", prompt + "\n") @@ -173,7 +178,7 @@ def create_preset_agent_session( "name": configuration.name, "model": getattr(configuration.model, "base", None) or getattr(configuration.model, "repo", None), - "max_trials": configuration.max_trials, + "trials_num": configuration.trials, "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "debug": debug, } @@ -285,7 +290,7 @@ def load_agent_session(preset_id: str) -> PresetAgentSession: def print_session_log(session: PresetAgentSession) -> None: - """Prints the session's redacted progress log verbatim (no markup).""" + """Prints the session's redacted progress log verbatim, no markup.""" try: content = session.log_path.read_text(encoding="utf-8") except OSError: @@ -428,10 +433,48 @@ def list_agent_sessions() -> list[dict[str, Any]]: entry["name"] = claimed_session_name(manifest) entry["status"] = status entry["trials"] = _summarize_session_trials(path / _TRIALS_FILENAME) + entry["verification"] = _read_last_session_verification(path / _VERIFICATIONS_FILENAME) + entry["constraints"] = _read_session_constraints(path) entries.append(entry) return entries +def _read_session_constraints(path: Path) -> dict[str, Any]: + """The objective the session was given. The session's own copy is read first: it + is written at creation and outlives the agent workspace, which is removed once + the session finishes. The workspace copy is the fallback, for sessions recorded + before the session-level copy existed.""" + for candidate in ( + path / _CONSTRAINTS_FILENAME, + path / "workspace" / "w" / _CONSTRAINTS_FILENAME, + ): + try: + data = json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(data, dict): + return data + return {} + + +def _read_last_session_verification(path: Path) -> Optional[dict[str, Any]]: + """The final service attempt in flight or last finished, from the session's + mirrored verification records. The last line wins: the agent appends one when + an attempt starts and another when it ends.""" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return None + for line in reversed(lines): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("status"), str): + return record + return None + + def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: """Best-so-far summary from a session's mirrored trial records.""" try: @@ -440,6 +483,14 @@ def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: lines = [] count = 0 best: Optional[dict[str, Any]] = None + # The fastest trial that broke a constraint, shown only when nothing passed. + best_failed: Optional[dict[str, Any]] = None + # One entry per trial in order, `None` for a trial that produced no benchmark. + series: list[Optional[float]] = [] + # Parallel to `series`: a trial that measured but broke a constraint. + failed: list[bool] = [] + # Kept outside `best` so a run where nothing passed still shows what it ran on. + gpu: Optional[str] = None for line in lines: try: record = json.loads(line) @@ -451,33 +502,75 @@ def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: # so task names must not be deduplicated. count += 1 benchmark = record.get("benchmark") + failed.append(bool(record.get("failed"))) + record_gpu = _format_trial_gpu(record) + if record_gpu: + gpu = record_gpu if not isinstance(benchmark, dict): + series.append(None) continue metrics = benchmark.get("metrics") or {} workload = benchmark.get("workload") or {} duration = metrics.get("duration_seconds") tokens = metrics.get("total_output_tokens") if not isinstance(duration, (int, float)) or duration <= 0: + series.append(None) continue if not isinstance(tokens, (int, float)): + series.append(None) continue tok_s = tokens / duration + series.append(tok_s) + # A failed trial keeps its benchmark — it is what the next trial learns + # from — but it is not a candidate for best, and promoting one would put + # a configuration that broke a constraint at the top of the listing. + if record.get("failed"): + if best_failed is None or tok_s > best_failed["tok_s"]: + best_failed = _trial_entry(tok_s, record, metrics, workload, record_gpu) + continue if best is None or tok_s > best["tok_s"]: - resources = record.get("resources") or {} - gpu = resources.get("gpu") if isinstance(resources, dict) else None - gpu_text = None - if isinstance(gpu, dict) and gpu.get("name"): - gpu_text = str(gpu["name"]) - if gpu.get("memory"): - gpu_text += f":{gpu['memory']}" - if gpu.get("count"): - gpu_text += f":{gpu['count']}" - best = { - "tok_s": tok_s, - "concurrency": workload.get("concurrency"), - "gpu": gpu_text, - } - return {"count": count, "best": best} + best = _trial_entry(tok_s, record, metrics, workload, record_gpu) + return { + "count": count, + "best": best, + "best_failed": best_failed, + "series": series, + "failed": failed, + "gpu": gpu, + } + + +def _trial_entry( + tok_s: float, + record: dict[str, Any], + metrics: dict[str, Any], + workload: dict[str, Any], + gpu: Optional[str], +) -> dict[str, Any]: + ttft = (metrics.get("ttft_ms") or {}).get("p50") + tpot = (metrics.get("tpot_ms") or {}).get("p50") + context_length = record.get("context_length") + return { + "tok_s": tok_s, + "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, + "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, + "context_length": context_length if isinstance(context_length, int) else None, + "concurrency": workload.get("concurrency"), + "gpu": gpu, + } + + +def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: + resources = record.get("resources") + gpu = resources.get("gpu") if isinstance(resources, dict) else None + if not isinstance(gpu, dict) or not gpu.get("name"): + return None + text = str(gpu["name"]) + if gpu.get("memory"): + text += f":{gpu['memory']}" + if gpu.get("count"): + text += f":{gpu['count']}" + return text def print_preset_progress(message: str, *, agent_session: PresetAgentSession) -> None: diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 2dd4c8005..13a87990b 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -92,8 +92,8 @@ def build_verified_preset( elif report.model != preset_configuration.model.exact_repo: raise CLIError("Claude changed an exact model request") if ( - preset_configuration.context_length is not None - and report.context_length < preset_configuration.context_length + preset_configuration.min_context_length is not None + and report.context_length < preset_configuration.min_context_length ): raise CLIError("Claude final service does not meet the requested context length") diff --git a/src/dstack/_internal/cli/services/presets/workspace.py b/src/dstack/_internal/cli/services/presets/workspace.py index 504a9baf0..54b24abf5 100644 --- a/src/dstack/_internal/cli/services/presets/workspace.py +++ b/src/dstack/_internal/cli/services/presets/workspace.py @@ -18,6 +18,7 @@ _PROGRESS_FILENAME, _RUNS_FILENAME, _TRIALS_FILENAME, + _VERIFICATIONS_FILENAME, PresetAgentSession, ) from dstack._internal.compat import IS_WINDOWS @@ -54,6 +55,10 @@ def runs_path(self) -> Path: def trials_path(self) -> Path: return self.path / _TRIALS_FILENAME + @property + def verifications_path(self) -> Path: + return self.path / _VERIFICATIONS_FILENAME + @property def constraints_path(self) -> Path: return self.path / _CONSTRAINTS_FILENAME @@ -170,6 +175,7 @@ def _prepare_workspace(workspace: PresetAgentWorkspace) -> None: workspace.progress_path, workspace.runs_path, workspace.trials_path, + workspace.verifications_path, ]: path.touch() workspace.bin_path.mkdir() diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index 84b88ae3d..84d8a6e28 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -40,7 +40,7 @@ class TestPresetLocalCommands: def test_handles_keyboard_interrupt(self, tmp_path, capsys): configuration_path = tmp_path / "preset.dstack.yml" configuration_path.write_text( - "type: preset\nname: qwen\nmodel:\n base: Qwen/Qwen3.5-27B\nmax_trials: 1\n" + "type: preset\nname: qwen\nmodel:\n base: Qwen/Qwen3.5-27B\ntrials: 1\nconcurrency: 8\nmax_ttft: 5000\nmin_context_length: 8192\n" ) with _patched_create_preset(side_effect=KeyboardInterrupt): @@ -59,7 +59,7 @@ def test_create_ends_quietly_when_stopped_from_another_cli(self, tmp_path, capsy configuration_path = tmp_path / "preset.dstack.yml" configuration_path.write_text( - "type: preset\nname: qwen\nmodel:\n base: Qwen/Qwen3.5-27B\nmax_trials: 1\n" + "type: preset\nname: qwen\nmodel:\n base: Qwen/Qwen3.5-27B\ntrials: 1\nconcurrency: 8\nmax_ttft: 5000\nmin_context_length: 8192\n" ) with _patched_create_preset(side_effect=CreationStopped): @@ -72,7 +72,7 @@ def test_create_ends_quietly_when_stopped_from_another_cli(self, tmp_path, capsy assert exit_code == 0 assert "Traceback" not in capsys.readouterr().err - def test_create_requires_max_trials(self, tmp_path, capsys): + def test_create_requires_trials(self, tmp_path, capsys): configuration_path = tmp_path / "preset.dstack.yml" configuration_path.write_text("type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\n") @@ -85,7 +85,7 @@ def test_create_requires_max_trials(self, tmp_path, capsys): assert exit_code == 1 captured = capsys.readouterr() - assert "max_trials is required" in captured.out + captured.err + assert "trials is required" in captured.out + captured.err create.assert_not_called() def _list_output(self, tmp_path, args, *, created_at): @@ -110,18 +110,24 @@ def test_lists_presets_without_api_client(self, tmp_path): assert "8f3a12c4" in output # The repo row is shown only in verbose mode. assert "repo=community/Qwen3.5-27B-GPTQ-Int4" not in output - # The context column is shown only in verbose mode. + # The benchmark always carries the context and workload that define its + # number. The context that was *required* waits for `-v`, and comes from + # the creation record, which a preset saved on its own does not have. assert "CONTEXT" not in output + assert "CONSTRAINTS" in output assert "BENCHMARK" in output - assert "32K" not in output - assert "42.1" in output - assert "con=1" in "".join(output.split()) - assert "tok/s" in output - assert "TTFT" in output - assert "108ms" in output + assert "ctx=32K" in "".join(output.split()) + assert "ctx>=" not in "".join(output.split()) + assert "io=1K/128" in "".join(output.split()) + # 42.1 is the aggregate, verbose-only; the default row shows per-user 1/TPOT. + assert "135" in output + assert "conc=1" in "".join(output.split()) + assert "tok/s/user=" in "".join(output.split()) + assert "ttft=" in "".join(output.split()) + assert "ttft=[/]108" in "".join(output.split()) or "ttft=108" in "".join(output.split()) assert "A6000:48GB:1" not in output - def test_verbose_list_adds_repo_and_context(self, tmp_path): + def test_verbose_list_adds_repo(self, tmp_path): preset = get_preset() PresetStore(tmp_path / ".dstack" / "presets").save(preset) @@ -131,10 +137,10 @@ def test_verbose_list_adds_repo_and_context(self, tmp_path): ).split() ) - # Verbose adds only the repo and the ctx= benchmark prefix. + # Verbose adds only the repo row. assert "repo=community/Qwen3.5-27B-GPTQ-Int4" in joined_verbose assert "ctx=32K" in joined_verbose - assert "con=1" in joined_verbose + assert "conc=1" in joined_verbose assert "hardware=" not in joined_verbose def test_deletes_preset_without_api_client(self, tmp_path): @@ -274,7 +280,10 @@ def test_merges_profile_configuration_and_cli_args(self, tmp_path): base: Qwen/Qwen3.5-27B regions: [file-region] max_price: 0.5 -max_trials: 1 +trials: 1 +concurrency: 8 +max_ttft: 5000 +min_context_length: 8192 env: - HF_TOKEN """ @@ -378,7 +387,7 @@ def test_create_detaches_the_name_from_the_old_preset(self, tmp_path): store.save(preset) configuration_path = tmp_path / "preset.dstack.yml" configuration_path.write_text( - "type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\nmax_trials: 1\n" + "type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\ntrials: 1\nconcurrency: 8\nmax_ttft: 5000\nmin_context_length: 8192\n" ) result = SimpleNamespace( preset=preset, path=tmp_path / "preset.yaml", final_run_name="qwen-1" @@ -401,7 +410,7 @@ def test_create_without_confirmation_exits_before_creating(self, tmp_path): store.save(preset) configuration_path = tmp_path / "preset.dstack.yml" configuration_path.write_text( - "type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\nmax_trials: 1\n" + "type: preset\nname: qwen\nbase: Qwen/Qwen3.5-27B\ntrials: 1\nconcurrency: 8\nmax_ttft: 5000\nmin_context_length: 8192\n" ) with ( @@ -430,7 +439,9 @@ def test_get_and_delete_resolve_names(self, tmp_path, capsys): def test_create_always_asks_even_without_a_name_conflict(self, tmp_path): configuration_path = tmp_path / "preset.dstack.yml" - configuration_path.write_text("type: preset\nbase: Qwen/Qwen3.5-27B\nmax_trials: 1\n") + configuration_path.write_text( + "type: preset\nbase: Qwen/Qwen3.5-27B\ntrials: 1\nconcurrency: 8\nmax_ttft: 5000\nmin_context_length: 8192\n" + ) with ( _patched_create_preset() as create, diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index e3b17396d..c3e9a44ca 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -29,6 +29,7 @@ ) from dstack._internal.cli.services.presets.session import ( PresetAgentSession, + _read_last_session_verification, _summarize_session_trials, create_preset_agent_session, load_resumable_agent_session, @@ -872,7 +873,50 @@ def test_counts_records_even_when_trials_share_a_task(self, tmp_path): # 4 records = 4 trials: one long-lived task commonly hosts several # trials, so shared task names must not collapse the count. assert summary["count"] == 4 - assert summary["best"] == {"tok_s": 2300.0, "concurrency": 8, "gpu": "A40:48GB:1"} + assert summary["best"] == { + "tok_s": 2300.0, + "tpot_ms": None, + "ttft_ms": None, + "context_length": None, + "concurrency": 8, + "gpu": "A40:48GB:1", + } + + +class TestReadLastSessionVerification: + def test_last_record_wins_and_a_missing_file_is_not_verifying(self, tmp_path): + path = tmp_path / "verifications.jsonl" + + assert _read_last_session_verification(path) is None + + path.write_text( + "\n".join( + json.dumps(entry) + for entry in [ + {"trial": 3, "run_name": "p-2", "status": "verifying"}, + {"trial": 3, "run_name": "p-2", "status": "failed", "reason": "probe"}, + {"trial": 2, "run_name": "p-3", "status": "verifying"}, + ] + ) + + "\n" + ) + + assert _read_last_session_verification(path) == { + "trial": 2, + "run_name": "p-3", + "status": "verifying", + } + + def test_skips_partial_trailing_lines(self, tmp_path): + # The mirror appends as the agent writes, so the file can be read + # mid-line. + path = tmp_path / "verifications.jsonl" + path.write_text( + json.dumps({"trial": 1, "run_name": "p-2", "status": "verifying"}) + + '\n{"trial": 1, "run_na' + ) + + assert _read_last_session_verification(path)["status"] == "verifying" class TestFileLineReader: diff --git a/src/tests/_internal/cli/services/presets/test_apply.py b/src/tests/_internal/cli/services/presets/test_apply.py index 919d2c02f..0597d3eab 100644 --- a/src/tests/_internal/cli/services/presets/test_apply.py +++ b/src/tests/_internal/cli/services/presets/test_apply.py @@ -22,7 +22,7 @@ def test_accepts_matching_base_model_and_context(self): configuration = PresetConfiguration( name="qwen", model={"base": "Qwen/Qwen3.5-27B"}, - context_length=8192, + min_context_length=8192, ) _validate_preset_matches(preset, configuration=configuration) @@ -32,7 +32,7 @@ def test_rejects_insufficient_context(self): configuration = PresetConfiguration( name="qwen", model={"base": "Qwen/Qwen3.5-27B"}, - context_length=8192, + min_context_length=8192, ) with pytest.raises(CLIError, match="context length"): @@ -124,7 +124,7 @@ def test_applies_the_referenced_preset(self, monkeypatch): configurator_args=service_args, plan_properties={ "Model": "Qwen/Qwen3.5-27B ([secondary]base[/])", - "Preset": "8f3a12c4 ([secondary]ctx=32K con=1 42.1 tok/s TTFT 108ms[/])", + "Preset": "8f3a12c4 ([secondary]io=1K/128 prefix=0% conc=1[/] tok/s/user=135 tok/s=42.1 ttft=108ms ctx=32K)", }, ) diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index 59f11968a..3297f21d9 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest +from pydantic import ValidationError from dstack._internal.cli.models.configurations import PresetConfiguration from dstack._internal.cli.services.presets.agent import ( @@ -88,16 +89,20 @@ def creation_context(tmp_path, monkeypatch): configuration = PresetConfiguration( name="qwen-build", model={"base": "Qwen/Qwen3.5-27B"}, - context_length=8192, - max_trials=1, + min_context_length=8192, + max_ttft=5000, + concurrency=8, + trials=1, fleets=["gpu-fleet"], env={"LICENSE": "license-secret", "TOKENIZERS_PARALLELISM": "false"}, ) source_configuration = PresetConfiguration( name="qwen-build", model={"base": "Qwen/Qwen3.5-27B"}, - context_length=8192, - max_trials=1, + min_context_length=8192, + max_ttft=5000, + concurrency=8, + trials=1, fleets=["gpu-fleet"], env=["LICENSE", "TOKENIZERS_PARALLELISM=false"], ) @@ -429,12 +434,112 @@ def stop(self, project, names, abort): self.run.status = RunStatus.TERMINATED +class TestFindingsInLogs: + def test_a_live_session_prints_the_log_alone(self, tmp_path, monkeypatch, capsys): + # Findings are written at the end, so there is nothing to append yet. + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + session = _agent_session(tmp_path) + print_preset_progress("provisioning", agent_session=session) + + print_session_log(session) + + out = capsys.readouterr().out + assert "provisioning" in out + assert "Findings" not in out + + +class TestFindings: + def test_passes_through_to_constraints(self): + configuration = PresetConfiguration( + name="qwen", + model={"base": "Qwen/Qwen3-32B"}, + max_ttft=5000, + min_context_length=8192, + concurrency=8, + trials=1, + input_tokens=8192, + shared_prefix_tokens=7424, + ) + + data = json.loads( + _build_constraints( + configuration=configuration, build_name="qwen-abc123", allowed_fleets=("a",) + ) + ) + + assert data["shared_prefix_tokens"] == 7424 + + def test_rejects_a_prefix_that_leaves_nothing_unique(self): + # Every request would be identical, which measures the cache rather than + # the configuration. + with pytest.raises(ValidationError, match="less than input_tokens"): + PresetConfiguration( + name="qwen", + model={"base": "Qwen/Qwen3-32B"}, + max_ttft=5000, + min_context_length=8192, + concurrency=8, + trials=1, + input_tokens=8192, + shared_prefix_tokens=8192, + ) + + def test_checks_against_the_default_input_tokens(self): + # `input_tokens` unset means 1024, so a larger prefix is still rejected. + with pytest.raises(ValidationError, match="less than input_tokens"): + PresetConfiguration( + name="qwen", + model={"base": "Qwen/Qwen3-32B"}, + max_ttft=5000, + min_context_length=8192, + concurrency=8, + trials=1, + shared_prefix_tokens=2048, + ) + + +class TestPerformanceConstraints: + def test_max_ttft_reaches_the_constraints(self): + configuration = PresetConfiguration( + name="qwen", + model={"base": "Qwen/Qwen3-32B"}, + min_context_length=8192, + concurrency=8, + trials=1, + max_ttft=10000, + ) + + data = json.loads( + _build_constraints( + configuration=configuration, build_name="qwen-abc123", allowed_fleets=("a",) + ) + ) + + assert data["max_ttft"] == 10000 + + def test_throughput_is_derived_not_read(self): + # A miscomputed field must not become the number we rank on. + preset = get_preset() + benchmark = preset.validations[0].benchmark + benchmark.metrics.output_tok_per_s = 999999.0 + benchmark.metrics.per_user_tok_per_s = 999999.0 + + expected = benchmark.metrics.total_output_tokens / benchmark.metrics.duration_seconds + assert benchmark.effective_output_tok_per_s == expected + # Per-user speed is the steady decode rate, not the aggregate over concurrency. + assert benchmark.effective_per_user_tok_per_s == 1000 / benchmark.metrics.tpot_ms.p50 + + class TestBuildConstraints: - def test_renders_all_fields_with_explicit_nulls_and_defaults(self): + def test_renders_defaults_for_the_optional_fields(self): configuration = PresetConfiguration( name="qwen", model={"base": "Qwen/Qwen3-32B"}, - max_trials=3, + concurrency=8, + trials=3, + max_ttft=5000, + min_context_length=32768, env=["HF_TOKEN"], ) @@ -448,9 +553,14 @@ def test_renders_all_fields_with_explicit_nulls_and_defaults(self): assert json.loads(text) == { "run_name_prefix": "qwen-abc123", "model": {"base": "Qwen/Qwen3-32B"}, - "context_length": None, - "max_trials": 3, + "min_context_length": 32768, + "max_ttft": 5000, + "trials_num": 3, "concurrency": 8, + "input_tokens": 1024, + "output_tokens": 1024, + "shared_prefix_tokens": 0, + "baseline": False, "fleets": ["gpu-fleet"], "env": ["HF_TOKEN"], } @@ -459,8 +569,9 @@ def test_renders_configured_values(self): configuration = PresetConfiguration( name="qwen", model={"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"}, - context_length=32768, - max_trials=10, + min_context_length=32768, + max_ttft=5000, + trials=10, concurrency=16, ) @@ -473,8 +584,8 @@ def test_renders_configured_values(self): ) assert data["model"] == {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"} - assert data["context_length"] == 32768 - assert data["max_trials"] == 10 + assert data["min_context_length"] == 32768 + assert data["trials_num"] == 10 assert data["concurrency"] == 16 assert data["fleets"] == ["a", "b"] diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 49cca6337..a46013a93 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -1,3 +1,4 @@ +import json from datetime import timedelta from io import StringIO @@ -33,11 +34,33 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): output = output_module.format_preset_benchmark(preset, verbose=True) - assert output.startswith("ctx=32K ") - assert "TTFT 8.15s" in output + # Per-user speed is 1/TPOT (p50 7.4ms), not the aggregate over concurrency. + assert output.startswith("tok/s/user=135 ") + assert output_module.format_preset_objective(preset).startswith("[secondary]io=1K/128 ") + assert "ctx=32K" in output + assert "ttft=8.15s" in output assert "e+03" not in output +class TestFormatPresetObjective: + def test_shows_the_shared_prefix_the_benchmark_actually_used(self): + preset = get_preset() + preset.validations[0].benchmark.workload.shared_prefix_tokens = 768 + + assert output_module.format_preset_objective(preset) == ( + "[secondary]io=1K/128 prefix=75% conc=1[/]" + ) + + def test_a_preset_saved_before_the_field_existed_still_loads(self): + # `shared_prefix_tokens` is absent from every preset saved so far. + preset = get_preset() + + assert preset.validations[0].benchmark.workload.shared_prefix_tokens == 0 + assert output_module.format_preset_objective(preset) == ( + "[secondary]io=1K/128 prefix=0% conc=1[/]" + ) + + class TestPrintPresets: def test_preserves_benchmark_concurrency_at_narrow_width(self, monkeypatch): output = StringIO() @@ -45,7 +68,7 @@ def test_preserves_benchmark_concurrency_at_narrow_width(self, monkeypatch): output_module.print_presets([get_preset()]) - assert "con=1" in "".join(output.getvalue().split()) + assert "conc=1" in "".join(output.getvalue().split()) def test_prints_submitted_column(self, monkeypatch): output = StringIO() @@ -60,7 +83,7 @@ def test_prints_submitted_column(self, monkeypatch): def _session_row(session: dict) -> dict: table = Table(box=None) - for column in ("BASE", "ID", "GPU", "BENCHMARK", "STATUS", "SUBMITTED"): + for column in ("BASE", "ID", "GPU", "CONSTRAINTS", "BENCHMARK", "", "STATUS", "SUBMITTED"): table.add_column(column) _add_session(table, session) return { @@ -74,39 +97,79 @@ def test_shows_progress_after_status_and_best_benchmark(self): { "id": "c7e18d52", "status": "running", - "max_trials": 3, + "trials_num": 3, "trials": { "count": 2, - "best": {"tok_s": 2339.0, "concurrency": 8, "gpu": "A40:48GB:1"}, + "best": { + "tok_s": 2339.0, + "tpot_ms": 3.42, + "concurrency": 8, + "gpu": "A40:48GB:1", + }, }, } ) - assert row["STATUS"] == "[bold sea_green3]clauding[/] [secondary](2/3)[/]" - assert row["BENCHMARK"] == "best trial: con=8 2339 tok/s" + assert row["STATUS"] == "[bold sea_green3]trialing[/] [secondary](2/3)[/]" + # Per-user speed is 1/TPOT, the same definition the preset row uses — not + # the aggregate over concurrency, which would read 292 here. + assert row["BENCHMARK"].startswith("tok/s/user=292") assert row["GPU"] == "A40:48GB:1" + def test_shows_the_shared_prefix_when_the_workload_has_one(self): + row = _session_row( + { + "id": "c7e18d52", + "status": "running", + "constraints": { + "input_tokens": 8192, + "output_tokens": 1024, + "concurrency": 162, + "shared_prefix_tokens": 7373, + }, + } + ) + + assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% conc=162[/]") + + def test_shows_the_shared_prefix_even_when_requests_are_fully_unique(self): + # `prefix=0%` is not noise: it decides how much of each request the engine + # can serve from cache, so a row without it cannot be compared to one with. + row = _session_row( + { + "id": "c7e18d52", + "status": "running", + "constraints": { + "input_tokens": 8192, + "output_tokens": 1024, + "concurrency": 162, + "shared_prefix_tokens": 0, + }, + } + ) + + assert "prefix=0%" in row["CONSTRAINTS"] + def test_shows_zero_progress_without_benchmark(self): row = _session_row( - {"id": "ab12cd34", "status": "running", "max_trials": 3, "trials": {"count": 0}} + {"id": "ab12cd34", "status": "running", "trials_num": 3, "trials": {"count": 0}} ) - assert row["STATUS"] == "[bold sea_green3]clauding[/] [secondary](0/3)[/]" - assert row["BENCHMARK"] == "" + assert row["STATUS"] == "[bold sea_green3]trialing[/] [secondary](0/3)[/]" def test_omits_progress_without_trials_data(self): row = _session_row({"id": "ab12cd34", "status": "interrupted"}) assert row["STATUS"] == "[bold gold1]interrupted[/]" - def test_counts_without_max_trials(self): + def test_counts_without_trials_num(self): row = _session_row({"id": "ab12cd34", "status": "interrupted", "trials": {"count": 2}}) assert row["STATUS"] == "[bold gold1]interrupted[/] [secondary](2)[/]" -class TestGroupOrdering: - def test_sorts_presets_and_sessions_newest_first(self, monkeypatch): +class TestOrdering: + def test_sorts_all_rows_newest_first_without_grouping(self, monkeypatch): buffer = StringIO() monkeypatch.setattr(output_module, "console", plain_console(buffer, width=200)) old = get_preset() @@ -131,8 +194,20 @@ def test_sorts_presets_and_sessions_newest_first(self, monkeypatch): output_module.print_presets([old, new], sessions=sessions) text = buffer.getvalue() + # Same contract as `dstack ps`: nothing is active here, so exactly one + # row is shown, the most recent. + assert "bbbbbbbb" in text + assert "aaaaaaaa" not in text + assert "11aa22bb" not in text and old.id not in text + + buffer.truncate(0) + buffer.seek(0) + output_module.print_presets([old, new], sessions=sessions, all_presets=True) + text = buffer.getvalue() + + # With -a: one flat list, newest first, presets and sessions interleaved. + assert text.index("bbbbbbbb") < text.index("aaaaaaaa") < text.index("11aa22bb") assert text.index("11aa22bb") < text.index(old.id) - assert text.index(old.id) < text.index("bbbbbbbb") < text.index("aaaaaaaa") class TestDoneProgress: @@ -145,7 +220,7 @@ def test_completed_creation_decorates_preset_row_without_extra_session_row(self, "id": preset.id, "status": "success", "model": preset.base, - "max_trials": 4, + "trials_num": 4, "trials": {"count": 3}, } ] @@ -153,21 +228,170 @@ def test_completed_creation_decorates_preset_row_without_extra_session_row(self, output_module.print_presets([preset], sessions=sessions) text = buffer.getvalue() - assert "verified (3/4)" in text + assert "(3/4)" in text + assert "verified" in text assert text.count(preset.id) == 1 class TestVerifyingStatus: - def test_running_session_with_exhausted_trials_shows_verifying(self): + def test_running_session_with_a_verification_record_shows_verifying(self): row = _session_row( - {"id": "ab12cd34", "status": "running", "max_trials": 2, "trials": {"count": 2}} + { + "id": "ab12cd34", + "status": "running", + "trials_num": 10, + "trials": {"count": 6}, + "verification": {"trial": 3, "run_name": "p-2", "status": "verifying"}, + } + ) + + # The trial budget is not spent, which the old inference required. + assert row["STATUS"] == "[bold deep_sky_blue1]verifying[/] [secondary](6/10)[/]" + + def test_a_failed_attempt_still_counts_as_verifying(self): + row = _session_row( + { + "id": "ab12cd34", + "status": "running", + "trials_num": 2, + "trials": {"count": 2}, + "verification": {"trial": 3, "status": "failed", "reason": "probe never passed"}, + } + ) + + assert row["STATUS"].startswith("[bold deep_sky_blue1]verifying[/]") + + def test_a_spent_trial_budget_alone_stays_trialing(self): + row = _session_row( + {"id": "ab12cd34", "status": "running", "trials_num": 2, "trials": {"count": 2}} + ) + + assert row["STATUS"].startswith("[bold sea_green3]trialing[/]") + + +class TestFailedTrials: + def test_a_failed_trial_keeps_its_benchmark_but_never_becomes_best(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + # The failed trial is the fastest. It broke `max_ttft`, so promoting it + # would put a non-compliant configuration at the top of the listing. + path = tmp_path / "trials.jsonl" + path.write_text( + "\n".join( + json.dumps( + { + "benchmark": { + "metrics": {"total_output_tokens": tokens, "duration_seconds": 1.0}, + "workload": {"concurrency": 8}, + }, + **({"failed": True} if failed else {}), + } + ) + for tokens, failed in ((100.0, False), (900.0, True), (300.0, False)) + ) + ) + + summary = _summarize_session_trials(path) + + assert summary["count"] == 3 + # Still charted: the trial happened and its number is real. + assert summary["series"] == [100.0, 900.0, 300.0] + assert summary["best"]["tok_s"] == 300.0 + assert summary["best_failed"]["tok_s"] == 900.0 + + def test_reports_the_gpu_when_no_trial_produced_a_benchmark(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + # Neither `best` nor `best_failed` can carry the hardware here, so this is + # the only thing left that knows what the run was on. + path = tmp_path / "trials.jsonl" + path.write_text( + json.dumps({"resources": {"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}}) ) - assert row["STATUS"] == "[bold deep_sky_blue1]verifying[/] [secondary](2/2)[/]" + summary = _summarize_session_trials(path) + + assert summary["best"] is None + assert summary["best_failed"] is None + assert summary["gpu"] == "MI300X:192GB:1" + + def test_the_fastest_failed_trial_is_kept_when_nothing_passed(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + path = tmp_path / "trials.jsonl" + path.write_text( + "\n".join( + json.dumps( + { + "benchmark": { + "metrics": { + "total_output_tokens": tokens, + "duration_seconds": 1.0, + "tpot_ms": {"p50": 34.4}, + "ttft_ms": {"p50": 4300.0}, + }, + "workload": {"concurrency": 4}, + }, + "failed": True, + } + ) + for tokens in (100.0, 300.0, 200.0) + ) + ) + + summary = _summarize_session_trials(path) - def test_running_session_with_remaining_trials_stays_clauding(self): + assert summary["best"] is None + assert summary["best_failed"]["tok_s"] == 300.0 + + def test_a_run_that_met_nothing_still_shows_what_it_measured(self): row = _session_row( - {"id": "ab12cd34", "status": "running", "max_trials": 2, "trials": {"count": 1}} + { + "id": "ab12cd34", + "status": "running", + "constraints": {"input_tokens": 10000, "output_tokens": 1500}, + "trials": { + "count": 1, + "series": [100.0], + "failed": [True], + "best": None, + "best_failed": { + "tok_s": 100.0, + "tpot_ms": 34.4, + "ttft_ms": 4300.0, + "context_length": 1048576, + "concurrency": 4, + "gpu": "MI300X:192GB:1", + }, + }, + } ) - assert row["STATUS"].startswith("[bold sea_green3]clauding[/]") + assert "ttft=4.3s" in row["BENCHMARK"] + assert row["GPU"] == "MI300X:192GB:1" + # Marked, not just dimmed, so it does not read as a met constraint where + # styling is absent or invisible to the reader. + assert "*tok/s/user" in row["BENCHMARK"] + + +class TestFailedTrialSpark: + def test_a_trial_that_broke_a_constraint_is_yellow_but_still_charted(self): + # Its number is real, so it earns a bar; the breach makes it yellow, not + # red — red is reserved for a trial that produced nothing. + session = { + "id": "ab12cd34", + "status": "running", + "trials": { + "count": 3, + "series": [100.0, 900.0, 300.0], + "failed": [False, True, False], + }, + } + + spark = output_module._format_trial_spark(session) + + assert spark.count("gold1") == 1 + assert "indian_red1" not in spark + assert "·" not in spark + # The failed trial is the highest number and must not be styled as best. + assert "sea_green3" not in spark diff --git a/src/tests/_internal/cli/services/presets/test_prompt.py b/src/tests/_internal/cli/services/presets/test_prompt.py index 111576340..fc1d8cf2c 100644 --- a/src/tests/_internal/cli/services/presets/test_prompt.py +++ b/src/tests/_internal/cli/services/presets/test_prompt.py @@ -13,7 +13,8 @@ def test_stays_byte_identical_without_user_prompt(self): assert text == get_preset_agent_system_prompt(None) == get_preset_agent_system_prompt("") assert "## Additional instructions" not in text - assert "\n" + "Plain and stay.\n" + ) + monkeypatch.setattr(prompt_module, "_SYSTEM_PROMPT_PATH", noted) + + text = get_preset_agent_system_prompt() + + assert "TODO" not in text + assert text == "Kept.\nPlain and stay." def test_rejects_unknown_directive_variables(self, tmp_path, monkeypatch): broken = tmp_path / "system_prompt.md" diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index fda1790b8..b3b27e954 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -49,7 +49,7 @@ def test_builds_portable_self_contained_preset(self): preset_configuration=PresetConfiguration( name="qwen-build", model={"base": "Qwen/Qwen3.5-27B"}, - context_length=8192, + min_context_length=8192, gateway="benchmark-gateway", env=["LICENSE", "TOKENIZERS_PARALLELISM=false"], ),