diff --git a/Cargo.lock b/Cargo.lock index 3c99a69128f7..4bc92f2b7916 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.13.0-dev.202610280000000086" +version = "0.13.0-dev.202610290000000087" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2149f65333729153bc6f89b30b2adca4cd58c7ddcc6f9ed6ce0a07cdf66f1cba" +checksum = "eb823782e806dd74cba719906a6b828b86f415aad54c6d5016c29ff278e89319" dependencies = [ "anyhow", "bincode 1.3.3", diff --git a/Cargo.toml b/Cargo.toml index 4e254e8d1d7f..27ef453c09e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"] [workspace.dependencies] dynamo-custom-policy-builtin = { path = "lib/router-plugins/builtin", version = "1.7.0" } # Published separately by https://github.com/ai-dynamo/aisimulate. -aisimulate-core = { version = "=0.13.0-dev.202610280000000086" } +aisimulate-core = { version = "=0.13.0-dev.202610290000000087" } # Local crates dynamo-runtime = { path = "lib/runtime", version = "1.7.0" } diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml index 1cacba7a9ff5..5760b9f24751 100644 --- a/benchmarks/pyproject.toml +++ b/benchmarks/pyproject.toml @@ -39,7 +39,7 @@ classifiers = [ ] dependencies = [ - "aisimulate==0.13.0.dev202610280000000086 ; python_version >= '3.11' and python_version < '3.14'", + "aisimulate==0.13.0.dev202610290000000087 ; python_version >= '3.11' and python_version < '3.14'", "aiperf==0.13.0", "matplotlib", "networkx", diff --git a/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py b/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py index 895cc3976932..0b33e2ec640f 100644 --- a/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py +++ b/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py @@ -6,11 +6,8 @@ from __future__ import annotations import importlib -from collections.abc import Mapping from typing import Any -import yaml - from dynamo.aisimulate.output.dgd.renderers.base import ( CandidateLike, CandidateMaterializationError, @@ -18,14 +15,6 @@ patch_dgd_manifest, ) -# AIC requires a workload ISL/OSL. With the evaluated engine limits preserved, -# the only engine field it still derives from them is TensorRT-LLM's -# disaggregated KV-transfer buffer. A trace has no single request length, so -# AIC gets placeholder lengths and the buffer size derived from them is dropped, -# leaving TensorRT-LLM's default, which covers max_seq_len. -_TRACE_PLACEHOLDER_LENGTH = 1 -_TRANSFER_BUFFER_ARG = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" - def _load_generator_api() -> tuple[Any, Any]: """Load the Sweeper-result bridge shipped by AISimulate.""" @@ -56,71 +45,6 @@ def _generator_overrides( } -def _workload_field(workload: Any, name: str) -> Any: - if isinstance(workload, Mapping): - return workload.get(name) - return getattr(workload, name, None) - - -def _is_trace_workload(workload: Any) -> bool: - return bool( - _workload_field(workload, "trace_paths") - or _workload_field(workload, "trace_path") - ) and ( - _workload_field(workload, "isl") is None - or _workload_field(workload, "osl") is None - ) - - -def _with_placeholder_lengths(workload: Any) -> dict[str, Any]: - """Copy a trace workload with the placeholder lengths AIC requires. - - The keys are overwritten rather than supplied as ``SlaConfig`` overrides - because AIC prefers a workload's present ``None`` values over overrides. - """ - if isinstance(workload, Mapping): - payload = dict(workload) - elif hasattr(workload, "model_dump"): - payload = workload.model_dump(mode="json") - else: - payload = dict(vars(workload)) - payload["isl"] = payload["osl"] = _TRACE_PLACEHOLDER_LENGTH - return payload - - -def _drop_transfer_buffer_size(rendered: str) -> str: - """Remove the KV-transfer buffer size AIC derived from placeholder lengths. - - Only the size goes; the transceiver backend stays, since TensorRT-LLM - disables KV transfer without one. - """ - from dynamo.profiler.utils.config import ( - break_arguments, - get_main_container_dict, - remove_all_argument_occurrences, - ) - - documents = [document for document in yaml.safe_load_all(rendered) if document] - changed = False - for document in documents: - if not isinstance(document, dict): - continue - for component in (document.get("spec") or {}).get("components") or []: - container = ( - get_main_container_dict(component) - if isinstance(component, dict) - else None - ) - if container is None: - continue - args = break_arguments(container.get("args")) - kept = remove_all_argument_occurrences(args, _TRANSFER_BUFFER_ARG) - if kept != args: - container["args"] = kept - changed = True - return yaml.safe_dump_all(documents, sort_keys=False) if changed else rendered - - def render( candidate: CandidateLike, workload: Any, @@ -130,10 +54,9 @@ def render( ) -> str: """Lower one Sweeper result through AIC and Dynamo's v1 templates.""" from_sweeper_candidate, generate_from_request = _load_generator_api() - trace = _is_trace_workload(workload) request = from_sweeper_candidate( candidate, - workload=_with_placeholder_lengths(workload) if trace else workload, + workload=workload, deployment_target="dynamo-python", generator_overrides=_generator_overrides(options, dgd_name=dgd_name), ) @@ -143,8 +66,6 @@ def render( raise CandidateMaterializationError( "AIC generator did not return k8s_deploy.yaml" ) - if trace: - rendered = _drop_transfer_buffer_size(rendered) return patch_dgd_manifest( rendered, candidate, diff --git a/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py b/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py index 9065272e26dd..53b1c9b0c3b2 100644 --- a/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py +++ b/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py @@ -263,50 +263,6 @@ def test_installed_aic_renders_real_disaggregated_candidate(monkeypatch) -> None assert workers["decode"]["replicas"] == 1 -@pytest.mark.parametrize( - "make_workload", [dict, lambda **fields: SimpleNamespace(**fields)] -) -def test_aic_renderer_gives_trace_workloads_placeholder_lengths( - monkeypatch, make_workload -) -> None: - captured = [] - - def fake_from_sweeper_candidate(candidate, *, workload, **_kwargs): - captured.append(workload) - return "request" - - monkeypatch.setattr( - aic_renderer, - "_load_generator_api", - lambda: ( - fake_from_sweeper_candidate, - lambda _request: {"k8s_deploy.yaml": "kind: DynamoGraphDeployment\n"}, - ), - ) - monkeypatch.setattr( - aic_renderer, "patch_dgd_manifest", lambda rendered, *_args, **_kwargs: rendered - ) - fields = { - "trace_path": "/traces/window.jsonl", - "trace_paths": ["/traces/window.jsonl"], - "trace_format": "dynamo", - "isl": None, - "osl": None, - } - workload = make_workload(**fields) - - render_dgd(_candidate(), workload, _options(), dgd_name="sweeper-dgd") - - assert captured == [{**fields, "isl": 1, "osl": 1}] - assert _workload_lengths(workload) == (None, None) - - -def _workload_lengths(workload): - if isinstance(workload, dict): - return workload["isl"], workload["osl"] - return workload.isl, workload.osl - - @dataclass(frozen=True) class _SweeperCandidate: config: dict @@ -362,6 +318,20 @@ def _transfer_buffer_args(rendered: str) -> dict[str, dict[str, str]]: return flags +def _without_transfer_buffer_size(rendered: str) -> dict: + buffer_arg = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" + dgd = yaml.safe_load(rendered) + for component in dgd["spec"]["components"]: + for container in ( + component.get("podTemplate", {}).get("spec", {}).get("containers", []) + ): + args = container.get("args") or [] + if buffer_arg in args: + start = args.index(buffer_arg) + del args[start : start + 2] + return dgd + + @pytest.mark.parametrize( ("backend", "backend_version", "image"), [ @@ -408,11 +378,9 @@ def render(workload) -> str: ) synthetic = render({"isl": 4000, "osl": 1000}) - # Apart from TensorRT-LLM's transfer buffer size, nothing may depend on - # the placeholder lengths a trace workload gives AIC. - assert yaml.safe_load(traced) == yaml.safe_load( - aic_renderer._drop_transfer_buffer_size(synthetic) - ) + # AIC derives only TensorRT-LLM's transfer buffer size from request + # lengths, so a trace DGD is the synthetic one without that size. + assert yaml.safe_load(traced) == _without_transfer_buffer_size(synthetic) if backend == "trtllm": # The trace keeps the transceiver backend; TensorRT-LLM sizes the buffer. assert _transfer_buffer_args(traced) == { @@ -425,63 +393,6 @@ def render(workload) -> str: ) -def test_dropping_transfer_buffer_size_keeps_other_arguments() -> None: - buffer_arg = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" - - def component(component_type, args): - return { - "type": component_type, - "podTemplate": {"spec": {"containers": [{"name": "main", "args": args}]}}, - } - - rendered = yaml.safe_dump( - { - "kind": "DynamoGraphDeployment", - "spec": { - "components": [ - {"name": "Frontend", "type": "frontend"}, - component( - "prefill", - [ - "--trtllm.cache_transceiver_config.backend", - "DEFAULT", - buffer_arg, - "6528", - "--max-num-tokens", - "4096", - ], - ), - component( - "decode", [f"{buffer_arg}=6528", "--max-batch-size", "64"] - ), - ] - }, - }, - sort_keys=False, - ) - unrelated = yaml.safe_dump( - {"spec": {"components": [component("worker", ["--block-size", "64"])]}} - ) - - dgd = yaml.safe_load(aic_renderer._drop_transfer_buffer_size(rendered)) - - args = { - component["type"]: component["podTemplate"]["spec"]["containers"][0]["args"] - for component in dgd["spec"]["components"] - if "podTemplate" in component - } - assert args == { - "prefill": [ - "--trtllm.cache_transceiver_config.backend", - "DEFAULT", - "--max-num-tokens", - "4096", - ], - "decode": ["--max-batch-size", "64"], - } - assert aic_renderer._drop_transfer_buffer_size(unrelated) is unrelated - - def test_materialize_direct_uses_config_modifiers(monkeypatch) -> None: pytest.importorskip("aisimulate_core") captured = {} diff --git a/container/deps/requirements.aisimulate.txt b/container/deps/requirements.aisimulate.txt index 4231f0448352..7c9c5210f7fe 100644 --- a/container/deps/requirements.aisimulate.txt +++ b/container/deps/requirements.aisimulate.txt @@ -3,4 +3,4 @@ # Published AISimulate wheel staged into planner and framework runtime wheelhouses. # Keep in sync with the aisimulate-core pins in Cargo.toml and lib/bindings/python/Cargo.toml. -aisimulate==0.13.0.dev202610280000000086 +aisimulate==0.13.0.dev202610290000000087 diff --git a/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md b/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md index 5d16a404e109..8eb9c89b3d29 100644 --- a/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md +++ b/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md @@ -86,7 +86,7 @@ kv_transfer: timing_mode: destination_missing ``` -The offline simulation uses a per-request line-rate model: +The offline simulation prices each transfer with a line-rate model: ```text transfer time = transferred KV bytes / kv_transfer_bandwidth @@ -95,7 +95,12 @@ transferred KV bytes = charged tokens * kv_bytes_per_token `timing_mode: full_prompt` charges the full logical prompt. `destination_missing` charges only the prompt footprint absent from the destination and can produce zero delay on a full destination hit. -Concurrent requests do not contend for a shared link. + +That time applies to a transfer alone on its links. In offline prediction, each prefill rank sends +over one link and each decode rank receives over one link, both at the configured bandwidth. +Transfers in flight at the same time share these links max-min fairly, so concurrent handoffs from +one prefill rank, or into one decode rank, take longer than a lone transfer. Live Mocker workers do +not share links: each transfer waits its own fixed delay. Set `bytes_per_token: auto` to derive the value from model metadata and parallelism, or provide a positive integer. Set a positive `bandwidth_gb_per_second` to apply the transfer delay. TensorRT-LLM diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 004e03cbf4b8..f0b72aa4310b 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -185,7 +185,7 @@ build, or mount a local copy before starting Planner. Use the generic AISimulate wheel; measurements do not belong inside the wheel. Use matching AISimulate Python and Rust releases that support execution profiles through the common forward-pass query API. This integration pins Python -`0.13.0.dev202610280000000086` and Rust `0.13.0-dev.202610280000000086`, +`0.13.0.dev202610290000000087` and Rust `0.13.0-dev.202610290000000087`, which include that support. Use version-2 execution profiles; version 1 retains its legacy direct-phase queries and must be explicitly migrated and repinned. diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 8d2e1191cbf6..cd3b2abd761d 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.13.0-dev.202610280000000086" +version = "0.13.0-dev.202610290000000087" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2149f65333729153bc6f89b30b2adca4cd58c7ddcc6f9ed6ce0a07cdf66f1cba" +checksum = "eb823782e806dd74cba719906a6b828b86f415aad54c6d5016c29ff278e89319" dependencies = [ "anyhow", "bincode 1.3.3", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index d4b90f6045cf..613f36c60080 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.13.0-dev.202610280000000086" +version = "0.13.0-dev.202610290000000087" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2149f65333729153bc6f89b30b2adca4cd58c7ddcc6f9ed6ce0a07cdf66f1cba" +checksum = "eb823782e806dd74cba719906a6b828b86f415aad54c6d5016c29ff278e89319" dependencies = [ "anyhow", "bincode 1.3.3", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 8805cdb72321..c1f6f3ad0e0b 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -61,7 +61,7 @@ request-trace-s3 = ["dynamo-llm/request-trace-s3"] # former AIC crate surface. The `python` feature is required for # AicEngineBuilder; this extension supplies the interpreter, so the dependency # must not enable `extension-module`. -aisimulate-core = { version = "=0.13.0-dev.202610280000000086", optional = true, features = ["python"] } +aisimulate-core = { version = "=0.13.0-dev.202610290000000087", optional = true, features = ["python"] } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } dynamo-parsers = { version = "=10.2.2" } diff --git a/pyproject.toml b/pyproject.toml index 9e0583619b80..811cf3b4ea3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ "ai-dynamo-runtime==1.7.0", - "aisimulate==0.13.0.dev202610280000000086 ; python_version >= '3.11' and python_version < '3.14'", + "aisimulate==0.13.0.dev202610290000000087 ; python_version >= '3.11' and python_version < '3.14'", "aiohttp>=3.14.3,<4.0", # Set the CA bundle floor for transitive HTTP clients outside containers too. "certifi>=2026.10.23",