From 52ff787f6d11ecec0b1e5865a5d0cfe5b8347e71 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 28 Oct 2026 10:50:20 -0700 Subject: [PATCH 1/4] fix(aisimulate): pass trace workloads straight to the AIC generator AISimulate's generator bridge now lowers a trace workload without request lengths and rejects placeholder ones (ai-dynamo/aisimulate#626), so the AIC renderer again passes the workload, and the manifest AIC returns, through unchanged. The renderer tests check that a trace DGD equals the synthetic one without TensorRT-LLM's KV-transfer buffer size. This needs an AISimulate nightly containing that change. The pin bump lands in this PR before it leaves draft. Signed-off-by: hongkuanz Co-Authored-By: Claude Opus 5.5 --- .../output/dgd/renderers/aic/renderer.py | 81 +----------- .../aisimulate/tests/dgd/test_renderers.py | 116 +++++------------- 2 files changed, 32 insertions(+), 165 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py b/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py index 895cc3976932..0b33e2ec640f 100644 --- a/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py +++ b/components/src/dynamo/aisimulate/output/dgd/renderers/aic/renderer.py @@ -6,11 +6,8 @@ from __future__ import annotations import importlib -from collections.abc import Mapping from typing import Any -import yaml - from dynamo.aisimulate.output.dgd.renderers.base import ( CandidateLike, CandidateMaterializationError, @@ -18,14 +15,6 @@ patch_dgd_manifest, ) -# AIC requires a workload ISL/OSL. With the evaluated engine limits preserved, -# the only engine field it still derives from them is TensorRT-LLM's -# disaggregated KV-transfer buffer. A trace has no single request length, so -# AIC gets placeholder lengths and the buffer size derived from them is dropped, -# leaving TensorRT-LLM's default, which covers max_seq_len. -_TRACE_PLACEHOLDER_LENGTH = 1 -_TRANSFER_BUFFER_ARG = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" - def _load_generator_api() -> tuple[Any, Any]: """Load the Sweeper-result bridge shipped by AISimulate.""" @@ -56,71 +45,6 @@ def _generator_overrides( } -def _workload_field(workload: Any, name: str) -> Any: - if isinstance(workload, Mapping): - return workload.get(name) - return getattr(workload, name, None) - - -def _is_trace_workload(workload: Any) -> bool: - return bool( - _workload_field(workload, "trace_paths") - or _workload_field(workload, "trace_path") - ) and ( - _workload_field(workload, "isl") is None - or _workload_field(workload, "osl") is None - ) - - -def _with_placeholder_lengths(workload: Any) -> dict[str, Any]: - """Copy a trace workload with the placeholder lengths AIC requires. - - The keys are overwritten rather than supplied as ``SlaConfig`` overrides - because AIC prefers a workload's present ``None`` values over overrides. - """ - if isinstance(workload, Mapping): - payload = dict(workload) - elif hasattr(workload, "model_dump"): - payload = workload.model_dump(mode="json") - else: - payload = dict(vars(workload)) - payload["isl"] = payload["osl"] = _TRACE_PLACEHOLDER_LENGTH - return payload - - -def _drop_transfer_buffer_size(rendered: str) -> str: - """Remove the KV-transfer buffer size AIC derived from placeholder lengths. - - Only the size goes; the transceiver backend stays, since TensorRT-LLM - disables KV transfer without one. - """ - from dynamo.profiler.utils.config import ( - break_arguments, - get_main_container_dict, - remove_all_argument_occurrences, - ) - - documents = [document for document in yaml.safe_load_all(rendered) if document] - changed = False - for document in documents: - if not isinstance(document, dict): - continue - for component in (document.get("spec") or {}).get("components") or []: - container = ( - get_main_container_dict(component) - if isinstance(component, dict) - else None - ) - if container is None: - continue - args = break_arguments(container.get("args")) - kept = remove_all_argument_occurrences(args, _TRANSFER_BUFFER_ARG) - if kept != args: - container["args"] = kept - changed = True - return yaml.safe_dump_all(documents, sort_keys=False) if changed else rendered - - def render( candidate: CandidateLike, workload: Any, @@ -130,10 +54,9 @@ def render( ) -> str: """Lower one Sweeper result through AIC and Dynamo's v1 templates.""" from_sweeper_candidate, generate_from_request = _load_generator_api() - trace = _is_trace_workload(workload) request = from_sweeper_candidate( candidate, - workload=_with_placeholder_lengths(workload) if trace else workload, + workload=workload, deployment_target="dynamo-python", generator_overrides=_generator_overrides(options, dgd_name=dgd_name), ) @@ -143,8 +66,6 @@ def render( raise CandidateMaterializationError( "AIC generator did not return k8s_deploy.yaml" ) - if trace: - rendered = _drop_transfer_buffer_size(rendered) return patch_dgd_manifest( rendered, candidate, diff --git a/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py b/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py index 9065272e26dd..60848569ba00 100644 --- a/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py +++ b/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py @@ -263,12 +263,7 @@ def test_installed_aic_renders_real_disaggregated_candidate(monkeypatch) -> None assert workers["decode"]["replicas"] == 1 -@pytest.mark.parametrize( - "make_workload", [dict, lambda **fields: SimpleNamespace(**fields)] -) -def test_aic_renderer_gives_trace_workloads_placeholder_lengths( - monkeypatch, make_workload -) -> None: +def test_aic_renderer_passes_trace_workloads_through(monkeypatch) -> None: captured = [] def fake_from_sweeper_candidate(candidate, *, workload, **_kwargs): @@ -286,25 +281,20 @@ def fake_from_sweeper_candidate(candidate, *, workload, **_kwargs): monkeypatch.setattr( aic_renderer, "patch_dgd_manifest", lambda rendered, *_args, **_kwargs: rendered ) - fields = { - "trace_path": "/traces/window.jsonl", - "trace_paths": ["/traces/window.jsonl"], - "trace_format": "dynamo", - "isl": None, - "osl": None, - } - workload = make_workload(**fields) - - render_dgd(_candidate(), workload, _options(), dgd_name="sweeper-dgd") - - assert captured == [{**fields, "isl": 1, "osl": 1}] - assert _workload_lengths(workload) == (None, None) + workload = SimpleNamespace( + trace_path="/traces/window.jsonl", + trace_paths=["/traces/window.jsonl"], + trace_format="dynamo", + isl=None, + osl=None, + ) + rendered = render_dgd(_candidate(), workload, _options(), dgd_name="sweeper-dgd") -def _workload_lengths(workload): - if isinstance(workload, dict): - return workload["isl"], workload["osl"] - return workload.isl, workload.osl + # AISimulate owns trace lowering, so the renderer neither fills in request + # lengths nor edits the manifest AIC returns. + assert len(captured) == 1 and captured[0] is workload + assert rendered == "kind: DynamoGraphDeployment\n" @dataclass(frozen=True) @@ -362,6 +352,21 @@ def _transfer_buffer_args(rendered: str) -> dict[str, dict[str, str]]: return flags +def _without_transfer_buffer_size(rendered: str) -> dict: + """Return the DGD without TensorRT-LLM's KV-transfer buffer size.""" + buffer_arg = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" + dgd = yaml.safe_load(rendered) + for component in dgd["spec"]["components"]: + for container in ( + component.get("podTemplate", {}).get("spec", {}).get("containers", []) + ): + args = container.get("args") or [] + if buffer_arg in args: + start = args.index(buffer_arg) + del args[start : start + 2] + return dgd + + @pytest.mark.parametrize( ("backend", "backend_version", "image"), [ @@ -408,11 +413,9 @@ def render(workload) -> str: ) synthetic = render({"isl": 4000, "osl": 1000}) - # Apart from TensorRT-LLM's transfer buffer size, nothing may depend on - # the placeholder lengths a trace workload gives AIC. - assert yaml.safe_load(traced) == yaml.safe_load( - aic_renderer._drop_transfer_buffer_size(synthetic) - ) + # AIC derives only TensorRT-LLM's transfer buffer size from request + # lengths, so a trace DGD is the synthetic one without that size. + assert yaml.safe_load(traced) == _without_transfer_buffer_size(synthetic) if backend == "trtllm": # The trace keeps the transceiver backend; TensorRT-LLM sizes the buffer. assert _transfer_buffer_args(traced) == { @@ -425,63 +428,6 @@ def render(workload) -> str: ) -def test_dropping_transfer_buffer_size_keeps_other_arguments() -> None: - buffer_arg = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" - - def component(component_type, args): - return { - "type": component_type, - "podTemplate": {"spec": {"containers": [{"name": "main", "args": args}]}}, - } - - rendered = yaml.safe_dump( - { - "kind": "DynamoGraphDeployment", - "spec": { - "components": [ - {"name": "Frontend", "type": "frontend"}, - component( - "prefill", - [ - "--trtllm.cache_transceiver_config.backend", - "DEFAULT", - buffer_arg, - "6528", - "--max-num-tokens", - "4096", - ], - ), - component( - "decode", [f"{buffer_arg}=6528", "--max-batch-size", "64"] - ), - ] - }, - }, - sort_keys=False, - ) - unrelated = yaml.safe_dump( - {"spec": {"components": [component("worker", ["--block-size", "64"])]}} - ) - - dgd = yaml.safe_load(aic_renderer._drop_transfer_buffer_size(rendered)) - - args = { - component["type"]: component["podTemplate"]["spec"]["containers"][0]["args"] - for component in dgd["spec"]["components"] - if "podTemplate" in component - } - assert args == { - "prefill": [ - "--trtllm.cache_transceiver_config.backend", - "DEFAULT", - "--max-num-tokens", - "4096", - ], - "decode": ["--max-batch-size", "64"], - } - assert aic_renderer._drop_transfer_buffer_size(unrelated) is unrelated - - def test_materialize_direct_uses_config_modifiers(monkeypatch) -> None: pytest.importorskip("aisimulate_core") captured = {} From 352dcc44576c0e5f233e3e15f3b1a9323b2a6758 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Thu, 29 Oct 2026 09:23:45 -0700 Subject: [PATCH 2/4] chore(deps): pin AISimulate nightly 87 for trace lowering Nightly 87 (0.13.0.dev202610290000000087, Rust 0.13.0-dev.202610290000000087) is built from AISimulate 03eadda3 and contains ai-dynamo/aisimulate#626, which the pass-through AIC renderer relies on. Update every pin and the three Cargo.lock files; only the aisimulate-core version and checksum change. Signed-off-by: hongkuanz Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- benchmarks/pyproject.toml | 2 +- container/deps/requirements.aisimulate.txt | 2 +- .../fern/pages/reference/components/planner-configuration.mdx | 2 +- lib/bindings/kvbm/Cargo.lock | 4 ++-- lib/bindings/python/Cargo.lock | 4 ++-- lib/bindings/python/Cargo.toml | 2 +- pyproject.toml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c99a69128f7..4bc92f2b7916 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.13.0-dev.202610280000000086" +version = "0.13.0-dev.202610290000000087" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2149f65333729153bc6f89b30b2adca4cd58c7ddcc6f9ed6ce0a07cdf66f1cba" +checksum = "eb823782e806dd74cba719906a6b828b86f415aad54c6d5016c29ff278e89319" dependencies = [ "anyhow", "bincode 1.3.3", diff --git a/Cargo.toml b/Cargo.toml index 4e254e8d1d7f..27ef453c09e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"] [workspace.dependencies] dynamo-custom-policy-builtin = { path = "lib/router-plugins/builtin", version = "1.7.0" } # Published separately by https://github.com/ai-dynamo/aisimulate. -aisimulate-core = { version = "=0.13.0-dev.202610280000000086" } +aisimulate-core = { version = "=0.13.0-dev.202610290000000087" } # Local crates dynamo-runtime = { path = "lib/runtime", version = "1.7.0" } diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml index 1cacba7a9ff5..5760b9f24751 100644 --- a/benchmarks/pyproject.toml +++ b/benchmarks/pyproject.toml @@ -39,7 +39,7 @@ classifiers = [ ] dependencies = [ - "aisimulate==0.13.0.dev202610280000000086 ; python_version >= '3.11' and python_version < '3.14'", + "aisimulate==0.13.0.dev202610290000000087 ; python_version >= '3.11' and python_version < '3.14'", "aiperf==0.13.0", "matplotlib", "networkx", diff --git a/container/deps/requirements.aisimulate.txt b/container/deps/requirements.aisimulate.txt index 4231f0448352..7c9c5210f7fe 100644 --- a/container/deps/requirements.aisimulate.txt +++ b/container/deps/requirements.aisimulate.txt @@ -3,4 +3,4 @@ # Published AISimulate wheel staged into planner and framework runtime wheelhouses. # Keep in sync with the aisimulate-core pins in Cargo.toml and lib/bindings/python/Cargo.toml. -aisimulate==0.13.0.dev202610280000000086 +aisimulate==0.13.0.dev202610290000000087 diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 004e03cbf4b8..f0b72aa4310b 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -185,7 +185,7 @@ build, or mount a local copy before starting Planner. Use the generic AISimulate wheel; measurements do not belong inside the wheel. Use matching AISimulate Python and Rust releases that support execution profiles through the common forward-pass query API. This integration pins Python -`0.13.0.dev202610280000000086` and Rust `0.13.0-dev.202610280000000086`, +`0.13.0.dev202610290000000087` and Rust `0.13.0-dev.202610290000000087`, which include that support. Use version-2 execution profiles; version 1 retains its legacy direct-phase queries and must be explicitly migrated and repinned. diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 8d2e1191cbf6..cd3b2abd761d 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.13.0-dev.202610280000000086" +version = "0.13.0-dev.202610290000000087" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2149f65333729153bc6f89b30b2adca4cd58c7ddcc6f9ed6ce0a07cdf66f1cba" +checksum = "eb823782e806dd74cba719906a6b828b86f415aad54c6d5016c29ff278e89319" dependencies = [ "anyhow", "bincode 1.3.3", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index d4b90f6045cf..613f36c60080 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.13.0-dev.202610280000000086" +version = "0.13.0-dev.202610290000000087" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2149f65333729153bc6f89b30b2adca4cd58c7ddcc6f9ed6ce0a07cdf66f1cba" +checksum = "eb823782e806dd74cba719906a6b828b86f415aad54c6d5016c29ff278e89319" dependencies = [ "anyhow", "bincode 1.3.3", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 8805cdb72321..c1f6f3ad0e0b 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -61,7 +61,7 @@ request-trace-s3 = ["dynamo-llm/request-trace-s3"] # former AIC crate surface. The `python` feature is required for # AicEngineBuilder; this extension supplies the interpreter, so the dependency # must not enable `extension-module`. -aisimulate-core = { version = "=0.13.0-dev.202610280000000086", optional = true, features = ["python"] } +aisimulate-core = { version = "=0.13.0-dev.202610290000000087", optional = true, features = ["python"] } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } dynamo-parsers = { version = "=10.2.2" } diff --git a/pyproject.toml b/pyproject.toml index 9e0583619b80..811cf3b4ea3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ "ai-dynamo-runtime==1.7.0", - "aisimulate==0.13.0.dev202610280000000086 ; python_version >= '3.11' and python_version < '3.14'", + "aisimulate==0.13.0.dev202610290000000087 ; python_version >= '3.11' and python_version < '3.14'", "aiohttp>=3.14.3,<4.0", # Set the CA bundle floor for transitive HTTP clients outside containers too. "certifi>=2026.10.23", From 99429c617ad9522243bae60f2ac7e058f3c8cae4 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Thu, 29 Oct 2026 09:23:46 -0700 Subject: [PATCH 3/4] docs(dynosim): describe KV transfer link sharing AISimulate nightly 87 (ai-dynamo/aisimulate#616) shares each prefill rank's sending link and each decode rank's receiving link max-min fairly among concurrent offline P/D transfers. The page still said concurrent requests do not contend for a shared link. Signed-off-by: hongkuanz Co-Authored-By: Claude Opus 5.5 --- .../simulation-with-dynosim/simulation-model.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md b/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md index 5d16a404e109..efcf60cd0c4a 100644 --- a/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md +++ b/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md @@ -86,7 +86,7 @@ kv_transfer: timing_mode: destination_missing ``` -The offline simulation uses a per-request line-rate model: +The offline simulation prices each transfer with a line-rate model: ```text transfer time = transferred KV bytes / kv_transfer_bandwidth @@ -95,7 +95,11 @@ transferred KV bytes = charged tokens * kv_bytes_per_token `timing_mode: full_prompt` charges the full logical prompt. `destination_missing` charges only the prompt footprint absent from the destination and can produce zero delay on a full destination hit. -Concurrent requests do not contend for a shared link. + +That time applies to a transfer alone on its links. Each prefill rank sends over one link and each +decode rank receives over one link, both at the configured bandwidth. Transfers in flight at the +same time share these links max-min fairly, so concurrent handoffs from one prefill rank, or into one +decode rank, take longer than a lone transfer. Set `bytes_per_token: auto` to derive the value from model metadata and parallelism, or provide a positive integer. Set a positive `bandwidth_gb_per_second` to apply the transfer delay. TensorRT-LLM From 88bca4d08f5b6a3b4186783b9a40c6c6458ce073 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Thu, 29 Oct 2026 10:04:07 -0700 Subject: [PATCH 4/4] fix(aisimulate): address review on trace pass-through - Scope KV transfer link sharing to offline prediction and note that live Mocker workers apply a fixed delay per transfer. - Drop the fake-bridge pass-through test; the real-bridge trace test already covers it in the dynamo-runtime pre-merge CPU lane. - Drop a docstring that repeated its function name. Signed-off-by: hongkuanz Co-Authored-By: Claude Opus 5.5 --- .../aisimulate/tests/dgd/test_renderers.py | 35 ------------------- .../simulation-model.md | 9 ++--- 2 files changed, 5 insertions(+), 39 deletions(-) diff --git a/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py b/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py index 60848569ba00..53b1c9b0c3b2 100644 --- a/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py +++ b/components/src/dynamo/aisimulate/tests/dgd/test_renderers.py @@ -263,40 +263,6 @@ def test_installed_aic_renders_real_disaggregated_candidate(monkeypatch) -> None assert workers["decode"]["replicas"] == 1 -def test_aic_renderer_passes_trace_workloads_through(monkeypatch) -> None: - captured = [] - - def fake_from_sweeper_candidate(candidate, *, workload, **_kwargs): - captured.append(workload) - return "request" - - monkeypatch.setattr( - aic_renderer, - "_load_generator_api", - lambda: ( - fake_from_sweeper_candidate, - lambda _request: {"k8s_deploy.yaml": "kind: DynamoGraphDeployment\n"}, - ), - ) - monkeypatch.setattr( - aic_renderer, "patch_dgd_manifest", lambda rendered, *_args, **_kwargs: rendered - ) - workload = SimpleNamespace( - trace_path="/traces/window.jsonl", - trace_paths=["/traces/window.jsonl"], - trace_format="dynamo", - isl=None, - osl=None, - ) - - rendered = render_dgd(_candidate(), workload, _options(), dgd_name="sweeper-dgd") - - # AISimulate owns trace lowering, so the renderer neither fills in request - # lengths nor edits the manifest AIC returns. - assert len(captured) == 1 and captured[0] is workload - assert rendered == "kind: DynamoGraphDeployment\n" - - @dataclass(frozen=True) class _SweeperCandidate: config: dict @@ -353,7 +319,6 @@ def _transfer_buffer_args(rendered: str) -> dict[str, dict[str, str]]: def _without_transfer_buffer_size(rendered: str) -> dict: - """Return the DGD without TensorRT-LLM's KV-transfer buffer size.""" buffer_arg = "--trtllm.cache_transceiver_config.max_tokens_in_buffer" dgd = yaml.safe_load(rendered) for component in dgd["spec"]["components"]: diff --git a/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md b/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md index efcf60cd0c4a..8eb9c89b3d29 100644 --- a/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md +++ b/docs/fern/pages/cli/operations/simulation-with-dynosim/simulation-model.md @@ -96,10 +96,11 @@ transferred KV bytes = charged tokens * kv_bytes_per_token `timing_mode: full_prompt` charges the full logical prompt. `destination_missing` charges only the prompt footprint absent from the destination and can produce zero delay on a full destination hit. -That time applies to a transfer alone on its links. Each prefill rank sends over one link and each -decode rank receives over one link, both at the configured bandwidth. Transfers in flight at the -same time share these links max-min fairly, so concurrent handoffs from one prefill rank, or into one -decode rank, take longer than a lone transfer. +That time applies to a transfer alone on its links. In offline prediction, each prefill rank sends +over one link and each decode rank receives over one link, both at the configured bandwidth. +Transfers in flight at the same time share these links max-min fairly, so concurrent handoffs from +one prefill rank, or into one decode rank, take longer than a lone transfer. Live Mocker workers do +not share links: each transfer waits its own fixed delay. Set `bytes_per_token: auto` to derive the value from model metadata and parallelism, or provide a positive integer. Set a positive `bandwidth_gb_per_second` to apply the transfer delay. TensorRT-LLM