From 2fdac40842ac7e76738ce76f4776c6f578f56863 Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Tue, 4 Aug 2026 14:24:09 +0200 Subject: [PATCH] [Breaking]: Drop gateway-based routers Drop the top-level `router` property from gateway and run configurations, deprecated in favor of replica-based routers since 0.20.17. New services and gateways with this configuration property will not be accepted. The behavior of existing services and gateways using this property is undefined. Users are advised to terminate such services and gateways (if any) prior to upgrading `dstack`. Some router implementation details still remain on the gateway and can be removed separately anytime. --- gateway/pyproject.toml | 1 + mkdocs/blog/posts/pd-disaggregation.md | 2 +- mkdocs/blog/posts/sglang-router.md | 3 + mkdocs/docs/concepts/gateways.md | 48 ----------- mkdocs/docs/concepts/services.md | 2 +- mkdocs/docs/reference/dstack.yml/gateway.md | 10 --- src/dstack/_internal/cli/commands/gateway.py | 15 ---- src/dstack/_internal/cli/commands/ps.py | 18 ---- .../cli/services/configurators/gateway.py | 7 -- .../cli/services/configurators/run.py | 8 -- .../_internal/core/backends/aws/compute.py | 4 +- .../_internal/core/backends/azure/compute.py | 4 +- .../_internal/core/backends/base/compute.py | 16 ++-- .../_internal/core/backends/gcp/compute.py | 4 +- .../core/backends/kubernetes/compute.py | 11 +-- .../_internal/core/compatibility/gateways.py | 2 - .../_internal/core/compatibility/runs.py | 6 -- .../_internal/core/models/configurations.py | 27 +----- src/dstack/_internal/core/models/gateways.py | 11 --- src/dstack/_internal/core/models/routers.py | 21 +---- src/dstack/_internal/proxy/lib/models.py | 1 + .../server/services/gateways/__init__.py | 19 +---- .../server/services/gateways/client.py | 3 - .../_internal/server/services/proxy/repo.py | 2 - .../server/services/services/__init__.py | 67 --------------- .../core/models/test_configurations.py | 21 ----- .../apply_gateway_plan_request.values.json | 1 - .../parsing/api_response/gateway.values.json | 1 - .../fixtures/parsing/config/gateway.input.yml | 5 +- .../parsing/config/gateway.types.json | 3 +- .../parsing/config/gateway.values.json | 4 - .../parsing/config/service.values.json | 1 - .../gateway_compute_configuration.values.json | 1 - .../db/gateway_configuration.values.json | 1 - .../fixtures/schema/configuration.json | 83 ------------------- .../apply_gateway_plan_request.json | 1 - .../serialization/api_response/gateway.json | 1 - .../db/gateway_compute_configuration.json | 1 - .../db/gateway_configuration.json | 1 - .../_internal/server/routers/test_gateways.py | 19 ----- 40 files changed, 23 insertions(+), 433 deletions(-) diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index 6c4d406a6f..81e5fda205 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ ] [project.optional-dependencies] +# TODO: drop, unused by the server since 0.21.0 sglang = ["sglang-router==0.3.2"] [tool.setuptools.package-data] diff --git a/mkdocs/blog/posts/pd-disaggregation.md b/mkdocs/blog/posts/pd-disaggregation.md index dd3f27c9e8..c885c6afc5 100644 --- a/mkdocs/blog/posts/pd-disaggregation.md +++ b/mkdocs/blog/posts/pd-disaggregation.md @@ -27,7 +27,7 @@ For inference, `dstack` provides a [services](../../docs/concepts/services.md) a > If you’re new to Prefill–Decode disaggregation, see the official [SGLang docs](https://docs.sglang.io/advanced_features/pd_disaggregation.html). !!! note "Deprecation notice" - Configuring the SGLang router in a gateway is deprecated and will be disallowed in a future release. To run router and workers as separate replica groups, see [SGLang PD disaggregation (router as replica group)](../../docs/examples/inference/sglang.md#pd-disaggregation). + The router configuration approach described in this blog post is no longer supported. For the modern approach, see [SGLang PD disaggregation (router as replica group)](../../docs/examples/inference/sglang.md#pd-disaggregation). ## Services diff --git a/mkdocs/blog/posts/sglang-router.md b/mkdocs/blog/posts/sglang-router.md index f33fd7e400..028e3a23da 100644 --- a/mkdocs/blog/posts/sglang-router.md +++ b/mkdocs/blog/posts/sglang-router.md @@ -125,6 +125,9 @@ After that, if you configure [replicas and scaling](../../docs/concepts/services ### Router +!!! note "Deprecation notice" + The router configuration approach described in this section is no longer supported. For the modern approach, see [SGLang PD disaggregation (router as replica group)](../../docs/examples/inference/sglang.md#pd-disaggregation). + By default, the gateway uses its built-in load balancer to route traffic across replicas. With the latest release, you can instead delegate traffic routing to the [SGLang Model Gateway](https://docs.sglang.ai/advanced_features/router.html) by setting the `router` property to `sglang`:
diff --git a/mkdocs/docs/concepts/gateways.md b/mkdocs/docs/concepts/gateways.md index 64e197fed1..c011486036 100644 --- a/mkdocs/docs/concepts/gateways.md +++ b/mkdocs/docs/concepts/gateways.md @@ -76,54 +76,6 @@ You can create gateways with the `aws`, `azure`, `gcp`, or `kubernetes` backends Gateways in `kubernetes` backend require an external load balancer. Managed Kubernetes solutions usually include a load balancer. For self-hosted Kubernetes, you must provide a load balancer by yourself. -### Router - -> In previous releases, `dstack` allowed configuring `router` the gateway, which was required for PD disaggregation. Since 0.20.17, the `router` configuration has moved to [services](services.md#pd-disaggregation), and the gateway no longer needs to configure router. - - - ### Certificate By default, when you run a service with a gateway, `dstack` provisions an SSL certificate via Let's Encrypt for the configured domain. This automatically enables HTTPS for the service endpoint. diff --git a/mkdocs/docs/concepts/services.md b/mkdocs/docs/concepts/services.md index 22116567cb..ea06d39be2 100644 --- a/mkdocs/docs/concepts/services.md +++ b/mkdocs/docs/concepts/services.md @@ -340,7 +340,7 @@ Setting the minimum number of replicas to `0` allows the service to scale down t ### PD disaggregation - + Since 0.20.17, `dstack` supports serving a model using Prefill-Decode disaggregation. To use it, configure three replica groups: one for the router, one for prefill workers, and one for decode workers. diff --git a/mkdocs/docs/reference/dstack.yml/gateway.md b/mkdocs/docs/reference/dstack.yml/gateway.md index 33fbeb4190..06d0433f41 100644 --- a/mkdocs/docs/reference/dstack.yml/gateway.md +++ b/mkdocs/docs/reference/dstack.yml/gateway.md @@ -10,16 +10,6 @@ The `gateway` configuration type allows creating and updating [gateways](../../c type: required: true -### `router` - -=== "SGLang Model Gateway" - - #SCHEMA# dstack._internal.core.models.routers.SGLangGatewayRouterConfig - overrides: - show_root_heading: false - type: - required: true - ### `certificate` Set to `null` to disable certificates (e.g. for [private gateways](../../concepts/gateways.md#public-ip)). diff --git a/src/dstack/_internal/cli/commands/gateway.py b/src/dstack/_internal/cli/commands/gateway.py index 2489e726b4..bd29431f20 100644 --- a/src/dstack/_internal/cli/commands/gateway.py +++ b/src/dstack/_internal/cli/commands/gateway.py @@ -19,7 +19,6 @@ ) from dstack._internal.core.errors import CLIError from dstack._internal.core.models.common import EntityReference -from dstack._internal.core.models.gateways import GatewayStatus from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -110,20 +109,6 @@ def _list(self, args: argparse.Namespace): raise CLIError("JSON output is not supported together with --watch") gateways = self.api.client.gateways.list(self.api.project, include_imported=True) - deprecated_router_gateways = [ - g.name - for g in gateways - if g.status != GatewayStatus.FAILED and g.configuration.router is not None - ] - if deprecated_router_gateways and args.format != "json": - logger.warning( - "Specifying `router` in gateway configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - " (affected gateways: %s)", - ", ".join(deprecated_router_gateways), - ) if not args.watch: if args.format == "json": print_gateways_json(gateways, project=self.api.project) diff --git a/src/dstack/_internal/cli/commands/ps.py b/src/dstack/_internal/cli/commands/ps.py index 0e8ab5d90b..0466081a32 100644 --- a/src/dstack/_internal/cli/commands/ps.py +++ b/src/dstack/_internal/cli/commands/ps.py @@ -11,7 +11,6 @@ console, ) from dstack._internal.core.errors import CLIError -from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -69,23 +68,6 @@ def _command(self, args: argparse.Namespace): # TODO: Add a `ps --json` option to control how many job submissions are returned. runs = self.api.runs.list(all=args.all, limit=args.last) - deprecated_router_runs = [ - run._run.run_spec.run_name - for run in runs - if not run.status.is_finished() - and isinstance(run._run.run_spec.configuration, ServiceConfiguration) - and run._run.run_spec.configuration.router is not None - and run._run.run_spec.run_name is not None - ] - if deprecated_router_runs and args.format != "json": - logger.warning( - "Specifying `router` in service configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - " (affected runs: %s)", - ", ".join(deprecated_router_runs), - ) if not args.watch: if args.format == "json": run_utils.print_runs_json(self.api.project, runs) diff --git a/src/dstack/_internal/cli/services/configurators/gateway.py b/src/dstack/_internal/cli/services/configurators/gateway.py index 4f78c11d88..76472b11f9 100644 --- a/src/dstack/_internal/cli/services/configurators/gateway.py +++ b/src/dstack/_internal/cli/services/configurators/gateway.py @@ -49,13 +49,6 @@ def apply_configuration( configuration=conf, configuration_path=configuration_path, ) - if spec.configuration.router is not None: - logger.warning( - "Specifying `router` in gateway configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - ) with console.status("Getting apply plan..."): try: plan = self.api.client.gateways.get_plan(project_name=self.api.project, spec=spec) diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index df2d0b35d4..36af8bd074 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -123,14 +123,6 @@ def get_plan( if conf.working_dir is not None and not is_absolute_posix_path(conf.working_dir): raise ConfigurationError("working_dir must be absolute") - if isinstance(conf, ServiceConfiguration) and conf.router is not None: - logger.warning( - "Specifying `router` in service configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - ) - repo = self.get_repo(conf, configuration_path, configurator_args) if repo is None: repo = init_default_virtual_repo(api=self.api) diff --git a/src/dstack/_internal/core/backends/aws/compute.py b/src/dstack/_internal/core/backends/aws/compute.py index 2e0a65b568..a0541c180a 100644 --- a/src/dstack/_internal/core/backends/aws/compute.py +++ b/src/dstack/_internal/core/backends/aws/compute.py @@ -569,9 +569,7 @@ def create_gateway( image_id=aws_resources.get_gateway_image_id(ec2_client), instance_type=configuration.instance_type or DEFAULT_GATEWAY_INSTANCE_TYPE, iam_instance_profile=None, - user_data=get_gateway_user_data( - configuration.ssh_key_pub, router=configuration.router - ), + user_data=get_gateway_user_data(configuration.ssh_key_pub), tags=tags, security_group_id=security_group_id, spot=False, diff --git a/src/dstack/_internal/core/backends/azure/compute.py b/src/dstack/_internal/core/backends/azure/compute.py index e8ed2d74b5..83a590a4b1 100644 --- a/src/dstack/_internal/core/backends/azure/compute.py +++ b/src/dstack/_internal/core/backends/azure/compute.py @@ -292,9 +292,7 @@ def create_gateway( image_reference=_get_gateway_image_ref(), vm_size=DEFAULT_GATEWAY_INSTANCE_TYPE, instance_name=instance_name, - user_data=get_gateway_user_data( - configuration.ssh_key_pub, router=configuration.router - ), + user_data=get_gateway_user_data(configuration.ssh_key_pub), ssh_pub_keys=[configuration.ssh_key_pub], spot=False, disk_size=30, diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index cf0e6a919a..8aae80e5ec 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -41,7 +41,6 @@ SSHKey, ) from dstack._internal.core.models.placement import PlacementGroup, PlacementGroupProvisioningData -from dstack._internal.core.models.routers import AnyGatewayRouterConfig from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run from dstack._internal.core.models.volumes import ( Volume, @@ -1089,9 +1088,7 @@ def get_run_shim_script( ] -def get_gateway_user_data( - authorized_key: str, router: Optional[AnyGatewayRouterConfig] = None -) -> str: +def get_gateway_user_data(authorized_key: str) -> str: return get_cloud_config( package_update=True, packages=[ @@ -1107,7 +1104,7 @@ def get_gateway_user_data( "s/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/", "/etc/nginx/nginx.conf", ], - ["su", "ubuntu", "-c", " && ".join(get_dstack_gateway_commands(router))], + ["su", "ubuntu", "-c", " && ".join(get_dstack_gateway_commands())], ], ssh_authorized_keys=[authorized_key], ) @@ -1207,22 +1204,19 @@ def get_latest_runner_build() -> Optional[str]: return None -def get_dstack_gateway_wheel(build: str, router: Optional[AnyGatewayRouterConfig] = None) -> str: +def get_dstack_gateway_wheel(build: str) -> str: channel = "release" if settings.DSTACK_RELEASE else "stgn" base_url = f"https://dstack-gateway-downloads.s3.amazonaws.com/{channel}" if build == "latest": build = _fetch_version(f"{base_url}/latest-version") or "latest" logger.debug("Found the latest gateway build: %s", build) wheel = f"{base_url}/dstack_gateway-{build}-py3-none-any.whl" - # Build package spec with extras if router is specified - if router: - return f"dstack-gateway[{router.type}] @ {wheel}" return f"dstack-gateway @ {wheel}" -def get_dstack_gateway_commands(router: Optional[AnyGatewayRouterConfig] = None) -> List[str]: +def get_dstack_gateway_commands() -> List[str]: build = get_dstack_runner_version() or "latest" - gateway_package = get_dstack_gateway_wheel(build, router) + gateway_package = get_dstack_gateway_wheel(build) return [ "mkdir -p /home/ubuntu/dstack", "python3 -m venv /home/ubuntu/dstack/blue", diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index b79857754d..9c94ad2ad2 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -606,9 +606,7 @@ def create_gateway( machine_type=configuration.instance_type or DEFAULT_GATEWAY_INSTANCE_TYPE, accelerators=[], spot=False, - user_data=get_gateway_user_data( - configuration.ssh_key_pub, router=configuration.router - ), + user_data=get_gateway_user_data(configuration.ssh_key_pub), authorized_keys=[configuration.ssh_key_pub], labels=labels, tags=[gcp_resources.DSTACK_GATEWAY_TAG], diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index 3fb02ce8f3..5b21377cd1 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -88,7 +88,6 @@ ) from dstack._internal.core.models.placement import PlacementGroup from dstack._internal.core.models.resources import GPUSpec -from dstack._internal.core.models.routers import AnyGatewayRouterConfig from dstack._internal.core.models.runs import ( Job, JobProvisioningData, @@ -502,9 +501,7 @@ def create_gateway( ) labels = filter_invalid_labels(labels) - commands = _get_gateway_commands( - authorized_keys=[configuration.ssh_key_pub], router=configuration.router - ) + commands = _get_gateway_commands(authorized_keys=[configuration.ssh_key_pub]) pod = client.V1Pod( metadata=client.V1ObjectMeta( name=instance_name, @@ -1375,11 +1372,9 @@ def _wait_for_load_balancer_address( time.sleep(1) -def _get_gateway_commands( - authorized_keys: List[str], router: Optional[AnyGatewayRouterConfig] = None -) -> List[str]: +def _get_gateway_commands(authorized_keys: List[str]) -> List[str]: authorized_keys_content = "\n".join(authorized_keys).strip() - gateway_commands = " && ".join(get_dstack_gateway_commands(router=router)) + gateway_commands = " && ".join(get_dstack_gateway_commands()) quoted_gateway_commands = shlex.quote(gateway_commands) commands = [ diff --git a/src/dstack/_internal/core/compatibility/gateways.py b/src/dstack/_internal/core/compatibility/gateways.py index 0a89e86113..8c0aa141c3 100644 --- a/src/dstack/_internal/core/compatibility/gateways.py +++ b/src/dstack/_internal/core/compatibility/gateways.py @@ -39,8 +39,6 @@ def _get_gateway_configuration_excludes( ) -> IncludeExcludeDictType: configuration_excludes: IncludeExcludeDictType = {} - if configuration.router is None: - configuration_excludes["router"] = True if configuration.replicas is None: configuration_excludes["replicas"] = True diff --git a/src/dstack/_internal/core/compatibility/runs.py b/src/dstack/_internal/core/compatibility/runs.py index 847e7be303..b8edc35c74 100644 --- a/src/dstack/_internal/core/compatibility/runs.py +++ b/src/dstack/_internal/core/compatibility/runs.py @@ -9,7 +9,6 @@ from dstack._internal.core.models.configurations import ( ServiceConfiguration, ) -from dstack._internal.core.models.routers import SGLangServiceRouterConfig from dstack._internal.core.models.runs import ( DEFAULT_PROBE_UNTIL_READY, DEFAULT_REPLICA_GROUP_NAME, @@ -108,11 +107,6 @@ def get_run_spec_excludes(run_spec: RunSpec) -> IncludeExcludeDictType: # Servers prior to 0.20.8 do not support probes=None configuration_excludes["probes"] = True - router = run_spec.configuration.router - if router is None: - configuration_excludes["router"] = True - elif isinstance(router, SGLangServiceRouterConfig) and router.pd_disaggregation is False: - configuration_excludes["router"] = {"pd_disaggregation": True} if run_spec.configuration.https is None: configuration_excludes["https"] = True diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index c05bd7f1db..be32988cde 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -39,7 +39,7 @@ SpotPolicy, ) from dstack._internal.core.models.resources import Range, ResourcesSpec -from dstack._internal.core.models.routers import AnyServiceRouterConfig, ReplicaGroupRouterConfig +from dstack._internal.core.models.routers import ReplicaGroupRouterConfig from dstack._internal.core.models.services import AnyModel, OpenAIChatModel from dstack._internal.core.models.unix import UnixUser from dstack._internal.core.models.volumes import ( @@ -1058,14 +1058,6 @@ class ServiceConfigurationParams(CoreModel): ) ), ] = None - router: Annotated[ - Optional[AnyServiceRouterConfig], - Field( - description=( - "Router configuration for the service. Requires a gateway with matching router enabled. " - ), - ), - ] = None @field_validator("port") @classmethod @@ -1366,23 +1358,6 @@ def validate_at_most_one_router_replica_group(self) -> Self: raise ValueError("For now replica group with `router` must have `count: 1`.") return self - @model_validator(mode="after") - def validate_replica_group_router_mutex(self) -> Self: - """ - When a replica group sets `router:`, service-level `router` must be omitted. - (Gateway-level SGLang is rejected at service registration when a gateway is selected.) - """ - replicas = self.replicas - if not isinstance(replicas, list): - return self - if not any(g.router is not None for g in replicas): - return self - if self.router is not None: - raise ValueError( - "Service-Level router configuration is not allowed together with replica-group `router`." - ) - return self - class ServiceConfiguration( ProfileParams, diff --git a/src/dstack/_internal/core/models/gateways.py b/src/dstack/_internal/core/models/gateways.py index 89da495743..4daf17dace 100644 --- a/src/dstack/_internal/core/models/gateways.py +++ b/src/dstack/_internal/core/models/gateways.py @@ -8,7 +8,6 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import ApplyAction, CoreModel -from dstack._internal.core.models.routers import AnyGatewayRouterConfig from dstack._internal.utils.tags import tags_validator GATEWAY_REPLICAS_DEFAULT = 1 @@ -69,15 +68,6 @@ class GatewayConfiguration(CoreModel): min_length=1, ), ] = None - router: Annotated[ - Optional[AnyGatewayRouterConfig], - Field( - description=( - "The router configuration for this gateway. " - "E.g. `{ type: sglang, policy: round_robin }`." - ), - ), - ] = None domain: Annotated[ Optional[str], Field( @@ -203,7 +193,6 @@ class GatewayComputeConfiguration(CoreModel): ssh_key_pub: str certificate: Annotated[Optional[AnyGatewayCertificate], Field(discriminator="type")] = None tags: Optional[Dict[str, str]] = None - router: Optional[AnyGatewayRouterConfig] = None class GatewayProvisioningData(CoreModel): diff --git a/src/dstack/_internal/core/models/routers.py b/src/dstack/_internal/core/models/routers.py index b1f189c522..59545c658a 100644 --- a/src/dstack/_internal/core/models/routers.py +++ b/src/dstack/_internal/core/models/routers.py @@ -12,25 +12,7 @@ class RouterType(str, Enum): DYNAMO = "dynamo" -class SGLangGatewayRouterConfig(CoreModel): - """Gateway-level router configuration. type and policy only. pd_disaggregation is service-level.""" - - type: Annotated[ - Literal["sglang"], - Field(description="The router type enabled on this gateway."), - ] = "sglang" - policy: Annotated[ - Literal["random", "round_robin", "cache_aware", "power_of_two"], - Field( - description=( - "The routing policy. Deprecated: prefer setting policy in the service's router config. " - "Options: `random`, `round_robin`, `cache_aware`, `power_of_two`" - ), - ), - ] = "cache_aware" - - -class SGLangServiceRouterConfig(CoreModel): +class SGLangServiceRouterConfig(CoreModel): # TODO: drop, unused by the server since 0.21.0 type: Annotated[Literal["sglang"], Field(description="The router type")] = "sglang" policy: Annotated[ Literal["random", "round_robin", "cache_aware", "power_of_two"], @@ -59,4 +41,3 @@ class ReplicaGroupRouterConfig(CoreModel): AnyServiceRouterConfig = SGLangServiceRouterConfig -AnyGatewayRouterConfig = SGLangGatewayRouterConfig diff --git a/src/dstack/_internal/proxy/lib/models.py b/src/dstack/_internal/proxy/lib/models.py index df025e6232..dbdc4d0381 100644 --- a/src/dstack/_internal/proxy/lib/models.py +++ b/src/dstack/_internal/proxy/lib/models.py @@ -64,6 +64,7 @@ class Service(ImmutableModel): replicas: tuple[Replica, ...] has_router_replica: bool = False router: Optional[AnyServiceRouterConfig] = None + """TODO: drop `router`, unused by the server since 0.21.0""" cors_enabled: bool = False # only used on gateways; enabled for openai-format models @property diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index d2fbadc5cc..225f7355d3 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -36,7 +36,6 @@ ) from dstack._internal.core.models.gateways import ( GATEWAY_REPLICAS_DEFAULT, - AnyGatewayRouterConfig, ApplyGatewayPlanInput, Gateway, GatewayComputeConfiguration, @@ -212,7 +211,6 @@ def create_gateway_compute_model( ssh_key_pub=gateway_ssh_public_key, certificate=configuration.certificate, tags=configuration.tags, - router=configuration.router, ) now = get_current_datetime() @@ -784,10 +782,9 @@ async def _update_gateway(gateway_compute_model: GatewayComputeModel, build: str gateway_compute_model.ssh_private_key, ) logger.debug("Updating gateway %s", connection.ip_address) - router = _get_gateway_compute_router_config(gateway_compute_model) # Build package spec with extras and wheel URL - gateway_package = get_dstack_gateway_wheel(build, router) + gateway_package = get_dstack_gateway_wheel(build) commands = [ # prevent update.sh from overwriting itself during execution "cp dstack/update.sh dstack/_update.sh", @@ -807,15 +804,6 @@ def _recently_updated(gateway_compute_model: GatewayComputeModel) -> bool: ) > get_current_datetime() - timedelta(seconds=60) -def _get_gateway_compute_router_config( - compute: GatewayComputeModel, -) -> Optional[AnyGatewayRouterConfig]: - if compute.configuration is None: # pre-0.18.2 gateway - return None # gateway routers introduced in 0.19.38 - compute_config = validate_json_extra_ignore(GatewayComputeConfiguration, compute.configuration) - return compute_config.router - - async def configure_gateway( connection: GatewayConnection, attempts: int = GATEWAY_CONFIGURE_ATTEMPTS, @@ -1156,8 +1144,3 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): if configuration.backend == BackendType.AWS: err += " or `certificate: { type: acm, arn: }` (AWS ACM)" raise ServerClientError(err) - - if configuration.router is not None and replicas > 1: - raise ServerClientError( - "The deprecated `router` property is not supported for multi-replica gateways" - ) diff --git a/src/dstack/_internal/server/services/gateways/client.py b/src/dstack/_internal/server/services/gateways/client.py index 01bebea1aa..10f2558327 100644 --- a/src/dstack/_internal/server/services/gateways/client.py +++ b/src/dstack/_internal/server/services/gateways/client.py @@ -10,7 +10,6 @@ from dstack._internal.core.models.common import validate_json_extra_ignore from dstack._internal.core.models.configurations import RateLimit from dstack._internal.core.models.instances import SSHConnectionParams -from dstack._internal.core.models.routers import AnyServiceRouterConfig from dstack._internal.core.models.runs import JobSpec, JobSubmission, Run, get_service_port from dstack._internal.proxy.gateway.schemas.services import ServiceListItem, ServiceListResponse from dstack._internal.proxy.gateway.schemas.stats import ServiceStats @@ -50,7 +49,6 @@ async def register_service( rate_limits: list[RateLimit], ssh_private_key: str, has_router_replica: bool = False, - router: Optional[AnyServiceRouterConfig] = None, ): if "openai" in options: entrypoint = f"gateway.{domain.split('.', maxsplit=1)[1]}" @@ -67,7 +65,6 @@ async def register_service( "rate_limits": [limit.model_dump() for limit in rate_limits], "ssh_private_key": ssh_private_key, "has_router_replica": has_router_replica, - "router": router.model_dump() if router is not None else None, } resp = await self._client.post( self._url(f"/api/registry/{project}/services/register"), json=payload diff --git a/src/dstack/_internal/server/services/proxy/repo.py b/src/dstack/_internal/server/services/proxy/repo.py index b1986522cf..5a0caffd7b 100644 --- a/src/dstack/_internal/server/services/proxy/repo.py +++ b/src/dstack/_internal/server/services/proxy/repo.py @@ -78,7 +78,6 @@ async def get_service(self, project_name: str, run_name: str) -> Optional[Servic None, ) has_router_replica = router_group is not None - router = run_spec.configuration.router replicas = [] for job in jobs: jpd = validate_json_extra_ignore( @@ -141,7 +140,6 @@ async def get_service(self, project_name: str, run_name: str) -> Optional[Servic strip_prefix=run_spec.configuration.strip_prefix, replicas=tuple(replicas), has_router_replica=has_router_replica, - router=router, ) async def list_models(self, project_name: str) -> List[ChatModel]: diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 0e2738832b..eaf878260e 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -3,7 +3,6 @@ """ from functools import partial -from typing import Optional import httpx from sqlalchemy.ext.asyncio import AsyncSession @@ -20,11 +19,6 @@ ServiceConfiguration, ) from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayStatus -from dstack._internal.core.models.routers import ( - AnyServiceRouterConfig, - RouterType, - SGLangServiceRouterConfig, -) from dstack._internal.core.models.runs import RunSpec, ServiceModelSpec, ServiceSpec from dstack._internal.core.models.services import OpenAIChatModel from dstack._internal.proxy.gateway.const import SERVICE_ALREADY_REGISTERED_ERROR_TEMPLATE @@ -115,21 +109,6 @@ async def _register_service_in_gateway( has_replica_group_router = any( g.router is not None for g in run_spec.configuration.replica_groups ) - if has_replica_group_router and _gateway_has_sglang_router(gateway_configuration): - raise ServerClientError( - "A replica-group `router:` cannot be used with a gateway that has router configuration." - ) - - # Check: service specifies SGLang router but gateway does not have it - service_router = run_spec.configuration.router - service_wants_sglang = service_router is not None and isinstance( - service_router, SGLangServiceRouterConfig - ) - if service_wants_sglang and not _gateway_has_sglang_router(gateway_configuration): - raise ServerClientError( - "Service requires gateway with SGLang router but gateway " - f"'{gateway.name}' does not have the SGLang router configured." - ) configure_service_https = _should_configure_service_https_on_gateway( run_spec, gateway_configuration @@ -152,8 +131,6 @@ async def _register_service_in_gateway( "Cannot run HTTPS service on gateway with no SSL certificates configured" ) - router = _build_service_router_config(gateway_configuration, run_spec.configuration) - gateway_https = _get_gateway_https(gateway_configuration) gateway_protocol = "https" if gateway_https else "http" @@ -198,7 +175,6 @@ async def _register_service_in_gateway( rate_limits=run_spec.configuration.rate_limits, ssh_private_key=run_model.project.ssh_private_key, has_router_replica=has_replica_group_router, - router=router, ) try: await do_register() @@ -240,14 +216,6 @@ async def _register_service_in_gateway( def _register_service_in_server(run_model: RunModel, run_spec: RunSpec) -> ServiceSpec: assert run_spec.configuration.type == "service" - if ( - run_spec.configuration.router is not None - and run_spec.configuration.router.type == RouterType.SGLANG - ): - raise ServerClientError( - "Service with SGLang router configuration requires a gateway. " - "Please configure a gateway with the SGLang router enabled." - ) if run_spec.configuration.https not in ( None, "auto", @@ -283,41 +251,6 @@ def _register_service_in_server(run_model: RunModel, run_spec: RunSpec) -> Servi ) -def _gateway_has_sglang_router(config: GatewayConfiguration) -> bool: - return config.router is not None and config.router.type == RouterType.SGLANG.value - - -def _build_service_router_config( - gateway_configuration: GatewayConfiguration, - service_configuration: ServiceConfiguration, -) -> Optional[AnyServiceRouterConfig]: - """ - Build router config from gateway (type, policy) + service (pd_disaggregation, policy override). - Service's policy overrides gateway's if present. Keeps backward compat: SGLang enabled - automatically when gateway has it configured. - """ - if not _gateway_has_sglang_router(gateway_configuration): - return None - - gateway_router = gateway_configuration.router - assert gateway_router is not None # ensured by _gateway_has_sglang_router - router_type = gateway_router.type - policy = gateway_router.policy - - service_router = service_configuration.router - if service_router is not None and isinstance(service_router, SGLangServiceRouterConfig): - policy = service_router.policy - pd_disaggregation = service_router.pd_disaggregation - else: - pd_disaggregation = False - - return SGLangServiceRouterConfig( - type=router_type, - policy=policy, - pd_disaggregation=pd_disaggregation, - ) - - def _get_service_spec( configuration: ServiceConfiguration, service_url: str, model_url: str ) -> ServiceSpec: diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index 98f6ac2b5b..3075b696b2 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -162,27 +162,6 @@ def test_replica_group_router(self): assert isinstance(router_g.router, ReplicaGroupRouterConfig) assert router_g.router.type == "sglang" - def test_replica_group_router_forbids_service_level_router(self): - conf = { - "type": "service", - "port": 8000, - "router": {"type": "sglang"}, - "replicas": [ - { - "name": "router", - "count": 1, - "commands": ["sglang serve"], - "router": {"type": "sglang"}, - }, - {"name": "worker", "count": 2, "commands": ["worker"]}, - ], - } - with pytest.raises( - ConfigurationError, - match="Service-Level router configuration is not allowed together with replica-group", - ): - parse_run_configuration(conf) - def test_spot_policy_set_at_both_service_and_group_rejected(self): with pytest.raises( ConfigurationError, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json index 431ae169f4..bee72a06d7 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json @@ -15,7 +15,6 @@ "public_ip": true, "region": "eu-west-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json index 0364f1826c..168e3bdfa1 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json @@ -12,7 +12,6 @@ "public_ip": true, "region": "eu-west-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml index 473703b10d..868d8e7f49 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml @@ -1,13 +1,10 @@ -# Exercises the gateway discriminator, its ACM certificate union arm, and the gateway-level router. +# Exercises the gateway discriminator and its ACM certificate union arm. type: gateway name: inference-gateway default: true backend: aws region: us-east-1 instance_type: t3.small -router: - type: sglang - policy: round_robin domain: example.com public_ip: false certificate: diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json index 83690c7e9c..cbae39cfde 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json @@ -1,6 +1,5 @@ { "/": "GatewayConfiguration", "/backend": "BackendType", - "/certificate": "ACMGatewayCertificate", - "/router": "SGLangGatewayRouterConfig" + "/certificate": "ACMGatewayCertificate" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json index c4a6c91adc..276f6b450f 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json @@ -11,10 +11,6 @@ "public_ip": false, "region": "us-east-1", "replicas": 2, - "router": { - "policy": "round_robin", - "type": "sglang" - }, "tags": { "env": "prod" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json index cf387dcd93..d6fb889e57 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json @@ -73,7 +73,6 @@ "shm_size": null }, "retry": null, - "router": null, "scaling": null, "schedule": null, "setup": [], diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json index fea497cc02..9c2f13c09e 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json @@ -8,7 +8,6 @@ "project_name": "main", "public_ip": true, "region": "eu-west-1", - "router": null, "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 gateway@example.com", "tags": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json index 2828b2021e..225d4a656f 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json @@ -10,7 +10,6 @@ "public_ip": true, "region": "eu-west-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json b/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json index 33d399b4b3..f0d5a56410 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json @@ -1736,18 +1736,6 @@ "description": "The number of gateway replicas. Defaults to `1`", "title": "Replicas" }, - "router": { - "anyOf": [ - { - "$ref": "#/$defs/SGLangGatewayRouterConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The router configuration for this gateway. E.g. `{ type: sglang, policy: round_robin }`." - }, "tags": { "anyOf": [ { @@ -3003,65 +2991,6 @@ "title": "RunpodVolumeConfiguration", "type": "object" }, - "SGLangGatewayRouterConfig": { - "additionalProperties": false, - "description": "Gateway-level router configuration. type and policy only. pd_disaggregation is service-level.", - "properties": { - "policy": { - "default": "cache_aware", - "description": "The routing policy. Deprecated: prefer setting policy in the service's router config. Options: `random`, `round_robin`, `cache_aware`, `power_of_two`", - "enum": [ - "random", - "round_robin", - "cache_aware", - "power_of_two" - ], - "title": "Policy", - "type": "string" - }, - "type": { - "const": "sglang", - "default": "sglang", - "description": "The router type enabled on this gateway.", - "title": "Type", - "type": "string" - } - }, - "title": "SGLangGatewayRouterConfig", - "type": "object" - }, - "SGLangServiceRouterConfig": { - "additionalProperties": false, - "properties": { - "pd_disaggregation": { - "default": false, - "description": "Enable PD disaggregation mode for the SGLang router", - "title": "Pd Disaggregation", - "type": "boolean" - }, - "policy": { - "default": "cache_aware", - "description": "The routing policy. Options: `random`, `round_robin`, `cache_aware`, `power_of_two`", - "enum": [ - "random", - "round_robin", - "cache_aware", - "power_of_two" - ], - "title": "Policy", - "type": "string" - }, - "type": { - "const": "sglang", - "default": "sglang", - "description": "The router type", - "title": "Type", - "type": "string" - } - }, - "title": "SGLangServiceRouterConfig", - "type": "object" - }, "SSHHostParams": { "additionalProperties": false, "properties": { @@ -4008,18 +3937,6 @@ "description": "The policy for resubmitting the run. Defaults to `false`", "title": "Retry" }, - "router": { - "anyOf": [ - { - "$ref": "#/$defs/SGLangServiceRouterConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Router configuration for the service. Requires a gateway with matching router enabled. " - }, "scaling": { "anyOf": [ { diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json index 16bdd691c8..25a3ef22db 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json @@ -15,7 +15,6 @@ "public_ip": true, "region": "us-east-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json index 3cacb9291b..f90d397cd1 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json @@ -12,7 +12,6 @@ "public_ip": true, "region": "us-east-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json index 7c78b5dd14..ea7179064c 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json @@ -6,7 +6,6 @@ "project_name": "test-project", "public_ip": true, "region": "us", - "router": null, "ssh_key_pub": "", "tags": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json index 37a4ff056a..5f8a0f6df9 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json @@ -10,7 +10,6 @@ "public_ip": true, "region": "us-east-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" } diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index c9c7c96386..19b39908d7 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -106,7 +106,6 @@ async def test_list( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": gateway.wildcard_domain, "default": False, "public_ip": True, @@ -193,7 +192,6 @@ async def test_get( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": gateway.wildcard_domain, "default": False, "public_ip": True, @@ -535,7 +533,6 @@ async def test_create_gateway(self, test_db, session: AsyncSession, client: Asyn "backend": backend.type.value, "region": "us", "instance_type": None, - "router": None, "domain": None, "default": True, "public_ip": True, @@ -626,7 +623,6 @@ async def test_create_gateway_without_name( "backend": backend.type.value, "region": "us", "instance_type": None, - "router": None, "domain": None, "default": True, "public_ip": True, @@ -746,19 +742,6 @@ async def test_create_gateway_with_invalid_domain_interpolation( " or `certificate: { type: acm, arn: }` (AWS ACM)", id="multi-replica-with-letsencrypt-cert", ), - pytest.param( - { - "type": "gateway", - "name": "test", - "backend": "aws", - "region": "us", - "certificate": None, - "router": {"type": "sglang"}, - "replicas": 2, - }, - "The deprecated `router` property is not supported for multi-replica gateways", - id="multi-replica-with-router", - ), pytest.param( { "type": "gateway", @@ -885,7 +868,6 @@ async def test_set_default_gateway( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": gateway.wildcard_domain, "default": True, "public_ip": True, @@ -1282,7 +1264,6 @@ async def test_set_wildcard_domain( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": "new.example", "default": False, "public_ip": True,