diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index 6c4d406a6f..81e5fda205 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ ] [project.optional-dependencies] +# TODO: drop, unused by the server since 0.21.0 sglang = ["sglang-router==0.3.2"] [tool.setuptools.package-data] diff --git a/mkdocs/blog/posts/pd-disaggregation.md b/mkdocs/blog/posts/pd-disaggregation.md index dd3f27c9e8..c885c6afc5 100644 --- a/mkdocs/blog/posts/pd-disaggregation.md +++ b/mkdocs/blog/posts/pd-disaggregation.md @@ -27,7 +27,7 @@ For inference, `dstack` provides a [services](../../docs/concepts/services.md) a > If you’re new to Prefill–Decode disaggregation, see the official [SGLang docs](https://docs.sglang.io/advanced_features/pd_disaggregation.html). !!! note "Deprecation notice" - Configuring the SGLang router in a gateway is deprecated and will be disallowed in a future release. To run router and workers as separate replica groups, see [SGLang PD disaggregation (router as replica group)](../../docs/examples/inference/sglang.md#pd-disaggregation). + The router configuration approach described in this blog post is no longer supported. For the modern approach, see [SGLang PD disaggregation (router as replica group)](../../docs/examples/inference/sglang.md#pd-disaggregation). ## Services diff --git a/mkdocs/blog/posts/sglang-router.md b/mkdocs/blog/posts/sglang-router.md index f33fd7e400..028e3a23da 100644 --- a/mkdocs/blog/posts/sglang-router.md +++ b/mkdocs/blog/posts/sglang-router.md @@ -125,6 +125,9 @@ After that, if you configure [replicas and scaling](../../docs/concepts/services ### Router +!!! note "Deprecation notice" + The router configuration approach described in this section is no longer supported. For the modern approach, see [SGLang PD disaggregation (router as replica group)](../../docs/examples/inference/sglang.md#pd-disaggregation). + By default, the gateway uses its built-in load balancer to route traffic across replicas. With the latest release, you can instead delegate traffic routing to the [SGLang Model Gateway](https://docs.sglang.ai/advanced_features/router.html) by setting the `router` property to `sglang`:
diff --git a/mkdocs/docs/concepts/gateways.md b/mkdocs/docs/concepts/gateways.md index 64e197fed1..c011486036 100644 --- a/mkdocs/docs/concepts/gateways.md +++ b/mkdocs/docs/concepts/gateways.md @@ -76,54 +76,6 @@ You can create gateways with the `aws`, `azure`, `gcp`, or `kubernetes` backends Gateways in `kubernetes` backend require an external load balancer. Managed Kubernetes solutions usually include a load balancer. For self-hosted Kubernetes, you must provide a load balancer by yourself. -### Router - -> In previous releases, `dstack` allowed configuring `router` the gateway, which was required for PD disaggregation. Since 0.20.17, the `router` configuration has moved to [services](services.md#pd-disaggregation), and the gateway no longer needs to configure router. - - - ### Certificate By default, when you run a service with a gateway, `dstack` provisions an SSL certificate via Let's Encrypt for the configured domain. This automatically enables HTTPS for the service endpoint. diff --git a/mkdocs/docs/concepts/services.md b/mkdocs/docs/concepts/services.md index 22116567cb..ea06d39be2 100644 --- a/mkdocs/docs/concepts/services.md +++ b/mkdocs/docs/concepts/services.md @@ -340,7 +340,7 @@ Setting the minimum number of replicas to `0` allows the service to scale down t ### PD disaggregation - + Since 0.20.17, `dstack` supports serving a model using Prefill-Decode disaggregation. To use it, configure three replica groups: one for the router, one for prefill workers, and one for decode workers. diff --git a/mkdocs/docs/reference/dstack.yml/gateway.md b/mkdocs/docs/reference/dstack.yml/gateway.md index 33fbeb4190..06d0433f41 100644 --- a/mkdocs/docs/reference/dstack.yml/gateway.md +++ b/mkdocs/docs/reference/dstack.yml/gateway.md @@ -10,16 +10,6 @@ The `gateway` configuration type allows creating and updating [gateways](../../c type: required: true -### `router` - -=== "SGLang Model Gateway" - - #SCHEMA# dstack._internal.core.models.routers.SGLangGatewayRouterConfig - overrides: - show_root_heading: false - type: - required: true - ### `certificate` Set to `null` to disable certificates (e.g. for [private gateways](../../concepts/gateways.md#public-ip)). diff --git a/src/dstack/_internal/cli/commands/gateway.py b/src/dstack/_internal/cli/commands/gateway.py index 2489e726b4..bd29431f20 100644 --- a/src/dstack/_internal/cli/commands/gateway.py +++ b/src/dstack/_internal/cli/commands/gateway.py @@ -19,7 +19,6 @@ ) from dstack._internal.core.errors import CLIError from dstack._internal.core.models.common import EntityReference -from dstack._internal.core.models.gateways import GatewayStatus from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -110,20 +109,6 @@ def _list(self, args: argparse.Namespace): raise CLIError("JSON output is not supported together with --watch") gateways = self.api.client.gateways.list(self.api.project, include_imported=True) - deprecated_router_gateways = [ - g.name - for g in gateways - if g.status != GatewayStatus.FAILED and g.configuration.router is not None - ] - if deprecated_router_gateways and args.format != "json": - logger.warning( - "Specifying `router` in gateway configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - " (affected gateways: %s)", - ", ".join(deprecated_router_gateways), - ) if not args.watch: if args.format == "json": print_gateways_json(gateways, project=self.api.project) diff --git a/src/dstack/_internal/cli/commands/ps.py b/src/dstack/_internal/cli/commands/ps.py index 0e8ab5d90b..0466081a32 100644 --- a/src/dstack/_internal/cli/commands/ps.py +++ b/src/dstack/_internal/cli/commands/ps.py @@ -11,7 +11,6 @@ console, ) from dstack._internal.core.errors import CLIError -from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -69,23 +68,6 @@ def _command(self, args: argparse.Namespace): # TODO: Add a `ps --json` option to control how many job submissions are returned. runs = self.api.runs.list(all=args.all, limit=args.last) - deprecated_router_runs = [ - run._run.run_spec.run_name - for run in runs - if not run.status.is_finished() - and isinstance(run._run.run_spec.configuration, ServiceConfiguration) - and run._run.run_spec.configuration.router is not None - and run._run.run_spec.run_name is not None - ] - if deprecated_router_runs and args.format != "json": - logger.warning( - "Specifying `router` in service configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - " (affected runs: %s)", - ", ".join(deprecated_router_runs), - ) if not args.watch: if args.format == "json": run_utils.print_runs_json(self.api.project, runs) diff --git a/src/dstack/_internal/cli/services/configurators/gateway.py b/src/dstack/_internal/cli/services/configurators/gateway.py index 4f78c11d88..76472b11f9 100644 --- a/src/dstack/_internal/cli/services/configurators/gateway.py +++ b/src/dstack/_internal/cli/services/configurators/gateway.py @@ -49,13 +49,6 @@ def apply_configuration( configuration=conf, configuration_path=configuration_path, ) - if spec.configuration.router is not None: - logger.warning( - "Specifying `router` in gateway configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - ) with console.status("Getting apply plan..."): try: plan = self.api.client.gateways.get_plan(project_name=self.api.project, spec=spec) diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index df2d0b35d4..36af8bd074 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -123,14 +123,6 @@ def get_plan( if conf.working_dir is not None and not is_absolute_posix_path(conf.working_dir): raise ConfigurationError("working_dir must be absolute") - if isinstance(conf, ServiceConfiguration) and conf.router is not None: - logger.warning( - "Specifying `router` in service configurations is deprecated" - " and will be disallowed in a future release." - " Please migrate to replica-based routers:" - " https://dstack.ai/docs/concepts/services/#pd-disaggregation" - ) - repo = self.get_repo(conf, configuration_path, configurator_args) if repo is None: repo = init_default_virtual_repo(api=self.api) diff --git a/src/dstack/_internal/core/backends/aws/compute.py b/src/dstack/_internal/core/backends/aws/compute.py index 2e0a65b568..a0541c180a 100644 --- a/src/dstack/_internal/core/backends/aws/compute.py +++ b/src/dstack/_internal/core/backends/aws/compute.py @@ -569,9 +569,7 @@ def create_gateway( image_id=aws_resources.get_gateway_image_id(ec2_client), instance_type=configuration.instance_type or DEFAULT_GATEWAY_INSTANCE_TYPE, iam_instance_profile=None, - user_data=get_gateway_user_data( - configuration.ssh_key_pub, router=configuration.router - ), + user_data=get_gateway_user_data(configuration.ssh_key_pub), tags=tags, security_group_id=security_group_id, spot=False, diff --git a/src/dstack/_internal/core/backends/azure/compute.py b/src/dstack/_internal/core/backends/azure/compute.py index e8ed2d74b5..83a590a4b1 100644 --- a/src/dstack/_internal/core/backends/azure/compute.py +++ b/src/dstack/_internal/core/backends/azure/compute.py @@ -292,9 +292,7 @@ def create_gateway( image_reference=_get_gateway_image_ref(), vm_size=DEFAULT_GATEWAY_INSTANCE_TYPE, instance_name=instance_name, - user_data=get_gateway_user_data( - configuration.ssh_key_pub, router=configuration.router - ), + user_data=get_gateway_user_data(configuration.ssh_key_pub), ssh_pub_keys=[configuration.ssh_key_pub], spot=False, disk_size=30, diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index cf0e6a919a..8aae80e5ec 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -41,7 +41,6 @@ SSHKey, ) from dstack._internal.core.models.placement import PlacementGroup, PlacementGroupProvisioningData -from dstack._internal.core.models.routers import AnyGatewayRouterConfig from dstack._internal.core.models.runs import Job, JobProvisioningData, Requirements, Run from dstack._internal.core.models.volumes import ( Volume, @@ -1089,9 +1088,7 @@ def get_run_shim_script( ] -def get_gateway_user_data( - authorized_key: str, router: Optional[AnyGatewayRouterConfig] = None -) -> str: +def get_gateway_user_data(authorized_key: str) -> str: return get_cloud_config( package_update=True, packages=[ @@ -1107,7 +1104,7 @@ def get_gateway_user_data( "s/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/", "/etc/nginx/nginx.conf", ], - ["su", "ubuntu", "-c", " && ".join(get_dstack_gateway_commands(router))], + ["su", "ubuntu", "-c", " && ".join(get_dstack_gateway_commands())], ], ssh_authorized_keys=[authorized_key], ) @@ -1207,22 +1204,19 @@ def get_latest_runner_build() -> Optional[str]: return None -def get_dstack_gateway_wheel(build: str, router: Optional[AnyGatewayRouterConfig] = None) -> str: +def get_dstack_gateway_wheel(build: str) -> str: channel = "release" if settings.DSTACK_RELEASE else "stgn" base_url = f"https://dstack-gateway-downloads.s3.amazonaws.com/{channel}" if build == "latest": build = _fetch_version(f"{base_url}/latest-version") or "latest" logger.debug("Found the latest gateway build: %s", build) wheel = f"{base_url}/dstack_gateway-{build}-py3-none-any.whl" - # Build package spec with extras if router is specified - if router: - return f"dstack-gateway[{router.type}] @ {wheel}" return f"dstack-gateway @ {wheel}" -def get_dstack_gateway_commands(router: Optional[AnyGatewayRouterConfig] = None) -> List[str]: +def get_dstack_gateway_commands() -> List[str]: build = get_dstack_runner_version() or "latest" - gateway_package = get_dstack_gateway_wheel(build, router) + gateway_package = get_dstack_gateway_wheel(build) return [ "mkdir -p /home/ubuntu/dstack", "python3 -m venv /home/ubuntu/dstack/blue", diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index b79857754d..9c94ad2ad2 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -606,9 +606,7 @@ def create_gateway( machine_type=configuration.instance_type or DEFAULT_GATEWAY_INSTANCE_TYPE, accelerators=[], spot=False, - user_data=get_gateway_user_data( - configuration.ssh_key_pub, router=configuration.router - ), + user_data=get_gateway_user_data(configuration.ssh_key_pub), authorized_keys=[configuration.ssh_key_pub], labels=labels, tags=[gcp_resources.DSTACK_GATEWAY_TAG], diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index 3fb02ce8f3..5b21377cd1 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -88,7 +88,6 @@ ) from dstack._internal.core.models.placement import PlacementGroup from dstack._internal.core.models.resources import GPUSpec -from dstack._internal.core.models.routers import AnyGatewayRouterConfig from dstack._internal.core.models.runs import ( Job, JobProvisioningData, @@ -502,9 +501,7 @@ def create_gateway( ) labels = filter_invalid_labels(labels) - commands = _get_gateway_commands( - authorized_keys=[configuration.ssh_key_pub], router=configuration.router - ) + commands = _get_gateway_commands(authorized_keys=[configuration.ssh_key_pub]) pod = client.V1Pod( metadata=client.V1ObjectMeta( name=instance_name, @@ -1375,11 +1372,9 @@ def _wait_for_load_balancer_address( time.sleep(1) -def _get_gateway_commands( - authorized_keys: List[str], router: Optional[AnyGatewayRouterConfig] = None -) -> List[str]: +def _get_gateway_commands(authorized_keys: List[str]) -> List[str]: authorized_keys_content = "\n".join(authorized_keys).strip() - gateway_commands = " && ".join(get_dstack_gateway_commands(router=router)) + gateway_commands = " && ".join(get_dstack_gateway_commands()) quoted_gateway_commands = shlex.quote(gateway_commands) commands = [ diff --git a/src/dstack/_internal/core/compatibility/gateways.py b/src/dstack/_internal/core/compatibility/gateways.py index 0a89e86113..8c0aa141c3 100644 --- a/src/dstack/_internal/core/compatibility/gateways.py +++ b/src/dstack/_internal/core/compatibility/gateways.py @@ -39,8 +39,6 @@ def _get_gateway_configuration_excludes( ) -> IncludeExcludeDictType: configuration_excludes: IncludeExcludeDictType = {} - if configuration.router is None: - configuration_excludes["router"] = True if configuration.replicas is None: configuration_excludes["replicas"] = True diff --git a/src/dstack/_internal/core/compatibility/runs.py b/src/dstack/_internal/core/compatibility/runs.py index 847e7be303..b8edc35c74 100644 --- a/src/dstack/_internal/core/compatibility/runs.py +++ b/src/dstack/_internal/core/compatibility/runs.py @@ -9,7 +9,6 @@ from dstack._internal.core.models.configurations import ( ServiceConfiguration, ) -from dstack._internal.core.models.routers import SGLangServiceRouterConfig from dstack._internal.core.models.runs import ( DEFAULT_PROBE_UNTIL_READY, DEFAULT_REPLICA_GROUP_NAME, @@ -108,11 +107,6 @@ def get_run_spec_excludes(run_spec: RunSpec) -> IncludeExcludeDictType: # Servers prior to 0.20.8 do not support probes=None configuration_excludes["probes"] = True - router = run_spec.configuration.router - if router is None: - configuration_excludes["router"] = True - elif isinstance(router, SGLangServiceRouterConfig) and router.pd_disaggregation is False: - configuration_excludes["router"] = {"pd_disaggregation": True} if run_spec.configuration.https is None: configuration_excludes["https"] = True diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index c05bd7f1db..be32988cde 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -39,7 +39,7 @@ SpotPolicy, ) from dstack._internal.core.models.resources import Range, ResourcesSpec -from dstack._internal.core.models.routers import AnyServiceRouterConfig, ReplicaGroupRouterConfig +from dstack._internal.core.models.routers import ReplicaGroupRouterConfig from dstack._internal.core.models.services import AnyModel, OpenAIChatModel from dstack._internal.core.models.unix import UnixUser from dstack._internal.core.models.volumes import ( @@ -1058,14 +1058,6 @@ class ServiceConfigurationParams(CoreModel): ) ), ] = None - router: Annotated[ - Optional[AnyServiceRouterConfig], - Field( - description=( - "Router configuration for the service. Requires a gateway with matching router enabled. " - ), - ), - ] = None @field_validator("port") @classmethod @@ -1366,23 +1358,6 @@ def validate_at_most_one_router_replica_group(self) -> Self: raise ValueError("For now replica group with `router` must have `count: 1`.") return self - @model_validator(mode="after") - def validate_replica_group_router_mutex(self) -> Self: - """ - When a replica group sets `router:`, service-level `router` must be omitted. - (Gateway-level SGLang is rejected at service registration when a gateway is selected.) - """ - replicas = self.replicas - if not isinstance(replicas, list): - return self - if not any(g.router is not None for g in replicas): - return self - if self.router is not None: - raise ValueError( - "Service-Level router configuration is not allowed together with replica-group `router`." - ) - return self - class ServiceConfiguration( ProfileParams, diff --git a/src/dstack/_internal/core/models/gateways.py b/src/dstack/_internal/core/models/gateways.py index 89da495743..4daf17dace 100644 --- a/src/dstack/_internal/core/models/gateways.py +++ b/src/dstack/_internal/core/models/gateways.py @@ -8,7 +8,6 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import ApplyAction, CoreModel -from dstack._internal.core.models.routers import AnyGatewayRouterConfig from dstack._internal.utils.tags import tags_validator GATEWAY_REPLICAS_DEFAULT = 1 @@ -69,15 +68,6 @@ class GatewayConfiguration(CoreModel): min_length=1, ), ] = None - router: Annotated[ - Optional[AnyGatewayRouterConfig], - Field( - description=( - "The router configuration for this gateway. " - "E.g. `{ type: sglang, policy: round_robin }`." - ), - ), - ] = None domain: Annotated[ Optional[str], Field( @@ -203,7 +193,6 @@ class GatewayComputeConfiguration(CoreModel): ssh_key_pub: str certificate: Annotated[Optional[AnyGatewayCertificate], Field(discriminator="type")] = None tags: Optional[Dict[str, str]] = None - router: Optional[AnyGatewayRouterConfig] = None class GatewayProvisioningData(CoreModel): diff --git a/src/dstack/_internal/core/models/routers.py b/src/dstack/_internal/core/models/routers.py index b1f189c522..59545c658a 100644 --- a/src/dstack/_internal/core/models/routers.py +++ b/src/dstack/_internal/core/models/routers.py @@ -12,25 +12,7 @@ class RouterType(str, Enum): DYNAMO = "dynamo" -class SGLangGatewayRouterConfig(CoreModel): - """Gateway-level router configuration. type and policy only. pd_disaggregation is service-level.""" - - type: Annotated[ - Literal["sglang"], - Field(description="The router type enabled on this gateway."), - ] = "sglang" - policy: Annotated[ - Literal["random", "round_robin", "cache_aware", "power_of_two"], - Field( - description=( - "The routing policy. Deprecated: prefer setting policy in the service's router config. " - "Options: `random`, `round_robin`, `cache_aware`, `power_of_two`" - ), - ), - ] = "cache_aware" - - -class SGLangServiceRouterConfig(CoreModel): +class SGLangServiceRouterConfig(CoreModel): # TODO: drop, unused by the server since 0.21.0 type: Annotated[Literal["sglang"], Field(description="The router type")] = "sglang" policy: Annotated[ Literal["random", "round_robin", "cache_aware", "power_of_two"], @@ -59,4 +41,3 @@ class ReplicaGroupRouterConfig(CoreModel): AnyServiceRouterConfig = SGLangServiceRouterConfig -AnyGatewayRouterConfig = SGLangGatewayRouterConfig diff --git a/src/dstack/_internal/proxy/lib/models.py b/src/dstack/_internal/proxy/lib/models.py index df025e6232..dbdc4d0381 100644 --- a/src/dstack/_internal/proxy/lib/models.py +++ b/src/dstack/_internal/proxy/lib/models.py @@ -64,6 +64,7 @@ class Service(ImmutableModel): replicas: tuple[Replica, ...] has_router_replica: bool = False router: Optional[AnyServiceRouterConfig] = None + """TODO: drop `router`, unused by the server since 0.21.0""" cors_enabled: bool = False # only used on gateways; enabled for openai-format models @property diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index d2fbadc5cc..225f7355d3 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -36,7 +36,6 @@ ) from dstack._internal.core.models.gateways import ( GATEWAY_REPLICAS_DEFAULT, - AnyGatewayRouterConfig, ApplyGatewayPlanInput, Gateway, GatewayComputeConfiguration, @@ -212,7 +211,6 @@ def create_gateway_compute_model( ssh_key_pub=gateway_ssh_public_key, certificate=configuration.certificate, tags=configuration.tags, - router=configuration.router, ) now = get_current_datetime() @@ -784,10 +782,9 @@ async def _update_gateway(gateway_compute_model: GatewayComputeModel, build: str gateway_compute_model.ssh_private_key, ) logger.debug("Updating gateway %s", connection.ip_address) - router = _get_gateway_compute_router_config(gateway_compute_model) # Build package spec with extras and wheel URL - gateway_package = get_dstack_gateway_wheel(build, router) + gateway_package = get_dstack_gateway_wheel(build) commands = [ # prevent update.sh from overwriting itself during execution "cp dstack/update.sh dstack/_update.sh", @@ -807,15 +804,6 @@ def _recently_updated(gateway_compute_model: GatewayComputeModel) -> bool: ) > get_current_datetime() - timedelta(seconds=60) -def _get_gateway_compute_router_config( - compute: GatewayComputeModel, -) -> Optional[AnyGatewayRouterConfig]: - if compute.configuration is None: # pre-0.18.2 gateway - return None # gateway routers introduced in 0.19.38 - compute_config = validate_json_extra_ignore(GatewayComputeConfiguration, compute.configuration) - return compute_config.router - - async def configure_gateway( connection: GatewayConnection, attempts: int = GATEWAY_CONFIGURE_ATTEMPTS, @@ -1156,8 +1144,3 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): if configuration.backend == BackendType.AWS: err += " or `certificate: { type: acm, arn: }` (AWS ACM)" raise ServerClientError(err) - - if configuration.router is not None and replicas > 1: - raise ServerClientError( - "The deprecated `router` property is not supported for multi-replica gateways" - ) diff --git a/src/dstack/_internal/server/services/gateways/client.py b/src/dstack/_internal/server/services/gateways/client.py index 01bebea1aa..10f2558327 100644 --- a/src/dstack/_internal/server/services/gateways/client.py +++ b/src/dstack/_internal/server/services/gateways/client.py @@ -10,7 +10,6 @@ from dstack._internal.core.models.common import validate_json_extra_ignore from dstack._internal.core.models.configurations import RateLimit from dstack._internal.core.models.instances import SSHConnectionParams -from dstack._internal.core.models.routers import AnyServiceRouterConfig from dstack._internal.core.models.runs import JobSpec, JobSubmission, Run, get_service_port from dstack._internal.proxy.gateway.schemas.services import ServiceListItem, ServiceListResponse from dstack._internal.proxy.gateway.schemas.stats import ServiceStats @@ -50,7 +49,6 @@ async def register_service( rate_limits: list[RateLimit], ssh_private_key: str, has_router_replica: bool = False, - router: Optional[AnyServiceRouterConfig] = None, ): if "openai" in options: entrypoint = f"gateway.{domain.split('.', maxsplit=1)[1]}" @@ -67,7 +65,6 @@ async def register_service( "rate_limits": [limit.model_dump() for limit in rate_limits], "ssh_private_key": ssh_private_key, "has_router_replica": has_router_replica, - "router": router.model_dump() if router is not None else None, } resp = await self._client.post( self._url(f"/api/registry/{project}/services/register"), json=payload diff --git a/src/dstack/_internal/server/services/proxy/repo.py b/src/dstack/_internal/server/services/proxy/repo.py index b1986522cf..5a0caffd7b 100644 --- a/src/dstack/_internal/server/services/proxy/repo.py +++ b/src/dstack/_internal/server/services/proxy/repo.py @@ -78,7 +78,6 @@ async def get_service(self, project_name: str, run_name: str) -> Optional[Servic None, ) has_router_replica = router_group is not None - router = run_spec.configuration.router replicas = [] for job in jobs: jpd = validate_json_extra_ignore( @@ -141,7 +140,6 @@ async def get_service(self, project_name: str, run_name: str) -> Optional[Servic strip_prefix=run_spec.configuration.strip_prefix, replicas=tuple(replicas), has_router_replica=has_router_replica, - router=router, ) async def list_models(self, project_name: str) -> List[ChatModel]: diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 0e2738832b..eaf878260e 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -3,7 +3,6 @@ """ from functools import partial -from typing import Optional import httpx from sqlalchemy.ext.asyncio import AsyncSession @@ -20,11 +19,6 @@ ServiceConfiguration, ) from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayStatus -from dstack._internal.core.models.routers import ( - AnyServiceRouterConfig, - RouterType, - SGLangServiceRouterConfig, -) from dstack._internal.core.models.runs import RunSpec, ServiceModelSpec, ServiceSpec from dstack._internal.core.models.services import OpenAIChatModel from dstack._internal.proxy.gateway.const import SERVICE_ALREADY_REGISTERED_ERROR_TEMPLATE @@ -115,21 +109,6 @@ async def _register_service_in_gateway( has_replica_group_router = any( g.router is not None for g in run_spec.configuration.replica_groups ) - if has_replica_group_router and _gateway_has_sglang_router(gateway_configuration): - raise ServerClientError( - "A replica-group `router:` cannot be used with a gateway that has router configuration." - ) - - # Check: service specifies SGLang router but gateway does not have it - service_router = run_spec.configuration.router - service_wants_sglang = service_router is not None and isinstance( - service_router, SGLangServiceRouterConfig - ) - if service_wants_sglang and not _gateway_has_sglang_router(gateway_configuration): - raise ServerClientError( - "Service requires gateway with SGLang router but gateway " - f"'{gateway.name}' does not have the SGLang router configured." - ) configure_service_https = _should_configure_service_https_on_gateway( run_spec, gateway_configuration @@ -152,8 +131,6 @@ async def _register_service_in_gateway( "Cannot run HTTPS service on gateway with no SSL certificates configured" ) - router = _build_service_router_config(gateway_configuration, run_spec.configuration) - gateway_https = _get_gateway_https(gateway_configuration) gateway_protocol = "https" if gateway_https else "http" @@ -198,7 +175,6 @@ async def _register_service_in_gateway( rate_limits=run_spec.configuration.rate_limits, ssh_private_key=run_model.project.ssh_private_key, has_router_replica=has_replica_group_router, - router=router, ) try: await do_register() @@ -240,14 +216,6 @@ async def _register_service_in_gateway( def _register_service_in_server(run_model: RunModel, run_spec: RunSpec) -> ServiceSpec: assert run_spec.configuration.type == "service" - if ( - run_spec.configuration.router is not None - and run_spec.configuration.router.type == RouterType.SGLANG - ): - raise ServerClientError( - "Service with SGLang router configuration requires a gateway. " - "Please configure a gateway with the SGLang router enabled." - ) if run_spec.configuration.https not in ( None, "auto", @@ -283,41 +251,6 @@ def _register_service_in_server(run_model: RunModel, run_spec: RunSpec) -> Servi ) -def _gateway_has_sglang_router(config: GatewayConfiguration) -> bool: - return config.router is not None and config.router.type == RouterType.SGLANG.value - - -def _build_service_router_config( - gateway_configuration: GatewayConfiguration, - service_configuration: ServiceConfiguration, -) -> Optional[AnyServiceRouterConfig]: - """ - Build router config from gateway (type, policy) + service (pd_disaggregation, policy override). - Service's policy overrides gateway's if present. Keeps backward compat: SGLang enabled - automatically when gateway has it configured. - """ - if not _gateway_has_sglang_router(gateway_configuration): - return None - - gateway_router = gateway_configuration.router - assert gateway_router is not None # ensured by _gateway_has_sglang_router - router_type = gateway_router.type - policy = gateway_router.policy - - service_router = service_configuration.router - if service_router is not None and isinstance(service_router, SGLangServiceRouterConfig): - policy = service_router.policy - pd_disaggregation = service_router.pd_disaggregation - else: - pd_disaggregation = False - - return SGLangServiceRouterConfig( - type=router_type, - policy=policy, - pd_disaggregation=pd_disaggregation, - ) - - def _get_service_spec( configuration: ServiceConfiguration, service_url: str, model_url: str ) -> ServiceSpec: diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index 98f6ac2b5b..3075b696b2 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -162,27 +162,6 @@ def test_replica_group_router(self): assert isinstance(router_g.router, ReplicaGroupRouterConfig) assert router_g.router.type == "sglang" - def test_replica_group_router_forbids_service_level_router(self): - conf = { - "type": "service", - "port": 8000, - "router": {"type": "sglang"}, - "replicas": [ - { - "name": "router", - "count": 1, - "commands": ["sglang serve"], - "router": {"type": "sglang"}, - }, - {"name": "worker", "count": 2, "commands": ["worker"]}, - ], - } - with pytest.raises( - ConfigurationError, - match="Service-Level router configuration is not allowed together with replica-group", - ): - parse_run_configuration(conf) - def test_spot_policy_set_at_both_service_and_group_rejected(self): with pytest.raises( ConfigurationError, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json index 431ae169f4..bee72a06d7 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json @@ -15,7 +15,6 @@ "public_ip": true, "region": "eu-west-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json index 0364f1826c..168e3bdfa1 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json @@ -12,7 +12,6 @@ "public_ip": true, "region": "eu-west-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml index 473703b10d..868d8e7f49 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml @@ -1,13 +1,10 @@ -# Exercises the gateway discriminator, its ACM certificate union arm, and the gateway-level router. +# Exercises the gateway discriminator and its ACM certificate union arm. type: gateway name: inference-gateway default: true backend: aws region: us-east-1 instance_type: t3.small -router: - type: sglang - policy: round_robin domain: example.com public_ip: false certificate: diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json index 83690c7e9c..cbae39cfde 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json @@ -1,6 +1,5 @@ { "/": "GatewayConfiguration", "/backend": "BackendType", - "/certificate": "ACMGatewayCertificate", - "/router": "SGLangGatewayRouterConfig" + "/certificate": "ACMGatewayCertificate" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json index c4a6c91adc..276f6b450f 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json @@ -11,10 +11,6 @@ "public_ip": false, "region": "us-east-1", "replicas": 2, - "router": { - "policy": "round_robin", - "type": "sglang" - }, "tags": { "env": "prod" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json index cf387dcd93..d6fb889e57 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json @@ -73,7 +73,6 @@ "shm_size": null }, "retry": null, - "router": null, "scaling": null, "schedule": null, "setup": [], diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json index fea497cc02..9c2f13c09e 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json @@ -8,7 +8,6 @@ "project_name": "main", "public_ip": true, "region": "eu-west-1", - "router": null, "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 gateway@example.com", "tags": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json index 2828b2021e..225d4a656f 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json @@ -10,7 +10,6 @@ "public_ip": true, "region": "eu-west-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json b/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json index 33d399b4b3..f0d5a56410 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json @@ -1736,18 +1736,6 @@ "description": "The number of gateway replicas. Defaults to `1`", "title": "Replicas" }, - "router": { - "anyOf": [ - { - "$ref": "#/$defs/SGLangGatewayRouterConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The router configuration for this gateway. E.g. `{ type: sglang, policy: round_robin }`." - }, "tags": { "anyOf": [ { @@ -3003,65 +2991,6 @@ "title": "RunpodVolumeConfiguration", "type": "object" }, - "SGLangGatewayRouterConfig": { - "additionalProperties": false, - "description": "Gateway-level router configuration. type and policy only. pd_disaggregation is service-level.", - "properties": { - "policy": { - "default": "cache_aware", - "description": "The routing policy. Deprecated: prefer setting policy in the service's router config. Options: `random`, `round_robin`, `cache_aware`, `power_of_two`", - "enum": [ - "random", - "round_robin", - "cache_aware", - "power_of_two" - ], - "title": "Policy", - "type": "string" - }, - "type": { - "const": "sglang", - "default": "sglang", - "description": "The router type enabled on this gateway.", - "title": "Type", - "type": "string" - } - }, - "title": "SGLangGatewayRouterConfig", - "type": "object" - }, - "SGLangServiceRouterConfig": { - "additionalProperties": false, - "properties": { - "pd_disaggregation": { - "default": false, - "description": "Enable PD disaggregation mode for the SGLang router", - "title": "Pd Disaggregation", - "type": "boolean" - }, - "policy": { - "default": "cache_aware", - "description": "The routing policy. Options: `random`, `round_robin`, `cache_aware`, `power_of_two`", - "enum": [ - "random", - "round_robin", - "cache_aware", - "power_of_two" - ], - "title": "Policy", - "type": "string" - }, - "type": { - "const": "sglang", - "default": "sglang", - "description": "The router type", - "title": "Type", - "type": "string" - } - }, - "title": "SGLangServiceRouterConfig", - "type": "object" - }, "SSHHostParams": { "additionalProperties": false, "properties": { @@ -4008,18 +3937,6 @@ "description": "The policy for resubmitting the run. Defaults to `false`", "title": "Retry" }, - "router": { - "anyOf": [ - { - "$ref": "#/$defs/SGLangServiceRouterConfig" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Router configuration for the service. Requires a gateway with matching router enabled. " - }, "scaling": { "anyOf": [ { diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json index 16bdd691c8..25a3ef22db 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json @@ -15,7 +15,6 @@ "public_ip": true, "region": "us-east-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json index 3cacb9291b..f90d397cd1 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json @@ -12,7 +12,6 @@ "public_ip": true, "region": "us-east-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json index 7c78b5dd14..ea7179064c 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json @@ -6,7 +6,6 @@ "project_name": "test-project", "public_ip": true, "region": "us", - "router": null, "ssh_key_pub": "", "tags": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json index 37a4ff056a..5f8a0f6df9 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json @@ -10,7 +10,6 @@ "public_ip": true, "region": "us-east-1", "replicas": null, - "router": null, "tags": null, "type": "gateway" } diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index c9c7c96386..19b39908d7 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -106,7 +106,6 @@ async def test_list( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": gateway.wildcard_domain, "default": False, "public_ip": True, @@ -193,7 +192,6 @@ async def test_get( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": gateway.wildcard_domain, "default": False, "public_ip": True, @@ -535,7 +533,6 @@ async def test_create_gateway(self, test_db, session: AsyncSession, client: Asyn "backend": backend.type.value, "region": "us", "instance_type": None, - "router": None, "domain": None, "default": True, "public_ip": True, @@ -626,7 +623,6 @@ async def test_create_gateway_without_name( "backend": backend.type.value, "region": "us", "instance_type": None, - "router": None, "domain": None, "default": True, "public_ip": True, @@ -746,19 +742,6 @@ async def test_create_gateway_with_invalid_domain_interpolation( " or `certificate: { type: acm, arn: }` (AWS ACM)", id="multi-replica-with-letsencrypt-cert", ), - pytest.param( - { - "type": "gateway", - "name": "test", - "backend": "aws", - "region": "us", - "certificate": None, - "router": {"type": "sglang"}, - "replicas": 2, - }, - "The deprecated `router` property is not supported for multi-replica gateways", - id="multi-replica-with-router", - ), pytest.param( { "type": "gateway", @@ -885,7 +868,6 @@ async def test_set_default_gateway( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": gateway.wildcard_domain, "default": True, "public_ip": True, @@ -1282,7 +1264,6 @@ async def test_set_wildcard_domain( "backend": backend.type.value, "region": gateway.region, "instance_type": None, - "router": None, "domain": "new.example", "default": False, "public_ip": True,