fix(ui): clarify Token Router identifiers and asset layout - #5393
fix(ui): clarify Token Router identifiers and asset layout#5393m199369309 wants to merge 32 commits into
Conversation
…deepseek-tokenizer-asset # Conflicts: # xinference/core/supervisor.py
…agent-orchestration # Conflicts: # xinference/api/tests/test_token_router_api.py # xinference/core/supervisor.py # xinference/router/app.py
…agent-orchestration
There was a problem hiding this comment.
Code Review
This pull request introduces the Token Router feature to Xinference, adding frontend components, routing editors, and management pages, alongside backend restful APIs, database stores, scheduling orchestration, and independent router service entry points. The review feedback highlights several critical improvements: importing the missing ShieldAlert component, defensively handling potential null/undefined values and non-array API responses to prevent runtime crashes, fixing a bug in the toNumber helper where empty strings bypass fallbacks, offloading synchronous database operations in the supervisor to a thread pool to keep the asyncio event loop responsive, and implementing bounded timeouts during agent shutdown to prevent excessive blocking.
| if (isRouterScope && clusterUIConfig?.token_router_enabled === false) { | ||
| return ( | ||
| <div className="flex min-h-[60vh] flex-col items-center justify-center gap-4 text-muted-foreground"> | ||
| <ShieldAlert className="h-16 w-16" /> |
| supervisors: response.filter( | ||
| (item): item is ClusterInfo => item.node_type === 'Supervisor' | ||
| ), | ||
| workers: response.filter((item): item is ClusterInfo => item.node_type === 'Worker'), | ||
| routers: tokenRouterEnabled | ||
| ? response.filter( | ||
| (item): item is RouterNodeClusterInfo => | ||
| item.node_type === 'Router' && | ||
| item.online && | ||
| item.connectivity_status === 'online' | ||
| ) | ||
| : [], |
There was a problem hiding this comment.
If the API response is not an array (e.g., due to an unexpected error or format), calling .filter directly on response will throw a TypeError and crash the page. It is safer to check if response is an array before filtering, which is also consistent with defensive patterns used elsewhere in the codebase.
const dataList = Array.isArray(response) ? response : [];
setData({
supervisors: dataList.filter(
(item): item is ClusterInfo => item.node_type === 'Supervisor'
),
workers: dataList.filter((item): item is ClusterInfo => item.node_type === 'Worker'),
routers: tokenRouterEnabled
? dataList.filter(
(item): item is RouterNodeClusterInfo =>
item.node_type === 'Router' &&
item.online &&
item.connectivity_status === 'online'
)
: [],
});
| const toNumber = (value: string, fallback = 0) => { | ||
| const parsed = Number(value); | ||
| return Number.isFinite(parsed) ? parsed : fallback; | ||
| }; |
There was a problem hiding this comment.
The toNumber helper function returns 0 when value is an empty string "" because Number("") evaluates to 0, which is a finite number. This bypasses the fallback value (e.g., 1 for max_context_tokens or priority), potentially leading to invalid values of 0 being set. Checking for empty or whitespace-only strings first ensures the fallback is correctly applied.
| const toNumber = (value: string, fallback = 0) => { | |
| const parsed = Number(value); | |
| return Number.isFinite(parsed) ? parsed : fallback; | |
| }; | |
| const toNumber = (value: string, fallback = 0) => { | |
| if (value.trim() === '') return fallback; | |
| const parsed = Number(value); | |
| return Number.isFinite(parsed) ? parsed : fallback; | |
| }; |
| const sourceChanged = values.tokenizer_source !== fromRouter(router).tokenizer_source; | ||
| const tokenizerChanged = | ||
| sourceChanged || | ||
| (values.tokenizer_source === 'asset' | ||
| ? values.tokenizer_asset_id.trim() !== (router.tokenizer_asset_id || '') | ||
| : values.tokenizer_path.trim() !== router.tokenizer_path); | ||
| if (tokenizerChanged && !window.confirm(t('tokenRouter.assetChangeWarning'))) return; |
There was a problem hiding this comment.
If values.tokenizer_asset_id or values.tokenizer_path is undefined or null, calling .trim() directly on them will throw a TypeError. Using optional chaining or a fallback empty string prevents potential runtime crashes during form submission.
| const sourceChanged = values.tokenizer_source !== fromRouter(router).tokenizer_source; | |
| const tokenizerChanged = | |
| sourceChanged || | |
| (values.tokenizer_source === 'asset' | |
| ? values.tokenizer_asset_id.trim() !== (router.tokenizer_asset_id || '') | |
| : values.tokenizer_path.trim() !== router.tokenizer_path); | |
| if (tokenizerChanged && !window.confirm(t('tokenRouter.assetChangeWarning'))) return; | |
| const sourceChanged = values.tokenizer_source !== fromRouter(router).tokenizer_source; | |
| const tokenizerChanged = | |
| sourceChanged || | |
| (values.tokenizer_source === 'asset' | |
| ? (values.tokenizer_asset_id || '').trim() !== (router.tokenizer_asset_id || '') | |
| : (values.tokenizer_path || '').trim() !== router.tokenizer_path); |
| async def _monitor_token_router_nodes(self) -> None: | ||
| while True: | ||
| try: | ||
| transitions = self._token_router_orchestration.sweep_nodes() |
There was a problem hiding this comment.
The sweep_nodes method performs synchronous database queries and updates (SQLite I/O). Calling it directly in the asynchronous _monitor_token_router_nodes loop blocks the asyncio event loop, which can degrade supervisor responsiveness and cause heartbeat timeouts. Offloading it to a thread pool using asyncio.to_thread keeps the event loop responsive.
transitions = await asyncio.to_thread(
self._token_router_orchestration.sweep_nodes
)| async def _stop_locked( | ||
| self, managed: ManagedRuntimeProcess, *, report: bool | ||
| ) -> None: |
There was a problem hiding this comment.
Add an optional timeout parameter to _stop_locked so that callers (like shutdown) can enforce a shorter, bounded timeout when stopping runtimes, rather than always waiting for the full drain_timeout_seconds (which can be up to 2 hours).
| async def _stop_locked( | |
| self, managed: ManagedRuntimeProcess, *, report: bool | |
| ) -> None: | |
| async def _stop_locked( | |
| self, managed: ManagedRuntimeProcess, *, report: bool, timeout: Optional[float] = None | |
| ) -> None: |
| try: | ||
| await asyncio.wait_for( | ||
| process.wait(), timeout=self.drain_timeout_seconds |
| async def shutdown(self) -> None: | ||
| self._stopping = True | ||
| async with self._lock: | ||
| runtimes = list(self._processes.values()) | ||
| self._processes.clear() | ||
| await asyncio.gather( | ||
| *(self._stop_locked(item, report=True) for item in runtimes), | ||
| return_exceptions=True, | ||
| ) |
There was a problem hiding this comment.
During Agent shutdown, calling _stop_locked with the full drain_timeout_seconds (which defaults to 7200 seconds) can block the shutdown process for an excessively long time. It is better to use a much shorter timeout (e.g., 10 seconds) during shutdown to ensure the Agent exits promptly when requested by the system.
| async def shutdown(self) -> None: | |
| self._stopping = True | |
| async with self._lock: | |
| runtimes = list(self._processes.values()) | |
| self._processes.clear() | |
| await asyncio.gather( | |
| *(self._stop_locked(item, report=True) for item in runtimes), | |
| return_exceptions=True, | |
| ) | |
| async def shutdown(self) -> None: | |
| self._stopping = True | |
| async with self._lock: | |
| runtimes = list(self._processes.values()) | |
| self._processes.clear() | |
| await asyncio.gather( | |
| *(self._stop_locked(item, report=True, timeout=10.0) for item in runtimes), | |
| return_exceptions=True, | |
| ) |
Summary
Dependency
Depends on #5380.
This PR is intentionally stacked on top of #5380 because the affected Token Router UI components are introduced there. Please merge #5380 first; after that merge, this PR can be rebased onto
main.Validation
git diff --checkpre-commit run --files <changed-files>npx prettier --check <changed-files>npx eslint <changed-files>npm run build