diff --git a/frontend/src/app/token-router/page.tsx b/frontend/src/app/token-router/page.tsx new file mode 100644 index 0000000000..98482ae686 --- /dev/null +++ b/frontend/src/app/token-router/page.tsx @@ -0,0 +1,10 @@ +import { PermissionGuard } from '@/components/auth/permission-guard'; +import TokenRouterPage from '@/components/pages/token-router'; + +export default function Page() { + return ( + + + + ); +} diff --git a/frontend/src/components/auth/permission-guard.tsx b/frontend/src/components/auth/permission-guard.tsx index 9b4049a57a..dfb233da96 100644 --- a/frontend/src/components/auth/permission-guard.tsx +++ b/frontend/src/components/auth/permission-guard.tsx @@ -6,7 +6,14 @@ import { useGlobal } from '@/contexts/global-context'; import { useMenuAuth } from '@/hooks/use-menu-auth'; interface PermissionGuardProps { - scope: 'monitor:view' | 'logs:list' | 'models:register'; + scope: + | 'monitor:view' + | 'logs:list' + | 'models:register' + | 'routers:list' + | 'routers:read' + | 'routers:write' + | 'routers:operate'; children: React.ReactNode; } @@ -41,6 +48,10 @@ export function PermissionGuard({ scope, children }: PermissionGuardProps) { 'monitor:view': auth.hasMonitorView, 'logs:list': auth.hasLogsList, 'models:register': auth.canRegisterModel, + 'routers:list': auth.hasRouterList, + 'routers:read': auth.hasRouterRead, + 'routers:write': auth.canWriteRouters, + 'routers:operate': auth.canOperateRouters, }; if (scopeMap[scope]) { diff --git a/frontend/src/components/layout/sidebar.tsx b/frontend/src/components/layout/sidebar.tsx index c74a8acb5b..9ca3203e6a 100644 --- a/frontend/src/components/layout/sidebar.tsx +++ b/frontend/src/components/layout/sidebar.tsx @@ -16,6 +16,7 @@ import { Globe, BotIcon, Rocket, + Route, Monitor, ScrollText, ShieldCheck, @@ -147,7 +148,15 @@ export function Sidebar() { const [collapsed, setCollapsed] = useState(false); const branding = getBrandingFromEnv(); const { clusterVersion, clusterAuth, clusterUIConfig } = useGlobal(); - const { isAdmin, usersManagePage, canAccessKeysPage, hasLogsList, hasMonitorView, canRegisterModel } = useMenuAuth(); + const { + isAdmin, + usersManagePage, + canAccessKeysPage, + hasLogsList, + hasMonitorView, + canRegisterModel, + canAccessRouterPage, + } = useMenuAuth(); const [token, setToken] = useState(); const showLoginOut = useMemo( () => Boolean(clusterAuth?.auth && token && token !== NO_AUTH), @@ -197,6 +206,13 @@ export function Sidebar() { Icon: Layers, Extra: ChevronRight, }, + { + path: '/token-router', + name: t('menu.tokenRouter'), + Icon: Route, + Extra: ChevronRight, + show: !clusterUIConfig?.auth_advanced || canAccessRouterPage, + }, { path: '/register-model', name: t('menu.registerModel'), @@ -307,7 +323,18 @@ export function Sidebar() { items: group.items.filter(({ show = true }) => show), })) .filter(({ items }) => items.length > 0); - }, [clusterUIConfig, locale, t, usersManagePage, canAccessKeysPage, isAdmin, hasLogsList, hasMonitorView, canRegisterModel]); + }, [ + clusterUIConfig, + locale, + t, + usersManagePage, + canAccessKeysPage, + isAdmin, + hasLogsList, + hasMonitorView, + canRegisterModel, + canAccessRouterPage, + ]); return (
void; +} + +const toNumber = (value: string, fallback = 0) => { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +}; + +const conditionOptions = (anyLabel: string, trueLabel: string, falseLabel: string) => [ + { value: 'any', label: anyLabel }, + { value: 'true', label: trueLabel }, + { value: 'false', label: falseLabel }, +]; + +export function AdvancedRoutingEditor({ value, candidates, onChange }: Props) { + const { t } = useI18n(); + const backendIds = value.backends.map((backend) => backend.id).filter(Boolean); + const candidateOptions = candidates.map((candidate) => ({ + value: candidate.model_uid, + label: `${candidate.model_uid}${candidate.model_engine ? ` · ${candidate.model_engine}` : ''}`, + description: candidate.eligible + ? candidate.compatibility_reason + : candidate.ineligible_reasons.join('; '), + disabled: !candidate.eligible, + })); + + const updateBackend = (index: number, patch: Partial) => { + const backends = value.backends.map((backend, itemIndex) => + itemIndex === index ? { ...backend, ...patch } : backend + ); + onChange({ ...value, backends }); + }; + + const removeBackend = (index: number) => { + if (value.backends.length <= 1) return; + onChange({ ...value, backends: value.backends.filter((_, itemIndex) => itemIndex !== index) }); + }; + + const addBackend = () => { + const suffix = value.backends.length + 1; + onChange({ + ...value, + backends: [ + ...value.backends, + { + id: `backend-${suffix}`, + model_uid: '', + max_context_tokens: 131072, + admission: { + max_active: 1, + max_queue: 4, + queue_timeout_seconds: 5, + retry_after_seconds: 1, + }, + }, + ], + }); + }; + + const updateRule = (index: number, patch: Partial) => { + const rules = value.rules.map((rule, itemIndex) => + itemIndex === index ? { ...rule, ...patch } : rule + ); + onChange({ ...value, rules }); + }; + + const updateRuleMatch = ( + index: number, + key: keyof TokenRouterRoutingRule['match'], + fieldValue: number | boolean | undefined + ) => { + const rule = value.rules[index]; + const match = { ...rule.match }; + if (fieldValue === undefined) delete match[key]; + else Object.assign(match, { [key]: fieldValue }); + updateRule(index, { match }); + }; + + const addRule = () => { + const suffix = value.rules.length + 1; + onChange({ + ...value, + rules: [ + ...value.rules, + { + id: `rule-${suffix}`, + priority: Math.max(1, 100 - value.rules.length * 10), + match: { total_tokens_gte: 0 }, + action: { type: 'route', backend_id: backendIds[0] || '' }, + }, + ], + }); + }; + + const removeRule = (index: number) => { + if (value.rules.length <= 1) return; + onChange({ ...value, rules: value.rules.filter((_, itemIndex) => itemIndex !== index) }); + }; + + const updateDefaultAction = (action: TokenRouterRoutingAction) => + onChange({ ...value, defaultAction: action }); + + return ( +
+
+ + + + + +
+ +
+
+
+

{t('tokenRouter.dynamicBackends')}

+

{t('tokenRouter.dynamicBackendsHint')}

+
+ +
+
+ {value.backends.map((backend, index) => { + const candidate = candidates.find((item) => item.model_uid === backend.model_uid); + return ( +
+
+
+ #{index + 1} + {backend.id || '—'} + {candidate && ( + + {candidate.compatibility_status} + + )} +
+ +
+
+ + updateBackend(index, { id: event.target.value })} + /> + + + + updateBackend(index, { + max_context_tokens: toNumber(event.target.value, 1), + }) + } + /> + + + + updateBackend(index, { + admission: { + ...backend.admission, + max_active: toNumber(event.target.value, 1), + }, + }) + } + /> + + + + updateBackend(index, { + admission: { + ...backend.admission, + max_queue: toNumber(event.target.value), + }, + }) + } + /> + +
+ {candidate ? ( + <> +
{candidate.model_name || '—'}
+
{candidate.model_format || '—'}
+
+ {candidate.model_ability.join(', ') || '—'} +
+ + ) : ( + t('tokenRouter.selectBackendCandidate') + )} +
+
+
+ ); + })} +
+
+ +
+
+
+

{t('tokenRouter.orderedRules')}

+

{t('tokenRouter.firstMatchHint')}

+
+ +
+
+ {[...value.rules] + .map((rule, sourceIndex) => ({ rule, sourceIndex })) + .sort((left, right) => right.rule.priority - left.rule.priority) + .map(({ rule, sourceIndex }, displayIndex) => ( +
+
+
+ {displayIndex + 1} + {rule.id || '—'} +
+ +
+
+ + updateRule(sourceIndex, { id: event.target.value })} + /> + + + + updateRule(sourceIndex, { priority: toNumber(event.target.value, 1) }) + } + /> + + + + updateRuleMatch( + sourceIndex, + 'total_tokens_gte', + event.target.value === '' ? undefined : toNumber(event.target.value) + ) + } + /> + + + + updateRuleMatch( + sourceIndex, + 'total_tokens_lte', + event.target.value === '' ? undefined : toNumber(event.target.value) + ) + } + /> + + {(['thinking', 'tools_present', 'stream'] as const).map((key) => ( + + + updateRule(sourceIndex, { + action: + actionType === 'reject' + ? { type: 'reject', reason: 'request_rejected' } + : { type: 'route', backend_id: backendIds[0] || '' }, + }) + } + /> + + {rule.action.type === 'route' ? ( + + + updateRule(sourceIndex, { + action: { type: 'reject', reason: event.target.value }, + }) + } + /> + + )} +
+
+ ))} +
+
+ +
+

{t('tokenRouter.defaultAction')}

+
+ + ({ value: backendId, label: backendId }))} + onChange={(backendId) => + updateDefaultAction({ type: 'route', backend_id: String(backendId || '') }) + } + /> + + ) : ( + + + updateDefaultAction({ type: 'reject', reason: event.target.value }) + } + /> + + )} +
+
+
+ ); +} + +function EditorField({ label, children }: { label: string; children: ReactNode }) { + return ( + + ); +} + +function ProfileItem({ + label, + value, + mono = false, +}: { + label: string; + value: string; + mono?: boolean; +}) { + return ( +
+
{label}
+
+ {value} +
+
+ ); +} diff --git a/frontend/src/components/pages/token-router/index.tsx b/frontend/src/components/pages/token-router/index.tsx new file mode 100644 index 0000000000..f25eb72e40 --- /dev/null +++ b/frontend/src/components/pages/token-router/index.tsx @@ -0,0 +1,368 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { + Eye, + Loader2, + Pencil, + Plus, + Power, + PowerOff, + RefreshCw, + Route, + ShieldCheck, + Trash2, +} from 'lucide-react'; +import { toast } from 'sonner'; + +import { Button } from '@/components/ui/button'; +import { ConfirmDialog } from '@/components/ui/confirm-dialog'; +import PageContainer from '@/components/ui/page-container'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { useI18n } from '@/contexts/i18n-context'; +import { useMenuAuth } from '@/hooks/use-menu-auth'; +import request from '@/lib/request'; +import type { TokenRouterItem } from '@/types/services'; +import { isTypedTokenRouter } from '@/types/services'; +import { RouterDetailDrawer } from './router-detail-drawer'; +import { RouterFormDialog } from './router-form-dialog'; +import { routerBackendList } from './router-config-normalizer'; +import { RouterStatusBadge } from './router-status-badge'; + +type RouterAction = 'enable' | 'disable' | 'validate' | 'delete'; + +type BusyAction = { + routerUid: string; + action: RouterAction; +} | null; + +export default function TokenRouterPage() { + const { t } = useI18n(); + const auth = useMenuAuth(); + const [routers, setRouters] = useState([]); + const [initialLoading, setInitialLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [formOpen, setFormOpen] = useState(false); + const [editing, setEditing] = useState(null); + const [details, setDetails] = useState(null); + const [deleting, setDeleting] = useState(null); + const [busyAction, setBusyAction] = useState(null); + + const load = useCallback(async (mode: 'initial' | 'manual' | 'silent' = 'silent') => { + if (mode === 'initial') setInitialLoading(true); + if (mode === 'manual') setRefreshing(true); + + try { + const data = await request.get('/v1/token_routers'); + setRouters(Array.isArray(data) ? data : []); + } finally { + if (mode === 'initial') setInitialLoading(false); + if (mode === 'manual') setRefreshing(false); + } + }, []); + + useEffect(() => { + void load('initial'); + const timer = window.setInterval(() => void load('silent'), 10000); + return () => window.clearInterval(timer); + }, [load]); + + const isBusy = useCallback( + (routerUid: string, action: RouterAction) => + busyAction?.routerUid === routerUid && busyAction.action === action, + [busyAction] + ); + + const runAction = async (router: TokenRouterItem, action: Exclude) => { + setBusyAction({ routerUid: router.router_uid, action }); + try { + const result = await request.post<{ valid?: boolean; errors?: string[] }>( + `/v1/token_routers/${router.router_uid}/${action}` + ); + + if (action === 'validate') { + if (result.valid) { + toast.success(t('tokenRouter.validationPassed')); + } else { + toast.error( + result.errors?.length ? result.errors.join('; ') : t('tokenRouter.validationFailed') + ); + } + } else { + toast.success( + t(action === 'enable' ? 'tokenRouter.enableSuccess' : 'tokenRouter.disableSuccess') + ); + } + + await load('silent'); + } finally { + setBusyAction(null); + } + }; + + const handleDelete = async () => { + if (!deleting) return; + + setBusyAction({ routerUid: deleting.router_uid, action: 'delete' }); + try { + await request.delete(`/v1/token_routers/${deleting.router_uid}`); + toast.success(t('tokenRouter.deleteSuccess')); + setDeleting(null); + await load('silent'); + } finally { + setBusyAction(null); + } + }; + + const deletingBusy = useMemo( + () => Boolean(deleting && isBusy(deleting.router_uid, 'delete')), + [deleting, isBusy] + ); + + return ( + + + {auth.canWriteRouters && ( + + )} +
+ } + > +
+ + + + + + + + + + + + + {t('tokenRouter.routerUid')} + + {t('tokenRouter.virtualModelUid')} + + {t('tokenRouter.backends')} + {t('tokenRouter.routeProfile')} + {t('tokenRouter.status')} + {t('tokenRouter.revision')} + + {t('common.operation')} + + + + + {initialLoading ? ( + + +
+ + {t('tokenRouter.loading')} +
+ + + ) : routers.length === 0 ? ( + + +
+ + {t('tokenRouter.empty')} +
+ + + ) : ( + routers.map((router) => { + const rowBusy = busyAction?.routerUid === router.router_uid; + const validating = isBusy(router.router_uid, 'validate'); + const toggling = isBusy(router.router_uid, router.enabled ? 'disable' : 'enable'); + + return ( + + +
{router.router_uid}
+ + +
{router.virtual_model_uid}
+ + + {routerBackendList(router) + .slice(0, 3) + .map((backend, index) => ( +
+ {backend.id}: {backend.model_uid} +
+ ))} + {routerBackendList(router).length > 3 && ( +
+ +{routerBackendList(router).length - 3} +
+ )} + + +
LLM / Chat
+
+ {isTypedTokenRouter(router) + ? t('tokenRouter.typedRulesSummary', { + count: router.routing.rules.length, + }) + : t('tokenRouter.tokenBudgetSummary', { + threshold: router.routing.short_threshold_tokens.toLocaleString(), + })} +
+ + +
+ +
+ {t('tokenRouter.onlineSummary', { + online: router.online_instances, + total: router.runtime_instances, + })} +
+
+ + {router.revision} + +
+ + {auth.canWriteRouters && ( + + )} + {auth.canOperateRouters && ( + + )} + {auth.canOperateRouters && ( + + )} + {auth.canWriteRouters && ( + + )} +
+ + + ); + }) + )} + +
+
+ + void load('silent')} + /> + !open && setDetails(null)} + /> + { + if (!open && !deletingBusy) setDeleting(null); + }} + onConfirm={handleDelete} + title={t('tokenRouter.deleteTitle')} + description={t('tokenRouter.confirmDelete', { + routerUid: deleting?.router_uid || '', + })} + confirmText={t('common.delete')} + confirmClassName="bg-destructive text-white hover:bg-destructive/90" + isLoading={deletingBusy} + /> + + ); +} diff --git a/frontend/src/components/pages/token-router/router-config-normalizer.ts b/frontend/src/components/pages/token-router/router-config-normalizer.ts new file mode 100644 index 0000000000..046697cebb --- /dev/null +++ b/frontend/src/components/pages/token-router/router-config-normalizer.ts @@ -0,0 +1,135 @@ +import type { + TokenRouterDynamicBackendConfig, + TokenRouterItem, + TokenRouterRoutingAction, + TokenRouterRoutingRule, + TokenRouterTypedItem, +} from '@/types/services'; +import { isTypedTokenRouter } from '@/types/services'; + +export type RouterMode = 'simple' | 'advanced'; + +export interface TypedRouterDraft { + backends: TokenRouterDynamicBackendConfig[]; + rules: TokenRouterRoutingRule[]; + defaultAction: TokenRouterRoutingAction; +} + +const admission = (maxActive: number, maxQueue: number) => ({ + max_active: maxActive, + max_queue: maxQueue, + queue_timeout_seconds: 5, + retry_after_seconds: 1, +}); + +export function createTypedDraft(): TypedRouterDraft { + return { + backends: [ + { + id: 'short', + model_uid: '', + max_context_tokens: 131072, + admission: admission(8, 32), + }, + { + id: 'long', + model_uid: '', + max_context_tokens: 1048576, + admission: admission(1, 2), + }, + ], + rules: [ + { + id: 'short-budget', + priority: 100, + match: { total_tokens_lte: 32768 }, + action: { type: 'route', backend_id: 'short' }, + }, + { + id: 'long-budget', + priority: 90, + match: { total_tokens_gte: 32769 }, + action: { type: 'route', backend_id: 'long' }, + }, + ], + defaultAction: { type: 'reject', reason: 'context_length_exceeded' }, + }; +} + +export function typedDraftFromRouter(router?: TokenRouterItem | null): TypedRouterDraft { + if (!router) return createTypedDraft(); + if (isTypedTokenRouter(router)) { + return { + backends: router.backends.map((backend) => ({ + ...backend, + admission: { ...backend.admission }, + })), + rules: router.routing.rules.map((rule) => ({ + ...rule, + match: { ...rule.match }, + action: { ...rule.action }, + })), + defaultAction: { ...router.routing.default_action }, + }; + } + + const thinkingRule: TokenRouterRoutingRule = { + id: 'thinking-policy', + priority: 110, + match: { thinking: true }, + action: + router.routing.thinking_policy === 'reject' + ? { type: 'reject', reason: 'thinking_not_supported' } + : { type: 'route', backend_id: router.routing.thinking_policy }, + }; + return { + backends: [ + { id: 'short', ...router.backends.short, admission: { ...router.backends.short.admission } }, + { id: 'long', ...router.backends.long, admission: { ...router.backends.long.admission } }, + ], + rules: [ + thinkingRule, + { + id: 'short-budget', + priority: 100, + match: { total_tokens_lte: router.routing.short_threshold_tokens }, + action: { type: 'route', backend_id: 'short' }, + }, + { + id: 'long-budget', + priority: 90, + match: { total_tokens_gte: router.routing.short_threshold_tokens + 1 }, + action: { type: 'route', backend_id: 'long' }, + }, + ], + defaultAction: { type: 'reject', reason: 'context_length_exceeded' }, + }; +} + +export function routerBackendList(router: TokenRouterItem): TokenRouterDynamicBackendConfig[] { + if (isTypedTokenRouter(router)) return router.backends; + return [ + { id: 'short', ...router.backends.short }, + { id: 'long', ...router.backends.long }, + ]; +} + +export function routerRuleList(router: TokenRouterItem): TokenRouterRoutingRule[] { + return typedDraftFromRouter(router).rules; +} + +export function routerContextReserve(router: TokenRouterItem): number { + return router.routing.context_reserve_tokens; +} + +export function routerDefaultOutput(router: TokenRouterItem): number { + return router.routing.default_output_tokens; +} + +export function routerMode(router?: TokenRouterItem | null): RouterMode { + return router && isTypedTokenRouter(router) ? 'advanced' : 'simple'; +} + +export function asTypedRouter(router: TokenRouterItem): TokenRouterTypedItem | null { + return isTypedTokenRouter(router) ? router : null; +} diff --git a/frontend/src/components/pages/token-router/router-detail-drawer.tsx b/frontend/src/components/pages/token-router/router-detail-drawer.tsx new file mode 100644 index 0000000000..5557b616ea --- /dev/null +++ b/frontend/src/components/pages/token-router/router-detail-drawer.tsx @@ -0,0 +1,1361 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'; +import { + Activity, + BarChart3, + Copy, + Gauge, + Loader2, + PackageCheck, + RefreshCw, + Server, + Settings2, + X, +} from 'lucide-react'; +import { toast } from 'sonner'; + +import { Button } from '@/components/ui/button'; +import { CollapsiblePanel } from '@/components/ui/collapsible'; +import { Sheet, SheetClose, SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet'; +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { useI18n, type TFunc } from '@/contexts/i18n-context'; +import request from '@/lib/request'; +import { cn } from '@/lib/utils'; +import type { + TokenizerAssetItem, + TokenRouterItem, + TokenRouterRoutingAction, + TokenRouterRuleMatch, + TokenRouterRuntimeInstance, +} from '@/types/services'; +import { isTypedTokenRouter } from '@/types/services'; +import { routerBackendList, routerRuleList } from './router-config-normalizer'; +import { RouterStatusBadge } from './router-status-badge'; + +interface Props { + router: TokenRouterItem | null; + open: boolean; + onOpenChange: (open: boolean) => void; +} + +type DetailTab = 'overview' | 'instances' | 'metrics'; +type LoadMode = 'initial' | 'manual' | 'silent'; +type MetricsResponse = Record; + +interface CapacitySummary { + active?: number; + waiting?: number; + maxActive?: number; + maxQueue?: number; +} + +interface MetricsSummary { + completedRequests?: number; + activeRequests?: number; + waitingRequests?: number; + onlineInstances?: number; + backendPools: Record; + tokenization: CapacitySummary; +} + +const POLL_INTERVAL_MS = 10000; + +export function RouterDetailDrawer({ router, open, onOpenChange }: Props) { + const { locale, t } = useI18n(); + const routerUid = router?.router_uid; + const tokenizerAssetId = router?.tokenizer_asset_id; + const [activeTab, setActiveTab] = useState('overview'); + const [instances, setInstances] = useState([]); + const [metrics, setMetrics] = useState({}); + const [tokenizerAsset, setTokenizerAsset] = useState(null); + const [tokenizerAssetLoading, setTokenizerAssetLoading] = useState(false); + const [tokenizerAssetLoadFailed, setTokenizerAssetLoadFailed] = useState(false); + const [instancesInitialLoading, setInstancesInitialLoading] = useState(false); + const [instancesRefreshing, setInstancesRefreshing] = useState(false); + const [metricsInitialLoading, setMetricsInitialLoading] = useState(false); + const [metricsRefreshing, setMetricsRefreshing] = useState(false); + const [instancesLoaded, setInstancesLoaded] = useState(false); + const [metricsLoaded, setMetricsLoaded] = useState(false); + const [instancesUpdatedAt, setInstancesUpdatedAt] = useState(null); + const [metricsUpdatedAt, setMetricsUpdatedAt] = useState(null); + const generationRef = useRef(0); + const instancesRequestRef = useRef(null); + const metricsRequestRef = useRef(null); + const activeRouterUidRef = useRef(open && routerUid ? routerUid : null); + + activeRouterUidRef.current = open && routerUid ? routerUid : null; + + const isCurrentRequest = useCallback((generation: number, uid: string) => { + return generationRef.current === generation && activeRouterUidRef.current === uid; + }, []); + + const loadInstances = useCallback( + async (mode: LoadMode = 'silent') => { + if (!open || !routerUid) return; + + const generation = generationRef.current; + const requestKey = `${generation}:${routerUid}`; + if (instancesRequestRef.current === requestKey) return; + instancesRequestRef.current = requestKey; + + if (mode === 'initial') setInstancesInitialLoading(true); + if (mode === 'manual') setInstancesRefreshing(true); + + try { + const runtimeData = await request.get( + `/v1/token_routers/${routerUid}/instances` + ); + if (!isCurrentRequest(generation, routerUid)) return; + + setInstances(Array.isArray(runtimeData) ? runtimeData : []); + setInstancesLoaded(true); + setInstancesUpdatedAt(new Date()); + } catch { + // The shared request interceptor displays the request error. Keep the last successful data. + } finally { + if (instancesRequestRef.current === requestKey) { + instancesRequestRef.current = null; + } + if (isCurrentRequest(generation, routerUid)) { + if (mode === 'initial') setInstancesInitialLoading(false); + if (mode === 'manual') setInstancesRefreshing(false); + } + } + }, + [isCurrentRequest, open, routerUid] + ); + + const loadMetrics = useCallback( + async (mode: LoadMode = 'silent') => { + if (!open || !routerUid) return; + + const generation = generationRef.current; + const requestKey = `${generation}:${routerUid}`; + if (metricsRequestRef.current === requestKey) return; + metricsRequestRef.current = requestKey; + + if (mode === 'initial') setMetricsInitialLoading(true); + if (mode === 'manual') setMetricsRefreshing(true); + + try { + const metricData = await request.get( + `/v1/token_routers/${routerUid}/metrics-summary` + ); + if (!isCurrentRequest(generation, routerUid)) return; + + setMetrics(isRecord(metricData) ? metricData : {}); + setMetricsLoaded(true); + setMetricsUpdatedAt(new Date()); + } catch { + // The shared request interceptor displays the request error. Keep the last successful data. + } finally { + if (metricsRequestRef.current === requestKey) { + metricsRequestRef.current = null; + } + if (isCurrentRequest(generation, routerUid)) { + if (mode === 'initial') setMetricsInitialLoading(false); + if (mode === 'manual') setMetricsRefreshing(false); + } + } + }, + [isCurrentRequest, open, routerUid] + ); + + const loadTokenizerAsset = useCallback(async () => { + if (!open || !routerUid || !tokenizerAssetId) return; + + const generation = generationRef.current; + setTokenizerAssetLoading(true); + setTokenizerAssetLoadFailed(false); + try { + const asset = await request.get( + `/v1/tokenizer_assets/${encodeURIComponent(tokenizerAssetId)}` + ); + if (!isCurrentRequest(generation, routerUid)) return; + setTokenizerAsset(asset); + } catch { + if (!isCurrentRequest(generation, routerUid)) return; + setTokenizerAssetLoadFailed(true); + } finally { + if (isCurrentRequest(generation, routerUid)) { + setTokenizerAssetLoading(false); + } + } + }, [isCurrentRequest, open, routerUid, tokenizerAssetId]); + + useEffect(() => { + generationRef.current += 1; + instancesRequestRef.current = null; + metricsRequestRef.current = null; + setActiveTab('overview'); + setInstances([]); + setMetrics({}); + setTokenizerAsset(null); + setTokenizerAssetLoading(false); + setTokenizerAssetLoadFailed(false); + setInstancesInitialLoading(false); + setInstancesRefreshing(false); + setMetricsInitialLoading(false); + setMetricsRefreshing(false); + setInstancesLoaded(false); + setMetricsLoaded(false); + setInstancesUpdatedAt(null); + setMetricsUpdatedAt(null); + + if (open && routerUid) { + void loadInstances('initial'); + if (tokenizerAssetId) void loadTokenizerAsset(); + } + + return () => { + generationRef.current += 1; + }; + }, [loadInstances, loadTokenizerAsset, open, routerUid, tokenizerAssetId]); + + useEffect(() => { + if (open && routerUid && activeTab === 'metrics' && !metricsLoaded) { + void loadMetrics('initial'); + } + }, [activeTab, loadMetrics, metricsLoaded, open, routerUid]); + + useEffect(() => { + if (!open || !routerUid) return; + + const timer = window.setInterval(() => { + if (activeTab === 'metrics') { + void loadMetrics('silent'); + } else { + void loadInstances('silent'); + } + }, POLL_INTERVAL_MS); + + return () => window.clearInterval(timer); + }, [activeTab, loadInstances, loadMetrics, open, routerUid]); + + const metricsSummary = useMemo(() => summarizeMetrics(metrics), [metrics]); + const hasMetrics = useMemo(() => containsMetricsData(metrics), [metrics]); + const hasKnownMetrics = hasMetricsSummary(metricsSummary); + const displayedInstanceCount = instancesLoaded + ? instances.length + : router?.runtime_instances || 0; + const displayedOnlineCount = instancesLoaded + ? instances.filter((item) => item.online).length + : router?.online_instances || 0; + const currentInitialLoading = + activeTab === 'metrics' ? metricsInitialLoading : instancesInitialLoading; + const currentRefreshing = activeTab === 'metrics' ? metricsRefreshing : instancesRefreshing; + const currentUpdatedAt = activeTab === 'metrics' ? metricsUpdatedAt : instancesUpdatedAt; + + const handleRefresh = () => { + if (activeTab === 'metrics') { + void loadMetrics('manual'); + } else { + void loadInstances('manual'); + if (activeTab === 'overview' && tokenizerAssetId) { + void loadTokenizerAsset(); + } + } + }; + + const copyValue = async (value: string) => { + try { + await writeClipboard(value); + toast.success(t('common.copySuccess')); + } catch { + toast.error(t('tokenRouter.copyFailed')); + } + }; + + const updatedAtText = currentUpdatedAt + ? formatDateTime(currentUpdatedAt, locale) + : t('tokenRouter.notUpdated'); + + return ( + + + +
+ {t('tokenRouter.details')} +
+ + + + +
+
+ + {router && ( +
+
+ + +
+ +
+ )} + + + setActiveTab(value as DetailTab)} + className="min-h-0 flex-1 gap-0" + > +
+ + + {t('tokenRouter.tabs.overview')} + + + {t('tokenRouter.tabs.instances')} + + + {t('tokenRouter.tabs.metrics')} + + +
+ + + {router && ( +
+
+ + + + + + +
+ +
+ }> + {t('tokenRouter.tokenizerAssetDetails')} + +
+ {router.tokenizer_asset_id ? ( +
+
+ + {tokenizerAsset && ( + + {t(`tokenRouter.assetStatuses.${tokenizerAsset.status}`)} + + )} +
+ + {tokenizerAssetLoading && !tokenizerAsset ? ( +
+ + {t('tokenRouter.loading')} +
+ ) : ( +
+ + + + + +
+ +
+
+ )} + + {tokenizerAssetLoadFailed && ( +
+ {t('tokenRouter.tokenizerAssetDetailsLoadFailed')} +
+ )} + {Boolean(tokenizerAsset?.errors.length) && ( +
+
+ {t('tokenRouter.tokenizerAssetErrors')} +
+
    + {tokenizerAsset?.errors.map((error) => ( +
  • + {error} +
    • + ))} +
+
+ )} +
+ ) : ( +
+
+ {t('tokenRouter.legacyCustomTokenizerPath')} +
+ +
+ )} +
+
+ +
+ }> + {t('tokenRouter.routingSummary')} + +
+
+
+ + + + +
+
+ {routerBackendList(router).map((backend) => ( + + ))} +
+ {isTypedTokenRouter(router) ? ( +
+
+ {t('tokenRouter.orderedRules')} +
+ {[...routerRuleList(router)] + .sort((left, right) => right.priority - left.priority) + .map((rule) => ( +
+ {rule.id} + P{rule.priority} + + {formatRuleMatch(rule.match)} + + + {formatRoutingAction(rule.action)} + +
+ ))} +
+ {t('tokenRouter.defaultAction')}:{' '} + {formatRoutingAction(router.routing.default_action)} +
+
+ ) : ( + + )} +
+
+
+ + } + contentClassName="space-y-5" + > + + + + + + + + + + + + + + + + + {isTypedTokenRouter(router) ? ( + <> + + + + ) : ( + <> + + + + )} + + + {routerBackendList(router).map((backend) => ( + + + + + + + + + ))} + + + + + + + + + + + + + + + + +
+ )} + + + +
+ + + {instancesInitialLoading && !instancesLoaded ? ( + + ) : instances.length === 0 ? ( + }> + {t('tokenRouter.noInstances')} + + ) : ( +
+ {instances.map((item) => ( +
+
+ + +
+ +
+ + +
+ +
+ +
+ + {item.process?.tokenizer_asset && ( +
+
+ {t('tokenRouter.runtimeTokenizerAsset')} +
+
+ + + +
+ +
+
+
+ )} + + {item.config_error && ( +
+
{t('tokenRouter.configError')}
+ {item.config_error} +
+ )} +
+ ))} +
+ )} +
+ + + +
+ + + {metricsInitialLoading && !metricsLoaded ? ( + + ) : !hasMetrics ? ( + }> + {t('tokenRouter.noMetrics')} + + ) : ( + <> + {hasKnownMetrics ? ( +
+
+ {metricsSummary.completedRequests !== undefined && ( + + )} + {metricsSummary.activeRequests !== undefined && ( + + )} + {metricsSummary.waitingRequests !== undefined && ( + + )} + {metricsSummary.onlineInstances !== undefined && ( + + )} +
+ +
+ {Object.entries(metricsSummary.backendPools).map( + ([backendId, summary]) => + hasCapacityData(summary) && ( + + ) + )} + {hasCapacityData(metricsSummary.tokenization) && ( + + )} +
+
+ ) : ( +
+ {t('tokenRouter.noKnownMetrics')} +
+ )} + + } + contentClassName="space-y-3" + > +
+ +
+ 
+                      {JSON.stringify(metrics, null, 2)}
+
+ + + )} +
+ + + + + {t('tokenRouter.lastUpdated')}: {updatedAtText} + + + + ); +} + +function SummaryCard({ + label, + value, + loading = false, + children, +}: { + label: string; + value?: string | number; + loading?: boolean; + children?: ReactNode; +}) { + return ( +
+
{label}
+
+ {loading ? ( + + ) : ( + children || value + )} +
+
+ ); +} + +function MetricCard({ label, value }: { label: string; value: number }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function CapacityCard({ + title, + summary, + activeLabel, + waitingLabel, +}: { + title: string; + summary: CapacitySummary; + activeLabel: string; + waitingLabel: string; +}) { + return ( +
+
+ + {title} +
+
+ {(summary.active !== undefined || summary.maxActive !== undefined) && ( + + )} + {(summary.waiting !== undefined || summary.maxQueue !== undefined) && ( + + )} +
+
+ ); +} + +function CapacityRow({ + label, + value, + maximum, +}: { + label: string; + value?: number; + maximum?: number; +}) { + const displayValue = + value !== undefined && maximum !== undefined + ? `${value} / ${maximum}` + : String(value ?? maximum ?? '—'); + + return ( +
+ {label} + {displayValue} +
+ ); +} + +function BackendCard({ + title, + modelUid, + contextTokens, + contextLabel, + copyLabel, + onCopy, +}: { + title: string; + modelUid: string; + contextTokens: number; + contextLabel: string; + copyLabel: string; + onCopy: (value: string) => Promise; +}) { + return ( +
+
{title}
+ +
+ +
+
+ ); +} + +function CopyableValue({ + value, + label, + copyLabel, + onCopy, + className, +}: { + value: string; + label: string; + copyLabel: string; + onCopy: (value: string) => Promise; + className?: string; +}) { + return ( +
+ + {value} + + +
+ ); +} + +function ConfigurationGroup({ title, children }: { title: string; children: ReactNode }) { + return ( +
+

+ {title} +

+
{children}
+
+ ); +} + +function SectionTitle({ icon, children }: { icon: ReactNode; children: ReactNode }) { + return ( +

+ {icon} + {children} +

+ ); +} + +function DetailItem({ + label, + value, + mono = false, + className, +}: { + label: string; + value: ReactNode; + mono?: boolean; + className?: string; +}) { + return ( +
+
{label}
+
+ {value} +
+
+ ); +} + +function TabMeta({ + count, + updatedAt, + locale, + autoRefreshingLabel, + lastUpdatedLabel, + notUpdatedLabel, +}: { + count?: number; + updatedAt: Date | null; + locale: string; + autoRefreshingLabel: string; + lastUpdatedLabel: string; + notUpdatedLabel: string; +}) { + return ( +
+
+ + + {autoRefreshingLabel} + + {count !== undefined && · {count}} +
+ + {lastUpdatedLabel}: {updatedAt ? formatDateTime(updatedAt, locale) : notUpdatedLabel} + +
+ ); +} + +function LoadingState({ label }: { label: string }) { + return ( +
+ + {label} +
+ ); +} + +function EmptyState({ icon, children }: { icon: ReactNode; children: ReactNode }) { + return ( +
+ {icon} + {children} +
+ ); +} + +function formatRuleMatch(match: TokenRouterRuleMatch): string { + const entries = Object.entries(match).filter(([, value]) => value !== undefined); + return entries.length + ? entries.map(([key, value]) => `${key}=${String(value)}`).join(' AND ') + : '—'; +} + +function formatRoutingAction(action: TokenRouterRoutingAction): string { + return action.type === 'route' + ? `route -> ${action.backend_id || '—'}` + : `reject: ${action.reason || '—'}`; +} + +function summarizeMetrics(metrics: MetricsResponse): MetricsSummary { + const summary: MetricsSummary = { + backendPools: {}, + tokenization: {}, + }; + const instances = Array.isArray(metrics.instances) ? metrics.instances : []; + let onlineSeen = false; + + for (const rawInstance of instances) { + const instance = asRecord(rawInstance); + if (!instance) continue; + + if (typeof instance.online === 'boolean') { + onlineSeen = true; + summary.onlineInstances = (summary.onlineInstances || 0) + (instance.online ? 1 : 0); + } + + const instanceMetrics = asRecord(instance.metrics); + const requests = asRecord(instanceMetrics?.requests); + if (requests) { + for (const [key, value] of Object.entries(requests)) { + if (key.split('/')[0] === 'completed') { + summary.completedRequests = addKnownNumber(summary.completedRequests, value); + } + } + } + + const process = asRecord(instance.process); + summary.activeRequests = addKnownNumber(summary.activeRequests, process?.active_requests); + + const pools = asRecord(process?.backends) || asRecord(process?.pools); + if (pools) { + for (const [backendId, rawPool] of Object.entries(pools)) { + const pool = asRecord(rawPool); + const capacity = (summary.backendPools[backendId] ||= {}); + addCapacity(capacity, pool); + summary.waitingRequests = addKnownNumber(summary.waitingRequests, pool?.waiting); + } + } + addCapacity(summary.tokenization, asRecord(process?.tokenization)); + } + + if (!onlineSeen) delete summary.onlineInstances; + return summary; +} + +function addCapacity(summary: CapacitySummary, source: Record | null) { + if (!source) return; + summary.active = addKnownNumber(summary.active, source.active); + summary.waiting = addKnownNumber(summary.waiting, source.waiting); + summary.maxActive = addKnownNumber(summary.maxActive, source.max_active); + summary.maxQueue = addKnownNumber(summary.maxQueue, source.max_queue); +} + +function addKnownNumber(current: number | undefined, value: unknown): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value)) return current; + return (current || 0) + value; +} + +function hasMetricsSummary(summary: MetricsSummary): boolean { + return ( + summary.completedRequests !== undefined || + summary.activeRequests !== undefined || + summary.waitingRequests !== undefined || + summary.onlineInstances !== undefined || + Object.values(summary.backendPools).some(hasCapacityData) || + hasCapacityData(summary.tokenization) + ); +} + +function hasCapacityData(summary: CapacitySummary): boolean { + return Object.values(summary).some((value) => value !== undefined); +} + +function containsMetricsData(metrics: MetricsResponse): boolean { + const instances = Array.isArray(metrics.instances) ? metrics.instances : []; + return instances.some((rawInstance) => { + const instance = asRecord(rawInstance); + return ( + hasObjectEntries(instance?.metrics) || + hasObjectEntries(instance?.backend_health) || + hasObjectEntries(instance?.process) + ); + }); +} + +function formatCapabilities(asset: TokenizerAssetItem | null, t: TFunc): string { + if (!asset) return '—'; + const capabilities = Object.entries(asset.capabilities || {}) + .filter(([, enabled]) => enabled) + .map(([name]) => t(`tokenRouter.capabilities.${name}`)); + return capabilities.length ? capabilities.join(', ') : '—'; +} + +function hasObjectEntries(value: unknown): boolean { + const record = asRecord(value); + return Boolean(record && Object.keys(record).length > 0); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asRecord(value: unknown): Record | null { + return isRecord(value) ? value : null; +} + +function formatDateTime(value: string | Date, locale: string): string { + const date = value instanceof Date ? value : new Date(value); + if (Number.isNaN(date.getTime())) return String(value); + + return new Intl.DateTimeFormat(locale, { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }).format(date); +} + +async function writeClipboard(value: string): Promise { + if (navigator.clipboard?.writeText) { + await navigator.clipboard.writeText(value); + return; + } + + const textarea = document.createElement('textarea'); + textarea.value = value; + textarea.style.position = 'fixed'; + textarea.style.opacity = '0'; + document.body.appendChild(textarea); + textarea.focus(); + textarea.select(); + const copied = document.execCommand('copy'); + document.body.removeChild(textarea); + if (!copied) throw new Error('Clipboard copy failed'); +} diff --git a/frontend/src/components/pages/token-router/router-form-dialog.tsx b/frontend/src/components/pages/token-router/router-form-dialog.tsx new file mode 100644 index 0000000000..bb6493c7e6 --- /dev/null +++ b/frontend/src/components/pages/token-router/router-form-dialog.tsx @@ -0,0 +1,920 @@ +'use client'; + +import { useEffect, useMemo, useState, type ReactNode } from 'react'; +import { toast } from 'sonner'; + +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { Form } from '@/components/ui/form'; +import { FormField } from '@/components/ui/form-field'; +import { Input } from '@/components/ui/input'; +import { Select } from '@/components/ui/select'; +import { Switch } from '@/components/ui/switch'; +import { useI18n, type TFunc } from '@/contexts/i18n-context'; +import { useForm } from '@/hooks/use-form'; +import request from '@/lib/request'; +import type { + RunningModelItem, + TokenizerAssetItem, + TokenizerAssetListResponse, + TokenRouterBackendCandidate, + TokenRouterBackendCandidateResponse, + TokenRouterDefaultsResponse, + TokenRouterItem, +} from '@/types/services'; +import { isTypedTokenRouter } from '@/types/services'; + +import { AdvancedRoutingEditor } from './advanced-routing-editor'; +import { + routerMode, + typedDraftFromRouter, + type RouterMode, + type TypedRouterDraft, +} from './router-config-normalizer'; + +interface Props { + open: boolean; + router?: TokenRouterItem | null; + onOpenChange: (open: boolean) => void; + onSaved: () => void; +} + +type FormState = { + router_uid: string; + virtual_model_uid: string; + tokenizer_source: 'asset' | 'custom'; + tokenizer_asset_id: string; + tokenizer_path: string; + backend_url: string; + model_aliases: string; + short_model_uid: string; + long_model_uid: string; + short_threshold_tokens: number; + short_max_context: number; + long_max_context: number; + context_reserve_tokens: number; + default_output_tokens: number; + thinking_policy: 'short' | 'long' | 'reject'; + request_timeout_seconds: number; + connect_timeout_seconds: number; + short_max_active: number; + short_max_queue: number; + long_max_active: number; + long_max_queue: number; + tokenization_workers: number; + tokenization_max_active: number; + tokenization_max_queue: number; +}; + +const EMPTY: FormState = { + router_uid: '', + virtual_model_uid: '', + tokenizer_source: 'asset', + tokenizer_asset_id: '', + tokenizer_path: '', + backend_url: '', + model_aliases: '', + short_model_uid: '', + long_model_uid: '', + short_threshold_tokens: 32768, + short_max_context: 131072, + long_max_context: 1048576, + context_reserve_tokens: 64, + default_output_tokens: 512, + thinking_policy: 'long', + request_timeout_seconds: 10800, + connect_timeout_seconds: 10, + short_max_active: 8, + short_max_queue: 32, + long_max_active: 1, + long_max_queue: 2, + tokenization_workers: 2, + tokenization_max_active: 2, + tokenization_max_queue: 8, +}; + +function fromRouter(router: TokenRouterItem): FormState { + const typed = isTypedTokenRouter(router); + const short = typed ? router.backends[0] : router.backends.short; + const long = typed ? router.backends[1] || router.backends[0] : router.backends.long; + return { + router_uid: router.router_uid, + virtual_model_uid: router.virtual_model_uid, + tokenizer_source: router.tokenizer_asset_id ? 'asset' : 'custom', + tokenizer_asset_id: router.tokenizer_asset_id || '', + tokenizer_path: router.tokenizer_path, + backend_url: router.backend_url, + model_aliases: router.model_aliases.join(', '), + short_model_uid: short?.model_uid || '', + long_model_uid: long?.model_uid || '', + short_threshold_tokens: typed + ? router.routing.rules.find((rule) => rule.match.total_tokens_lte !== undefined)?.match + .total_tokens_lte || 32768 + : router.routing.short_threshold_tokens, + short_max_context: short?.max_context_tokens || 131072, + long_max_context: long?.max_context_tokens || 1048576, + context_reserve_tokens: router.routing.context_reserve_tokens, + default_output_tokens: router.routing.default_output_tokens, + thinking_policy: 'long', + request_timeout_seconds: router.request_timeout_seconds, + connect_timeout_seconds: router.connect_timeout_seconds, + short_max_active: short?.admission.max_active || 1, + short_max_queue: short?.admission.max_queue || 0, + long_max_active: long?.admission.max_active || 1, + long_max_queue: long?.admission.max_queue || 0, + tokenization_workers: router.tokenization.max_workers, + tokenization_max_active: router.tokenization.max_active, + tokenization_max_queue: router.tokenization.max_queue, + }; +} + +const normalizeNumber = (value: unknown) => (value === '' ? '' : Number(value)); +const isPositiveInteger = (value: unknown) => Number.isInteger(Number(value)) && Number(value) > 0; +const isNonNegativeInteger = (value: unknown) => + Number.isInteger(Number(value)) && Number(value) >= 0; + +function normalizeBackendUrl(value: string) { + return value.trim().replace(/\/+$/, ''); +} + +function isBackendUrl(value: unknown) { + if (typeof value !== 'string') return false; + try { + const url = new URL(value.trim()); + return ( + (url.protocol === 'http:' || url.protocol === 'https:') && + Boolean(url.hostname) && + !url.username && + !url.password && + (url.pathname === '' || url.pathname === '/') && + !url.search && + !url.hash + ); + } catch { + return false; + } +} + +export function RouterFormDialog({ open, router, onOpenChange, onSaved }: Props) { + const { t } = useI18n(); + const [saving, setSaving] = useState(false); + const [models, setModels] = useState([]); + const [candidates, setCandidates] = useState([]); + const [mode, setMode] = useState(routerMode(router)); + const [typedDraft, setTypedDraft] = useState(() => + typedDraftFromRouter(router) + ); + const [assets, setAssets] = useState([]); + const [allowCustomPath, setAllowCustomPath] = useState(false); + const [assetsLoadFailed, setAssetsLoadFailed] = useState(false); + const [backendDefaults, setBackendDefaults] = useState< + TokenRouterDefaultsResponse['backend'] | null + >(null); + const [backendDefaultsLoadFailed, setBackendDefaultsLoadFailed] = useState(false); + const [useCustomBackend, setUseCustomBackend] = useState(false); + const [tokenizerSource, setTokenizerSource] = useState<'asset' | 'custom'>('asset'); + const [selectedAssetId, setSelectedAssetId] = useState(''); + const [form] = useForm(); + const editing = Boolean(router); + const canUseCustomPath = allowCustomPath || Boolean(router && !router.tokenizer_asset_id); + const assetOptions = useMemo(() => { + if (!router?.tokenizer_asset_id) return assets; + if (assets.some((asset) => asset.asset_id === router.tokenizer_asset_id)) return assets; + return [ + ...assets, + { + asset_id: router.tokenizer_asset_id, + origin: router.tokenizer_asset_origin || 'external', + display_name: router.tokenizer_asset_id, + model_family: '', + model_name: '', + revision: router.tokenizer_asset_revision || '', + encoding_type: '', + compatible_models: [], + capabilities: {}, + enabled: true, + status: 'invalid' as const, + valid: false, + fingerprint: router.tokenizer_asset_fingerprint || '', + errors: [], + }, + ]; + }, [assets, router]); + const selectedAsset = assetOptions.find((asset) => asset.asset_id === selectedAssetId); + + useEffect(() => { + if (!open) return; + let cancelled = false; + + setAssets([]); + setAllowCustomPath(false); + setAssetsLoadFailed(false); + setBackendDefaults(null); + setBackendDefaultsLoadFailed(false); + setUseCustomBackend(false); + + const load = async () => { + const [modelsResult, assetsResult, defaultsResult] = await Promise.allSettled([ + request.get>('/v1/models'), + request.get('/v1/tokenizer_assets'), + request.get('/v1/token_routers/defaults'), + ]); + if (cancelled) return; + + if (modelsResult.status === 'fulfilled') { + const items = Array.isArray(modelsResult.value) + ? modelsResult.value + : Object.values(modelsResult.value || {}); + setModels( + items + .filter((item) => item.model_type === 'LLM') + .map((item) => item.id) + .filter(Boolean) + ); + } else { + setModels([]); + } + + const assetResponse = + assetsResult.status === 'fulfilled' + ? assetsResult.value + : { items: [], allow_custom_path: false }; + const availableAssets = assetResponse.items || []; + setAssets(availableAssets); + setAllowCustomPath(assetResponse.allow_custom_path); + setAssetsLoadFailed( + assetsResult.status === 'rejected' || Boolean(assetResponse.config_error) + ); + + const defaults = defaultsResult.status === 'fulfilled' ? defaultsResult.value.backend : null; + const defaultBackendUrl = + defaults?.available && defaults.backend_url + ? normalizeBackendUrl(defaults.backend_url) + : ''; + setBackendDefaults(defaults); + setBackendDefaultsLoadFailed(defaultsResult.status === 'rejected'); + + const values: FormState = router + ? fromRouter(router) + : { ...EMPTY, backend_url: defaultBackendUrl }; + const customBackend = router + ? !defaultBackendUrl || normalizeBackendUrl(router.backend_url) !== defaultBackendUrl + : !defaultBackendUrl; + setUseCustomBackend(customBackend); + if (!router) { + const defaultAsset = availableAssets.find((asset) => asset.status === 'available'); + if (defaultAsset) { + values.tokenizer_source = 'asset'; + values.tokenizer_asset_id = defaultAsset.asset_id; + } else { + values.tokenizer_source = assetResponse.allow_custom_path ? 'custom' : 'asset'; + } + } + setMode(routerMode(router)); + setTypedDraft(typedDraftFromRouter(router)); + setTokenizerSource(values.tokenizer_source); + setSelectedAssetId(values.tokenizer_asset_id); + form.initialValues.current = values; + form.resetFields(); + form.setFieldsValue(values); + }; + + void load(); + return () => { + cancelled = true; + }; + }, [form, open, router]); + + useEffect(() => { + if (!open) return; + let cancelled = false; + const assetQuery = + tokenizerSource === 'asset' && selectedAssetId + ? `?tokenizer_asset_id=${encodeURIComponent(selectedAssetId)}` + : ''; + + setCandidates([]); + void request + .get(`/v1/token_routers/backend-candidates${assetQuery}`) + .then((response) => { + if (!cancelled) setCandidates(response.items || []); + }) + .catch(() => { + if (!cancelled) setCandidates([]); + }); + + return () => { + cancelled = true; + }; + }, [open, selectedAssetId, tokenizerSource]); + + const requiredRule = { required: true, message: t('tokenRouter.validation.required') } as const; + const positiveIntegerRule = { + validator: isPositiveInteger, + message: t('tokenRouter.validation.positiveInteger'), + } as const; + const nonNegativeIntegerRule = { + validator: isNonNegativeInteger, + message: t('tokenRouter.validation.nonNegativeInteger'), + } as const; + + const handleSave = async (rawValues: Record) => { + const values = rawValues as FormState; + if (router) { + const sourceChanged = values.tokenizer_source !== fromRouter(router).tokenizer_source; + const tokenizerChanged = + sourceChanged || + (values.tokenizer_source === 'asset' + ? values.tokenizer_asset_id.trim() !== (router.tokenizer_asset_id || '') + : values.tokenizer_path.trim() !== router.tokenizer_path); + if (tokenizerChanged && !window.confirm(t('tokenRouter.assetChangeWarning'))) return; + } + + setSaving(true); + + const admission = (max_active: number, max_queue: number) => ({ + max_active, + max_queue, + queue_timeout_seconds: 5, + retry_after_seconds: 1, + }); + if (mode === 'advanced') { + const backendIds = typedDraft.backends.map((backend) => backend.id.trim()); + const ruleIds = typedDraft.rules.map((rule) => rule.id.trim()); + const priorities = typedDraft.rules.map((rule) => rule.priority); + const invalidRule = typedDraft.rules.some((rule) => { + const hasCondition = Object.values(rule.match).some( + (value) => value !== undefined && value !== null + ); + const invalidTokenRange = + rule.match.total_tokens_gte !== undefined && + rule.match.total_tokens_lte !== undefined && + rule.match.total_tokens_gte > rule.match.total_tokens_lte; + const invalidAction = + rule.action.type === 'route' + ? !backendIds.includes(rule.action.backend_id) + : !rule.action.reason.trim(); + return !hasCondition || invalidTokenRange || invalidAction; + }); + if ( + typedDraft.backends.length < 1 || + typedDraft.backends.length > 16 || + typedDraft.rules.length < 1 || + typedDraft.rules.length > 64 || + typedDraft.backends.some( + (backend) => + !/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/.test(backend.id.trim()) || + !backend.model_uid.trim() || + backend.max_context_tokens < 1 || + backend.admission.max_active < 1 || + backend.admission.max_queue < 0 + ) || + new Set(backendIds).size !== backendIds.length || + typedDraft.rules.some( + (rule) => + !/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/.test(rule.id.trim()) || + rule.priority < 1 || + rule.priority > 10000 + ) || + new Set(ruleIds).size !== ruleIds.length || + new Set(priorities).size !== priorities.length || + invalidRule || + (typedDraft.defaultAction.type === 'route' + ? !backendIds.includes(typedDraft.defaultAction.backend_id) + : !typedDraft.defaultAction.reason.trim()) + ) { + toast.error(t('tokenRouter.validation.invalidAdvancedConfig')); + setSaving(false); + return; + } + } + + const commonPayload = { + virtual_model_uid: values.virtual_model_uid.trim(), + model_type: 'LLM' as const, + route_profile: 'llm_chat' as const, + ...(values.tokenizer_source === 'asset' + ? { tokenizer_asset_id: values.tokenizer_asset_id.trim() } + : { tokenizer_path: values.tokenizer_path.trim() }), + backend_url: values.backend_url.trim(), + model_aliases: values.model_aliases + .split(',') + .map((item) => item.trim()) + .filter(Boolean), + request_timeout_seconds: values.request_timeout_seconds, + connect_timeout_seconds: values.connect_timeout_seconds, + tokenization: { + executor: 'process' as const, + multiprocessing_start_method: 'spawn' as const, + max_workers: values.tokenization_workers, + max_active: values.tokenization_max_active, + max_queue: values.tokenization_max_queue, + queue_timeout_seconds: 5, + retry_after_seconds: 1, + }, + }; + const payload = + mode === 'advanced' + ? { + ...commonPayload, + config_version: 2 as const, + strategy: 'typed_rules' as const, + backends: typedDraft.backends.map((backend) => ({ + ...backend, + id: backend.id.trim(), + model_uid: backend.model_uid.trim(), + })), + routing: { + evaluation_mode: 'first_match' as const, + context_reserve_tokens: values.context_reserve_tokens, + default_output_tokens: values.default_output_tokens, + rules: typedDraft.rules.map((rule) => ({ ...rule, id: rule.id.trim() })), + default_action: typedDraft.defaultAction, + }, + } + : { + ...commonPayload, + config_version: 1 as const, + strategy: 'token_budget' as const, + backends: { + short: { + model_uid: values.short_model_uid.trim(), + max_context_tokens: values.short_max_context, + admission: admission(values.short_max_active, values.short_max_queue), + }, + long: { + model_uid: values.long_model_uid.trim(), + max_context_tokens: values.long_max_context, + admission: admission(values.long_max_active, values.long_max_queue), + }, + }, + routing: { + short_threshold_tokens: values.short_threshold_tokens, + context_reserve_tokens: values.context_reserve_tokens, + default_output_tokens: values.default_output_tokens, + thinking_policy: values.thinking_policy, + overflow_policy: 'reject' as const, + }, + }; + + try { + if (router) { + await request.put(`/v1/token_routers/${router.router_uid}`, { + ...payload, + revision: router.revision, + }); + toast.success(t('tokenRouter.updateSuccess')); + } else { + await request.post('/v1/token_routers', { + ...payload, + router_uid: values.router_uid.trim(), + }); + toast.success(t('tokenRouter.createSuccess')); + } + onOpenChange(false); + onSaved(); + } finally { + setSaving(false); + } + }; + + const numberField = ( + name: keyof FormState, + label: string, + allowZero = false, + extraRules: Array<{ validator: (value: unknown) => boolean; message: string }> = [] + ) => ( + + + + ); + + return ( + !saving && onOpenChange(nextOpen)}> + + + {editing ? t('tokenRouter.edit') : t('tokenRouter.create')} + + + + {models.map((uid) => ( + + +
+ + + + + + + + + + allowClear={false} + options={[ + { value: 'asset', label: t('tokenRouter.tokenizerAsset') }, + { + value: 'custom', + label: t('tokenRouter.customTokenizerPath'), + disabled: !canUseCustomPath, + }, + ]} + onChange={(value) => setTokenizerSource(value || 'asset')} + /> + + {tokenizerSource === 'asset' ? ( + + + + )} + {assetsLoadFailed && tokenizerSource === 'asset' && ( +
+ {t('tokenRouter.tokenizerAssetsLoadFailed')} +
+ )} + {assets.length === 0 && tokenizerSource === 'asset' && ( +
+ {t('tokenRouter.noTokenizerAssets')} +
+ )} + {tokenizerSource === 'asset' && selectedAsset && ( + + )} + {tokenizerSource === 'custom' && ( +
+
{t('tokenRouter.legacyCustomTokenizerPath')}
+
{t('tokenRouter.customTokenizerPathRisk')}
+
+ )} +
+
+
+
{t('tokenRouter.currentSupervisor')}
+
+ {t('tokenRouter.currentSupervisorDescription')} +
+
+
+ + {t('tokenRouter.useCustomBackend')} + + { + setUseCustomBackend(checked); + if (!checked && backendDefaults?.backend_url) { + form.setFieldValue( + 'backend_url', + normalizeBackendUrl(backendDefaults.backend_url) + ); + } + }} + /> +
+
+ {backendDefaults?.available && backendDefaults.backend_url && ( +
+
+ {t('tokenRouter.backendAddress')} +
+
+ {backendDefaults.backend_url} +
+
+ )} + {!useCustomBackend && !backendDefaults && !backendDefaultsLoadFailed && ( +
+ {t('tokenRouter.backendDefaultsLoading')} +
+ )} + {backendDefaultsLoadFailed && ( +
+ {t('tokenRouter.backendDefaultsLoadFailed')} +
+ )} + {backendDefaults && !backendDefaults.available && ( +
+ {t('tokenRouter.backendDefaultsUnavailable')} +
+ )} +
+ {t('tokenRouter.customBackendHint')} +
+
+ {useCustomBackend && ( + + + + )} + + + + + + +
+ + +
+ + + {mode === 'simple' ? ( + <> + + + + + + String(value).trim() !== + String(form.getFieldValue('short_model_uid') || '').trim(), + message: t('tokenRouter.validation.backendsMustDiffer'), + }, + ]} + > + + + {numberField('short_max_context', t('tokenRouter.shortContext'))} + {numberField('long_max_context', t('tokenRouter.longContext'), false, [ + { + validator: (value) => + Number(value) >= Number(form.getFieldValue('short_max_context')), + message: t('tokenRouter.validation.shortContextExceedsLongContext'), + }, + ])} + + + + {numberField('short_threshold_tokens', t('tokenRouter.threshold'), false, [ + { + validator: (value) => + Number(value) <= Number(form.getFieldValue('short_max_context')), + message: t('tokenRouter.validation.thresholdExceedsShortContext'), + }, + ])} + {numberField('context_reserve_tokens', t('tokenRouter.reserve'), true)} + {numberField('default_output_tokens', t('tokenRouter.defaultOutput'))} + + + allowClear={false} + options={[ + { value: 'long', label: t('tokenRouter.thinkingPolicies.long') }, + { value: 'short', label: t('tokenRouter.thinkingPolicies.short') }, + { value: 'reject', label: t('tokenRouter.thinkingPolicies.reject') }, + ]} + /> + + + + + {numberField('short_max_active', t('tokenRouter.shortActive'))} + {numberField('short_max_queue', t('tokenRouter.shortQueue'), true)} + {numberField('long_max_active', t('tokenRouter.longActive'))} + {numberField('long_max_queue', t('tokenRouter.longQueue'), true)} + + + ) : ( + + {numberField('context_reserve_tokens', t('tokenRouter.reserve'), true)} + {numberField('default_output_tokens', t('tokenRouter.defaultOutput'))} + + + )} + + + {numberField('tokenization_workers', t('tokenRouter.tokenWorkers'))} + {numberField('tokenization_max_active', t('tokenRouter.tokenActive'), false, [ + { + validator: (value) => + Number(value) >= Number(form.getFieldValue('tokenization_workers')), + message: t('tokenRouter.validation.tokenActiveBelowWorkers'), + }, + ])} + {numberField('tokenization_max_queue', t('tokenRouter.tokenQueue'), true)} + + + + {numberField('request_timeout_seconds', t('tokenRouter.requestTimeout'))} + {numberField('connect_timeout_seconds', t('tokenRouter.connectTimeout'))} + + + + + + +
+ + + ); +} + +function FormSection({ title, children }: { title: string; children: ReactNode }) { + return ( +
+

{title}

+
{children}
+
+ ); +} + +function TokenizerAssetSummary({ asset, t }: { asset: TokenizerAssetItem; t: TFunc }) { + const capabilityNames = Object.entries(asset.capabilities || {}) + .filter(([, enabled]) => enabled) + .map(([name]) => t(`tokenRouter.capabilities.${name}`)); + + return ( +
+ + + + + +
+ ); +} + +function AssetSummaryItem({ + label, + value, + mono = false, +}: { + label: string; + value: string; + mono?: boolean; +}) { + return ( +
+
{label}
+
{value}
+
+ ); +} diff --git a/frontend/src/components/pages/token-router/router-status-badge.tsx b/frontend/src/components/pages/token-router/router-status-badge.tsx new file mode 100644 index 0000000000..8da0bcca57 --- /dev/null +++ b/frontend/src/components/pages/token-router/router-status-badge.tsx @@ -0,0 +1,22 @@ +import { Badge } from '@/components/ui/badge'; +import { useI18n } from '@/contexts/i18n-context'; +import { cn } from '@/lib/utils'; + +const styles: Record = { + ready: 'border-emerald-500/30 bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + syncing: 'border-blue-500/30 bg-blue-500/15 text-blue-700 dark:text-blue-300', + degraded: 'border-amber-500/30 bg-amber-500/15 text-amber-700 dark:text-amber-300', + offline: 'border-slate-500/30 bg-slate-500/15 text-slate-700 dark:text-slate-300', + error: 'border-red-500/30 bg-red-500/15 text-red-700 dark:text-red-300', + disabled: 'border-zinc-500/30 bg-zinc-500/15 text-zinc-700 dark:text-zinc-300', + draft: 'border-purple-500/30 bg-purple-500/15 text-purple-700 dark:text-purple-300', +}; + +const knownStatuses = new Set(Object.keys(styles)); + +export function RouterStatusBadge({ status }: { status: string }) { + const { t } = useI18n(); + const label = knownStatuses.has(status) ? t(`tokenRouter.statuses.${status}`) : status; + + return {label}; +} diff --git a/frontend/src/components/pages/user-management/permissions.tsx b/frontend/src/components/pages/user-management/permissions.tsx index bc6c6c7e01..911368b8ef 100644 --- a/frontend/src/components/pages/user-management/permissions.tsx +++ b/frontend/src/components/pages/user-management/permissions.tsx @@ -15,6 +15,10 @@ export const ALL_PERMISSIONS = [ 'models:read', 'models:write', 'models:register', + 'routers:list', + 'routers:read', + 'routers:write', + 'routers:operate', 'keys:create', 'keys:manage', 'users:manage', @@ -35,6 +39,10 @@ const PERMISSION_GROUPS = [ key: 'models', permissions: ['models:list', 'models:read', 'models:write', 'models:register'], }, + { + key: 'routers', + permissions: ['routers:list', 'routers:read', 'routers:write', 'routers:operate'], + }, { key: 'keys', permissions: ['keys:create', 'keys:manage'], diff --git a/frontend/src/hooks/use-menu-auth.ts b/frontend/src/hooks/use-menu-auth.ts index 1460442385..a54ba6dc14 100644 --- a/frontend/src/hooks/use-menu-auth.ts +++ b/frontend/src/hooks/use-menu-auth.ts @@ -25,5 +25,10 @@ export function useMenuAuth() { hasLogsList: hasScope('admin', 'logs:list'), hasMonitorView: hasScope('admin', 'monitor:view'), canRegisterModel: hasScope('admin', 'models:register', 'models:add', 'models:unregister'), + hasRouterList: hasScope('admin', 'routers:list'), + hasRouterRead: hasScope('admin', 'routers:read'), + canWriteRouters: hasScope('admin', 'routers:write'), + canOperateRouters: hasScope('admin', 'routers:operate'), + canAccessRouterPage: hasScope('admin', 'routers:list', 'routers:read'), }; } diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index 3ada27b327..e67abc4675 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -5,6 +5,7 @@ const en = { systemManagement: 'System Management', resourcesAndSupport: 'Resources & Support', launchModel: 'Launch Model', + tokenRouter: 'Token Router', runningModels: 'Running Models', registerModel: 'Register Model', clusterInfo: 'Cluster Information', @@ -19,6 +20,258 @@ const en = { documentation: 'Documentation', website: 'Official Website', }, + tokenRouter: { + title: 'Token Router', + description: 'Manage token-budget routing between short- and long-context model backends.', + create: 'Create Router', + edit: 'Edit Router', + routerUid: 'Router UID', + virtualModelUid: 'Virtual Model UID', + tokenizerPath: 'Tokenizer Path', + tokenizerSource: 'Tokenizer Source', + tokenizerAsset: 'Tokenizer Asset', + customTokenizerPath: 'Custom Tokenizer Path', + customTokenizerPathHint: + 'Use a server-local path only when custom paths are enabled or when editing a legacy configuration.', + noTokenizerAssets: + 'No available Tokenizer assets are registered. Contact an administrator or enable a permitted custom path.', + tokenizerAssetsLoadFailed: + 'Tokenizer assets could not be loaded. Asset selection remains fail-closed; refresh the page or contact an administrator.', + customTokenizerPathRisk: + 'A custom server path bypasses registered asset metadata and checksum controls. Use it only for a retained legacy configuration or an explicitly approved exception.', + legacyCustomTokenizerPath: 'Legacy Custom Tokenizer Path', + assetChangeWarning: + 'Changing the Tokenizer source, asset, or path will reload the Router configuration and restart its Tokenization Workers. Continue?', + tokenizerAssetDetails: 'Tokenizer Asset Details', + tokenizerAssetDetailsLoadFailed: + 'Asset details could not be loaded. The Router configuration values are shown as a fallback.', + tokenizerAssetStatus: 'Asset Status', + tokenizerAssetOrigin: 'Tokenizer Asset Origin', + assetOrigins: { + builtin: 'Built-in', + external: 'External', + }, + compatibleModels: 'Compatible Models', + tokenizerCapabilities: 'Capabilities', + tokenizerAssetValidatedAt: 'Last Validation', + tokenizerAssetErrors: 'Asset Validation Errors', + runtimeTokenizerAsset: 'Runtime Loaded Tokenizer Asset', + tokenizerAssetRevision: 'Tokenizer Asset Revision', + tokenizerAssetFingerprint: 'Tokenizer Asset Fingerprint', + assetStatuses: { + available: 'Available', + invalid: 'Invalid', + disabled: 'Disabled', + missing: 'Missing', + validating: 'Validating', + }, + capabilities: { + chat: 'Chat', + tools: 'Tools', + thinking: 'Thinking', + }, + backendUrl: 'Backend URL', + currentSupervisor: 'Current Cluster Supervisor (Auto-detected)', + currentSupervisorDescription: + 'Token Router calls the actual backend models through the current Supervisor REST API.', + useCustomBackend: 'Use a custom backend URL', + backendAddress: 'Internal Service URL', + backendDefaultsLoading: 'Detecting the current Supervisor...', + backendDefaultsLoadFailed: + 'Unable to load the current Supervisor URL. Use a custom backend URL.', + backendDefaultsUnavailable: + 'The current Supervisor has no internal URL configured for Token Router. Use a custom URL.', + customBackendHint: + 'Usually no change is needed. The custom URL must be a Supervisor REST API reachable by Token Router, not a Worker, model instance, or Token Router URL.', + aliases: 'Model Aliases', + thinkingPolicy: 'Thinking Request Policy', + shortBackend: 'Short Backend Model UID', + longBackend: 'Long Backend Model UID', + shortBackendDisplay: 'Short', + longBackendDisplay: 'Long', + threshold: 'Short Threshold Tokens', + reserve: 'Context Reserve Tokens', + shortContext: 'Short Max Context Tokens', + longContext: 'Long Max Context Tokens', + defaultOutput: 'Default Output Tokens', + requestTimeout: 'Request Timeout (seconds)', + connectTimeout: 'Connect Timeout (seconds)', + shortActive: 'Short Max Active', + shortQueue: 'Short Max Queue', + longActive: 'Long Max Active', + longQueue: 'Long Max Queue', + tokenWorkers: 'Tokenization Workers', + tokenActive: 'Tokenization Max Active', + tokenQueue: 'Tokenization Max Queue', + details: 'Router Details', + configuration: 'Configuration Summary', + revision: 'Revision', + runtimeInstances: 'Runtime Instances', + noInstances: 'No runtime instances are registered.', + metrics: 'Metrics Summary', + noMetrics: 'No metrics are available.', + endpoint: 'Endpoint', + configError: 'Configuration Error', + backends: 'Backends', + status: 'Status', + empty: 'No token routers configured.', + loading: 'Loading...', + onlineSummary: '{{online}}/{{total}} online', + ackRevision: 'ACK revision: {{revision}}', + heartbeatAge: 'Since last heartbeat: {{seconds}}s', + closeDetails: 'Close router details', + tabs: { + overview: 'Overview', + instances: 'Instances', + metrics: 'Metrics', + }, + routingSummary: 'Routing Summary', + advancedConfiguration: 'Advanced Configuration', + advancedConfigurationDescription: + 'View routing, admission, tokenization, and timeout settings.', + advancedDiagnostics: 'Advanced Diagnostics', + rawMetrics: 'Raw Metrics JSON', + copyValue: 'Copy', + copyMetrics: 'Copy JSON', + copyFailed: 'Failed to copy.', + lastUpdated: 'Last updated', + autoRefreshing: 'Auto-refresh every 10 seconds', + notUpdated: 'Not updated yet', + runtimeInstanceCount: 'Runtime Instances', + onlineInstanceCount: 'Online Instances', + instanceId: 'Instance ID', + ackedRevision: 'Acknowledged Revision', + heartbeatAgeLabel: 'Since Last Heartbeat', + secondsValue: '{{seconds}}s', + modelType: 'Model Type', + strategy: 'Strategy', + overflowPolicy: 'Overflow Policy', + createdAt: 'Created At', + updatedAt: 'Updated At', + shortAdmission: 'Short Admission', + longAdmission: 'Long Admission', + queueTimeout: 'Queue Timeout', + retryAfter: 'Retry After', + tokenizationExecutor: 'Tokenization Executor', + multiprocessingStartMethod: 'Multiprocessing Start Method', + completedRequests: 'Completed Requests', + activeRequests: 'Active Requests', + waitingRequests: 'Waiting Requests', + shortPool: 'Short Pool', + longPool: 'Long Pool', + tokenizationPool: 'Tokenization', + activeCapacity: 'Active / Capacity', + waitingCapacity: 'Waiting / Queue Capacity', + noKnownMetrics: 'No recognized summary metrics are available. View the raw JSON for details.', + validationPassed: 'Router validation passed.', + validationFailed: 'Router validation failed.', + validate: 'Validate', + disable: 'Disable', + enable: 'Enable', + confirmDelete: 'Delete router {{routerUid}}? This action cannot be undone.', + deleteTitle: 'Delete Token Router', + deleteDisabledHint: 'Disable this Router before deleting it.', + createSuccess: 'Token Router created successfully.', + updateSuccess: 'Token Router updated successfully.', + deleteSuccess: 'Token Router deleted successfully.', + enableSuccess: 'Token Router enabled successfully.', + disableSuccess: 'Token Router disabled successfully.', + configurationMode: 'Configuration Mode', + simpleMode: 'Simple Mode', + simpleModeDescription: + 'Use short and long request backends with a token threshold while preserving existing configurations.', + advancedMode: 'Advanced Rules', + advancedModeDescription: + 'Use dynamic backends and ordered routing rules for multiple backends and complex request routing.', + routeProfile: 'Route Profile', + typedRulesSummary: '{{count}} rules', + tokenBudgetSummary: 'Threshold {{threshold}}', + dynamicBackends: 'Dynamic Backends', + dynamicBackendsHint: + 'Only physical models compatible with LLM Chat, the engine, and the tokenizer are selectable.', + addBackend: 'Add Backend', + backendId: 'Backend ID', + backendModel: 'Backend Model', + maxContext: 'Maximum Context Tokens', + maxActive: 'Maximum Active', + maxQueue: 'Maximum Queue', + selectBackendCandidate: 'Select a compatible backend candidate.', + orderedRules: 'Ordered Routing Rules', + firstMatchHint: 'Rules run from highest to lowest priority and stop at the first match.', + addRule: 'Add Rule', + ruleId: 'Rule ID', + priority: 'Priority', + tokenGte: 'Total Tokens (>=)', + tokenLte: 'Total Tokens (<=)', + conditions: { + thinking: 'Thinking', + tools_present: 'Tools Present', + stream: 'Streaming', + }, + conditionValues: { + any: 'Any', + true: 'Yes', + false: 'No', + }, + actionType: 'Action Type', + actionTypes: { + route: 'Route', + reject: 'Reject', + }, + targetBackend: 'Target Backend', + rejectReason: 'Reject Reason', + defaultAction: 'Default Action', + evaluationMode: 'Evaluation Mode', + backendAdmission: 'Backend Admission', + profile: { + modelType: 'Model Type', + task: 'Task', + protocol: 'API Protocol', + endpoint: 'Endpoint', + strategy: 'Routing Strategy', + }, + statuses: { + ready: 'Ready', + syncing: 'Syncing', + degraded: 'Degraded', + offline: 'Offline', + error: 'Error', + disabled: 'Disabled', + draft: 'Draft', + }, + thinkingPolicies: { + long: 'Long request', + short: 'Short request', + reject: 'Reject', + }, + strategies: { + typedRules: 'Typed Rules', + tokenBudget: 'Token Budget', + }, + overflowPolicies: { + reject: 'Reject', + }, + sections: { + basic: 'Basic Configuration', + backends: 'Backend Models', + routing: 'Routing Policy', + admission: 'Admission Control', + tokenization: 'Tokenization', + timeouts: 'Timeouts', + }, + validation: { + required: 'This field is required.', + positiveInteger: 'Enter a positive integer.', + nonNegativeInteger: 'Enter a non-negative integer.', + backendsMustDiffer: 'Short and long backend models must be different.', + invalidUrl: 'Enter an HTTP or HTTPS URL.', + thresholdExceedsShortContext: 'The short threshold cannot exceed the short context limit.', + shortContextExceedsLongContext: + 'The short context limit cannot exceed the long context limit.', + tokenActiveBelowWorkers: 'Tokenization max active must be at least the worker count.', + invalidAdvancedConfig: 'The advanced backend and routing rule configuration is invalid.', + }, + }, model: { languageModels: 'Language Models', embeddingModels: 'Embedding Models', @@ -515,6 +768,7 @@ const en = { virtualenv: 'virtualenv', logs: 'logs', monitor: 'monitor', + routers: 'routers', }, permissionLabels: { admin: 'Administrator', @@ -531,6 +785,10 @@ const en = { 'virtualenv:delete': 'Delete virtual environments', 'logs:list': 'View logs', 'monitor:view': 'View monitoring', + 'routers:list': 'View token router list', + 'routers:read': 'Read token router details', + 'routers:write': 'Create, update, and delete token routers', + 'routers:operate': 'Validate, enable, and disable token routers', }, }, apiKey: { @@ -777,6 +1035,8 @@ const en = { cancel: 'Cancel', confirm: 'Confirm', update: 'Update', + refresh: 'Refresh', + save: 'Save', deploy: 'Deploy', stop: 'Stop', packUp: 'Collapse', diff --git a/frontend/src/i18n/locales/ja.ts b/frontend/src/i18n/locales/ja.ts index b7a64df626..a10c14bc3f 100644 --- a/frontend/src/i18n/locales/ja.ts +++ b/frontend/src/i18n/locales/ja.ts @@ -5,6 +5,7 @@ const ja = { systemManagement: 'システム管理', resourcesAndSupport: 'リソースとサポート', launchModel: 'モデル起動', + tokenRouter: 'Token ルーター', runningModels: '実行中のモデル', registerModel: 'モデル登録', clusterInfo: 'クラスター情報', @@ -19,6 +20,247 @@ const ja = { documentation: 'ドキュメント', website: '公式サイト', }, + tokenRouter: { + title: 'Token Router', + description: + '短いコンテキストと長いコンテキストのモデル間のトークン予算ルーティングを管理します。', + create: 'Router を作成', + edit: 'Router を編集', + routerUid: 'Router UID', + virtualModelUid: '仮想モデル UID', + tokenizerPath: 'Tokenizer パス', + tokenizerSource: 'Tokenizer ソース', + tokenizerAsset: 'Tokenizer アセット', + customTokenizerPath: 'カスタム Tokenizer パス', + customTokenizerPathHint: + 'カスタムパスが許可されている場合、または既存の旧設定を編集する場合のみ、サーバーのローカルパスを使用します。', + noTokenizerAssets: + '利用可能な Tokenizer アセットが登録されていません。管理者に連絡するか、許可されたカスタムパスを有効にしてください。', + tokenizerAssetsLoadFailed: + 'Tokenizer アセットを読み込めませんでした。アセット選択はフェイルクローズのままです。ページを更新するか、管理者に連絡してください。', + customTokenizerPathRisk: + 'カスタムのサーバーパスは、登録済みアセットのメタデータとチェックサム管理を迂回します。既存設定の維持または明示的に承認された例外にのみ使用してください。', + legacyCustomTokenizerPath: '旧カスタム Tokenizer パス', + assetChangeWarning: + 'Tokenizer のソース、アセット、またはパスを変更すると Router 設定が再読み込みされ、Tokenization Workers が再起動されます。続行しますか？', + tokenizerAssetDetails: 'Tokenizer アセット詳細', + tokenizerAssetDetailsLoadFailed: + 'アセット詳細を読み込めませんでした。Router 設定値を代替表示しています。', + tokenizerAssetStatus: 'アセット状態', + tokenizerAssetOrigin: 'Tokenizer アセットの由来', + assetOrigins: { + builtin: '組み込み', + external: '外部', + }, + compatibleModels: '互換モデル', + tokenizerCapabilities: '機能', + tokenizerAssetValidatedAt: '最終検証日時', + tokenizerAssetErrors: 'アセット検証エラー', + runtimeTokenizerAsset: 'Runtime が読み込んだ Tokenizer アセット', + tokenizerAssetRevision: 'Tokenizer アセットのリビジョン', + tokenizerAssetFingerprint: 'Tokenizer アセットのフィンガープリント', + assetStatuses: { + available: '利用可能', + invalid: '無効', + disabled: '無効化済み', + missing: '見つかりません', + validating: '検証中', + }, + capabilities: { + chat: 'チャット', + tools: 'ツール', + thinking: '思考モード', + }, + backendUrl: 'バックエンド URL', + currentSupervisor: '現在のクラスタ Supervisor（自動検出）', + currentSupervisorDescription: + 'Token Router は現在の Supervisor REST API を通じて実際のバックエンドモデルを呼び出します。', + useCustomBackend: 'カスタムバックエンド URL を使用', + backendAddress: '内部サービス URL', + backendDefaultsLoading: '現在の Supervisor を検出しています...', + backendDefaultsLoadFailed: + '現在の Supervisor URL を取得できません。カスタムバックエンド URL を使用してください。', + backendDefaultsUnavailable: + '現在の Supervisor には Token Router からアクセス可能な内部 URL が設定されていません。カスタム URL を使用してください。', + customBackendHint: + '通常は変更不要です。カスタム URL には Token Router からアクセス可能な Supervisor REST API を指定し、Worker、モデルインスタンス、Token Router の URL は指定しないでください。', + aliases: 'モデルエイリアス', + thinkingPolicy: '推論リクエストポリシー', + shortBackend: '短文バックエンドモデル UID', + longBackend: '長文バックエンドモデル UID', + shortBackendDisplay: '短文', + longBackendDisplay: '長文', + threshold: '短文しきい値トークン数', + reserve: 'コンテキスト予約トークン数', + shortContext: '短文最大コンテキスト', + longContext: '長文最大コンテキスト', + defaultOutput: 'デフォルト出力トークン数', + requestTimeout: 'リクエストタイムアウト（秒）', + connectTimeout: '接続タイムアウト（秒）', + shortActive: '短文最大アクティブ数', + shortQueue: '短文最大キュー数', + longActive: '長文最大アクティブ数', + longQueue: '長文最大キュー数', + tokenWorkers: 'Tokenization ワーカー数', + tokenActive: 'Tokenization 最大アクティブ数', + tokenQueue: 'Tokenization 最大キュー数', + details: 'Router 詳細', + configuration: '設定概要', + revision: 'リビジョン', + runtimeInstances: 'ランタイムインスタンス', + noInstances: '登録済みランタイムインスタンスはありません。', + metrics: 'メトリクス概要', + noMetrics: '利用可能なメトリクスはありません。', + endpoint: 'エンドポイント', + configError: '設定エラー', + backends: 'バックエンド', + status: 'ステータス', + empty: 'Token Router は設定されていません。', + loading: '読み込み中...', + onlineSummary: '{{online}}/{{total}} オンライン', + ackRevision: 'ACK リビジョン：{{revision}}', + heartbeatAge: '最終ハートビートから：{{seconds}} 秒', + closeDetails: 'Router 詳細を閉じる', + tabs: { + overview: '概要', + instances: 'インスタンス', + metrics: 'メトリクス', + }, + routingSummary: 'ルーティング概要', + advancedConfiguration: '詳細設定', + advancedConfigurationDescription: + 'ルーティング、Admission、Tokenization、タイムアウト設定を表示します。', + advancedDiagnostics: '高度な診断', + rawMetrics: 'Raw Metrics JSON', + copyValue: 'コピー', + copyMetrics: 'JSON をコピー', + copyFailed: 'コピーに失敗しました。', + lastUpdated: '最終更新', + autoRefreshing: '10 秒ごとに自動更新', + notUpdated: '未更新', + runtimeInstanceCount: '実行中インスタンス数', + onlineInstanceCount: 'オンラインインスタンス数', + instanceId: 'インスタンス ID', + ackedRevision: '確認済みリビジョン', + heartbeatAgeLabel: '最終ハートビートからの経過時間', + secondsValue: '{{seconds}} 秒', + modelType: 'モデルタイプ', + strategy: 'ストラテジー', + overflowPolicy: 'オーバーフローポリシー', + createdAt: '作成日時', + updatedAt: '更新日時', + shortAdmission: '短文 Admission', + longAdmission: '長文 Admission', + queueTimeout: 'キュータイムアウト', + retryAfter: '再試行間隔', + tokenizationExecutor: 'Tokenization Executor', + multiprocessingStartMethod: 'マルチプロセス開始方式', + completedRequests: '完了リクエスト', + activeRequests: 'アクティブリクエスト', + waitingRequests: '待機中リクエスト', + shortPool: '短文プール', + longPool: '長文プール', + tokenizationPool: 'Tokenization', + activeCapacity: 'アクティブ数 / 容量', + waitingCapacity: '待機数 / キュー容量', + noKnownMetrics: '認識可能な概要メトリクスがありません。Raw JSON で詳細を確認してください。', + validationPassed: 'Router の検証に成功しました。', + validationFailed: 'Router の検証に失敗しました。', + validate: '検証', + disable: '無効化', + enable: '有効化', + confirmDelete: 'Router {{routerUid}} を削除しますか？この操作は元に戻せません。', + deleteTitle: 'Token Router を削除', + deleteDisabledHint: '削除する前に Router を無効化してください。', + createSuccess: 'Token Router を作成しました。', + updateSuccess: 'Token Router を更新しました。', + deleteSuccess: 'Token Router を削除しました。', + enableSuccess: 'Token Router を有効化しました。', + disableSuccess: 'Token Router を無効化しました。', + configurationMode: '設定モード', + simpleMode: 'シンプルモード', + simpleModeDescription: + '短いリクエスト用と長いリクエスト用のバックエンド、およびトークンしきい値を使用し、既存設定との互換性を維持します。', + advancedMode: '高度なルールモード', + advancedModeDescription: + '動的バックエンドと順序付きルーティングルールを使用し、複数バックエンドや複雑なリクエスト振り分けに対応します。', + routeProfile: 'Route Profile', + typedRulesSummary: '{{count}} ルール', + tokenBudgetSummary: 'しきい値 {{threshold}}', + dynamicBackends: '動的バックエンド', + dynamicBackendsHint: 'LLM Chat、Engine、Tokenizer の互換性確認済み物理モデルのみ選択できます。', + addBackend: 'バックエンドを追加', + backendId: 'Backend ID', + backendModel: 'バックエンドモデル', + maxContext: '最大コンテキストトークン', + maxActive: '最大アクティブ数', + maxQueue: '最大キュー数', + selectBackendCandidate: '互換性のあるバックエンド候補を選択してください。', + orderedRules: '順序付きルーティングルール', + firstMatchHint: '優先度の高い順に評価し、最初に一致したルールで停止します。', + addRule: 'ルールを追加', + ruleId: 'Rule ID', + priority: '優先度', + tokenGte: '総トークン下限（>=）', + tokenLte: '総トークン上限（<=）', + conditions: { thinking: 'Thinking', tools_present: 'Tools あり', stream: 'Streaming' }, + conditionValues: { any: '指定なし', true: 'はい', false: 'いいえ' }, + actionType: 'アクションタイプ', + actionTypes: { route: 'ルーティング', reject: '拒否' }, + targetBackend: '対象 Backend', + rejectReason: '拒否理由', + defaultAction: 'デフォルトアクション', + evaluationMode: '評価モード', + backendAdmission: 'Backend Admission', + profile: { + modelType: 'モデルタイプ', + task: 'タスク', + protocol: 'API プロトコル', + endpoint: 'Endpoint', + strategy: 'ルーティング戦略', + }, + statuses: { + ready: '準備完了', + syncing: '同期中', + degraded: '縮退', + offline: 'オフライン', + error: 'エラー', + disabled: '無効', + draft: '下書き', + }, + thinkingPolicies: { + long: '長文リクエスト', + short: '短文リクエスト', + reject: '拒否', + }, + strategies: { + typedRules: 'Typed Rules', + tokenBudget: 'Token Budget', + }, + overflowPolicies: { + reject: '拒否', + }, + sections: { + basic: '基本設定', + backends: 'バックエンドモデル', + routing: 'ルーティングポリシー', + admission: 'Admission 制御', + tokenization: 'Tokenization', + timeouts: 'タイムアウト', + }, + validation: { + required: 'この項目は必須です。', + positiveInteger: '正の整数を入力してください。', + nonNegativeInteger: '0 以上の整数を入力してください。', + backendsMustDiffer: '短文と長文のバックエンドモデルは異なる必要があります。', + invalidUrl: 'HTTP または HTTPS の URL を入力してください。', + thresholdExceedsShortContext: '短文しきい値は短文コンテキスト上限を超えられません。', + shortContextExceedsLongContext: + '短文コンテキスト上限は長文コンテキスト上限を超えられません。', + tokenActiveBelowWorkers: 'Tokenization 最大アクティブ数はワーカー数以上にしてください。', + invalidAdvancedConfig: '高度なバックエンドおよびルーティングルールの設定が無効です。', + }, + }, model: { languageModels: '言語モデル', embeddingModels: '埋め込みモデル', @@ -515,6 +757,7 @@ const ja = { virtualenv: 'virtualenv', logs: 'logs', monitor: 'monitor', + routers: 'ルーター', }, permissionLabels: { admin: '管理者', @@ -531,6 +774,10 @@ const ja = { 'virtualenv:delete': '仮想環境を削除', 'logs:list': 'ログを表示', 'monitor:view': '監視を表示', + 'routers:list': 'Token Router 一覧を表示', + 'routers:read': 'Token Router の詳細を読み取り', + 'routers:write': 'Token Router を作成、更新、削除', + 'routers:operate': 'Token Router を検証、有効化、無効化', }, }, apiKey: { @@ -777,6 +1024,8 @@ const ja = { cancel: 'キャンセル', confirm: '確認', update: '更新', + refresh: '更新', + save: '保存', deploy: '配備', stop: '停止', packUp: '折りたたむ', diff --git a/frontend/src/i18n/locales/ko.ts b/frontend/src/i18n/locales/ko.ts index 558e34c7d9..f106b82ca3 100644 --- a/frontend/src/i18n/locales/ko.ts +++ b/frontend/src/i18n/locales/ko.ts @@ -5,6 +5,7 @@ const ko = { systemManagement: '시스템 관리', resourcesAndSupport: '리소스 및 지원', launchModel: '모델 실행', + tokenRouter: 'Token 라우터', runningModels: '실행 중인 모델', registerModel: '모델 등록', clusterInfo: '클러스터 정보', @@ -19,6 +20,246 @@ const ko = { documentation: '문서', website: '공식 웹사이트', }, + tokenRouter: { + title: 'Token Router', + description: '짧은 컨텍스트와 긴 컨텍스트 모델 백엔드 간의 토큰 예산 라우팅을 관리합니다.', + create: 'Router 생성', + edit: 'Router 편집', + routerUid: 'Router UID', + virtualModelUid: '가상 모델 UID', + tokenizerPath: 'Tokenizer 경로', + tokenizerSource: 'Tokenizer 소스', + tokenizerAsset: 'Tokenizer 자산', + customTokenizerPath: '사용자 지정 Tokenizer 경로', + customTokenizerPathHint: + '사용자 지정 경로가 허용되거나 기존 레거시 구성을 편집할 때만 서버 로컬 경로를 사용합니다.', + noTokenizerAssets: + '사용 가능한 Tokenizer 자산이 등록되지 않았습니다. 관리자에게 문의하거나 허용된 사용자 지정 경로를 활성화하세요.', + tokenizerAssetsLoadFailed: + 'Tokenizer 자산을 불러오지 못했습니다. 자산 선택은 실패 시 닫힘 상태로 유지됩니다. 페이지를 새로 고치거나 관리자에게 문의하세요.', + customTokenizerPathRisk: + '사용자 지정 서버 경로는 등록된 자산 메타데이터와 체크섬 통제를 우회합니다. 기존 구성을 유지하거나 명시적으로 승인된 예외에만 사용하세요.', + legacyCustomTokenizerPath: '레거시 사용자 지정 Tokenizer 경로', + assetChangeWarning: + 'Tokenizer 소스, 자산 또는 경로를 변경하면 Router 구성이 다시 로드되고 Tokenization Workers가 재시작됩니다. 계속하시겠습니까?', + tokenizerAssetDetails: 'Tokenizer 자산 세부 정보', + tokenizerAssetDetailsLoadFailed: + '자산 세부 정보를 불러오지 못했습니다. Router 구성 값을 대신 표시합니다.', + tokenizerAssetStatus: '자산 상태', + tokenizerAssetOrigin: 'Tokenizer 자산 출처', + assetOrigins: { + builtin: '기본 제공', + external: '외부', + }, + compatibleModels: '호환 모델', + tokenizerCapabilities: '기능', + tokenizerAssetValidatedAt: '최근 검증 시간', + tokenizerAssetErrors: '자산 검증 오류', + runtimeTokenizerAsset: 'Runtime이 실제로 로드한 Tokenizer 자산', + tokenizerAssetRevision: 'Tokenizer 자산 리비전', + tokenizerAssetFingerprint: 'Tokenizer 자산 지문', + assetStatuses: { + available: '사용 가능', + invalid: '유효하지 않음', + disabled: '비활성화됨', + missing: '누락됨', + validating: '검증 중', + }, + capabilities: { + chat: '채팅', + tools: '도구 호출', + thinking: '사고 모드', + }, + backendUrl: '백엔드 URL', + currentSupervisor: '현재 클러스터 Supervisor(자동 감지)', + currentSupervisorDescription: + 'Token Router는 현재 Supervisor REST API를 통해 실제 백엔드 모델을 호출합니다.', + useCustomBackend: '사용자 지정 백엔드 URL 사용', + backendAddress: '내부 서비스 URL', + backendDefaultsLoading: '현재 Supervisor를 감지하는 중...', + backendDefaultsLoadFailed: + '현재 Supervisor URL을 불러올 수 없습니다. 사용자 지정 백엔드 URL을 사용하세요.', + backendDefaultsUnavailable: + '현재 Supervisor에 Token Router가 접근할 수 있는 내부 URL이 구성되어 있지 않습니다. 사용자 지정 URL을 사용하세요.', + customBackendHint: + '일반적으로 변경할 필요가 없습니다. 사용자 지정 URL은 Token Router에서 접근 가능한 Supervisor REST API여야 하며 Worker, 모델 인스턴스 또는 Token Router URL이면 안 됩니다.', + aliases: '모델 별칭', + thinkingPolicy: '추론 요청 정책', + shortBackend: '짧은 요청 백엔드 모델 UID', + longBackend: '긴 요청 백엔드 모델 UID', + shortBackendDisplay: '짧은 요청', + longBackendDisplay: '긴 요청', + threshold: '짧은 요청 임계 토큰 수', + reserve: '컨텍스트 예약 토큰 수', + shortContext: '짧은 요청 최대 컨텍스트', + longContext: '긴 요청 최대 컨텍스트', + defaultOutput: '기본 출력 토큰 수', + requestTimeout: '요청 제한 시간(초)', + connectTimeout: '연결 제한 시간(초)', + shortActive: '짧은 요청 최대 활성 수', + shortQueue: '짧은 요청 최대 대기 수', + longActive: '긴 요청 최대 활성 수', + longQueue: '긴 요청 최대 대기 수', + tokenWorkers: 'Tokenization 워커 수', + tokenActive: 'Tokenization 최대 활성 수', + tokenQueue: 'Tokenization 최대 대기 수', + details: 'Router 세부 정보', + configuration: '구성 요약', + revision: '리비전', + runtimeInstances: '런타임 인스턴스', + noInstances: '등록된 런타임 인스턴스가 없습니다.', + metrics: '메트릭 요약', + noMetrics: '사용 가능한 메트릭이 없습니다.', + endpoint: '엔드포인트', + configError: '구성 오류', + backends: '백엔드', + status: '상태', + empty: 'Token Router가 구성되지 않았습니다.', + loading: '로딩 중...', + onlineSummary: '{{online}}/{{total}} 온라인', + ackRevision: 'ACK 리비전: {{revision}}', + heartbeatAge: '마지막 하트비트 이후: {{seconds}}초', + closeDetails: 'Router 세부 정보 닫기', + tabs: { + overview: '개요', + instances: '인스턴스', + metrics: '메트릭', + }, + routingSummary: '라우팅 요약', + advancedConfiguration: '고급 구성', + advancedConfigurationDescription: + '라우팅, Admission, Tokenization 및 제한 시간 구성을 확인합니다.', + advancedDiagnostics: '고급 진단', + rawMetrics: '원시 Metrics JSON', + copyValue: '복사', + copyMetrics: 'JSON 복사', + copyFailed: '복사하지 못했습니다.', + lastUpdated: '마지막 업데이트', + autoRefreshing: '10초마다 자동 새로 고침', + notUpdated: '아직 업데이트되지 않음', + runtimeInstanceCount: '런타임 인스턴스 수', + onlineInstanceCount: '온라인 인스턴스 수', + instanceId: '인스턴스 ID', + ackedRevision: '확인된 리비전', + heartbeatAgeLabel: '마지막 하트비트 이후', + secondsValue: '{{seconds}}초', + modelType: '모델 유형', + strategy: '전략', + overflowPolicy: '오버플로 정책', + createdAt: '생성 시간', + updatedAt: '업데이트 시간', + shortAdmission: '짧은 요청 Admission', + longAdmission: '긴 요청 Admission', + queueTimeout: '대기열 제한 시간', + retryAfter: '재시도 간격', + tokenizationExecutor: 'Tokenization 실행기', + multiprocessingStartMethod: '멀티프로세싱 시작 방식', + completedRequests: '완료된 요청', + activeRequests: '활성 요청', + waitingRequests: '대기 요청', + shortPool: '짧은 요청 풀', + longPool: '긴 요청 풀', + tokenizationPool: 'Tokenization', + activeCapacity: '활성 수 / 용량', + waitingCapacity: '대기 수 / 대기열 용량', + noKnownMetrics: '인식 가능한 요약 메트릭이 없습니다. 원시 JSON에서 세부 정보를 확인하세요.', + validationPassed: 'Router 검증을 통과했습니다.', + validationFailed: 'Router 검증에 실패했습니다.', + validate: '검증', + disable: '비활성화', + enable: '활성화', + confirmDelete: 'Router {{routerUid}}를 삭제하시겠습니까? 이 작업은 되돌릴 수 없습니다.', + deleteTitle: 'Token Router 삭제', + deleteDisabledHint: '삭제하기 전에 Router를 비활성화하세요.', + createSuccess: 'Token Router를 생성했습니다.', + updateSuccess: 'Token Router를 업데이트했습니다.', + deleteSuccess: 'Token Router를 삭제했습니다.', + enableSuccess: 'Token Router를 활성화했습니다.', + disableSuccess: 'Token Router를 비활성화했습니다.', + configurationMode: '구성 모드', + simpleMode: '간단 모드', + simpleModeDescription: + '짧은 요청 및 긴 요청 백엔드와 토큰 임계값을 사용하며 기존 구성과의 호환성을 유지합니다.', + advancedMode: '고급 규칙 모드', + advancedModeDescription: + '동적 백엔드와 순서가 있는 라우팅 규칙을 사용하여 여러 백엔드와 복잡한 요청 분기를 지원합니다.', + routeProfile: 'Route Profile', + typedRulesSummary: '{{count}}개 규칙', + tokenBudgetSummary: '임계값 {{threshold}}', + dynamicBackends: '동적 백엔드', + dynamicBackendsHint: + 'LLM Chat, Engine 및 Tokenizer 호환성이 확인된 물리 모델만 선택할 수 있습니다.', + addBackend: '백엔드 추가', + backendId: 'Backend ID', + backendModel: '백엔드 모델', + maxContext: '최대 컨텍스트 토큰', + maxActive: '최대 활성 수', + maxQueue: '최대 대기열 수', + selectBackendCandidate: '호환 가능한 백엔드 후보를 선택하세요.', + orderedRules: '순서가 있는 라우팅 규칙', + firstMatchHint: '우선순위가 높은 순서로 평가하고 첫 번째 일치 규칙에서 중지합니다.', + addRule: '규칙 추가', + ruleId: 'Rule ID', + priority: '우선순위', + tokenGte: '총 토큰 하한(>=)', + tokenLte: '총 토큰 상한(<=)', + conditions: { thinking: 'Thinking', tools_present: 'Tools 포함', stream: 'Streaming' }, + conditionValues: { any: '제한 없음', true: '예', false: '아니요' }, + actionType: '작업 유형', + actionTypes: { route: '라우팅', reject: '거부' }, + targetBackend: '대상 Backend', + rejectReason: '거부 사유', + defaultAction: '기본 작업', + evaluationMode: '평가 모드', + backendAdmission: 'Backend Admission', + profile: { + modelType: '모델 유형', + task: '작업', + protocol: 'API 프로토콜', + endpoint: 'Endpoint', + strategy: '라우팅 전략', + }, + statuses: { + ready: '준비됨', + syncing: '동기화 중', + degraded: '성능 저하', + offline: '오프라인', + error: '오류', + disabled: '비활성화됨', + draft: '초안', + }, + thinkingPolicies: { + long: '긴 요청', + short: '짧은 요청', + reject: '거부', + }, + strategies: { + typedRules: 'Typed Rules', + tokenBudget: 'Token Budget', + }, + overflowPolicies: { + reject: '거부', + }, + sections: { + basic: '기본 구성', + backends: '백엔드 모델', + routing: '라우팅 정책', + admission: 'Admission 제어', + tokenization: 'Tokenization', + timeouts: '제한 시간', + }, + validation: { + required: '필수 입력 항목입니다.', + positiveInteger: '양의 정수를 입력하세요.', + nonNegativeInteger: '0 이상의 정수를 입력하세요.', + backendsMustDiffer: '짧은 요청과 긴 요청 백엔드 모델은 서로 달라야 합니다.', + invalidUrl: 'HTTP 또는 HTTPS URL을 입력하세요.', + thresholdExceedsShortContext: '짧은 요청 임계값은 짧은 컨텍스트 한도를 초과할 수 없습니다.', + shortContextExceedsLongContext: '짧은 컨텍스트 한도는 긴 컨텍스트 한도를 초과할 수 없습니다.', + tokenActiveBelowWorkers: 'Tokenization 최대 활성 수는 워커 수 이상이어야 합니다.', + invalidAdvancedConfig: '고급 백엔드 및 라우팅 규칙 구성이 올바르지 않습니다.', + }, + }, model: { languageModels: '언어 모델', embeddingModels: '임베딩 모델', @@ -510,6 +751,7 @@ const ko = { virtualenv: 'virtualenv', logs: 'logs', monitor: 'monitor', + routers: '라우터', }, permissionLabels: { admin: '관리자', @@ -526,6 +768,10 @@ const ko = { 'virtualenv:delete': '가상 환경 삭제', 'logs:list': '로그 보기', 'monitor:view': '모니터링 보기', + 'routers:list': 'Token Router 목록 보기', + 'routers:read': 'Token Router 세부 정보 읽기', + 'routers:write': 'Token Router 생성, 수정 및 삭제', + 'routers:operate': 'Token Router 검증, 활성화 및 비활성화', }, }, apiKey: { @@ -768,6 +1014,8 @@ const ko = { cancel: '취소', confirm: '확인', update: '업데이트', + refresh: '새로 고침', + save: '저장', deploy: '배포', stop: '중지', packUp: '접기', diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts index e2fc4955c1..2b7c8f8d01 100644 --- a/frontend/src/i18n/locales/zh.ts +++ b/frontend/src/i18n/locales/zh.ts @@ -5,6 +5,7 @@ const zh = { systemManagement: '系统管理', resourcesAndSupport: '资源与支持', launchModel: '启动模型', + tokenRouter: 'Token 路由', runningModels: '运行模型', registerModel: '注册模型', clusterInfo: '集群信息', @@ -19,6 +20,249 @@ const zh = { documentation: '文档', website: '官网', }, + tokenRouter: { + title: 'Token Router', + description: '管理短上下文与长上下文模型后端之间的 Token 预算路由。', + create: '创建 Router', + edit: '编辑 Router', + routerUid: 'Router UID', + virtualModelUid: '虚拟模型 UID', + tokenizerPath: 'Tokenizer 路径', + tokenizerSource: 'Tokenizer 来源', + tokenizerAsset: 'Tokenizer 资产', + customTokenizerPath: '自定义 Tokenizer 路径', + customTokenizerPathHint: '仅在服务端允许自定义路径，或编辑历史路径配置时使用服务器本地路径。', + noTokenizerAssets: '当前没有已注册的可用 Tokenizer 资产，请联系管理员或启用受控的自定义路径。', + tokenizerAssetsLoadFailed: + 'Tokenizer 资产加载失败，资产选择将保持关闭状态。请刷新页面或联系管理员。', + customTokenizerPathRisk: + '自定义服务器路径会绕过已注册资产的元数据和校验和管控，仅应用于保留历史配置或经过明确审批的例外场景。', + legacyCustomTokenizerPath: '历史自定义 Tokenizer 路径', + assetChangeWarning: + '修改 Tokenizer 来源、资产或路径会重新加载 Router 配置，并重启其 Tokenization Workers。是否继续？', + tokenizerAssetDetails: 'Tokenizer 资产详情', + tokenizerAssetDetailsLoadFailed: + 'Tokenizer 资产详情加载失败，当前使用 Router 配置中的信息作为回退展示。', + tokenizerAssetStatus: '资产状态', + tokenizerAssetOrigin: 'Tokenizer 资产来源', + assetOrigins: { + builtin: '内置', + external: '外部', + }, + compatibleModels: '兼容模型', + tokenizerCapabilities: '能力', + tokenizerAssetValidatedAt: '最近校验时间', + tokenizerAssetErrors: '资产校验错误', + runtimeTokenizerAsset: 'Runtime 实际加载的 Tokenizer 资产', + tokenizerAssetRevision: 'Tokenizer 资产版本', + tokenizerAssetFingerprint: 'Tokenizer 资产指纹', + assetStatuses: { + available: '可用', + invalid: '无效', + disabled: '已禁用', + missing: '缺失', + validating: '校验中', + }, + capabilities: { + chat: '对话', + tools: '工具调用', + thinking: '思考模式', + }, + backendUrl: '后端地址', + currentSupervisor: '当前集群 Supervisor（自动识别）', + currentSupervisorDescription: 'Token Router 将通过当前 Supervisor REST API 调用实际后端模型。', + useCustomBackend: '使用自定义后端地址', + backendAddress: '内部服务地址', + backendDefaultsLoading: '正在识别当前 Supervisor...', + backendDefaultsLoadFailed: '无法加载当前 Supervisor 地址，请使用自定义后端地址。', + backendDefaultsUnavailable: + '当前 Supervisor 未配置可供 Token Router 访问的内部地址，请使用自定义地址。', + customBackendHint: + '通常无需修改。自定义地址必须是 Token Router 可访问的 Supervisor REST API，不是 Worker、模型实例或 Token Router 地址。', + aliases: '模型别名', + thinkingPolicy: '推理请求策略', + shortBackend: '短请求后端模型 UID', + longBackend: '长请求后端模型 UID', + shortBackendDisplay: '短请求', + longBackendDisplay: '长请求', + threshold: '短请求阈值 Token 数', + reserve: '上下文预留 Token 数', + shortContext: '短请求最大上下文 Token 数', + longContext: '长请求最大上下文 Token 数', + defaultOutput: '默认输出 Token 数', + requestTimeout: '请求超时（秒）', + connectTimeout: '连接超时（秒）', + shortActive: '短请求最大活跃数', + shortQueue: '短请求最大排队数', + longActive: '长请求最大活跃数', + longQueue: '长请求最大排队数', + tokenWorkers: 'Tokenization 进程数', + tokenActive: 'Tokenization 最大活跃数', + tokenQueue: 'Tokenization 最大排队数', + details: 'Router 详情', + configuration: '配置摘要', + revision: '配置版本', + runtimeInstances: '运行实例', + noInstances: '暂无已注册的运行实例。', + metrics: '指标摘要', + noMetrics: '暂无指标数据。', + endpoint: '访问地址', + configError: '配置错误', + backends: '后端', + status: '状态', + empty: '尚未配置 Token Router。', + loading: '正在加载...', + onlineSummary: '{{online}}/{{total}} 个在线', + ackRevision: '已确认版本：{{revision}}', + heartbeatAge: '距上次心跳：{{seconds}} 秒', + closeDetails: '关闭 Router 详情', + tabs: { + overview: '概览', + instances: '运行实例', + metrics: '指标', + }, + routingSummary: '路由摘要', + advancedConfiguration: '高级配置', + advancedConfigurationDescription: '查看路由、Admission、Tokenization 和超时配置。', + advancedDiagnostics: '高级诊断', + rawMetrics: '原始 Metrics JSON', + copyValue: '复制', + copyMetrics: '复制 JSON', + copyFailed: '复制失败。', + lastUpdated: '最后更新时间', + autoRefreshing: '每 10 秒自动刷新', + notUpdated: '尚未更新', + runtimeInstanceCount: '运行实例数', + onlineInstanceCount: '在线实例数', + instanceId: '实例 ID', + ackedRevision: '已确认版本', + heartbeatAgeLabel: '距上次心跳', + secondsValue: '{{seconds}} 秒', + modelType: '模型类型', + strategy: '路由策略', + overflowPolicy: '溢出策略', + createdAt: '创建时间', + updatedAt: '更新时间', + shortAdmission: '短请求 Admission', + longAdmission: '长请求 Admission', + queueTimeout: '排队超时', + retryAfter: '建议重试间隔', + tokenizationExecutor: 'Tokenization 执行器', + multiprocessingStartMethod: '多进程启动方式', + completedRequests: '已完成请求', + activeRequests: '活跃请求', + waitingRequests: '等待请求', + shortPool: '短请求池', + longPool: '长请求池', + tokenizationPool: 'Tokenization', + activeCapacity: '活跃数 / 容量', + waitingCapacity: '等待数 / 队列容量', + noKnownMetrics: '暂无可识别的摘要指标，请展开原始 JSON 查看详细信息。', + validationPassed: 'Router 配置校验通过。', + validationFailed: 'Router 配置校验未通过。', + validate: '校验', + disable: '停用', + enable: '启用', + confirmDelete: '确定删除 Router {{routerUid}} 吗？此操作不可撤销。', + deleteTitle: '删除 Token Router', + deleteDisabledHint: '请先停用该 Router，再执行删除。', + createSuccess: 'Token Router 创建成功。', + updateSuccess: 'Token Router 更新成功。', + deleteSuccess: 'Token Router 删除成功。', + enableSuccess: 'Token Router 启用成功。', + disableSuccess: 'Token Router 停用成功。', + configurationMode: '配置模式', + simpleMode: '简单模式', + simpleModeDescription: '使用短请求、长请求双后端和 Token 阈值，兼容现有配置。', + advancedMode: '高级规则模式', + advancedModeDescription: '使用动态后端列表和有序路由规则，适用于多后端和复杂请求分流。', + routeProfile: 'Route Profile', + typedRulesSummary: '{{count}} 条规则', + tokenBudgetSummary: '阈值 {{threshold}}', + dynamicBackends: '动态后端', + dynamicBackendsHint: '仅可选择通过 LLM Chat、Engine 和 Tokenizer 兼容检查的物理模型。', + addBackend: '添加后端', + backendId: 'Backend ID', + backendModel: '后端模型', + maxContext: '最大上下文 Token 数', + maxActive: '最大活跃数', + maxQueue: '最大排队数', + selectBackendCandidate: '请选择兼容的后端候选模型。', + orderedRules: '有序路由规则', + firstMatchHint: '按优先级从高到低执行，命中第一条规则后停止。', + addRule: '添加规则', + ruleId: 'Rule ID', + priority: '优先级', + tokenGte: '总 Token 下限（>=）', + tokenLte: '总 Token 上限（<=）', + conditions: { + thinking: 'Thinking', + tools_present: '包含 Tools', + stream: 'Streaming', + }, + conditionValues: { + any: '不限', + true: '是', + false: '否', + }, + actionType: '动作类型', + actionTypes: { + route: '路由', + reject: '拒绝', + }, + targetBackend: '目标 Backend', + rejectReason: '拒绝原因', + defaultAction: '默认动作', + evaluationMode: '规则执行模式', + backendAdmission: 'Backend Admission', + profile: { + modelType: '模型类型', + task: '任务', + protocol: 'API 协议', + endpoint: 'Endpoint', + strategy: '路由策略', + }, + statuses: { + ready: '已就绪', + syncing: '同步中', + degraded: '降级', + offline: '离线', + error: '异常', + disabled: '已停用', + draft: '草稿', + }, + thinkingPolicies: { + long: '长请求', + short: '短请求', + reject: '拒绝', + }, + strategies: { + typedRules: 'Typed Rules', + tokenBudget: 'Token 预算', + }, + overflowPolicies: { + reject: '拒绝', + }, + sections: { + basic: '基础配置', + backends: '后端模型', + routing: '路由策略', + admission: 'Admission 配置', + tokenization: 'Tokenization 配置', + timeouts: '超时配置', + }, + validation: { + required: '此字段不能为空。', + positiveInteger: '请输入正整数。', + nonNegativeInteger: '请输入非负整数。', + backendsMustDiffer: '短请求和长请求后端模型不能相同。', + invalidUrl: '请输入以 HTTP 或 HTTPS 开头的有效地址。', + thresholdExceedsShortContext: '短请求阈值不能超过短请求最大上下文。', + shortContextExceedsLongContext: '短请求最大上下文不能超过长请求最大上下文。', + tokenActiveBelowWorkers: 'Tokenization 最大活跃数不能小于进程数。', + invalidAdvancedConfig: '高级后端和路由规则配置无效。', + }, + }, model: { languageModels: '语言模型', embeddingModels: '嵌入模型', @@ -532,6 +776,7 @@ const zh = { virtualenv: 'virtualenv', logs: 'logs', monitor: 'monitor', + routers: '路由管理', }, permissionLabels: { admin: '管理员', @@ -548,6 +793,10 @@ const zh = { 'virtualenv:delete': '删除虚拟环境', 'logs:list': '查看日志', 'monitor:view': '查看监控', + 'routers:list': '查看 Token Router 列表', + 'routers:read': '读取 Token Router 详情', + 'routers:write': '创建、修改和删除 Token Router', + 'routers:operate': '校验、启用和停用 Token Router', }, }, apiKey: { @@ -756,6 +1005,8 @@ const zh = { cancel: '取消', confirm: '确定', update: '更新', + refresh: '刷新', + save: '保存', deploy: '部署', stop: '停止', packUp: '收起', diff --git a/frontend/src/types/services.ts b/frontend/src/types/services.ts index e53bc6ee2c..342841ccaa 100644 --- a/frontend/src/types/services.ts +++ b/frontend/src/types/services.ts @@ -281,3 +281,193 @@ export interface UserItem { permissions: string[]; created_at: string | null; } + +export interface TokenizerAssetItem { + asset_id: string; + origin: 'builtin' | 'external'; + display_name: string; + model_family: string; + model_name: string; + revision: string; + encoding_type: string; + compatible_models: string[]; + capabilities: Record; + enabled: boolean; + status: 'available' | 'invalid' | 'disabled' | 'missing' | 'validating'; + valid: boolean; + fingerprint: string; + errors: string[]; + checks?: Record; + validated_at?: string; +} + +export interface TokenizerAssetListResponse { + items: TokenizerAssetItem[]; + allow_custom_path: boolean; + config_error?: string; +} + +export interface TokenRouterAdmissionConfig { + max_active: number; + max_queue: number; + queue_timeout_seconds: number; + retry_after_seconds: number; +} + +export interface TokenRouterBackendConfig { + model_uid: string; + max_context_tokens: number; + admission: TokenRouterAdmissionConfig; +} + +export interface TokenRouterDynamicBackendConfig extends TokenRouterBackendConfig { + id: string; +} + +export interface TokenRouterRuleMatch { + total_tokens_gte?: number; + total_tokens_lte?: number; + thinking?: boolean; + tools_present?: boolean; + stream?: boolean; +} + +export type TokenRouterRouteAction = { + type: 'route'; + backend_id: string; +}; + +export type TokenRouterRejectAction = { + type: 'reject'; + reason: string; +}; + +export type TokenRouterRoutingAction = TokenRouterRouteAction | TokenRouterRejectAction; + +export interface TokenRouterRoutingRule { + id: string; + priority: number; + match: TokenRouterRuleMatch; + action: TokenRouterRoutingAction; +} + +interface TokenRouterItemBase { + router_uid: string; + virtual_model_uid: string; + model_type: 'LLM'; + route_profile?: 'llm_chat'; + tokenizer_asset_id?: string; + tokenizer_asset_origin?: 'builtin' | 'external'; + tokenizer_path: string; + tokenizer_asset_revision?: string; + tokenizer_asset_fingerprint?: string; + backend_url: string; + model_aliases: string[]; + request_timeout_seconds: number; + connect_timeout_seconds: number; + tokenization: { + executor: 'process'; + multiprocessing_start_method: 'spawn'; + max_workers: number; + max_active: number; + max_queue: number; + queue_timeout_seconds: number; + retry_after_seconds: number; + }; + enabled: boolean; + revision: number; + status: 'draft' | 'disabled' | 'syncing' | 'ready' | 'degraded' | 'offline' | 'error'; + runtime_instances: number; + online_instances: number; + created_at: string; + updated_at: string; +} + +export interface TokenRouterLegacyItem extends TokenRouterItemBase { + config_version?: 1; + strategy: 'token_budget'; + backends: { + short: TokenRouterBackendConfig; + long: TokenRouterBackendConfig; + }; + routing: { + short_threshold_tokens: number; + context_reserve_tokens: number; + default_output_tokens: number; + thinking_policy: 'short' | 'long' | 'reject'; + overflow_policy: 'reject'; + }; +} + +export interface TokenRouterTypedItem extends TokenRouterItemBase { + config_version: 2; + route_profile: 'llm_chat'; + strategy: 'typed_rules'; + backends: TokenRouterDynamicBackendConfig[]; + routing: { + evaluation_mode: 'first_match'; + context_reserve_tokens: number; + default_output_tokens: number; + rules: TokenRouterRoutingRule[]; + default_action: TokenRouterRoutingAction; + }; +} + +export type TokenRouterItem = TokenRouterLegacyItem | TokenRouterTypedItem; + +export interface TokenRouterBackendCandidate { + model_uid: string; + model_name: string; + model_type?: string; + model_engine: string; + model_format: string; + model_ability: string[]; + context_length?: number; + compatibility_status: 'Verified' | 'Unsupported' | 'Unknown'; + compatibility_reason: string; + eligible: boolean; + ineligible_reasons: string[]; +} + +export interface TokenRouterBackendCandidateResponse { + items: TokenRouterBackendCandidate[]; + errors: string[]; +} + +export interface TokenRouterDefaultsResponse { + backend: { + mode: 'current_supervisor'; + display_name: string; + backend_url: string | null; + source: 'server_config' | 'rest_endpoint' | 'unavailable'; + available: boolean; + error?: string; + }; +} + +export function isTypedTokenRouter(router: TokenRouterItem): router is TokenRouterTypedItem { + return router.config_version === 2 || Array.isArray(router.backends); +} + +export interface TokenRouterRuntimeInstance { + router_uid: string; + instance_id: string; + endpoint: string; + status?: string; + online: boolean; + acked_revision: number; + heartbeat_age_seconds: number; + last_heartbeat: number; + config_error?: string; + metrics?: Record; + backend_health?: Record; + process?: { + tokenizer_asset?: { + asset_id?: string; + origin?: 'builtin' | 'external'; + revision?: string; + fingerprint?: string; + }; + [key: string]: unknown; + }; +} diff --git a/xinference/api/routers/token_routers.py b/xinference/api/routers/token_routers.py index 2496dfd0f2..d83dc098cf 100644 --- a/xinference/api/routers/token_routers.py +++ b/xinference/api/routers/token_routers.py @@ -5,6 +5,7 @@ import hmac import os +from ipaddress import ip_address from typing import TYPE_CHECKING, Any, Callable, Dict, Optional from fastapi import Depends, Header, HTTPException, Query, Request, Security @@ -29,6 +30,19 @@ def _username(user: Optional[dict]) -> str: return user.get("username", "") if user else "" +def _is_non_routable_bind_host(host: str) -> bool: + normalized = host.strip().strip("[]").casefold() + if normalized in {"localhost", "*"}: + return True + try: + address = ip_address(normalized) + except ValueError: + # A hostname may identify an address reachable by the Token Router; + # only reject addresses that are unambiguously local or wildcard. + return False + return address.is_loopback or address.is_unspecified + + def _internal_authorized(authorization: str) -> bool: expected = os.environ.get("XINFERENCE_TOKEN_ROUTER_INTERNAL_TOKEN", "") if not expected or not authorization.startswith("Bearer "): @@ -61,12 +75,49 @@ async def list_routers(api: "RESTfulAPI") -> JSONResponse: return JSONResponse(content=await (await _supervisor(api)).list_token_routers()) -async def list_backend_candidates(api: "RESTfulAPI") -> JSONResponse: +async def list_backend_candidates( + tokenizer_asset_id: Optional[str], api: "RESTfulAPI" +) -> JSONResponse: return JSONResponse( - content=await (await _supervisor(api)).list_token_router_backend_candidates() + content=await (await _supervisor(api)).list_token_router_backend_candidates( + tokenizer_asset_id=tokenizer_asset_id or None + ) ) +async def get_backend_defaults(api: "RESTfulAPI") -> JSONResponse: + host = str(getattr(api, "_host", "") or "").strip() + port = getattr(api, "_port", None) + backend: Dict[str, Any] = { + "mode": "current_supervisor", + "display_name": "Current Supervisor", + "backend_url": None, + "source": "unavailable", + "available": False, + } + if ( + host + and isinstance(port, int) + and 0 < port <= 65535 + and not _is_non_routable_bind_host(host) + ): + # Bracket IPv6 literals so the generated URL remains valid. + if ":" in host and not (host.startswith("[") and host.endswith("]")): + host = f"[{host}]" + backend.update( + backend_url=f"http://{host}:{port}", + source="rest_endpoint", + available=True, + ) + elif host and _is_non_routable_bind_host(host): + backend["error"] = ( + "REST API bind address is not reachable by a separate Token Router" + ) + else: + backend["error"] = "REST API host and port are unavailable" + return JSONResponse(content={"backend": backend}) + + async def list_tokenizer_assets(api: "RESTfulAPI") -> JSONResponse: return JSONResponse(content=await (await _supervisor(api)).list_tokenizer_assets()) @@ -336,6 +387,15 @@ async def anonymous( list_backend_candidates, ["GET"], "routers:list", + query_params=("tokenizer_asset_id",), + ) + # Keep this static route before /{router_uid}; otherwise "defaults" is + # interpreted as a router UID by the dynamic route. + route( + "/v1/token_routers/defaults", + get_backend_defaults, + ["GET"], + "routers:list", ) route( "/v1/token_routers", diff --git a/xinference/api/tests/test_token_router_api.py b/xinference/api/tests/test_token_router_api.py index c8e63cf00b..1a1978325b 100644 --- a/xinference/api/tests/test_token_router_api.py +++ b/xinference/api/tests/test_token_router_api.py @@ -145,6 +145,16 @@ def match_path(self, tokenizer_path: str): def validate_path(self, tokenizer_path: str, *, smoke_test: bool) -> dict: return {"valid": True, "errors": []} + def get_asset(self, asset_id: str) -> dict: + if asset_id != "test-asset": + raise KeyError(asset_id) + return { + "asset_id": asset_id, + "status": "available", + "valid": True, + "compatible_models": ["DeepSeek-V4-Flash-0731"], + } + def make_supervisor(tmp_path): supervisor = SupervisorActor.__new__(SupervisorActor) @@ -185,11 +195,21 @@ async def list_models(_self): class FakeAPI: - def __init__(self, supervisor, *, authenticated: bool, auth_service) -> None: + def __init__( + self, + supervisor, + *, + authenticated: bool, + auth_service, + host: str = "127.0.0.1", + port: int = 9997, + ) -> None: self._router = APIRouter() self._supervisor = supervisor self._authenticated = authenticated self._auth_service = auth_service + self._host = host + self._port = port def is_authenticated(self) -> bool: return self._authenticated @@ -202,11 +222,20 @@ async def unused_auth(): return {"username": "anonymous"} -def create_app(supervisor, *, authenticated: bool = False, auth_service=unused_auth): +def create_app( + supervisor, + *, + authenticated: bool = False, + auth_service=unused_auth, + host: str = "127.0.0.1", + port: int = 9997, +): api = FakeAPI( supervisor, authenticated=authenticated, auth_service=auth_service, + host=host, + port=port, ) register_routes(api) # type: ignore[arg-type] app = FastAPI() @@ -720,6 +749,9 @@ async def list_models(_self): transport = httpx.ASGITransport(app=app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: response = await client.get("/v1/token_routers/backend-candidates") + asset_response = await client.get( + "/v1/token_routers/backend-candidates?tokenizer_asset_id=test-asset" + ) assert response.status_code == 200 candidates = {item["model_uid"]: item for item in response.json()["items"]} @@ -730,8 +762,62 @@ async def list_models(_self): assert candidates["completion-only"]["eligible"] is False assert candidates["nested-router"]["compatibility_status"] == "Unsupported" assert candidates["unsupported-engine"]["eligible"] is False + # Without an asset selection, candidate discovery retains the legacy behavior. assert candidates["asset-mismatch"]["eligible"] is True + assert asset_response.status_code == 200 + asset_candidates = { + item["model_uid"]: item for item in asset_response.json()["items"] + } + assert asset_candidates["eligible-vllm"]["eligible"] is True + assert asset_candidates["asset-mismatch"]["eligible"] is False + assert any( + "not compatible with Tokenizer asset test-asset" in reason + for reason in asset_candidates["asset-mismatch"]["ineligible_reasons"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("host", ["127.0.0.1", "0.0.0.0", "::1", "::", "localhost"]) +async def test_token_router_defaults_rejects_non_routable_bind_address( + tmp_path, host +) -> None: + app = create_app(make_supervisor(tmp_path), host=host) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/v1/token_routers/defaults") + + assert response.status_code == 200 + assert response.json() == { + "backend": { + "mode": "current_supervisor", + "display_name": "Current Supervisor", + "backend_url": None, + "source": "unavailable", + "available": False, + "error": ( + "REST API bind address is not reachable by a separate Token Router" + ), + } + } + + +@pytest.mark.asyncio +async def test_token_router_defaults_returns_routable_rest_endpoint(tmp_path) -> None: + app = create_app(make_supervisor(tmp_path), host="xinference-supervisor") + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/v1/token_routers/defaults") + + assert response.status_code == 200 + assert response.json()["backend"] == { + "mode": "current_supervisor", + "display_name": "Current Supervisor", + "backend_url": "http://xinference-supervisor:9997", + "source": "rest_endpoint", + "available": True, + } + @pytest.mark.asyncio async def test_v2_validation_checks_tools_and_thinking_capabilities(tmp_path) -> None: diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index 5be6fc2077..d6104b300d 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -5095,8 +5095,38 @@ def _token_router_engine_compatibility( "reason": f"Engine {engine} has not been validated for this Router", } - async def list_token_router_backend_candidates(self) -> Dict[str, Any]: + async def list_token_router_backend_candidates( + self, tokenizer_asset_id: Optional[str] = None + ) -> Dict[str, Any]: running_models = await self.list_models() + asset_id = str(tokenizer_asset_id or "").strip() + asset_compatibility: Optional[set[str]] = None + asset_error: Optional[str] = None + if asset_id: + try: + asset = await self._call_tokenizer_asset_registry_async( + "get_asset", asset_id + ) + compatible_models = asset.get("compatible_models", []) + if asset.get("status") != "available": + asset_error = f"Tokenizer asset is not available: {asset_id}" + elif not isinstance(compatible_models, list) or not compatible_models: + asset_error = ( + f"Tokenizer asset has no compatible model metadata: {asset_id}" + ) + else: + asset_compatibility = { + str(name).strip().casefold() + for name in compatible_models + if str(name).strip() + } + if not asset_compatibility: + asset_error = f"Tokenizer asset has no compatible model metadata: {asset_id}" + except KeyError: + asset_error = f"Tokenizer asset is not registered: {asset_id}" + except Exception as exc: + asset_error = f"Unable to inspect Tokenizer asset {asset_id}: {exc}" + items: List[Dict[str, Any]] = [] for model_uid, model_info in running_models.items(): if not isinstance(model_info, dict): @@ -5113,6 +5143,20 @@ async def list_token_router_backend_candidates(self) -> Dict[str, Any]: reasons.append("model_ability must include chat") if compatibility["status"] in {"Unsupported", "Unknown"}: reasons.append(compatibility["reason"]) + if asset_error: + reasons.append(asset_error) + elif asset_compatibility is not None: + model_name = str(model_info.get("model_name") or "").strip() + if not model_name: + reasons.append( + "model_name is required for Tokenizer asset compatibility " + "validation" + ) + elif model_name.casefold() not in asset_compatibility: + reasons.append( + f"model_name is not compatible with Tokenizer asset " + f"{asset_id}: {model_name}" + ) items.append( { "model_uid": model_uid, @@ -5129,7 +5173,10 @@ async def list_token_router_backend_candidates(self) -> Dict[str, Any]: } ) items.sort(key=lambda item: (not item["eligible"], item["model_uid"])) - return {"items": items, "errors": []} + return { + "items": items, + "errors": [asset_error] if asset_error else [], + } async def validate_token_router(self, router_uid: str) -> Optional[Dict[str, Any]]: config = self._token_router_store.get(router_uid)