From 6132c7bf70f6a8ef5522b588a16c32166f396f4d Mon Sep 17 00:00:00 2001 From: forthfate Date: Sun, 27 Sep 2026 01:40:33 +0900 Subject: [PATCH 1/5] feat: add AI usage and rate limit monitoring --- backend/app/main.py | 10 ++ backend/app/providers.py | 82 +++++++++++++--- backend/app/store.py | 98 +++++++++++++++++++ backend/tests/test_api.py | 52 ++++++++++ .../features/settings/ai-usage-monitor.tsx | 60 ++++++++++++ frontend/src/features/settings/page.tsx | 2 + frontend/src/locales/languages/en.json | 8 ++ frontend/src/locales/languages/ja.json | 8 ++ frontend/src/locales/languages/ko.json | 8 ++ frontend/src/theme-overrides.css | 1 + 10 files changed, 313 insertions(+), 16 deletions(-) create mode 100644 frontend/src/features/settings/ai-usage-monitor.tsx diff --git a/backend/app/main.py b/backend/app/main.py index 4f0e6423..7dbb7efc 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1116,6 +1116,11 @@ def telemetry(): return store.telemetry() +@app.get("/api/ai-usage") +def ai_usage(minutes: int = 60): + return store.ai_usage(minutes) + + @app.get("/api/orbit-logs") def orbit_logs(): return store.orbit_logs() @@ -1637,6 +1642,11 @@ def list_telemetry_v1(): return store.telemetry() +@app.get("/api/v1/ai-usage", tags=["Observability"], operation_id="getAiUsage") +def ai_usage_v1(minutes: int = 60): + return store.ai_usage(minutes) + + @app.get("/api/v1/logs", tags=["Observability"], operation_id="listLogs") def list_logs_v1(): return store.orbit_logs() diff --git a/backend/app/providers.py b/backend/app/providers.py index da1942da..9744750e 100644 --- a/backend/app/providers.py +++ b/backend/app/providers.py @@ -8,12 +8,15 @@ import json import os import subprocess +import time from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any, Callable import requests +from opentelemetry import trace +from opentelemetry.trace import Status, StatusCode @dataclass(frozen=True) @@ -27,20 +30,74 @@ class ModelSettings: class AzureOpenAIProvider: + @staticmethod + def _post(settings: ModelSettings, key: str, payload: dict[str, Any]) -> dict[str, Any]: + started = time.perf_counter() + with trace.get_tracer("orbit.providers").start_as_current_span("gen_ai.azure.responses") as span: + input_value = payload.get("input", "") + input_chars = ( + len(input_value) + if isinstance(input_value, str) + else len(json.dumps(input_value, ensure_ascii=False)) + ) + tools = payload.get("tools") + span.set_attributes( + { + "gen_ai.system": "azure_openai", + "gen_ai.request.model": settings.model, + "gen_ai.request.input_chars": input_chars, + "gen_ai.request.tool_count": len(tools) if isinstance(tools, list) else 0, + } + ) + try: + response = requests.post( + f"{settings.endpoint.rstrip('/')}/responses", + headers={"api-key": key, "Content-Type": "application/json"}, + json=payload, + timeout=180, + ) + span.set_attribute("http.response.status_code", response.status_code) + for name in ( + "x-ratelimit-limit-requests", + "x-ratelimit-remaining-requests", + "x-ratelimit-limit-tokens", + "x-ratelimit-remaining-tokens", + "retry-after", + ): + if response.headers.get(name): + span.set_attribute(f"azure.{name.replace('-', '_')}", response.headers[name]) + response.raise_for_status() + body = response.json() + usage = body.get("usage", {}) if isinstance(body, dict) else {} + for source, target in ( + ("input_tokens", "gen_ai.usage.input_tokens"), + ("output_tokens", "gen_ai.usage.output_tokens"), + ("total_tokens", "gen_ai.usage.total_tokens"), + ): + if isinstance(usage.get(source), int): + span.set_attribute(target, usage[source]) + output = body.get("output_text", "") if isinstance(body, dict) else "" + if isinstance(output, str): + span.set_attribute("gen_ai.response.output_chars", len(output)) + return body + except requests.RequestException as error: + span.set_attribute("error.type", type(error).__name__) + span.set_attribute("error.message", str(error)[:500]) + span.record_exception(error) + span.set_status(Status(StatusCode.ERROR)) + raise + finally: + span.set_attribute( + "gen_ai.request.duration_ms", round((time.perf_counter() - started) * 1000) + ) + def complete(self, settings: ModelSettings, prompt: str) -> str: key = os.environ.get(settings.secret_env, "") if not key or not settings.endpoint or not settings.model: raise RuntimeError( "Azure OpenAI provider requires endpoint, model, and configured secret environment variable." ) - response = requests.post( - f"{settings.endpoint.rstrip('/')}/responses", - headers={"api-key": key, "Content-Type": "application/json"}, - json={"model": settings.model, "input": prompt}, - timeout=180, - ) - response.raise_for_status() - payload = response.json() + payload = self._post(settings, key, {"model": settings.model, "input": prompt}) output_text = payload.get("output_text") if isinstance(output_text, str) and output_text: return output_text @@ -78,14 +135,7 @@ def complete_with_tools( payload: dict[str, Any] = {"model": settings.model, "input": input_value, "tools": definitions} if response_id: payload["previous_response_id"] = response_id - response = requests.post( - f"{settings.endpoint.rstrip('/')}/responses", - headers={"api-key": key, "Content-Type": "application/json"}, - json=payload, - timeout=180, - ) - response.raise_for_status() - body = response.json() + body = self._post(settings, key, payload) response_id = body.get("id") calls = [ item diff --git a/backend/app/store.py b/backend/app/store.py index a3441710..89eeb378 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -2981,6 +2981,104 @@ def telemetry(self) -> list[dict[str, Any]]: lines = TELEMETRY.read_text(encoding="utf-8").splitlines()[-100:] return [json.loads(line) for line in reversed(lines)] + def ai_usage(self, minutes: int = 60) -> dict[str, Any]: + """Summarize locally retained provider spans without retaining prompt text.""" + minutes = max(15, min(minutes, 24 * 60)) + now = datetime.now(UTC) + cutoff = now.timestamp() - minutes * 60 + events: list[dict[str, Any]] = [] + if TELEMETRY.exists(): + for line in TELEMETRY.read_text(encoding="utf-8").splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + for resource in record.get("resourceSpans", []): + for scope in resource.get("scopeSpans", []): + for span in scope.get("spans", []): + if span.get("name") != "gen_ai.azure.responses": + continue + timestamp = float(span.get("startTime", 0)) / 1_000_000_000 + if timestamp < cutoff: + continue + a = span.get("attributes", {}) + events.append( + { + "id": span.get("spanId", ""), + "time": datetime.fromtimestamp(timestamp, UTC).isoformat(), + "model": a.get("gen_ai.request.model", ""), + "status": int(a.get("http.response.status_code", 0) or 0), + "failed": span.get("status") == "ERROR", + "input_tokens": int(a.get("gen_ai.usage.input_tokens", 0) or 0), + "output_tokens": int(a.get("gen_ai.usage.output_tokens", 0) or 0), + "total_tokens": int(a.get("gen_ai.usage.total_tokens", 0) or 0), + "input_chars": int(a.get("gen_ai.request.input_chars", 0) or 0), + "output_chars": int(a.get("gen_ai.response.output_chars", 0) or 0), + "tool_count": int(a.get("gen_ai.request.tool_count", 0) or 0), + "duration_ms": int(a.get("gen_ai.request.duration_ms", 0) or 0), + "error": a.get("error.message", ""), + "limit_requests": a.get("azure.x_ratelimit_limit_requests"), + "remaining_requests": a.get("azure.x_ratelimit_remaining_requests"), + "limit_tokens": a.get("azure.x_ratelimit_limit_tokens"), + "remaining_tokens": a.get("azure.x_ratelimit_remaining_tokens"), + "retry_after": a.get("azure.retry_after"), + } + ) + events.sort(key=lambda item: item["time"], reverse=True) + chronological = list(reversed(events)) + bucket_seconds = 60 if minutes <= 60 else 5 * 60 if minutes <= 6 * 60 else 60 * 60 + bucket_start = int(cutoff // bucket_seconds) * bucket_seconds + buckets: dict[int, dict[str, int | str]] = {} + while bucket_start <= int(now.timestamp()): + buckets[bucket_start] = { + "time": datetime.fromtimestamp(bucket_start, UTC).isoformat(), + "requests": 0, + "tokens": 0, + } + bucket_start += bucket_seconds + for event in chronological: + stamp = datetime.fromisoformat(event["time"]).timestamp() + bucket = int(stamp // bucket_seconds) * bucket_seconds + point = buckets.setdefault( + bucket, {"time": datetime.fromtimestamp(bucket, UTC).isoformat(), "requests": 0, "tokens": 0} + ) + point["requests"] = int(point["requests"]) + 1 + point["tokens"] = int(point["tokens"]) + event["total_tokens"] + requests = len(events) + errors = sum(event["failed"] or event["status"] >= 400 for event in events) + last = events[0] if events else {} + recent = [ + event + for event in events + if datetime.fromisoformat(event["time"]).timestamp() >= now.timestamp() - 60 + ] + return { + "minutes": minutes, + "summary": { + "requests": requests, + "tokens": sum(event["total_tokens"] for event in events), + "errors": errors, + "success_rate": round((requests - errors) / requests * 100, 1) if requests else 100, + "average_duration_ms": round(sum(event["duration_ms"] for event in events) / requests) + if requests + else 0, + "rpm": len(recent), + "tpm": sum(event["total_tokens"] for event in recent), + }, + "limits": { + key: last.get(key) + for key in ( + "limit_requests", + "remaining_requests", + "limit_tokens", + "remaining_tokens", + "retry_after", + ) + }, + "timeline": list(buckets.values()), + "events": events[:100], + } + @staticmethod def _prompt_snapshot_content(directory: Path, state: object) -> str | None: if not isinstance(state, dict) or not state.get("exists"): diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 55806053..29809ad1 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1499,6 +1499,7 @@ def test_v1_read_only_control_room_resources_are_available(): "/api/v1/application-settings", "/api/v1/dashboard", "/api/v1/telemetry", + "/api/v1/ai-usage", "/api/v1/logs", "/api/v1/improvements", "/api/v1/reported-issues", @@ -1506,6 +1507,57 @@ def test_v1_read_only_control_room_resources_are_available(): assert client.get(path).status_code == 200 +def test_ai_usage_summarizes_provider_spans_without_prompt_content(monkeypatch, tmp_path): + telemetry = tmp_path / "telemetry.jsonl" + timestamp = int(datetime.now(UTC).timestamp() * 1_000_000_000) + telemetry.write_text( + json.dumps( + { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "name": "gen_ai.azure.responses", + "spanId": "call-1", + "startTime": timestamp, + "status": "ERROR", + "attributes": { + "gen_ai.request.model": "gpt-test", + "http.response.status_code": 429, + "gen_ai.usage.total_tokens": 17, + "gen_ai.request.duration_ms": 120, + "azure.x_ratelimit_remaining_requests": "0", + }, + } + ] + } + ] + } + ] + } + ) + + "\n", + encoding="utf-8", + ) + monkeypatch.setattr(store_module, "TELEMETRY", telemetry) + + result = store_module.ConsoleStore().ai_usage(15) + + assert result["summary"] == { + "requests": 1, + "tokens": 17, + "errors": 1, + "success_rate": 0.0, + "average_duration_ms": 120, + "rpm": 1, + "tpm": 17, + } + assert result["limits"]["remaining_requests"] == "0" + assert result["events"][0]["model"] == "gpt-test" + + def test_supervisor_result_requires_the_two_template_return_keys(): assert store_module.ConsoleStore._validated_supervisor_result( '{"improvements": [], "reported_issues": []}' diff --git a/frontend/src/features/settings/ai-usage-monitor.tsx b/frontend/src/features/settings/ai-usage-monitor.tsx new file mode 100644 index 00000000..bf76be69 --- /dev/null +++ b/frontend/src/features/settings/ai-usage-monitor.tsx @@ -0,0 +1,60 @@ +import { useEffect, useMemo, useState } from "react"; +import { AlertTriangle, RefreshCw } from "lucide-react"; +import { Area, AreaChart, CartesianGrid, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; +import type { Locale } from "../../locales"; +import { intlLocales, localeMessages } from "../../locales"; +import { DataTable, type Column } from "../../components/ui/data-table"; +import { PanelHeader } from "../../components/ui/page-header"; +import { SectionInfo } from "../../components/ui/section-info"; +import { api } from "../../services/api"; + +type UsageEvent = { id: string; time: string; model: string; status: number; failed: boolean; input_tokens: number; output_tokens: number; total_tokens: number; input_chars: number; output_chars: number; tool_count: number; duration_ms: number; error: string; limit_requests?: string; remaining_requests?: string; limit_tokens?: string; remaining_tokens?: string; retry_after?: string }; +type AiUsage = { minutes: number; summary: { requests: number; tokens: number; errors: number; success_rate: number; average_duration_ms: number; rpm: number; tpm: number }; limits: Record; timeline: { time: string; requests: number; tokens: number }[]; events: UsageEvent[] }; +type Copy = { title: string; description: string; range: string; refresh: string; rpm: string; tpm: string; successRate: string; averageLatency: string; requests: string; tokens: string; limits: string; remainingRequests: string; remainingTokens: string; retryAfter: string; requestTrend: string; tokenTrend: string; recentCalls: string; time: string; model: string; status: string; usage: string; latency: string; details: string; input: string; output: string; tools: string; empty: string; unavailable: string; noLimitData: string }; + +const format = (value: number) => new Intl.NumberFormat().format(value); +const tick = (value: string, locale: Locale) => new Date(value).toLocaleTimeString(intlLocales[locale], { hour: "2-digit", minute: "2-digit" }); + +export function AiUsageMonitor({ locale }: { locale: Locale }) { + const t = localeMessages(locale, "aiUsage"); + const [minutes, setMinutes] = useState(60), [data, setData] = useState(), [selected, setSelected] = useState(), [loading, setLoading] = useState(false); + const load = () => { + setLoading(true); + api(`/api/ai-usage?minutes=${minutes}`).then(setData).catch(() => setData(undefined)).finally(() => setLoading(false)); + }; + useEffect(() => { + const refresh = () => { + api(`/api/ai-usage?minutes=${minutes}`).then(setData).catch(() => setData(undefined)); + }; + const start = window.setTimeout(refresh, 0); + const id = window.setInterval(refresh, 15_000); + return () => { window.clearTimeout(start); window.clearInterval(id); }; + }, [minutes]); + const columns = useMemo[]>(() => [ + { id: "time", header: t.time, render: (row) => new Date(row.time).toLocaleTimeString(intlLocales[locale]), sortValue: (row) => row.time }, + { id: "model", header: t.model, render: (row) => row.model || "—", sortValue: (row) => row.model }, + { id: "status", header: t.status, render: (row) => = 400 ? "ai-usage-error" : ""}>{row.status || (row.failed ? "Error" : "—")}, sortValue: (row) => row.status }, + { id: "usage", header: t.usage, render: (row) => `${format(row.input_tokens)} / ${format(row.output_tokens)}`, sortValue: (row) => row.total_tokens }, + { id: "latency", header: t.latency, render: (row) => `${format(row.duration_ms)} ms`, sortValue: (row) => row.duration_ms }, + ], [locale, t]); + const limits = data?.limits; + return
+ } action={
} /> + {!data ?

{t.unavailable}

: <> +
+ + 0} /> +
+ {data.summary.errors > 0 &&

{data.summary.errors} {t.requests}

} +
+

{t.requestTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />
+

{t.tokenTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />
+
+
{t.limits}{limits?.remaining_requests || limits?.remaining_tokens ? {t.remainingRequests}: {limits.remaining_requests ?? "—"}{limits.limit_requests ? ` / ${limits.limit_requests}` : ""} · {t.remainingTokens}: {limits.remaining_tokens ?? "—"}{limits.limit_tokens ? ` / ${limits.limit_tokens}` : ""}{limits.retry_after ? ` · ${t.retryAfter}: ${limits.retry_after}s` : ""} : {t.noLimitData}}
+

{t.recentCalls}

+ {selected &&
{t.details}{t.input}: {format(selected.input_tokens)} tokens / {format(selected.input_chars)} chars · {t.output}: {format(selected.output_tokens)} tokens / {format(selected.output_chars)} chars · {t.tools}: {selected.tool_count}{selected.error && {selected.error}}
} + } +
; +} + +function Metric({ label, value, warning = false }: { label: string; value: string; warning?: boolean }) { return
{label}{value}
; } diff --git a/frontend/src/features/settings/page.tsx b/frontend/src/features/settings/page.tsx index 7b647b53..38b3ad53 100644 --- a/frontend/src/features/settings/page.tsx +++ b/frontend/src/features/settings/page.tsx @@ -11,6 +11,7 @@ import { api } from "../../services/api"; import { useToast } from "../../components/ui/toast-context"; import { ProfileCatalog } from "../assets/page"; import { OrbitLogs } from "./orbit-logs"; +import { AiUsageMonitor } from "./ai-usage-monitor"; import { useAssistantUiBridge } from "../../components/assistant-ui-bridge"; import { JsonEditor } from "../../components/ui/json-editor"; @@ -424,6 +425,7 @@ export function SettingsPage({

{prompt || l.empty}

+ setMcpOpen(false)}>
diff --git a/frontend/src/locales/languages/en.json b/frontend/src/locales/languages/en.json index 6402eb32..eeb98f2d 100644 --- a/frontend/src/locales/languages/en.json +++ b/frontend/src/locales/languages/en.json @@ -489,6 +489,14 @@ "description": "Review recorded Orbit events, including operational messages and errors.", "empty": "No Orbit events recorded yet." }, + "aiUsage": { + "title": "AI usage & rate limits", + "description": "Monitor local Azure OpenAI request metadata and rate-limit headers. Prompt and response text are not retained.", + "range": "Range", "refresh": "Refresh", "rpm": "RPM (last minute)", "tpm": "TPM (last minute)", "successRate": "Success rate", "averageLatency": "Average latency", + "requests": "Requests", "tokens": "Tokens", "limits": "Latest Azure limit headers", "remainingRequests": "Requests remaining", "remainingTokens": "Tokens remaining", "retryAfter": "Retry after", + "requestTrend": "Request trend", "tokenTrend": "Token trend", "recentCalls": "Recent model calls", "time": "Time", "model": "Model", "status": "Status", "usage": "Input / output", "latency": "Latency", "details": "Selected call", "input": "Input", "output": "Output", "tools": "Tools", + "empty": "No model calls were retained in this period.", "unavailable": "Usage data could not be loaded. Start or refresh the local Orbit server.", "noLimitData": "No Azure rate-limit headers have been received yet." + }, "improvements": "Improvements", "dashboard": "Dashboard", "assets": "Assets", diff --git a/frontend/src/locales/languages/ja.json b/frontend/src/locales/languages/ja.json index 8612fa33..f5d9e9a4 100644 --- a/frontend/src/locales/languages/ja.json +++ b/frontend/src/locales/languages/ja.json @@ -483,6 +483,14 @@ "description": "運用メッセージとエラーを含む Orbit イベントの記録を確認します。", "empty": "記録された Orbit イベントはまだありません。" }, + "aiUsage": { + "title": "AI 使用量とレート制限", + "description": "ローカルの Azure OpenAI リクエストメタデータとレート制限ヘッダーを確認します。プロンプトと応答本文は保存しません。", + "range": "期間", "refresh": "更新", "rpm": "RPM（直近1分）", "tpm": "TPM（直近1分）", "successRate": "成功率", "averageLatency": "平均レイテンシー", + "requests": "リクエスト", "tokens": "トークン", "limits": "最新の Azure 制限ヘッダー", "remainingRequests": "残りリクエスト", "remainingTokens": "残りトークン", "retryAfter": "再試行まで", + "requestTrend": "リクエスト推移", "tokenTrend": "トークン推移", "recentCalls": "最近のモデル呼び出し", "time": "時刻", "model": "モデル", "status": "状態", "usage": "入力 / 出力", "latency": "レイテンシー", "details": "選択した呼び出し", "input": "入力", "output": "出力", "tools": "ツール", + "empty": "この期間に保存されたモデル呼び出しはありません。", "unavailable": "使用量データを読み込めません。ローカル Orbit サーバーを開始または更新してください。", "noLimitData": "Azure のレート制限ヘッダーはまだ受信していません。" + }, "improvements": "改善", "dashboard": "ダッシュボード", "assets": "アセット", diff --git a/frontend/src/locales/languages/ko.json b/frontend/src/locales/languages/ko.json index 6ea541c9..15b329a4 100644 --- a/frontend/src/locales/languages/ko.json +++ b/frontend/src/locales/languages/ko.json @@ -488,6 +488,14 @@ "description": "운영 메시지와 오류를 포함한 Orbit 이벤트 기록을 확인합니다.", "empty": "아직 기록된 Orbit 이벤트가 없습니다." }, + "aiUsage": { + "title": "AI 사용량 및 속도 제한", + "description": "로컬 Azure OpenAI 요청 메타데이터와 속도 제한 헤더를 확인합니다. 프롬프트와 응답 원문은 보관하지 않습니다.", + "range": "기간", "refresh": "새로고침", "rpm": "RPM (최근 1분)", "tpm": "TPM (최근 1분)", "successRate": "성공률", "averageLatency": "평균 지연 시간", + "requests": "요청", "tokens": "토큰", "limits": "최근 Azure 제한 헤더", "remainingRequests": "남은 요청", "remainingTokens": "남은 토큰", "retryAfter": "재시도 후", + "requestTrend": "요청 추이", "tokenTrend": "토큰 추이", "recentCalls": "최근 모델 호출", "time": "시간", "model": "모델", "status": "상태", "usage": "입력 / 출력", "latency": "지연 시간", "details": "선택한 호출", "input": "입력", "output": "출력", "tools": "도구", + "empty": "이 기간에 보관된 모델 호출이 없습니다.", "unavailable": "사용량 데이터를 불러올 수 없습니다. 로컬 Orbit 서버를 시작하거나 새로고침하세요.", "noLimitData": "아직 Azure 속도 제한 헤더를 받지 못했습니다." + }, "improvements": "향상", "dashboard": "대시보드", "assets": "자산", diff --git a/frontend/src/theme-overrides.css b/frontend/src/theme-overrides.css index 523bca00..f0c56589 100644 --- a/frontend/src/theme-overrides.css +++ b/frontend/src/theme-overrides.css @@ -112,6 +112,7 @@ footer { display:flex; align-items:center; justify-content:space-between; margin .dashboard-hero-actions { display:flex; margin-top:6px; }.dashboard-hero-actions .approve { padding:9px 13px; font-size:11px; font-weight:700; } .dashboard-overview .panel-head { margin-bottom:5px; }.dashboard-overview > .hint { margin:0 0 18px; } .dashboard-feedback-trends { grid-template-columns:repeat(2,minmax(0,1fr)); }.dashboard-feedback-trends .trend-head,.dashboard-feedback-trends > .analytics-chart:last-child { grid-column:1/-1; }.dashboard-feedback-trends .trend-head:has(.section-description),.evaluation-feedback-trends .trend-head:has(.section-description) { margin-bottom:-6px; }.dashboard-feedback-trends > .analytics-grid { display:contents; } @media(max-width:720px){.dashboard-feedback-trends { grid-template-columns:1fr; }} +.ai-usage-metrics { display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:10px; margin:14px 0; }.ai-usage-metrics>div,.ai-usage-limits,.ai-usage-detail { padding:11px; border:1px solid var(--line); border-radius:8px; background:var(--bg); }.ai-usage-metrics span,.ai-usage-limits span,.ai-usage-detail span { display:block; color:var(--muted); font-size:12px; }.ai-usage-metrics strong { display:block; margin-top:4px; font-size:18px; }.ai-usage-error { color:#c7514a; font-weight:700; }.ai-usage-warning { margin:10px 0; }.ai-usage-limits { display:flex; gap:10px; align-items:baseline; margin:14px 0; }.ai-usage-table h3 { margin:14px 0 8px; font-size:13px; }.ai-usage-detail { display:grid; gap:5px; margin-top:10px; }.spin { animation:orbit-spin 1s linear infinite; } @keyframes orbit-spin { to { transform:rotate(360deg); } } @media(max-width:720px){.ai-usage-metrics { grid-template-columns:repeat(2,minmax(0,1fr)); }.ai-usage-limits { display:grid; }} .build-table .tr.th,.active-evaluation-table .tr.th { margin-bottom:6px; border-top:0; border-bottom:1px solid var(--line); }.build-table .tr:not(.th):hover { background:var(--surface-raised); }.table-link { border:0; padding:0; color:inherit; background:transparent; font:inherit; font-weight:700; text-align:left; cursor:pointer; } .build-name { display:flex; align-items:center; gap:7px; min-width:0; }.build-star { display:grid; flex:none; place-items:center; width:24px; height:24px; padding:0; border:0; border-radius:5px; color:var(--muted); background:transparent; cursor:pointer; }.build-star:hover,.build-star:focus-visible { color:#f1c85b; background:color-mix(in srgb,#f1c85b 13%,transparent); outline:0; }.build-star--active { color:#f1c85b; } .app-content { padding-inline:32px; } From 8655098276378d05e027dec145105a97c034d2fe Mon Sep 17 00:00:00 2001 From: forthfate Date: Sun, 27 Sep 2026 01:49:00 +0900 Subject: [PATCH 2/5] fix: refine AI usage monitor empty state --- frontend/src/features/settings/ai-usage-monitor.tsx | 2 +- frontend/src/theme-overrides.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/settings/ai-usage-monitor.tsx b/frontend/src/features/settings/ai-usage-monitor.tsx index bf76be69..8030a4a6 100644 --- a/frontend/src/features/settings/ai-usage-monitor.tsx +++ b/frontend/src/features/settings/ai-usage-monitor.tsx @@ -50,7 +50,7 @@ export function AiUsageMonitor({ locale }: { locale: Locale }) {

{t.requestTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />

{t.tokenTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />
-
{t.limits}{limits?.remaining_requests || limits?.remaining_tokens ? {t.remainingRequests}: {limits.remaining_requests ?? "—"}{limits.limit_requests ? ` / ${limits.limit_requests}` : ""} · {t.remainingTokens}: {limits.remaining_tokens ?? "—"}{limits.limit_tokens ? ` / ${limits.limit_tokens}` : ""}{limits.retry_after ? ` · ${t.retryAfter}: ${limits.retry_after}s` : ""} : {t.noLimitData}}
+ {limits?.remaining_requests || limits?.remaining_tokens ?
{t.limits}{t.remainingRequests}: {limits.remaining_requests ?? "—"}{limits.limit_requests ? ` / ${limits.limit_requests}` : ""} · {t.remainingTokens}: {limits.remaining_tokens ?? "—"}{limits.limit_tokens ? ` / ${limits.limit_tokens}` : ""}{limits.retry_after ? ` · ${t.retryAfter}: ${limits.retry_after}s` : ""}
:

{t.noLimitData}

}

{t.recentCalls}

{selected &&
{t.details}{t.input}: {format(selected.input_tokens)} tokens / {format(selected.input_chars)} chars · {t.output}: {format(selected.output_tokens)} tokens / {format(selected.output_chars)} chars · {t.tools}: {selected.tool_count}{selected.error && {selected.error}}
} } diff --git a/frontend/src/theme-overrides.css b/frontend/src/theme-overrides.css index f0c56589..49c0a4f3 100644 --- a/frontend/src/theme-overrides.css +++ b/frontend/src/theme-overrides.css @@ -112,7 +112,7 @@ footer { display:flex; align-items:center; justify-content:space-between; margin .dashboard-hero-actions { display:flex; margin-top:6px; }.dashboard-hero-actions .approve { padding:9px 13px; font-size:11px; font-weight:700; } .dashboard-overview .panel-head { margin-bottom:5px; }.dashboard-overview > .hint { margin:0 0 18px; } .dashboard-feedback-trends { grid-template-columns:repeat(2,minmax(0,1fr)); }.dashboard-feedback-trends .trend-head,.dashboard-feedback-trends > .analytics-chart:last-child { grid-column:1/-1; }.dashboard-feedback-trends .trend-head:has(.section-description),.evaluation-feedback-trends .trend-head:has(.section-description) { margin-bottom:-6px; }.dashboard-feedback-trends > .analytics-grid { display:contents; } @media(max-width:720px){.dashboard-feedback-trends { grid-template-columns:1fr; }} -.ai-usage-metrics { display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:10px; margin:14px 0; }.ai-usage-metrics>div,.ai-usage-limits,.ai-usage-detail { padding:11px; border:1px solid var(--line); border-radius:8px; background:var(--bg); }.ai-usage-metrics span,.ai-usage-limits span,.ai-usage-detail span { display:block; color:var(--muted); font-size:12px; }.ai-usage-metrics strong { display:block; margin-top:4px; font-size:18px; }.ai-usage-error { color:#c7514a; font-weight:700; }.ai-usage-warning { margin:10px 0; }.ai-usage-limits { display:flex; gap:10px; align-items:baseline; margin:14px 0; }.ai-usage-table h3 { margin:14px 0 8px; font-size:13px; }.ai-usage-detail { display:grid; gap:5px; margin-top:10px; }.spin { animation:orbit-spin 1s linear infinite; } @keyframes orbit-spin { to { transform:rotate(360deg); } } @media(max-width:720px){.ai-usage-metrics { grid-template-columns:repeat(2,minmax(0,1fr)); }.ai-usage-limits { display:grid; }} +.ai-usage-metrics { display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:10px; margin:14px 0; }.ai-usage-metrics>div,.ai-usage-limits,.ai-usage-detail { padding:11px; border:1px solid var(--line); border-radius:8px; background:var(--bg); }.ai-usage-metrics span,.ai-usage-limits span,.ai-usage-detail span { display:block; color:var(--muted); font-size:12px; }.ai-usage-metrics strong { display:block; margin-top:4px; font-size:18px; }.ai-usage-error { color:#c7514a; font-weight:700; }.ai-usage-warning { margin:10px 0; }.ai-usage-limits { display:flex; gap:10px; align-items:baseline; margin:14px 0; }.ai-usage-monitor .panel-head .setting-actions>label { display:inline-flex; align-items:center; gap:8px; white-space:nowrap; }.ai-usage-table h3 { margin:14px 0 8px; font-size:13px; }.ai-usage-detail { display:grid; gap:5px; margin-top:10px; }.spin { animation:orbit-spin 1s linear infinite; } @keyframes orbit-spin { to { transform:rotate(360deg); } } @media(max-width:720px){.ai-usage-metrics { grid-template-columns:repeat(2,minmax(0,1fr)); }.ai-usage-limits { display:grid; }} .build-table .tr.th,.active-evaluation-table .tr.th { margin-bottom:6px; border-top:0; border-bottom:1px solid var(--line); }.build-table .tr:not(.th):hover { background:var(--surface-raised); }.table-link { border:0; padding:0; color:inherit; background:transparent; font:inherit; font-weight:700; text-align:left; cursor:pointer; } .build-name { display:flex; align-items:center; gap:7px; min-width:0; }.build-star { display:grid; flex:none; place-items:center; width:24px; height:24px; padding:0; border:0; border-radius:5px; color:var(--muted); background:transparent; cursor:pointer; }.build-star:hover,.build-star:focus-visible { color:#f1c85b; background:color-mix(in srgb,#f1c85b 13%,transparent); outline:0; }.build-star--active { color:#f1c85b; } .app-content { padding-inline:32px; } From ca2ac2faf70e846d4e569ef5f7e02f1e5d28db1f Mon Sep 17 00:00:00 2001 From: forthfate Date: Sun, 27 Sep 2026 01:54:01 +0900 Subject: [PATCH 3/5] fix: clarify AI usage rates by profile --- backend/app/providers.py | 2 ++ backend/app/store.py | 25 +++++++++++++------ backend/tests/test_api.py | 5 ++-- .../features/settings/ai-usage-monitor.tsx | 17 +++++++------ frontend/src/locales/languages/en.json | 2 +- frontend/src/locales/languages/ja.json | 2 +- frontend/src/locales/languages/ko.json | 2 +- frontend/src/theme-overrides.css | 1 + 8 files changed, 37 insertions(+), 19 deletions(-) diff --git a/backend/app/providers.py b/backend/app/providers.py index 9744750e..252bbd8f 100644 --- a/backend/app/providers.py +++ b/backend/app/providers.py @@ -21,6 +21,7 @@ @dataclass(frozen=True) class ModelSettings: + profile_name: str = "" provider: str = "azure-openai" model: str = "" endpoint: str = "" @@ -44,6 +45,7 @@ def _post(settings: ModelSettings, key: str, payload: dict[str, Any]) -> dict[st span.set_attributes( { "gen_ai.system": "azure_openai", + "gen_ai.request.profile": settings.profile_name or settings.model, "gen_ai.request.model": settings.model, "gen_ai.request.input_chars": input_chars, "gen_ai.request.tool_count": len(tools) if isinstance(tools, list) else 0, diff --git a/backend/app/store.py b/backend/app/store.py index 89eeb378..d3a6392e 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -3007,6 +3007,8 @@ def ai_usage(self, minutes: int = 60) -> dict[str, Any]: "id": span.get("spanId", ""), "time": datetime.fromtimestamp(timestamp, UTC).isoformat(), "model": a.get("gen_ai.request.model", ""), + "profile": a.get("gen_ai.request.profile") + or a.get("gen_ai.request.model", ""), "status": int(a.get("http.response.status_code", 0) or 0), "failed": span.get("status") == "ERROR", "input_tokens": int(a.get("gen_ai.usage.input_tokens", 0) or 0), @@ -3047,11 +3049,18 @@ def ai_usage(self, minutes: int = 60) -> dict[str, Any]: requests = len(events) errors = sum(event["failed"] or event["status"] >= 400 for event in events) last = events[0] if events else {} - recent = [ - event - for event in events - if datetime.fromisoformat(event["time"]).timestamp() >= now.timestamp() - 60 - ] + profiles = sorted({str(event["profile"]) for event in events if event["profile"]}) + for point in buckets.values(): + for profile_name in profiles: + point[profile_name] = 0 + point[f"tokens:{profile_name}"] = 0 + for event in chronological: + stamp = datetime.fromisoformat(event["time"]).timestamp() + bucket = int(stamp // bucket_seconds) * bucket_seconds + point = buckets[bucket] + point[event["profile"]] = int(point[event["profile"]]) + 1 + token_key = f"tokens:{event['profile']}" + point[token_key] = int(point[token_key]) + event["total_tokens"] return { "minutes": minutes, "summary": { @@ -3062,8 +3071,9 @@ def ai_usage(self, minutes: int = 60) -> dict[str, Any]: "average_duration_ms": round(sum(event["duration_ms"] for event in events) / requests) if requests else 0, - "rpm": len(recent), - "tpm": sum(event["total_tokens"] for event in recent), + "rpm": round(requests / minutes, 2), + "tpm": round(sum(event["total_tokens"] for event in events) / minutes, 2), + "rate_interval_seconds": bucket_seconds, }, "limits": { key: last.get(key) @@ -3076,6 +3086,7 @@ def ai_usage(self, minutes: int = 60) -> dict[str, Any]: ) }, "timeline": list(buckets.values()), + "profiles": profiles, "events": events[:100], } diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 29809ad1..2e61a0f2 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1551,8 +1551,9 @@ def test_ai_usage_summarizes_provider_spans_without_prompt_content(monkeypatch, "errors": 1, "success_rate": 0.0, "average_duration_ms": 120, - "rpm": 1, - "tpm": 17, + "rpm": 0.07, + "tpm": 1.13, + "rate_interval_seconds": 60, } assert result["limits"]["remaining_requests"] == "0" assert result["events"][0]["model"] == "gpt-test" diff --git a/frontend/src/features/settings/ai-usage-monitor.tsx b/frontend/src/features/settings/ai-usage-monitor.tsx index 8030a4a6..3bca6b8f 100644 --- a/frontend/src/features/settings/ai-usage-monitor.tsx +++ b/frontend/src/features/settings/ai-usage-monitor.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from "react"; import { AlertTriangle, RefreshCw } from "lucide-react"; -import { Area, AreaChart, CartesianGrid, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; +import { Area, AreaChart, CartesianGrid, Legend, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; import type { Locale } from "../../locales"; import { intlLocales, localeMessages } from "../../locales"; import { DataTable, type Column } from "../../components/ui/data-table"; @@ -9,11 +9,13 @@ import { SectionInfo } from "../../components/ui/section-info"; import { api } from "../../services/api"; type UsageEvent = { id: string; time: string; model: string; status: number; failed: boolean; input_tokens: number; output_tokens: number; total_tokens: number; input_chars: number; output_chars: number; tool_count: number; duration_ms: number; error: string; limit_requests?: string; remaining_requests?: string; limit_tokens?: string; remaining_tokens?: string; retry_after?: string }; -type AiUsage = { minutes: number; summary: { requests: number; tokens: number; errors: number; success_rate: number; average_duration_ms: number; rpm: number; tpm: number }; limits: Record; timeline: { time: string; requests: number; tokens: number }[]; events: UsageEvent[] }; +type AiUsage = { minutes: number; summary: { requests: number; tokens: number; errors: number; success_rate: number; average_duration_ms: number; rpm: number; tpm: number; rate_interval_seconds: number }; limits: Record; timeline: ({ time: string; requests: number; tokens: number } & Record)[]; profiles: string[]; events: UsageEvent[] }; type Copy = { title: string; description: string; range: string; refresh: string; rpm: string; tpm: string; successRate: string; averageLatency: string; requests: string; tokens: string; limits: string; remainingRequests: string; remainingTokens: string; retryAfter: string; requestTrend: string; tokenTrend: string; recentCalls: string; time: string; model: string; status: string; usage: string; latency: string; details: string; input: string; output: string; tools: string; empty: string; unavailable: string; noLimitData: string }; const format = (value: number) => new Intl.NumberFormat().format(value); const tick = (value: string, locale: Locale) => new Date(value).toLocaleTimeString(intlLocales[locale], { hour: "2-digit", minute: "2-digit" }); +const duration = (seconds: number) => seconds >= 3600 ? `${seconds / 3600}h` : `${seconds / 60}m`; +const profileColors = ["#8fb8ff", "#79c99e", "#f1d292", "#d7b7ff", "#eaa89f"]; export function AiUsageMonitor({ locale }: { locale: Locale }) { const t = localeMessages(locale, "aiUsage"); @@ -38,23 +40,24 @@ export function AiUsageMonitor({ locale }: { locale: Locale }) { { id: "latency", header: t.latency, render: (row) => `${format(row.duration_ms)} ms`, sortValue: (row) => row.duration_ms }, ], [locale, t]); const limits = data?.limits; + const rateDetail = data ? `${duration(data.summary.rate_interval_seconds)} aggregation · ${duration(data.minutes * 60)} window` : ""; return
} action={
} /> {!data ?

{t.unavailable}

: <>
- + 0} />
{data.summary.errors > 0 &&

{data.summary.errors} {t.requests}

}
-

{t.requestTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />
-

{t.tokenTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />
+

{t.requestTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />{data.profiles.map((profile, index) => )}
+

{t.tokenTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />{data.profiles.map((profile, index) => )}
{limits?.remaining_requests || limits?.remaining_tokens ?
{t.limits}{t.remainingRequests}: {limits.remaining_requests ?? "—"}{limits.limit_requests ? ` / ${limits.limit_requests}` : ""} · {t.remainingTokens}: {limits.remaining_tokens ?? "—"}{limits.limit_tokens ? ` / ${limits.limit_tokens}` : ""}{limits.retry_after ? ` · ${t.retryAfter}: ${limits.retry_after}s` : ""}
:

{t.noLimitData}

} -

{t.recentCalls}

+

{t.recentCalls}

{selected &&
{t.details}{t.input}: {format(selected.input_tokens)} tokens / {format(selected.input_chars)} chars · {t.output}: {format(selected.output_tokens)} tokens / {format(selected.output_chars)} chars · {t.tools}: {selected.tool_count}{selected.error && {selected.error}}
} }
; } -function Metric({ label, value, warning = false }: { label: string; value: string; warning?: boolean }) { return
{label}{value}
; } +function Metric({ label, value, detail, warning = false }: { label: string; value: string; detail?: string; warning?: boolean }) { return
{label}{value}{detail && {detail}}
; } diff --git a/frontend/src/locales/languages/en.json b/frontend/src/locales/languages/en.json index eeb98f2d..dc5ce8e0 100644 --- a/frontend/src/locales/languages/en.json +++ b/frontend/src/locales/languages/en.json @@ -492,7 +492,7 @@ "aiUsage": { "title": "AI usage & rate limits", "description": "Monitor local Azure OpenAI request metadata and rate-limit headers. Prompt and response text are not retained.", - "range": "Range", "refresh": "Refresh", "rpm": "RPM (last minute)", "tpm": "TPM (last minute)", "successRate": "Success rate", "averageLatency": "Average latency", + "range": "Range", "refresh": "Refresh", "rpm": "Average RPM", "tpm": "Average TPM", "successRate": "Success rate", "averageLatency": "Average latency", "requests": "Requests", "tokens": "Tokens", "limits": "Latest Azure limit headers", "remainingRequests": "Requests remaining", "remainingTokens": "Tokens remaining", "retryAfter": "Retry after", "requestTrend": "Request trend", "tokenTrend": "Token trend", "recentCalls": "Recent model calls", "time": "Time", "model": "Model", "status": "Status", "usage": "Input / output", "latency": "Latency", "details": "Selected call", "input": "Input", "output": "Output", "tools": "Tools", "empty": "No model calls were retained in this period.", "unavailable": "Usage data could not be loaded. Start or refresh the local Orbit server.", "noLimitData": "No Azure rate-limit headers have been received yet." diff --git a/frontend/src/locales/languages/ja.json b/frontend/src/locales/languages/ja.json index f5d9e9a4..06a3501c 100644 --- a/frontend/src/locales/languages/ja.json +++ b/frontend/src/locales/languages/ja.json @@ -486,7 +486,7 @@ "aiUsage": { "title": "AI 使用量とレート制限", "description": "ローカルの Azure OpenAI リクエストメタデータとレート制限ヘッダーを確認します。プロンプトと応答本文は保存しません。", - "range": "期間", "refresh": "更新", "rpm": "RPM（直近1分）", "tpm": "TPM（直近1分）", "successRate": "成功率", "averageLatency": "平均レイテンシー", + "range": "期間", "refresh": "更新", "rpm": "平均 RPM", "tpm": "平均 TPM", "successRate": "成功率", "averageLatency": "平均レイテンシー", "requests": "リクエスト", "tokens": "トークン", "limits": "最新の Azure 制限ヘッダー", "remainingRequests": "残りリクエスト", "remainingTokens": "残りトークン", "retryAfter": "再試行まで", "requestTrend": "リクエスト推移", "tokenTrend": "トークン推移", "recentCalls": "最近のモデル呼び出し", "time": "時刻", "model": "モデル", "status": "状態", "usage": "入力 / 出力", "latency": "レイテンシー", "details": "選択した呼び出し", "input": "入力", "output": "出力", "tools": "ツール", "empty": "この期間に保存されたモデル呼び出しはありません。", "unavailable": "使用量データを読み込めません。ローカル Orbit サーバーを開始または更新してください。", "noLimitData": "Azure のレート制限ヘッダーはまだ受信していません。" diff --git a/frontend/src/locales/languages/ko.json b/frontend/src/locales/languages/ko.json index 15b329a4..85203987 100644 --- a/frontend/src/locales/languages/ko.json +++ b/frontend/src/locales/languages/ko.json @@ -491,7 +491,7 @@ "aiUsage": { "title": "AI 사용량 및 속도 제한", "description": "로컬 Azure OpenAI 요청 메타데이터와 속도 제한 헤더를 확인합니다. 프롬프트와 응답 원문은 보관하지 않습니다.", - "range": "기간", "refresh": "새로고침", "rpm": "RPM (최근 1분)", "tpm": "TPM (최근 1분)", "successRate": "성공률", "averageLatency": "평균 지연 시간", + "range": "기간", "refresh": "새로고침", "rpm": "평균 RPM", "tpm": "평균 TPM", "successRate": "성공률", "averageLatency": "평균 지연 시간", "requests": "요청", "tokens": "토큰", "limits": "최근 Azure 제한 헤더", "remainingRequests": "남은 요청", "remainingTokens": "남은 토큰", "retryAfter": "재시도 후", "requestTrend": "요청 추이", "tokenTrend": "토큰 추이", "recentCalls": "최근 모델 호출", "time": "시간", "model": "모델", "status": "상태", "usage": "입력 / 출력", "latency": "지연 시간", "details": "선택한 호출", "input": "입력", "output": "출력", "tools": "도구", "empty": "이 기간에 보관된 모델 호출이 없습니다.", "unavailable": "사용량 데이터를 불러올 수 없습니다. 로컬 Orbit 서버를 시작하거나 새로고침하세요.", "noLimitData": "아직 Azure 속도 제한 헤더를 받지 못했습니다." diff --git a/frontend/src/theme-overrides.css b/frontend/src/theme-overrides.css index 49c0a4f3..d3fb3c6d 100644 --- a/frontend/src/theme-overrides.css +++ b/frontend/src/theme-overrides.css @@ -113,6 +113,7 @@ footer { display:flex; align-items:center; justify-content:space-between; margin .dashboard-overview .panel-head { margin-bottom:5px; }.dashboard-overview > .hint { margin:0 0 18px; } .dashboard-feedback-trends { grid-template-columns:repeat(2,minmax(0,1fr)); }.dashboard-feedback-trends .trend-head,.dashboard-feedback-trends > .analytics-chart:last-child { grid-column:1/-1; }.dashboard-feedback-trends .trend-head:has(.section-description),.evaluation-feedback-trends .trend-head:has(.section-description) { margin-bottom:-6px; }.dashboard-feedback-trends > .analytics-grid { display:contents; } @media(max-width:720px){.dashboard-feedback-trends { grid-template-columns:1fr; }} .ai-usage-metrics { display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:10px; margin:14px 0; }.ai-usage-metrics>div,.ai-usage-limits,.ai-usage-detail { padding:11px; border:1px solid var(--line); border-radius:8px; background:var(--bg); }.ai-usage-metrics span,.ai-usage-limits span,.ai-usage-detail span { display:block; color:var(--muted); font-size:12px; }.ai-usage-metrics strong { display:block; margin-top:4px; font-size:18px; }.ai-usage-error { color:#c7514a; font-weight:700; }.ai-usage-warning { margin:10px 0; }.ai-usage-limits { display:flex; gap:10px; align-items:baseline; margin:14px 0; }.ai-usage-monitor .panel-head .setting-actions>label { display:inline-flex; align-items:center; gap:8px; white-space:nowrap; }.ai-usage-table h3 { margin:14px 0 8px; font-size:13px; }.ai-usage-detail { display:grid; gap:5px; margin-top:10px; }.spin { animation:orbit-spin 1s linear infinite; } @keyframes orbit-spin { to { transform:rotate(360deg); } } @media(max-width:720px){.ai-usage-metrics { grid-template-columns:repeat(2,minmax(0,1fr)); }.ai-usage-limits { display:grid; }} +.ai-usage-metrics small { display:block; margin-top:4px; color:var(--muted); font-size:10px; }.ai-usage-table-scroll { max-height:360px; overflow-y:auto; border-bottom:1px solid var(--line); } .build-table .tr.th,.active-evaluation-table .tr.th { margin-bottom:6px; border-top:0; border-bottom:1px solid var(--line); }.build-table .tr:not(.th):hover { background:var(--surface-raised); }.table-link { border:0; padding:0; color:inherit; background:transparent; font:inherit; font-weight:700; text-align:left; cursor:pointer; } .build-name { display:flex; align-items:center; gap:7px; min-width:0; }.build-star { display:grid; flex:none; place-items:center; width:24px; height:24px; padding:0; border:0; border-radius:5px; color:var(--muted); background:transparent; cursor:pointer; }.build-star:hover,.build-star:focus-visible { color:#f1c85b; background:color-mix(in srgb,#f1c85b 13%,transparent); outline:0; }.build-star--active { color:#f1c85b; } .app-content { padding-inline:32px; } From 462d3b69047a5d6d6dc594310a99cb7a0f880f93 Mon Sep 17 00:00:00 2001 From: forthfate Date: Sun, 27 Sep 2026 02:02:32 +0900 Subject: [PATCH 4/5] feat: expose model call character volume --- backend/app/providers.py | 1 + backend/app/store.py | 1 + frontend/src/features/settings/ai-usage-monitor.tsx | 9 +++++---- frontend/src/locales/languages/en.json | 2 +- frontend/src/locales/languages/ja.json | 2 +- frontend/src/locales/languages/ko.json | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/backend/app/providers.py b/backend/app/providers.py index 252bbd8f..d6035706 100644 --- a/backend/app/providers.py +++ b/backend/app/providers.py @@ -47,6 +47,7 @@ def _post(settings: ModelSettings, key: str, payload: dict[str, Any]) -> dict[st "gen_ai.system": "azure_openai", "gen_ai.request.profile": settings.profile_name or settings.model, "gen_ai.request.model": settings.model, + "gen_ai.request.mode": "tool_result" if payload.get("previous_response_id") else "prompt", "gen_ai.request.input_chars": input_chars, "gen_ai.request.tool_count": len(tools) if isinstance(tools, list) else 0, } diff --git a/backend/app/store.py b/backend/app/store.py index d3a6392e..ed366250 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -3009,6 +3009,7 @@ def ai_usage(self, minutes: int = 60) -> dict[str, Any]: "model": a.get("gen_ai.request.model", ""), "profile": a.get("gen_ai.request.profile") or a.get("gen_ai.request.model", ""), + "request_mode": a.get("gen_ai.request.mode", "prompt"), "status": int(a.get("http.response.status_code", 0) or 0), "failed": span.get("status") == "ERROR", "input_tokens": int(a.get("gen_ai.usage.input_tokens", 0) or 0), diff --git a/frontend/src/features/settings/ai-usage-monitor.tsx b/frontend/src/features/settings/ai-usage-monitor.tsx index 3bca6b8f..2ecc5b21 100644 --- a/frontend/src/features/settings/ai-usage-monitor.tsx +++ b/frontend/src/features/settings/ai-usage-monitor.tsx @@ -8,9 +8,9 @@ import { PanelHeader } from "../../components/ui/page-header"; import { SectionInfo } from "../../components/ui/section-info"; import { api } from "../../services/api"; -type UsageEvent = { id: string; time: string; model: string; status: number; failed: boolean; input_tokens: number; output_tokens: number; total_tokens: number; input_chars: number; output_chars: number; tool_count: number; duration_ms: number; error: string; limit_requests?: string; remaining_requests?: string; limit_tokens?: string; remaining_tokens?: string; retry_after?: string }; +type UsageEvent = { id: string; time: string; model: string; request_mode: string; status: number; failed: boolean; input_tokens: number; output_tokens: number; total_tokens: number; input_chars: number; output_chars: number; tool_count: number; duration_ms: number; error: string; limit_requests?: string; remaining_requests?: string; limit_tokens?: string; remaining_tokens?: string; retry_after?: string }; type AiUsage = { minutes: number; summary: { requests: number; tokens: number; errors: number; success_rate: number; average_duration_ms: number; rpm: number; tpm: number; rate_interval_seconds: number }; limits: Record; timeline: ({ time: string; requests: number; tokens: number } & Record)[]; profiles: string[]; events: UsageEvent[] }; -type Copy = { title: string; description: string; range: string; refresh: string; rpm: string; tpm: string; successRate: string; averageLatency: string; requests: string; tokens: string; limits: string; remainingRequests: string; remainingTokens: string; retryAfter: string; requestTrend: string; tokenTrend: string; recentCalls: string; time: string; model: string; status: string; usage: string; latency: string; details: string; input: string; output: string; tools: string; empty: string; unavailable: string; noLimitData: string }; +type Copy = { title: string; description: string; range: string; refresh: string; rpm: string; tpm: string; successRate: string; averageLatency: string; requests: string; tokens: string; limits: string; remainingRequests: string; remainingTokens: string; retryAfter: string; requestTrend: string; tokenTrend: string; recentCalls: string; time: string; model: string; status: string; usage: string; characters: string; latency: string; details: string; requestType: string; promptRequest: string; toolResultRequest: string; input: string; output: string; tools: string; empty: string; unavailable: string; noLimitData: string }; const format = (value: number) => new Intl.NumberFormat().format(value); const tick = (value: string, locale: Locale) => new Date(value).toLocaleTimeString(intlLocales[locale], { hour: "2-digit", minute: "2-digit" }); @@ -37,6 +37,7 @@ export function AiUsageMonitor({ locale }: { locale: Locale }) { { id: "model", header: t.model, render: (row) => row.model || "—", sortValue: (row) => row.model }, { id: "status", header: t.status, render: (row) => = 400 ? "ai-usage-error" : ""}>{row.status || (row.failed ? "Error" : "—")}, sortValue: (row) => row.status }, { id: "usage", header: t.usage, render: (row) => `${format(row.input_tokens)} / ${format(row.output_tokens)}`, sortValue: (row) => row.total_tokens }, + { id: "characters", header: t.characters, render: (row) => `${format(row.input_chars)} / ${format(row.output_chars)}`, sortValue: (row) => row.input_chars + row.output_chars }, { id: "latency", header: t.latency, render: (row) => `${format(row.duration_ms)} ms`, sortValue: (row) => row.duration_ms }, ], [locale, t]); const limits = data?.limits; @@ -54,8 +55,8 @@ export function AiUsageMonitor({ locale }: { locale: Locale }) {

{t.tokenTrend}

tick(String(value), locale)} /> new Date(String(value)).toLocaleString(intlLocales[locale])} />{data.profiles.map((profile, index) => )}
{limits?.remaining_requests || limits?.remaining_tokens ?
{t.limits}{t.remainingRequests}: {limits.remaining_requests ?? "—"}{limits.limit_requests ? ` / ${limits.limit_requests}` : ""} · {t.remainingTokens}: {limits.remaining_tokens ?? "—"}{limits.limit_tokens ? ` / ${limits.limit_tokens}` : ""}{limits.retry_after ? ` · ${t.retryAfter}: ${limits.retry_after}s` : ""}
:

{t.noLimitData}

} -

{t.recentCalls}

- {selected &&
{t.details}{t.input}: {format(selected.input_tokens)} tokens / {format(selected.input_chars)} chars · {t.output}: {format(selected.output_tokens)} tokens / {format(selected.output_chars)} chars · {t.tools}: {selected.tool_count}{selected.error && {selected.error}}
} +

{t.recentCalls}

+ {selected &&
{t.details}{t.requestType}: {selected.request_mode === "tool_result" ? t.toolResultRequest : t.promptRequest} · {t.input}: {format(selected.input_tokens)} tokens / {format(selected.input_chars)} chars · {t.output}: {format(selected.output_tokens)} tokens / {format(selected.output_chars)} chars · {t.tools}: {selected.tool_count}{selected.error && {selected.error}}
} } ; } diff --git a/frontend/src/locales/languages/en.json b/frontend/src/locales/languages/en.json index dc5ce8e0..51e2b10c 100644 --- a/frontend/src/locales/languages/en.json +++ b/frontend/src/locales/languages/en.json @@ -494,7 +494,7 @@ "description": "Monitor local Azure OpenAI request metadata and rate-limit headers. Prompt and response text are not retained.", "range": "Range", "refresh": "Refresh", "rpm": "Average RPM", "tpm": "Average TPM", "successRate": "Success rate", "averageLatency": "Average latency", "requests": "Requests", "tokens": "Tokens", "limits": "Latest Azure limit headers", "remainingRequests": "Requests remaining", "remainingTokens": "Tokens remaining", "retryAfter": "Retry after", - "requestTrend": "Request trend", "tokenTrend": "Token trend", "recentCalls": "Recent model calls", "time": "Time", "model": "Model", "status": "Status", "usage": "Input / output", "latency": "Latency", "details": "Selected call", "input": "Input", "output": "Output", "tools": "Tools", + "requestTrend": "Request trend", "tokenTrend": "Token trend", "recentCalls": "Recent model calls", "time": "Time", "model": "Model", "status": "Status", "usage": "Input / output", "characters": "Input / output chars", "latency": "Latency", "details": "Selected call", "requestType": "Request type", "promptRequest": "Prompt request", "toolResultRequest": "Tool-result continuation", "input": "Input", "output": "Output", "tools": "Tools", "empty": "No model calls were retained in this period.", "unavailable": "Usage data could not be loaded. Start or refresh the local Orbit server.", "noLimitData": "No Azure rate-limit headers have been received yet." }, "improvements": "Improvements", diff --git a/frontend/src/locales/languages/ja.json b/frontend/src/locales/languages/ja.json index 06a3501c..3cd27986 100644 --- a/frontend/src/locales/languages/ja.json +++ b/frontend/src/locales/languages/ja.json @@ -488,7 +488,7 @@ "description": "ローカルの Azure OpenAI リクエストメタデータとレート制限ヘッダーを確認します。プロンプトと応答本文は保存しません。", "range": "期間", "refresh": "更新", "rpm": "平均 RPM", "tpm": "平均 TPM", "successRate": "成功率", "averageLatency": "平均レイテンシー", "requests": "リクエスト", "tokens": "トークン", "limits": "最新の Azure 制限ヘッダー", "remainingRequests": "残りリクエスト", "remainingTokens": "残りトークン", "retryAfter": "再試行まで", - "requestTrend": "リクエスト推移", "tokenTrend": "トークン推移", "recentCalls": "最近のモデル呼び出し", "time": "時刻", "model": "モデル", "status": "状態", "usage": "入力 / 出力", "latency": "レイテンシー", "details": "選択した呼び出し", "input": "入力", "output": "出力", "tools": "ツール", + "requestTrend": "リクエスト推移", "tokenTrend": "トークン推移", "recentCalls": "最近のモデル呼び出し", "time": "時刻", "model": "モデル", "status": "状態", "usage": "入力 / 出力", "characters": "入力 / 出力文字数", "latency": "レイテンシー", "details": "選択した呼び出し", "requestType": "リクエスト種別", "promptRequest": "プロンプトリクエスト", "toolResultRequest": "ツール結果の後続リクエスト", "input": "入力", "output": "出力", "tools": "ツール", "empty": "この期間に保存されたモデル呼び出しはありません。", "unavailable": "使用量データを読み込めません。ローカル Orbit サーバーを開始または更新してください。", "noLimitData": "Azure のレート制限ヘッダーはまだ受信していません。" }, "improvements": "改善", diff --git a/frontend/src/locales/languages/ko.json b/frontend/src/locales/languages/ko.json index 85203987..ccc6817a 100644 --- a/frontend/src/locales/languages/ko.json +++ b/frontend/src/locales/languages/ko.json @@ -493,7 +493,7 @@ "description": "로컬 Azure OpenAI 요청 메타데이터와 속도 제한 헤더를 확인합니다. 프롬프트와 응답 원문은 보관하지 않습니다.", "range": "기간", "refresh": "새로고침", "rpm": "평균 RPM", "tpm": "평균 TPM", "successRate": "성공률", "averageLatency": "평균 지연 시간", "requests": "요청", "tokens": "토큰", "limits": "최근 Azure 제한 헤더", "remainingRequests": "남은 요청", "remainingTokens": "남은 토큰", "retryAfter": "재시도 후", - "requestTrend": "요청 추이", "tokenTrend": "토큰 추이", "recentCalls": "최근 모델 호출", "time": "시간", "model": "모델", "status": "상태", "usage": "입력 / 출력", "latency": "지연 시간", "details": "선택한 호출", "input": "입력", "output": "출력", "tools": "도구", + "requestTrend": "요청 추이", "tokenTrend": "토큰 추이", "recentCalls": "최근 모델 호출", "time": "시간", "model": "모델", "status": "상태", "usage": "입력 / 출력", "characters": "입력 / 출력 문자", "latency": "지연 시간", "details": "선택한 호출", "requestType": "요청 유형", "promptRequest": "프롬프트 요청", "toolResultRequest": "도구 결과 후속 요청", "input": "입력", "output": "출력", "tools": "도구", "empty": "이 기간에 보관된 모델 호출이 없습니다.", "unavailable": "사용량 데이터를 불러올 수 없습니다. 로컬 Orbit 서버를 시작하거나 새로고침하세요.", "noLimitData": "아직 Azure 속도 제한 헤더를 받지 못했습니다." }, "improvements": "향상", From 9b881c6097af15c3fae2c1e09b0a833250db3c12 Mon Sep 17 00:00:00 2001 From: forthfate Date: Sun, 27 Sep 2026 12:17:09 +0900 Subject: [PATCH 5/5] fix: bound MCP pipeline tool responses --- backend/app/mcp_server.py | 132 +++++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 8 deletions(-) diff --git a/backend/app/mcp_server.py b/backend/app/mcp_server.py index 0cc4c394..ba4273d1 100644 --- a/backend/app/mcp_server.py +++ b/backend/app/mcp_server.py @@ -22,6 +22,112 @@ def _json(value: Any) -> Any: return value.model_dump(mode="json") if hasattr(value, "model_dump") else value +def _text(value: object, limit: int = 500) -> str: + """Keep MCP summaries bounded without losing a useful error description.""" + return str(value or "")[:limit] + + +def _pipeline_summary(value: Any) -> dict[str, Any]: + run = _json(value) + steps = run.get("step_results", []) if isinstance(run, dict) else [] + supervisor = run.get("supervisor_results", []) if isinstance(run, dict) else [] + files = [ + file + for step in steps + if isinstance(step, dict) + for file in step.get("data_files", []) + if isinstance(file, dict) + ] + return { + key: run.get(key) + for key in ( + "id", + "build_id", + "build_name", + "workflow_id", + "workflow_name", + "status", + "current_phase", + "current_step", + "execution_mode", + "execution_type", + "created_at", + "updated_at", + "finished_at", + "loop_limit", + "approval_score", + "supervisor_status", + "supervisor_error", + ) + } | { + "step_count": len(steps), + "supervisor_result_count": len(supervisor), + "evidence_file_count": len(files), + } + + +def _pipeline_detail(value: Any, detail: str, max_items: int) -> dict[str, Any]: + run = _json(value) + result = _pipeline_summary(run) + if detail == "summary": + return result + limit = max(1, min(max_items, 50)) + if detail == "steps": + result["steps"] = [ + { + key: _text(step.get(key)) + if key in {"id", "phase", "name", "status", "error"} + else step.get(key) + for key in ( + "id", + "phase", + "name", + "loop_index", + "status", + "started_at", + "ended_at", + "exit_code", + "error", + ) + } + for step in run.get("step_results", [])[:limit] + if isinstance(step, dict) + ] + elif detail == "supervision": + records = run.get("supervisor_results", []) or [] + result["supervision"] = [ + { + "iteration": record.get("iteration"), + "stage": record.get("stage"), + "status": record.get("status"), + "recorded_at": record.get("recorded_at"), + "error": _text(record.get("error")), + "score": (record.get("response") or {}).get("evaluation", {}).get("score") + if isinstance(record.get("response"), dict) + else None, + "approval": (record.get("response") or {}).get("evaluation", {}).get("approval") + if isinstance(record.get("response"), dict) + else None, + } + for record in records[:limit] + if isinstance(record, dict) + ] + elif detail == "evidence": + files = [ + file + for step in run.get("step_results", []) + if isinstance(step, dict) + for file in step.get("data_files", []) + if isinstance(file, dict) + ] + result["evidence"] = [ + {key: file.get(key) for key in ("label", "filename", "relative_path", "size", "content_type")} + for file in files[:limit] + ] + result["evidence_truncated"] = len(files) > limit + return result + + def create_mcp_server( get_store: Callable[[], ConsoleStore], openapi: Callable[[], dict[str, Any]] ) -> FastMCP: @@ -53,10 +159,14 @@ def openapi_schema() -> str: ) def get_status() -> dict[str, Any]: store = get_store() + dashboard = store.dashboard() return { "health": {"status": "ok"}, - "dashboard": _json(store.dashboard()), - "active_pipelines": _json(store.active_evaluations()), + "dashboard": { + "metrics": dashboard["metrics"], + "active_build_count": len(dashboard["active_builds"]), + }, + "active_pipelines": [_pipeline_summary(run) for run in store.active_evaluations()[:20]], "docker": preflight_docker(), } @@ -71,19 +181,25 @@ def get_project(project_id: str) -> dict[str, Any]: return _json(get_store().build(project_id)) @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) - def list_pipelines(project_id: str | None = None, status: str | None = None) -> list[dict[str, Any]]: - """List retained pipeline runs, optionally filtered by project or status.""" + def list_pipelines( + project_id: str | None = None, status: str | None = None, limit: int = 20 + ) -> list[dict[str, Any]]: + """List compact pipeline summaries; use get_pipeline for a bounded detail section.""" pipelines = get_store().runs() if project_id: pipelines = [pipeline for pipeline in pipelines if pipeline.build_id == project_id] if status: pipelines = [pipeline for pipeline in pipelines if pipeline.status == status] - return [_json(pipeline) for pipeline in pipelines] + return [_pipeline_summary(pipeline) for pipeline in pipelines[: max(1, min(limit, 50))]] @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) - def get_pipeline(pipeline_id: str) -> dict[str, Any]: - """Get a pipeline's current state, phases, evidence, and supervisor results.""" - return _json(get_store().run(pipeline_id)) + def get_pipeline( + pipeline_id: str, + detail: Literal["summary", "steps", "supervision", "evidence"] = "summary", + max_items: int = 20, + ) -> dict[str, Any]: + """Get one bounded pipeline section; summary is the default and never returns raw logs or HTML.""" + return _pipeline_detail(get_store().run(pipeline_id), detail, max_items) @mcp.tool( description="Start a project pipeline. Use test mode for a one-off safe validation run.",