diff --git a/apps/daemon/internal/agent/codex/error_classification.go b/apps/daemon/internal/agent/codex/error_classification.go index 14670ec16..31e570505 100644 --- a/apps/daemon/internal/agent/codex/error_classification.go +++ b/apps/daemon/internal/agent/codex/error_classification.go @@ -2,6 +2,7 @@ package codex import ( "encoding/json" + "strings" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" ) @@ -14,6 +15,18 @@ func classifyTurnError(failure *TurnError) proto.ErrorPayload { } var variant string var code string + // Some native HTTP failures arrive as "other" without typed status. Only + // the exact native terminal HTTP envelope is recognized, never accumulated + // output, notification text or provider response-body keywords. + if len(failure.CodexErrorInfo) == 0 || string(failure.CodexErrorInfo) == "null" || + (json.Unmarshal(failure.CodexErrorInfo, &variant) == nil && variant == "other") { + const prefix = "unexpected status 402 Payment Required" + message := failure.Message + if !strings.ContainsAny(message, "\r\n") && (message == prefix || strings.HasPrefix(message, prefix+": ") || strings.HasPrefix(message, prefix+", ")) { + status := 402 + return proto.ErrorPayload{Code: "provider_billing_unavailable", HTTPStatus: &status, UpstreamRequestID: nativeRequestID(message)} + } + } if json.Unmarshal(failure.CodexErrorInfo, &variant) == nil { switch variant { case "unauthorized": @@ -53,8 +66,21 @@ func classifyTurnError(failure *TurnError) proto.ErrorPayload { code = "rate_limit_exceeded" } code, status = proto.NormalizeEngineFailure(code, status) - return proto.ErrorPayload{Code: code, HTTPStatus: status} + return proto.ErrorPayload{Code: code, HTTPStatus: status, UpstreamRequestID: nativeRequestID(failure.Message)} } } return proto.ErrorPayload{} } + +// The native transport places its request ID last. Ignore arbitrary body text +// and reject multiline messages or anything other than a UUID suffix. +func nativeRequestID(message string) string { + if !strings.HasPrefix(message, "unexpected status ") || strings.ContainsAny(message, "\r\n") { + return "" + } + _, id, ok := strings.Cut(message, ", request id: ") + if !ok { + return "" + } + return proto.NormalizeProviderRequestID(id) +} diff --git a/apps/daemon/internal/agent/codex/error_classification_test.go b/apps/daemon/internal/agent/codex/error_classification_test.go index 6a5a04014..6e4b8d94b 100644 --- a/apps/daemon/internal/agent/codex/error_classification_test.go +++ b/apps/daemon/internal/agent/codex/error_classification_test.go @@ -93,3 +93,29 @@ func TestClassificationOnlyFromRootTerminalError(t *testing.T) { }) } } + +func TestTerminalProviderBillingEnvelope(t *testing.T) { + const id = "88a8d808-59c1-4bf3-8950-b34ce57e028b" + message := "unexpected status 402 Payment Required: secret-body-canary, url: https://example.invalid/v1/responses, request id: " + id + for _, raw := range []json.RawMessage{nil, json.RawMessage(`null`), json.RawMessage(`"other"`), json.RawMessage(`{"httpConnectionFailed":{"httpStatusCode":402}}`)} { + got := classifyTurnError(&TurnError{Message: message, CodexErrorInfo: raw}) + if got.Code != "provider_billing_unavailable" || got.HTTPStatus == nil || *got.HTTPStatus != 402 || got.UpstreamRequestID != id { + t.Fatalf("%s: %+v", raw, got) + } + } + for _, message := range []string{"quoted: " + message, "unexpected status 402 Payment Requiredish", "insufficient credits", "unexpected status 402 Payment Required\nquoted next line"} { + got := classifyTurnError(&TurnError{Message: message, CodexErrorInfo: json.RawMessage(`"other"`)}) + if got.Code != "" || got.UpstreamRequestID != "" { + t.Fatal("non-native envelope classified", got) + } + } + for _, raw := range []string{`"unauthorized"`, `"unknownFuture"`, `{"httpConnectionFailed":{"httpStatusCode":"402"}}`, `{"other":true}`} { + got := classifyTurnError(&TurnError{Message: message, CodexErrorInfo: json.RawMessage(raw)}) + if got.Code == "provider_billing_unavailable" { + t.Fatal("overrode typed error", got) + } + } + if nativeRequestID(message+" secret-token") != "" || nativeRequestID("quoted: "+message) != "" { + t.Fatal("unsafe request ID") + } +} diff --git a/apps/daemon/internal/agent/codex/session.go b/apps/daemon/internal/agent/codex/session.go index 044c66ae0..14950eafa 100644 --- a/apps/daemon/internal/agent/codex/session.go +++ b/apps/daemon/internal/agent/codex/session.go @@ -198,7 +198,7 @@ func (s *Session) onTurnCompleted(raw json.RawMessage) { "status", p.Turn.Status, "final_text_len", len(finalText), "buffered_err_text_len", len(errText), - "raw_payload", string(raw)) + "has_error", p.Turn.Error != nil) if status == "failed" { // Body precedence on failure: // 1. agent's final text (rare on hard failures but exists for @@ -311,7 +311,7 @@ func (s *Session) onErrorNotif(raw json.RawMessage) { s.cfg.logger.Warn("codex: error notification received", "run_id", s.runID, "thread_id", s.currentThreadID(), - "message", p.Message) + "message_len", len(p.Message)) s.finalTextMu.Lock() s.lastErrText = p.Message s.finalTextMu.Unlock() @@ -376,15 +376,15 @@ func (s *Session) emitTerminalFailure(message string, asError bool, failure prot } s.stopSteering() s.stopFunctionCalls() - // Always log: this is the only place the daemon decides "the prompt is - // over, here's what went wrong (if anything)". Without this, post- - // mortem requires correlating server-side TypeError frames against - // daemon timestamps with no message body anywhere. + // Keep safe correlation at terminal settlement; native bodies stay private. if asError { s.cfg.logger.Warn("codex: emitting terminal error", "run_id", s.runID, "thread_id", s.currentThreadID(), - "message", message) + "error_code", failure.Code, + "http_status", failure.HTTPStatus, + "upstream_request_id", proto.NormalizeProviderRequestID(failure.UpstreamRequestID), + "message_len", len(message)) } else { s.cfg.logger.Info("codex: emitting terminal done", "run_id", s.runID, diff --git a/apps/daemon/internal/agent/codex/session_log_test.go b/apps/daemon/internal/agent/codex/session_log_test.go index d0ec7977c..6b40c8a00 100644 --- a/apps/daemon/internal/agent/codex/session_log_test.go +++ b/apps/daemon/internal/agent/codex/session_log_test.go @@ -12,16 +12,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" ) -// TestEmitTerminal_LogsErrorMessage pins the diagnostic invariant: -// every time the codex session decides "this prompt is over with an -// error", the error message MUST land in the daemon's structured log. -// -// Before this fix, emitTerminal silently sent a TypeError envelope to -// the upstream channel. If the upstream connector logged only the -// envelope type (not the body), debugging required guessing what the -// session had decided to surface. With the fix the daemon log carries -// the exact message string, the run_id, and the thread_id at the same -// timestamp the TypeError frame was emitted. +// Terminal logs retain safe metadata while the private runtime frame retains the native error. func TestEmitTerminal_LogsErrorMessage(t *testing.T) { var buf bytes.Buffer logger := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) @@ -42,11 +33,14 @@ func TestEmitTerminal_LogsErrorMessage(t *testing.T) { s.emitTerminal(message, true) logs := buf.String() + if strings.Contains(logs, message) { + t.Fatal("native error body leaked to logs") + } for _, want := range []string{ `"msg":"codex: emitting terminal error"`, `"run_id":"run-test-123"`, `"thread_id":"thread-abc"`, - `"message":"` + message + `"`, + `"message_len":`, `"level":"WARN"`, } { if !strings.Contains(logs, want) { @@ -129,10 +123,13 @@ func TestOnErrorNotif_LogsAndBuffers(t *testing.T) { t.Fatalf("lastErrText not buffered: %q", got) } logs := buf.String() + if strings.Contains(logs, "401 from gateway") { + t.Fatal("notification body leaked") + } for _, want := range []string{ `"msg":"codex: error notification received"`, `"thread_id":"thread-y"`, - `401 from gateway`, + `"message_len":`, } { if !strings.Contains(logs, want) { t.Errorf("log missing %q\n--- log ---\n%s", want, logs) @@ -254,3 +251,20 @@ func drainEnvelopes(out <-chan proto.Envelope) []proto.Envelope { } } } + +func TestTerminalBillingLogsOnlySafeCorrelation(t *testing.T) { + var buf bytes.Buffer + out := make(chan proto.Envelope, 4) + s := &Session{runID: "run", cfg: sessionConfig{logger: slog.New(slog.NewJSONHandler(&buf, nil))}, out: out, cancelCtx: context.Background()} + message := "unexpected status 402 Payment Required: secret-canary, url: https://private.invalid, request id: 88a8d808-59c1-4bf3-8950-b34ce57e028b" + s.emitTerminalFailure(message, true, classifyTurnError(&TurnError{Message: message})) + logs := buf.String() + for _, want := range []string{`"error_code":"provider_billing_unavailable"`, `"http_status":402`, `"upstream_request_id":"88a8d808-59c1-4bf3-8950-b34ce57e028b"`} { + if !strings.Contains(logs, want) { + t.Fatal("missing safe correlation", logs) + } + } + if strings.Contains(logs, "secret-canary") || strings.Contains(logs, "private.invalid") { + t.Fatal("native body leaked") + } +} diff --git a/apps/web/src/features/sessions/session-diagnostics.test.tsx b/apps/web/src/features/sessions/session-diagnostics.test.tsx index d7c8edc77..c5b794b6e 100644 --- a/apps/web/src/features/sessions/session-diagnostics.test.tsx +++ b/apps/web/src/features/sessions/session-diagnostics.test.tsx @@ -78,3 +78,12 @@ describe("Core diagnostics in the console", () => { client.clear(); }); }); + +it.each(["en", "zh-CN"])("renders billing failure rather than internal error in %s", async (language) => { + await i18n.changeLanguage(language); + const billing: DiagnosticFailure = { code: "provider_billing_unavailable", params: { http_status: 402 }, failed_at: null }; + for (const html of [render("session", { ...sessionData, failure: { ...billing, source: "turn", turn_id: turn.id } }), render("turn", { ...turnData, failure: billing })]) { + expect(html).toContain(language === "en" ? "billing or usage-limit issue" : "账单或使用额度问题"); + expect(html).not.toContain(i18n.getFixedT(language, "diagnostics")("failure.internal_error")); + } +}); diff --git a/apps/web/src/i18n/locales/en/diagnostics.ts b/apps/web/src/i18n/locales/en/diagnostics.ts index 03d741940..eab0681cc 100644 --- a/apps/web/src/i18n/locales/en/diagnostics.ts +++ b/apps/web/src/i18n/locales/en/diagnostics.ts @@ -10,6 +10,7 @@ export const diagnostics = { providerHelp: "Core records outcomes from hosted Turns using this deployment default. Records are best effort and may lag; missing records do not prove the provider is ready or unused. Replacing the default starts a new observation history.", providerStaleHelp: "The latest read failed. These are the last confirmed observations.", failure: { + provider_billing_unavailable: "The model service rejected this request because of a billing or usage-limit issue. Contact support to check the model service account before trying again.", authentication_error: "The model provider rejected authentication. Check its API key.", rate_limit_exceeded: "The model provider's rate limit was reached.", usage_limit_exceeded: "The model provider's usage limit was reached. Check its quota.", diff --git a/apps/web/src/i18n/locales/zh-CN/diagnostics.ts b/apps/web/src/i18n/locales/zh-CN/diagnostics.ts index e0a5e1769..b2db1b3dd 100644 --- a/apps/web/src/i18n/locales/zh-CN/diagnostics.ts +++ b/apps/web/src/i18n/locales/zh-CN/diagnostics.ts @@ -10,6 +10,7 @@ export const diagnostics = { providerHelp: "Core 记录使用此部署默认配置的托管 Turn 结果。记录可能延迟或缺失;暂无记录不代表服务已就绪或从未使用。替换默认配置后会重新记录。", providerStaleHelp: "最新读取失败,当前显示上次确认的记录。", failure: { + provider_billing_unavailable: "模型服务因账单或使用额度问题拒绝了请求。请联系支持人员检查模型服务账户后再尝试。", authentication_error: "模型服务认证失败,请检查 API key。", rate_limit_exceeded: "已达到模型服务的请求频率上限。", usage_limit_exceeded: "已达到模型服务的用量上限,请检查额度。", diff --git a/contracts/agents-api/core-errors.md b/contracts/agents-api/core-errors.md index eb51930cb..c6bec2e66 100644 --- a/contracts/agents-api/core-errors.md +++ b/contracts/agents-api/core-errors.md @@ -114,6 +114,7 @@ The [Session and Turn diagnostics reads](./session-diagnostics.md) return these | `authentication_error` | Native provider authentication rejected | | `rate_limit_exceeded` | Native rate limit classification | | `usage_limit_exceeded` | Native billing or usage limit classification | +| `provider_billing_unavailable` | Model endpoint HTTP 402; billing correction is required, not automatic retry; params contain `http_status` 402 or null | | `server_overloaded` | Native overload classification | | `server_error` | Native server failure classification | | `invalid_request` | Native request rejection | @@ -138,4 +139,4 @@ The [Session and Turn diagnostics reads](./session-diagnostics.md) return these A database failure is an error, never an empty or healthy snapshot. Provisioning reasons and native messages are never parsed for categories or parameters. -Native categories apply only to a failed Turn whose outcome has `error_code: engine_failed`. Core accepts only the listed `engine_error_code` values; an unknown, malformed or absent value stays `harness_error`. Only `connection_failed` uses `engine_http_status`. Nested metadata and provider text never classify a failure. Core storage, incomplete-stream and cancellation failures take precedence, and cancelled or completed Turns have no failure. [Native error classification](../../docs/runtime-protocol.md#native-failure-classification) lists which adapters report each category. +Native categories apply only to a failed Turn whose outcome has `error_code: engine_failed`. Core accepts only the listed `engine_error_code` values; an unknown, malformed or absent value stays `harness_error`. `connection_failed` and `provider_billing_unavailable` use their validated `engine_http_status`. Nested metadata and provider text never classify a failure. Core storage, incomplete-stream and cancellation failures take precedence, and cancelled or completed Turns have no failure. [Native error classification](../../docs/runtime-protocol.md#native-failure-classification) lists which adapters report each category. diff --git a/contracts/agents-api/core-metrics.md b/contracts/agents-api/core-metrics.md index 5ad43d4b8..7bded7852 100644 --- a/contracts/agents-api/core-metrics.md +++ b/contracts/agents-api/core-metrics.md @@ -63,3 +63,9 @@ Core resolves its own cgroup, including nested and subtree mounts, and reports t - The Runtime sampler's `processed` and `failed` count the targets its last sweep observed and failed to observe. - A cleanup job's `processed` counts the rows it removed. - For the scheduler and the cleanup jobs, a failed pass sets `processed` to null and `failed` to 1; `failed` never estimates lost rows or failed Turns. + +## Terminal Turn statistics + +`execution.terminal_turns` is a nullable database-backed summary over completed root Turns in the requested `[range.start, range.end)` interval, selected by `completed_at`. It contains `total`, `completed`, `failed`, `cancelled` and `failures` (`code`, `source: turn`, `count`). `total` includes all three terminal statuses; the failure proportion is `failed / total` when total is nonzero. The classification is identical to Project diagnostics, including `unknown`. Empty successful reads contain zeros and an empty failures array. Unavailable history yields null, never measured zero. + +These are window aggregates of retained authoritative terminal rows, not process counters. Retried reads, concurrent writers and service restarts cannot add duplicate counts. Deleted Sessions' retained Turns are included. Environment failures without a Turn are excluded from both numerator and denominator. No Project, model, Session or Turn identifiers become dimensions. Aggregation shares the bounded repeatable-read history snapshot and its three-second request budget; no retry or execution mutation occurs. diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 29303854d..4346a670d 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -369,6 +369,7 @@ definitions: - authentication_error - rate_limit_exceeded - usage_limit_exceeded + - provider_billing_unavailable - server_overloaded - server_error - invalid_request @@ -750,6 +751,7 @@ definitions: - authentication_error - rate_limit_exceeded - usage_limit_exceeded + - provider_billing_unavailable - server_overloaded - server_error - invalid_request @@ -890,6 +892,10 @@ definitions: type: integer slots_total: type: integer + terminal_turns: + allOf: + - $ref: '#/definitions/coremetrics.TerminalTurns' + x-nullable: true unavailable: type: integer x-nullable: true @@ -906,6 +912,7 @@ definitions: - series - slots_in_use - slots_total + - terminal_turns - unavailable - waiting_for_daemon type: object @@ -928,6 +935,17 @@ definitions: - queued - start type: object + coremetrics.FailureCount: + properties: + code: + type: string + count: + type: integer + source: + enum: + - turn + type: string + type: object coremetrics.Job: properties: failed: @@ -1078,6 +1096,21 @@ definitions: x-enum-varnames: - ServiceRunning - ServiceDegraded + coremetrics.TerminalTurns: + properties: + cancelled: + type: integer + completed: + type: integer + failed: + type: integer + failures: + items: + $ref: '#/definitions/coremetrics.FailureCount' + type: array + total: + type: integer + type: object coremetrics.View: properties: database: diff --git a/contracts/agents-api/index.md b/contracts/agents-api/index.md index f5eed41ba..e299348c5 100644 --- a/contracts/agents-api/index.md +++ b/contracts/agents-api/index.md @@ -16,7 +16,7 @@ Core targets the complete OpenAI Agents API as pinned below ([public API rule](h Run `make openapi` to regenerate the public Go types, the Agent request shapes, the route inventory and all three OpenAPI documents. `scripts/generate-public-api.py` reads the checked-in, checksum-verified official source without network access. It selects Agents, Vaults, Files and Skills and follows their schema references, preserving union types, nullability, required fields and constraints. The request shapes in `services/core/internal/api/official_shapes.gen.go` project `CreateAgentParams`, `UpdateAgentParams` and `SessionAgentConfigParam`; Core checks request bodies against them before it reads an Agent configuration. Core's extension types in `v1/` remain authored in Go and are added to the public schema during generation. The TypeScript client's types, enum values and field names in `packages/agents-client/src/generated/public-api.ts` are generated from the resulting public schema. The internal `/core/v1` and `/api/v1` documents come from handler annotations; the same generator projects the `/core/v1` document into `packages/agents-client/src/generated/core-api.ts`, which imports the public types it references from `public-api.ts`. In those annotations a response field Core always sends carries `binding:"required"`, a field that can be null carries `extensions:"x-nullable"`, and a Core-owned closed set uses a named Go type with typed constants in its producing package. Swag derives one enum definition from that type, shared by every wire field. Single-value discriminators keep `enums:` tags; fields mirroring official or Runtime-owned sets retain tags checked against their owners by contract tests. These annotations only shape the documents, and the client rejects a response that breaks them. `make check-openapi` checks freshness and the generator; it also runs through `make check-go`. -The public contract is the official API plus Core extensions. Standard fields are generated into `v1/official.gen.go`; `go-bindings.json` lists types consumed by Core and overrides only the Go representation or field order that existing storage or custom JSON encoding requires. Unspecified fields follow the official schema; shared shapes use one Go type. Selected discriminated unions also generate JSON serializers to retain required nullable fields for each variant. Other union serializers, Core's local limits, execution admission and state transitions remain implementation code. Contract tests verify that the public schema preserves the official definitions, extensions remain in `x_agents_core`, and all documents match registered routes. Official-client and raw HTTP tests verify behavior. Schema generation does not qualify an unimplemented feature; the gaps below still apply. Upstream upgrades update the OpenAPI and SDK pins together after comparison and compatibility tests. +The public contract is the official API plus Core extensions. Standard fields are generated into `v1/official.gen.go`; `go-bindings.json` lists types consumed by Core and overrides only the Go representation or field order that existing storage or custom JSON encoding requires. Unspecified fields follow the official schema; shared shapes use one Go type. Selected discriminated unions also generate JSON serializers to retain required nullable fields for each variant. Other union serializers, Core's local limits, execution admission and state transitions remain implementation code. Contract tests verify that the public schema preserves the official definitions, field extensions remain in `x_agents_core` and the two [Project diagnostics routes](./session-diagnostics.md#project-diagnostics-extension) are explicitly marked Core extensions, and all documents match registered routes. Official-client and raw HTTP tests verify behavior. Schema generation does not qualify an unimplemented feature; the gaps below still apply. Upstream upgrades update the OpenAPI and SDK pins together after comparison and compatibility tests. The official source and existing service have these recorded differences: Agents authentication errors can return a null `code`; empty Files pages return null `first_id` and `last_id`; File resources can return null `expires_at` and `status_details`. The source declares those fields non-null. The official-client response validator allows null only for these named fields and otherwise validates OpenAPI 3.1 response schemas. Files and Skills operations omit error responses in the source, so those error bodies use the upstream shared `ErrorResponse` schema. [Wire semantics](./wire-semantics.md) and raw HTTP tests qualify service behavior; the published schema retains the official definitions. diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index 43e25a570..57282438e 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -6329,6 +6329,199 @@ "name": "Delete a vault credential" } } + }, + "/agents/sessions/{session_id}/diagnostics": { + "get": { + "description": "OpenAgentCore extension. Project key only. Safe committed failure classification without private execution details.", + "parameters": [ + { + "description": "Session ID", + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "OpenAI-Beta", + "in": "header", + "required": true, + "schema": { + "type": "string", + "const": "agents=v1" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/v1.SessionDiagnostics" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "503": { + "description": "Service Unavailable", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + } + }, + "summary": "Retrieve Project Session diagnostics", + "tags": [ + "Diagnostics" + ], + "x-agents-core-extension": true + } + }, + "/agents/sessions/{session_id}/turns/{turn_id}/diagnostics": { + "get": { + "description": "OpenAgentCore extension. Project key only. Safe committed failure classification without private execution details.", + "parameters": [ + { + "description": "Session ID", + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Root Turn ID", + "in": "path", + "name": "turn_id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "OpenAI-Beta", + "in": "header", + "required": true, + "schema": { + "type": "string", + "const": "agents=v1" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/v1.TurnDiagnostics" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + }, + "503": { + "description": "Service Unavailable", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse-2" + } + } + } + } + }, + "summary": "Retrieve Project Turn diagnostics", + "tags": [ + "Diagnostics" + ], + "x-agents-core-extension": true + } } }, "webhooks": { @@ -17715,6 +17908,65 @@ }, "type": "object" }, + "v1.ExecutionDiagnostic": { + "properties": { + "code": { + "enum": [ + "unknown", + "harness_error", + "model_provider_required", + "runtime_unavailable", + "runtime_disconnected", + "runtime_preparation_failed", + "execution_interrupted", + "delivery_unconfirmed", + "input_rejected", + "executor_protocol_error", + "core_storage_failed", + "environment_connection_timeout", + "environment_unavailable", + "environment_provisioning_failed", + "authentication_error", + "rate_limit_exceeded", + "usage_limit_exceeded", + "provider_billing_unavailable", + "server_overloaded", + "server_error", + "invalid_request", + "resource_not_found", + "request_timeout", + "context_length_exceeded", + "cyber_policy", + "connection_failed" + ], + "type": "string" + }, + "failed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "source": { + "enum": [ + "turn", + "environment", + "environment_input" + ], + "type": "string" + } + }, + "required": [ + "code", + "failed_at", + "source" + ], + "type": "object" + }, "v1.InstallationStatus": { "enum": [ "available", @@ -17833,6 +18085,65 @@ }, "type": "object" }, + "v1.SessionDiagnostics": { + "properties": { + "diagnostic": { + "anyOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/v1.ExecutionDiagnostic" + } + ] + }, + { + "type": "null" + } + ] + }, + "input_available": { + "description": "InputAvailable is advisory at this snapshot; submission retains all atomic admission checks.", + "type": "boolean" + }, + "input_unavailable_reason": { + "enum": [ + "busy", + "requires_action", + "environment_unavailable", + "configuration_required" + ], + "type": "string" + }, + "object": { + "enum": [ + "agent.session_diagnostics" + ], + "type": "string" + }, + "session_id": { + "type": "string" + }, + "status": { + "enum": [ + "idle", + "in_progress", + "requires_action", + "failed" + ], + "type": "string" + }, + "turn_id": { + "type": "string" + } + }, + "required": [ + "diagnostic", + "object", + "session_id", + "status" + ], + "type": "object" + }, "v1.SessionExecutionInput": { "properties": { "environment": { @@ -17847,6 +18158,55 @@ } }, "type": "object" + }, + "v1.TurnDiagnostics": { + "properties": { + "diagnostic": { + "anyOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/v1.ExecutionDiagnostic" + } + ] + }, + { + "type": "null" + } + ] + }, + "object": { + "enum": [ + "agent.turn_diagnostics" + ], + "type": "string" + }, + "session_id": { + "type": "string" + }, + "status": { + "enum": [ + "queued", + "in_progress", + "waiting", + "completed", + "failed", + "cancelled" + ], + "type": "string" + }, + "turn_id": { + "type": "string" + } + }, + "required": [ + "diagnostic", + "object", + "session_id", + "status", + "turn_id" + ], + "type": "object" } }, "responses": { diff --git a/contracts/agents-api/session-diagnostics.md b/contracts/agents-api/session-diagnostics.md index d56d595de..9885af5eb 100644 --- a/contracts/agents-api/session-diagnostics.md +++ b/contracts/agents-api/session-diagnostics.md @@ -44,3 +44,13 @@ The public completion time of a successful Turn may come from the native executo ## Client `AdminClient.retrieveSessionDiagnostics(projectId, sessionId, options)` and `AdminClient.retrieveTurnDiagnostics(projectId, sessionId, turnId, options)` in `packages/agents-client` support request cancellation and validate scope, categories, nullability and safe parameter values. + +## Project diagnostics extension + +Applications use a Project API key and `OpenAI-Beta: agents=v1` with `GET /v1/agents/sessions/{session_id}/diagnostics` or `GET /v1/agents/sessions/{session_id}/turns/{turn_id}/diagnostics`. These two read-only routes are OpenAgentCore extensions, not official OpenAI Agents API operations. The generated public OpenAPI marks them `x-agents-core-extension: true`. Existing official Session and Turn responses and their error enums do not change. + +The objects are `agent.session_diagnostics` and `agent.turn_diagnostics`. Both return `session_id`, `status` and nullable `diagnostic`; Turn diagnostics also return `turn_id`. Session diagnostics include `turn_id` only when the failure source is a Turn. `diagnostic` contains only `code`, `source` and nullable RFC 3339 `failed_at`. The fixed categories and snapshot precedence above apply; the catch-all `internal_error` is exposed as `unknown`. No `params`, native text, timing Items, device identities or credential data are exposed. `diagnostic` is null for nonfailed status. Authentication and resource isolation use the requesting Project, including for deleted or foreign resources. Administrator keys are rejected. + +The reads share the administrator snapshot and five-second budget and return `Cache-Control: no-store`. A storage or timeout error is an HTTP error, not a failed execution snapshot; callers must distinguish diagnostic unavailability from an execution with an unknown cause. Reads do not change execution or counters and do not authorize automatic retry. + +Session diagnostics also expose optional `input_available` and, when false, `input_unavailable_reason` (`busy`, `requires_action`, `environment_unavailable`, or `configuration_required`). This is an advisory snapshot for an explicit new message, not a reservation or a promise of execution. An active Turn, pending input, required action, unavailable Environment or missing model provider blocks that suggestion; a failed Turn by itself does not. Clients must invalidate the suggestion when state changes and reread authoritative state. Missing capability data is not permission to resume a failed Session. Submission retains every transactional admission and idempotency check. A new user message uses a new idempotency key and Turn; historical failures remain unchanged, and no diagnostic read resends work. diff --git a/contracts/agents-api/v1/diagnostics.go b/contracts/agents-api/v1/diagnostics.go new file mode 100644 index 000000000..6867b209c --- /dev/null +++ b/contracts/agents-api/v1/diagnostics.go @@ -0,0 +1,32 @@ +package v1 + +import "time" + +// ExecutionDiagnostic is the safe failure projection of a committed snapshot. +// It deliberately excludes native messages, parameters and executor identities. +type ExecutionDiagnostic struct { + Code string `json:"code" binding:"required" enums:"unknown,harness_error,model_provider_required,runtime_unavailable,runtime_disconnected,runtime_preparation_failed,execution_interrupted,delivery_unconfirmed,input_rejected,executor_protocol_error,core_storage_failed,environment_connection_timeout,environment_unavailable,environment_provisioning_failed,authentication_error,rate_limit_exceeded,usage_limit_exceeded,provider_billing_unavailable,server_overloaded,server_error,invalid_request,resource_not_found,request_timeout,context_length_exceeded,cyber_policy,connection_failed"` + Source string `json:"source" binding:"required" enums:"turn,environment,environment_input"` + FailedAt *time.Time `json:"failed_at" binding:"required" extensions:"x-nullable"` +} + +// SessionDiagnostics is an OpenAgentCore extension, not an official OpenAI DTO. +type SessionDiagnostics struct { + Object string `json:"object" binding:"required" enums:"agent.session_diagnostics"` + SessionID string `json:"session_id" binding:"required"` + TurnID string `json:"turn_id,omitempty"` + Status string `json:"status" binding:"required" enums:"idle,in_progress,requires_action,failed"` + Diagnostic *ExecutionDiagnostic `json:"diagnostic" binding:"required" extensions:"x-nullable"` + // InputAvailable is advisory at this snapshot; submission retains all atomic admission checks. + InputAvailable *bool `json:"input_available,omitempty"` + InputUnavailableReason string `json:"input_unavailable_reason,omitempty" enums:"busy,requires_action,environment_unavailable,configuration_required"` +} + +// TurnDiagnostics is an OpenAgentCore extension, not an official OpenAI DTO. +type TurnDiagnostics struct { + Object string `json:"object" binding:"required" enums:"agent.turn_diagnostics"` + SessionID string `json:"session_id" binding:"required"` + TurnID string `json:"turn_id" binding:"required"` + Status string `json:"status" binding:"required" enums:"queued,in_progress,waiting,completed,failed,cancelled"` + Diagnostic *ExecutionDiagnostic `json:"diagnostic" binding:"required" extensions:"x-nullable"` +} diff --git a/contracts/agents-api/v1/upstream_contract_test.go b/contracts/agents-api/v1/upstream_contract_test.go index 140dd2c75..36dfc60e5 100644 --- a/contracts/agents-api/v1/upstream_contract_test.go +++ b/contracts/agents-api/v1/upstream_contract_test.go @@ -80,6 +80,9 @@ func TestPublicOperationsPreserveOfficialContract(t *testing.T) { t.Errorf("missing official path %s", path) continue } + if len(actual) != len(item.(map[string]any)) { + t.Errorf("unexpected methods on official path %s", path) + } for method, raw := range item.(map[string]any) { expected := raw.(map[string]any) operation := actual[method].(map[string]any) @@ -104,6 +107,26 @@ func TestPublicOperationsPreserveOfficialContract(t *testing.T) { } delete(published, path) } + // These two GET operations are Core extensions, never official operations. + // Keep this allowlist exact so another path or method requires review. + for _, path := range []string{ + "/agents/sessions/{session_id}/diagnostics", + "/agents/sessions/{session_id}/turns/{turn_id}/diagnostics", + } { + item, ok := published[path].(map[string]any) + if !ok { + t.Errorf("missing Core diagnostic extension %s", path) + continue + } + operation, ok := item["get"].(map[string]any) + if !ok || operation["x-agents-core-extension"] != true { + t.Errorf("GET %s must explicitly identify a Core extension", path) + } + if len(item) != 1 { + t.Errorf("unexpected methods on Core diagnostic extension %s", path) + } + delete(published, path) + } if count != 58 || len(published) != 0 { t.Fatalf("expected pinned 58 operations, got %d, extra paths %v", count, published) } diff --git a/contracts/agents-api/zh/core-errors.md b/contracts/agents-api/zh/core-errors.md index 95262bdba..666dd4ded 100644 --- a/contracts/agents-api/zh/core-errors.md +++ b/contracts/agents-api/zh/core-errors.md @@ -1,7 +1,7 @@ --- title: "Core 管理错误" source: contracts/agents-api/core-errors.md -source_hash: 71088f9154793bd72bdccb199f1ac6da6da4097da05bed828299011eb5dfe12d +source_hash: 35ab3bfb8cec674766bbcc9dc2f0342045539ed4bf443c532c9a1271e7ec8c3e --- `/core/v1` 上的错误使用此封装结构。`message` 是安全的英文文本;`code` 和 `param` 可以为 null。客户端依据稳定的 `code` 和可选的 `param` 进行处理,对未知代码显示 `message`,绝不解析消息,也绝不自动重试被拒绝的写操作。 @@ -116,6 +116,7 @@ Web 的控制台服务器在 `/core` 路径上发生自身故障时使用此封 | `authentication_error` | 原生提供商拒绝了身份验证 | | `rate_limit_exceeded` | 原生速率限制分类 | | `usage_limit_exceeded` | 原生计费或使用量限制分类 | +| `provider_billing_unavailable` | 模型端点 HTTP 402,需要修正计费而非自动重试;params 中 `http_status` 为 402 或 null | | `server_overloaded` | 原生过载分类 | | `server_error` | 原生服务器故障分类 | | `invalid_request` | 原生请求被拒绝 | @@ -140,4 +141,4 @@ Web 的控制台服务器在 `/core` 路径上发生自身故障时使用此封 数据库故障属于错误,绝不会产生空快照或健康快照。绝不会解析预置原因或原生消息以确定类别或参数。 -原生类别仅适用于 outcome 中含有 `error_code: engine_failed` 的失败 Turn。Core 仅接受列出的 `engine_error_code` 值;如果该值未知、格式错误或缺失,则仍归为 `harness_error`。只有 `connection_failed` 使用 `engine_http_status`。绝不会根据嵌套元数据或提供商文本来划分失败类别。Core 存储、流不完整和取消故障具有更高优先级;已取消或已完成的 Turn 没有故障。[Native error classification](../../../docs/zh/runtime-protocol.md#native-failure-classification) 列出了各适配器会报告哪些类别。 +原生类别仅适用于 outcome 中含有 `error_code: engine_failed` 的失败 Turn。Core 仅接受列出的 `engine_error_code` 值;如果该值未知、格式错误或缺失,则仍归为 `harness_error`。`connection_failed` 与 `provider_billing_unavailable` 使用各自经校验的 `engine_http_status`。绝不会根据嵌套元数据或提供商文本来划分失败类别。Core 存储、流不完整和取消故障具有更高优先级;已取消或已完成的 Turn 没有故障。[Native error classification](../../../docs/zh/runtime-protocol.md#native-failure-classification) 列出了各适配器会报告哪些类别。 diff --git a/contracts/agents-api/zh/core-metrics.md b/contracts/agents-api/zh/core-metrics.md index 540750d78..de5d9a8cf 100644 --- a/contracts/agents-api/zh/core-metrics.md +++ b/contracts/agents-api/zh/core-metrics.md @@ -1,7 +1,7 @@ --- title: "Core 运行指标" source: contracts/agents-api/core-metrics.md -source_hash: f94805c348f60947ac0a61de898539d982e529226c7f07bc73d2a36a3a641821 +source_hash: 8e5922f7df4fa1cbfb59251815eb2f59ee35119560057448fe4ad5b7f3c1e6fa --- `GET /core/v1/metrics?range=1h|6h|24h|7d` 报告 Core 自身的健康状况:进程、执行队列与槽位、PostgreSQL 和后台任务。它要求 Core 密钥([Core 管理 API](admin-api.md))。 @@ -65,3 +65,9 @@ Core 解析自身的 cgroup,包括嵌套与子树挂载,并报告该 cgroup - Runtime sampler 的 `processed` 和 `failed` 统计上次扫描成功观察和未能观察的目标。 - 清理任务的 `processed` 统计删除的行数。 - scheduler 和清理任务失败时,`processed` 为 null,`failed` 为 1;`failed` 不估算丢失行数或失败 Turn 数量。 + +## Terminal Turn statistics + +`execution.terminal_turns` 是基于数据库的可空汇总,按 `completed_at` 选择请求 `[range.start, range.end)` 窗口内的 root Turn 终态,包含 `total`、`completed`、`failed`、`cancelled` 和 `failures`(`code`、`source: turn`、`count`)。`total` 包含三种终态;非零时失败率为 `failed / total`。分类与 Project 诊断一致,含 `unknown`。查询成功但没有记录时返回零与空数组;历史不可用时为 null,而非零。 + +这是保留的权威终态记录的窗口聚合,不是进程计数器。重复读取、并发写入及服务重启不会重复计数;包含已删除 Session 的保留 Turn。尚未形成 Turn 的环境失败不进入分子或分母。维度不包含 Project、模型、Session 或 Turn 标识。聚合共享有界的 repeatable-read 历史快照和三秒请求预算,不触发重试或执行变更。 diff --git a/contracts/agents-api/zh/index.md b/contracts/agents-api/zh/index.md index 9d55afff7..75d1eef36 100644 --- a/contracts/agents-api/zh/index.md +++ b/contracts/agents-api/zh/index.md @@ -1,7 +1,7 @@ --- title: "Agents API 覆盖台账" source: contracts/agents-api/index.md -source_hash: 2394633ee55b7f86d374d5ab3827d0168d3dcbae8b962115063613e288b4cfda +source_hash: afbd6749108365984e99ff42975b969658efc627bd4dd524e5c5a53f279e3715 --- Core 旨在以下方固定版本为准支持完整的 OpenAI Agents API([public API rule](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#public-api))。本台账记录 Core 对各项资源实现了哪些内容、哪些契约保存其详细信息,并列出相对于 OpenAI 服务的所有已知差异和所有未解决缺口。[API namespaces and credentials](../../../docs/zh/api/index.md) 说明谁调用哪些 API;[Agents API guide](../../../docs/zh/api/public-agent-api.md) 介绍使用方法。 @@ -18,7 +18,7 @@ Core 旨在以下方固定版本为准支持完整的 OpenAI Agents API([publi 运行 `make openapi` 重新生成公共 Go 类型、Agent 请求结构、路由清单和三个 OpenAPI 文档。`scripts/generate-public-api.py` 读取仓库内经过校验和验证的官方源文件,无需网络。它选择 Agents、Vaults、Files 和 Skills,并跟随 schema 引用,保留联合类型、可空性、必填字段和约束。`services/core/internal/api/official_shapes.gen.go` 中的请求结构投影 `CreateAgentParams`、`UpdateAgentParams` 和 `SessionAgentConfigParam`;Core 在读取 Agent 配置前先按它们检查请求体。Core 在 `v1/` 中的扩展类型继续由 Go 定义,在生成时加入公共 schema。TypeScript 客户端在 `packages/agents-client/src/generated/public-api.ts` 中的类型、枚举值和字段名由生成后的公共 schema 生成。内部 `/core/v1` 和 `/api/v1` 文档由处理函数注解生成;同一生成器把 `/core/v1` 文档投影为 `packages/agents-client/src/generated/core-api.ts`,其中引用的公共类型从 `public-api.ts` 导入。在这些注解里,Core 总会发送的响应字段带 `binding:"required"`,可以为 null 的字段带 `extensions:"x-nullable"`,Core 拥有的封闭集合使用生产方包中的具名 Go 类型和带类型的常量。Swag 从该类型生成一个枚举定义,供所有线上字段共享。单值判别字段保留 `enums:` 标签;映射官方或 Runtime 所有集合的字段保留标签,并通过契约测试与其所有者核对。这些注解只影响文档,客户端会拒绝不符合它们的响应。`make check-openapi` 检查生成结果是否最新并测试生成器;`make check-go` 也会运行此检查。 -公共契约是官方 API 加上 Core 扩展。标准字段生成到 `v1/official.gen.go`;`go-bindings.json` 只列出 Core 使用的类型,仅在已有存储或自定义 JSON 编码需要时覆盖 Go 表示或字段顺序。未覆盖的字段遵循官方 schema,相同结构复用同一个 Go 类型。部分带判别字段的联合类型也从 schema 生成 JSON 序列化代码,保留每个分支必需的可空字段。其他联合类型序列化、Core 的本地限制、执行准入和状态转换仍由实现代码负责。契约测试验证公共 schema 保留官方定义、扩展位于 `x_agents_core` 中,且所有文档与注册路由一致。官方客户端和原始 HTTP 测试验证行为。生成 schema 不代表某个尚未实现的功能已经得到验证;下方缺口仍然适用。升级上游时,在比对和兼容性测试后一起更新 OpenAPI 和 SDK 固定版本。 +公共契约是官方 API 加上 Core 扩展。标准字段生成到 `v1/official.gen.go`;`go-bindings.json` 只列出 Core 使用的类型,仅在已有存储或自定义 JSON 编码需要时覆盖 Go 表示或字段顺序。未覆盖的字段遵循官方 schema,相同结构复用同一个 Go 类型。部分带判别字段的联合类型也从 schema 生成 JSON 序列化代码,保留每个分支必需的可空字段。其他联合类型序列化、Core 的本地限制、执行准入和状态转换仍由实现代码负责。契约测试验证公共 schema 保留官方定义、字段扩展位于 `x_agents_core` 中,两条 [Project 诊断路由](./session-diagnostics.md#project-diagnostics-extension) 明确标记为 Core 扩展,且所有文档与注册路由一致。官方客户端和原始 HTTP 测试验证行为。生成 schema 不代表某个尚未实现的功能已经得到验证;下方缺口仍然适用。升级上游时,在比对和兼容性测试后一起更新 OpenAPI 和 SDK 固定版本。 官方源文件与已有服务存在以下已记录的差异:Agents 鉴权错误的 `code` 可以为 null;Files 空页的 `first_id` 和 `last_id` 为 null;File 资源的 `expires_at` 和 `status_details` 可以为 null。源文件将这些字段声明为非空。官方客户端响应验证器只对这些指定字段允许 null,其余部分按 OpenAPI 3.1 响应 schema 验证。源文件中的 Files 和 Skills 操作未声明错误响应,因此这些错误体使用上游共享的 `ErrorResponse` schema。[传输语义](wire-semantics.md)和原始 HTTP 测试验证服务行为;发布的 schema 保留官方定义。 diff --git a/contracts/agents-api/zh/session-diagnostics.md b/contracts/agents-api/zh/session-diagnostics.md index 6a823c16b..7eefba97d 100644 --- a/contracts/agents-api/zh/session-diagnostics.md +++ b/contracts/agents-api/zh/session-diagnostics.md @@ -1,7 +1,7 @@ --- title: "根 Session 与 Turn 诊断" source: contracts/agents-api/session-diagnostics.md -source_hash: d84a678de2a21f66e2090afff19aea5cff5d3f11d5c32306ebd4387b015cb416 +source_hash: f6c2bf7d41c48795074e26f5df2393723033cb53533c115581fdfcfb720784b9 --- 这些只读路由要求 Core 密钥。Project ID 选择目标 Project,不用于认证。两者都返回 `Cache-Control: no-store`: @@ -46,3 +46,13 @@ source_hash: d84a678de2a21f66e2090afff19aea5cff5d3f11d5c32306ebd4387b015cb416 ## 客户端 {#client} `packages/agents-client` 中的 `AdminClient.retrieveSessionDiagnostics(projectId, sessionId, options)` 和 `AdminClient.retrieveTurnDiagnostics(projectId, sessionId, turnId, options)` 支持请求取消,并验证范围、分类、可空性和安全参数值。 + +## Project diagnostics extension + +应用使用 Project API key 和 `OpenAI-Beta: agents=v1` 调用 `GET /v1/agents/sessions/{session_id}/diagnostics` 或 `GET /v1/agents/sessions/{session_id}/turns/{turn_id}/diagnostics`。这两条只读路由是 OpenAgentCore 扩展,并非官方 OpenAI Agents API 操作;生成的公共 OpenAPI 标记 `x-agents-core-extension: true`。既有官方 Session、Turn 响应及其错误枚举保持不变。 + +对象名为 `agent.session_diagnostics` 和 `agent.turn_diagnostics`,都返回 `session_id`、`status` 和可空的 `diagnostic`。Turn 诊断必有 `turn_id`,Session 诊断仅在失败来自 Turn 时包含 `turn_id`。`diagnostic` 只包含 `code`、`source` 和可空的 RFC 3339 `failed_at`。分类及快照优先级遵循上文,兜底的 `internal_error` 对应用显示为 `unknown`。不暴露 `params`、原生文本、Item 时序、设备身份或凭据。非失败状态的 `diagnostic` 为 null。鉴权和资源隔离以请求 Project 为准,外租户、删除资源不可见,管理员 key 不被接受。 + +读取共享管理接口的快照和五秒预算,返回 `Cache-Control: no-store`。存储或超时错误是 HTTP 错误,而非执行失败快照;调用方必须区分诊断不可用与执行原因未知。读取不会改变执行或计数,也不代表允许自动重试。 + +Session 诊断还提供可选的 `input_available`,为 false 时提供 `input_unavailable_reason`(`busy`、`requires_action`、`environment_unavailable` 或 `configuration_required`)。这是用户显式发送新消息的快照建议,不是预约或执行承诺。活动 Turn、待处理输入、待操作、不可用环境或缺少模型供应商配置会阻止该建议;单独的历史 Turn 失败不会阻止。状态变化后客户端必须废弃建议并重新读取权威状态;能力缺失不等于允许恢复失败 Session。正式提交保留全部事务准入与幂等检查。用户新消息使用新的幂等键并创建新 Turn,历史失败保持不变,诊断读取不得自动重发工作。 diff --git a/deploy/kubernetes/BETA_CHANGELOG.md b/deploy/kubernetes/BETA_CHANGELOG.md index 4ad373b12..eca9dd012 100644 --- a/deploy/kubernetes/BETA_CHANGELOG.md +++ b/deploy/kubernetes/BETA_CHANGELOG.md @@ -148,3 +148,21 @@ The beta adapter retains its existing build-script layout; the main-only builder | Runtime/guest observations | Require a separately built, qualified and activated template containing this revision | The [operator guide](../../docs/getting-started/operations.md#runtime-startup-latency) defines native version, catalog and initialize boundaries; the [E2B helper guide](../../services/core/tools/e2b-provider/README.md#observations) defines helper and guest stages. Core/Web deployment alone enables helper-side stream timing and guest record filtering, but cannot add new instrumentation to existing Runtime images. No remote calls or mandatory checks are removed, and this change does not establish a production latency improvement. Preserve the previous images and template selection for rollback. + +## 2026-10-11 — Project failure diagnostics and explicit continuation + +| Item | Value | +| --- | --- | +| Deployed production baseline | `11956c3c78d84e3edfabf43d13bc8497f4f59c25` | +| Feature source | `0ebfe7932035dccb89966fbd4ba8013156ec9a87` (including `00b62751` and `c263a9c7`) | +| Scope | Safe Project diagnostics, terminal failure statistics, typed provider billing failures and input availability | +| Schema migrations / DDL | None; the release keeps the deployed schema and does not include later beta migrations | +| Required assets | Matching Core/Web and combined E2B Runtime template | + +The backport preserves the deployed Store and Swagger generation architecture. Failed Turn diagnostics remain immutable; explicit new input creates a new Turn only when the existing admission checks allow it. Diagnostic reads never retry model execution. Private provider messages and request identifiers remain outside the Project API. + +Build and qualify the matching Runtime template, then select it with the supported generation-checked administrative deployment flow. The new template applies only to new allocations; existing allocations retain their current Runtime. Historical failures are not reclassified or backfilled. Source validation and local database fixtures do not establish production model-call acceptance. + +### Release commit and forward integration + +The exact production release target is `51fa68f640f7e4fb05eec6279dd521a0f2122f3d`, based on deployed commit `11956c3c78d84e3edfabf43d13bc8497f4f59c25`; this target adds no DDL. The normal merge of that release history into beta preserves beta’s newer typed schemas, generated clients and persistence architecture. The resulting beta HEAD also contains eight existing migrations absent from the deployed baseline. Do not deploy that beta HEAD under this release authorization: build and deploy the exact narrow release commit above. This forward integration records ancestry and does not qualify those eight migrations for production. diff --git a/docs/api/index.md b/docs/api/index.md index 0a6d3cf8c..e8858d888 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -6,7 +6,7 @@ Core serves three namespaces. Each has one kind of caller and its own credential | Namespace | Caller | Credential | Contents | Owner | | --- | --- | --- | --- | --- | -| `/v1` | Applications: business systems and the official OpenAI SDK | Project API key | Exactly the 58 method and path pairs of the pinned official Agents API, listed in [upstream-routes.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/upstream-routes.json). Core-only fields sit inside `x_agents_core`: `harness`, `model_provider`, `harness_config`, `environment`, and the read-only Session `installation` | [Agents API guide](./public-agent-api.md) | +| `/v1` | Applications: business systems and the official OpenAI SDK | Project API key | The 58 official method and path pairs of the pinned official Agents API, listed in [upstream-routes.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/upstream-routes.json). Two [Project diagnostics extension routes](../../contracts/agents-api/session-diagnostics.md#project-diagnostics-extension) provide safe failure snapshots. Core-only fields on official objects sit inside `x_agents_core`: `harness`, `model_provider`, `harness_config`, `environment`, and the read-only Session `installation` | [Agents API guide](./public-agent-api.md) | | `/core/v1` | Web's console server and operator scripts | [Core key](../getting-started/operations.md#core-key) | Installation facts, Projects and keys, resource reads and deletion, Session archive, executor credentials, default models, metrics, audit, sandbox deployment and nodes, independent workspace storage configuration | [Core administration API](../../contracts/agents-api/admin-api.md) | | `/api/v1` | Nodes, Runtime daemons, self-hosted executors and their installers | Machine credentials: node enrollment tokens and node credentials, installation grants, executor credentials, and daemon credentials. Each works only on its own routes | Machine bootstrap and connections under `/api/v1/sandbox-node/*` and `/api/v1/agent-daemon/*`, including WebSockets, and the public native installer downloads | [Machine connection API](../../contracts/agents-api/machine-api.md) | diff --git a/docs/api/public-agent-api.md b/docs/api/public-agent-api.md index 2528cca2b..668c95da2 100644 --- a/docs/api/public-agent-api.md +++ b/docs/api/public-agent-api.md @@ -536,7 +536,7 @@ The HTTP path is `/vaults`, with the Beta header. A Session selects credentials 1. Read the Session's `status` and `error`, and the latest Turn's `error`. A failed Turn reports only a generic `internal_error`. 2. Check that the Environment is connected and its harness is available. 3. Check the harness, model and tool combination in [Harness capabilities](../../contracts/agents-api/harness-capabilities.md). -4. Ask the administrator for the Session's [diagnostics](../../contracts/agents-api/session-diagnostics.md), which name the failure category, and to check [troubleshooting](../getting-started/operations.md#troubleshooting) for service logs, credentials and node readiness. +4. Read the Session's [Project diagnostics extension](../../contracts/agents-api/session-diagnostics.md#project-diagnostics-extension) for its safe failure category; ask the administrator to check [troubleshooting](../getting-started/operations.md#troubleshooting) for service logs, credentials and node readiness. A 401 usually means a key from another namespace; see [API namespaces and credentials](./index.md). diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index 67f00d5c8..0996bc09f 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -182,9 +182,9 @@ Retry only an operation whose own contract makes retry safe. A retired preparati An adapter may add `code` and `http_status` to a Run's `error` frame. They are optional neutral metadata, not a terminal event or a public error contract; the frame's text, Usage, `done`, native Session identity and cancellation receipts keep their order and meaning. -The accepted codes are `authentication_error`, `rate_limit_exceeded`, `usage_limit_exceeded`, `server_overloaded`, `server_error`, `invalid_request`, `resource_not_found`, `request_timeout`, `context_length_exceeded`, `cyber_policy` and `connection_failed` ([`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go)). Only `connection_failed` keeps `http_status`, and only an integer from 100 to 599; every other status is discarded. A missing, malformed or unknown value leaves the error unclassified without discarding Usage or `done`. +The accepted codes are `authentication_error`, `rate_limit_exceeded`, `usage_limit_exceeded`, `server_overloaded`, `server_error`, `invalid_request`, `resource_not_found`, `request_timeout`, `context_length_exceeded`, `cyber_policy`, `provider_billing_unavailable` and `connection_failed` ([`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go)). `connection_failed` keeps an integer `http_status` from 100 to 599; status 402 normalizes to `provider_billing_unavailable`, which keeps only status 402. This category describes the model endpoint billing rejection, not the application user balance, and does not authorize automatic retries. Every other status is discarded. A missing, malformed or unknown value leaves the error unclassified without discarding Usage or `done`. -Core stores accepted values in the Turn outcome as `engine_error_code` and `engine_http_status`. The classification is subordinate to the terminal status and Core's `error_code`: it cannot turn a completed or cancelled Turn into a failure, hide an incomplete event stream, or override a persistence or cancellation-receipt failure. Normal delivery and terminal journal draining use the same extraction. [Session diagnostics](../contracts/agents-api/session-diagnostics.md) expose the category only for a failed Turn whose Core error is `engine_failed`. The [Codex](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/codex/README.md) and [Claude Code](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/claude/README.md) adapter guides give each Harness's mapping; an adapter never classifies error prose. +Core stores accepted values in the Turn outcome as `engine_error_code` and `engine_http_status`. The optional `upstream_request_id` is accepted only as a UUID and stored privately as `engine_request_id`; it never enters Project diagnostics, public Session/Turn responses or SSE. The classification is subordinate to the terminal status and Core's `error_code`: it cannot turn a completed or cancelled Turn into a failure, hide an incomplete event stream, or override a persistence or cancellation-receipt failure. Normal delivery and terminal journal draining use the same extraction. [Session diagnostics](../contracts/agents-api/session-diagnostics.md) expose the category only for a failed Turn whose Core error is `engine_failed`. The [Codex](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/codex/README.md) and [Claude Code](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/claude/README.md) adapter guides give each Harness's mapping; adapters never classify arbitrary error prose. Codex alone recognizes its exact terminal HTTP 402 envelope when structured information is absent or `other`; see its adapter guide for the bounded rule. ## Workspace operations diff --git a/docs/zh/api/index.md b/docs/zh/api/index.md index 8e74b6ee9..18c8171b7 100644 --- a/docs/zh/api/index.md +++ b/docs/zh/api/index.md @@ -1,14 +1,14 @@ --- title: "API 命名空间和凭据" source: docs/api/index.md -source_hash: cb8f94a1fc272ce57b319fa375c9a82cc450092526492e2141d08ecca1cf42f2 +source_hash: 2c1e3e3be14472d15ef5b655d7a32a97d8d5261b8bbd28ce6e9b1109ee5ad90d --- Core 提供三个命名空间。每个命名空间都有一种调用方及其独立凭据,凭据只能在其所属命名空间中使用。 | 命名空间 | 调用方 | 凭据 | 内容 | 所有者 | | --- | --- | --- | --- | --- | -| `/v1` | 应用程序:业务系统和官方 OpenAI SDK | Project API key | 固定版本官方 Agents API 中全部且仅有的 58 个方法和路径对,列于 [upstream-routes.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/upstream-routes.json)。仅属于 Core 的字段位于 `x_agents_core` 中:`harness`、`model_provider`、`harness_config`、`environment`,以及只读的 Session `installation` | [Agents API 指南](public-agent-api.md) | +| `/v1` | 应用程序:业务系统和官方 OpenAI SDK | Project API key | 固定版本官方 Agents API 中 58 个官方方法和路径对,列于 [upstream-routes.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/upstream-routes.json)。另有两条 [Project 诊断扩展路由](../../../contracts/agents-api/zh/session-diagnostics.md#project-diagnostics-extension) 提供安全失败快照。官方对象中仅属于 Core 的字段位于 `x_agents_core` 中:`harness`、`model_provider`、`harness_config`、`environment`,以及只读的 Session `installation` | [Agents API 指南](public-agent-api.md) | | `/core/v1` | Web 的控制台服务器和操作员脚本 | [Core key](../getting-started/operations.md#core-key) | 安装信息、Project 和密钥、资源读取和删除、Session 归档、执行器凭据、默认模型、指标、审计、沙箱部署和节点、独立工作区存储配置 | [Core 管理 API](../../../contracts/agents-api/zh/admin-api.md) | | `/api/v1` | 节点、Runtime 守护进程、自托管执行器及其安装程序 | 机器凭据:节点注册令牌和节点凭据、安装授权、执行器凭据和守护进程凭据。每种凭据只能用于其各自的路由 | `/api/v1/sandbox-node/*` 和 `/api/v1/agent-daemon/*` 下的机器初始化与连接(包括 WebSockets),以及公共原生安装程序下载 | [机器连接 API](../../../contracts/agents-api/zh/machine-api.md) | diff --git a/docs/zh/api/public-agent-api.md b/docs/zh/api/public-agent-api.md index 698df8c07..ae42920d5 100644 --- a/docs/zh/api/public-agent-api.md +++ b/docs/zh/api/public-agent-api.md @@ -1,7 +1,7 @@ --- title: "Agents API 指南" source: docs/api/public-agent-api.md -source_hash: 4f830cdc1d1a73646d466b7de49496b72701b2f23f4842a384116cb75a91e5e5 +source_hash: 3210c3c7d871de2dc2fa8601d213bb93eacc830ee1643e56fd308673d2b68bf8 --- Core 在 `/v1` 提供 [OpenAI Agents API](https://platform.openai.com/docs/api-reference)。可以使用官方 OpenAI SDK 或普通 HTTP。本指南针对每项常见操作同时展示这两种方式,并说明 Core 与 OpenAI 存在差异的地方。 @@ -538,7 +538,7 @@ HTTP 路径为 `/vaults`,需要 Beta 请求头。Session 从其 `vault_ids` 1. 读取 Session 的 `status` 和 `error`,以及最新 Turn 的 `error`。失败的 Turn 只会报告通用的 `internal_error`。 2. 检查 Environment 是否已连接,以及其 harness 是否可用。 3. 在 [Harness capabilities](../../../contracts/agents-api/zh/harness-capabilities.md) 中检查 harness、模型和工具的组合。 -4. 向管理员索取 Session 的[诊断信息](../../../contracts/agents-api/zh/session-diagnostics.md),其中会指出故障类别;同时请查看[故障排除](../getting-started/operations.md#troubleshooting),了解服务日志、凭据和节点就绪状态。 +4. 读取 Session 的 [Project 诊断扩展](../../../contracts/agents-api/zh/session-diagnostics.md#project-diagnostics-extension) 获取安全故障分类,再请管理员查看[故障排除](../getting-started/operations.md#troubleshooting),了解服务日志、凭据和节点就绪状态。 401 通常表示使用了其他命名空间中的密钥;请参阅 [API 命名空间与凭据](index.md)。 diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index 6cb249e5d..8f95ce807 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: a3a06d0f5db73286e87f8784df45e304cb18685c43ede7c90880b127cacc453c +source_hash: a55eaf44f8d0240e41ce3b7f1ef76d7dcb0e0e0b8ebc5d41d0d3c2511fe5d88c --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon,定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次;Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们,因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -185,9 +185,9 @@ transport 与 execution 结果分开。Core 唯一 Run 订阅入口是 `Subscrib adapter 可以给 Run 的 `error` frame 添加 `code` 和 `http_status`。它们是可选的中性元数据,不是终结事件或公开错误契约;frame 文本、Usage、`done`、原生 Session 身份和取消回执保持原顺序与含义。 -接受的 code 为 `authentication_error`、`rate_limit_exceeded`、`usage_limit_exceeded`、`server_overloaded`、`server_error`、`invalid_request`、`resource_not_found`、`request_timeout`、`context_length_exceeded`、`cyber_policy` 和 `connection_failed`([`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go))。仅 `connection_failed` 保留 `http_status`,且只能是 100 到 599 的整数;其他 status 都丢弃。值缺失、格式错误或未知时保持未分类错误,不丢弃 Usage 或 `done`。 +接受的 code 为 `authentication_error`、`rate_limit_exceeded`、`usage_limit_exceeded`、`server_overloaded`、`server_error`、`invalid_request`、`resource_not_found`、`request_timeout`、`context_length_exceeded`、`cyber_policy`、`provider_billing_unavailable` 和 `connection_failed`([`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go))。`connection_failed` 保留 100 到 599 的整数 `http_status`;402 归一为 `provider_billing_unavailable`,此分类仅保留状态 402。它表示模型端点的计费拒绝,不代表应用用户余额不足,也不授权自动重试。其他 status 都丢弃。值缺失、格式错误或未知时保持未分类错误,不丢弃 Usage 或 `done`。 -Core 在 Turn outcome 中将接受的值保存为 `engine_error_code` 和 `engine_http_status`。分类从属于终结状态和 Core 的 `error_code`:不能将已完成或已取消 Turn 变成失败、隐藏不完整事件流,或覆盖持久化或取消回执失败。正常 delivery 和 terminal journal draining 使用同一提取逻辑。[Session 诊断](../../contracts/agents-api/zh/session-diagnostics.md)仅为 Core error 为 `engine_failed` 的失败 Turn 暴露类别。[Codex](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/codex/README.md) 和 [Claude Code](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/claude/README.md) adapter 指南给出各 Harness 的映射;adapter 不依据错误文字分类。 +Core 在 Turn outcome 中将接受的值保存为 `engine_error_code` 和 `engine_http_status`。可选 `upstream_request_id` 仅接受 UUID,并私有保存为 `engine_request_id`;不得进入 Project 诊断、公开 Session/Turn 响应或 SSE。分类从属于终结状态和 Core 的 `error_code`:不能将已完成或已取消 Turn 变成失败、隐藏不完整事件流,或覆盖持久化或取消回执失败。正常 delivery 和 terminal journal draining 使用同一提取逻辑。[Session 诊断](../../contracts/agents-api/zh/session-diagnostics.md)仅为 Core error 为 `engine_failed` 的失败 Turn 暴露类别。[Codex](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/codex/README.md) 和 [Claude Code](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/claude/README.md) adapter 指南给出各 Harness 的映射;adapter 不依据任意错误文字分类。仅 Codex 在结构化信息缺失或为 `other` 时识别准确的终态 HTTP 402 封装;严格边界见该适配器指南。 ## 工作区操作 {#workspace-operations} diff --git a/internal/agentdaemon/proto/engine_failure.go b/internal/agentdaemon/proto/engine_failure.go index 03574f499..0239089df 100644 --- a/internal/agentdaemon/proto/engine_failure.go +++ b/internal/agentdaemon/proto/engine_failure.go @@ -1,6 +1,19 @@ package proto -import "encoding/json" +import ( + "encoding/json" + "regexp" +) + +var providerRequestIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// NormalizeProviderRequestID permits only UUID correlation identifiers, never arbitrary headers. +func NormalizeProviderRequestID(id string) string { + if providerRequestIDPattern.MatchString(id) { + return id + } + return "" +} // NormalizeEngineFailure accepts only neutral, finite classifications. Optional // metadata from newer or malformed peers never changes terminal/usage decoding. @@ -10,6 +23,14 @@ func NormalizeEngineFailure(code string, status *int) (string, *int) { return code, nil case "connection_failed": if status != nil && *status >= 100 && *status <= 599 { + if *status == 402 { + return "provider_billing_unavailable", status + } + return code, status + } + return code, nil + case "provider_billing_unavailable": + if status != nil && *status == 402 { return code, status } return code, nil @@ -23,6 +44,7 @@ func (p *ErrorPayload) UnmarshalJSON(raw []byte) error { Error string `json:"error"` Code json.RawMessage `json:"code"` HTTPStatus json.RawMessage `json:"http_status"` + RequestID json.RawMessage `json:"upstream_request_id"` } if err := json.Unmarshal(raw, &wire); err != nil { return err @@ -33,5 +55,8 @@ func (p *ErrorPayload) UnmarshalJSON(raw []byte) error { _ = json.Unmarshal(wire.HTTPStatus, &status) p.Error = wire.Error p.Code, p.HTTPStatus = NormalizeEngineFailure(code, status) + var requestID string + _ = json.Unmarshal(wire.RequestID, &requestID) + p.UpstreamRequestID = NormalizeProviderRequestID(requestID) return nil } diff --git a/internal/agentdaemon/proto/engine_failure_test.go b/internal/agentdaemon/proto/engine_failure_test.go index 29b6bf38b..df4436866 100644 --- a/internal/agentdaemon/proto/engine_failure_test.go +++ b/internal/agentdaemon/proto/engine_failure_test.go @@ -40,3 +40,22 @@ func TestErrorClassificationCompatibility(t *testing.T) { } } } + +func TestProviderBillingClassificationAndCorrelation(t *testing.T) { + for _, code := range []string{"connection_failed", "provider_billing_unavailable"} { + raw := `{"error":"private-body","code":"` + code + `","http_status":402,"upstream_request_id":"88a8d808-59c1-4bf3-8950-b34ce57e028b"}` + var p ErrorPayload + if err := json.Unmarshal([]byte(raw), &p); err != nil { + t.Fatal(err) + } + if p.Code != "provider_billing_unavailable" || p.HTTPStatus == nil || *p.HTTPStatus != 402 || p.UpstreamRequestID == "" { + t.Fatal(p) + } + } + for _, id := range []string{`"Bearer secret"`, `123`, `{"token":"secret"}`, `null`} { + var p ErrorPayload + if err := json.Unmarshal([]byte(`{"error":"unchanged","upstream_request_id":`+id+`}`), &p); err != nil || p.UpstreamRequestID != "" || p.Error != "unchanged" { + t.Fatal(err, p) + } + } +} diff --git a/internal/agentdaemon/proto/inbound.go b/internal/agentdaemon/proto/inbound.go index dfbc36f2d..989e00d60 100644 --- a/internal/agentdaemon/proto/inbound.go +++ b/internal/agentdaemon/proto/inbound.go @@ -126,9 +126,10 @@ type UsagePayload struct { // ErrorPayload reports a prompt-level failure. type ErrorPayload struct { - Error string `json:"error"` - Code string `json:"code,omitempty"` - HTTPStatus *int `json:"http_status,omitempty"` + Error string `json:"error"` + Code string `json:"code,omitempty"` + HTTPStatus *int `json:"http_status,omitempty"` + UpstreamRequestID string `json:"upstream_request_id,omitempty"` } // DonePayload mirrors connector.PromptOutput shape. Redeclared (not diff --git a/packages/agents-client/src/core-metrics.test.ts b/packages/agents-client/src/core-metrics.test.ts index b160cf6c8..143e0094f 100644 --- a/packages/agents-client/src/core-metrics.test.ts +++ b/packages/agents-client/src/core-metrics.test.ts @@ -23,6 +23,17 @@ describe("Core metrics client", () => { }); describe("Core metrics projection", () => { + it("preserves measured terminal failures and rejects malformed counts", () => { + const terminal = { total: 3, completed: 1, failed: 2, cancelled: 0, failures: [{ code: "provider_billing_unavailable", source: "turn", count: 2 }] }; + const body = { object: "core.metrics", range: { resolution_seconds: 60 }, service: { status: "running" }, execution: { slots_in_use: 0, slots_total: 4, connected_daemons: 0, terminal_turns: terminal } }; + expect(projectCoreMetrics(body).execution.terminal_turns).toEqual(terminal); + for (const total of [-1, 1.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => projectCoreMetrics({ ...body, execution: { ...body.execution, terminal_turns: { ...terminal, total } } })).toThrow(); + } + expect(() => projectCoreMetrics({ ...body, execution: { ...body.execution, terminal_turns: { ...terminal, total: "3" } } })).toThrow(); + expect(projectCoreMetrics({ ...body, execution: { ...body.execution, terminal_turns: null } }).execution.terminal_turns).toBeNull(); + }); + it("keeps unmeasured figures null instead of zero", () => { const metrics = projectCoreMetrics({ object: "core.metrics", diff --git a/packages/agents-client/src/core-metrics.ts b/packages/agents-client/src/core-metrics.ts index f6859cfd1..632639f2c 100644 --- a/packages/agents-client/src/core-metrics.ts +++ b/packages/agents-client/src/core-metrics.ts @@ -2,7 +2,7 @@ import { AgentCoreError } from "./client"; import { CoreRequester, type CoreClientOptions } from "./core-request"; import { isOneOf } from "./response-projection"; import type { ReadOptions } from "./types"; -import { jobStatusValues, serviceStatusValues, type CoremetricsView, type JobStatus, type Latency, type ServiceState, type ServiceStatus } from "./generated/core-api"; +import { jobStatusValues, serviceStatusValues, type CoremetricsView, type JobStatus, type Latency, type ServiceState, type ServiceStatus, type TerminalTurns } from "./generated/core-api"; function invalidCoreMetrics(): never { throw new AgentCoreError("Core metrics: the response is not JSON.", 0, "invalid_response"); @@ -37,9 +37,8 @@ function number(value: unknown): number | null { } function count(value: unknown, path: string): number { - const result = number(value); - if (result === null) throw new AgentCoreError(`Core metrics: ${path} is missing.`, 0, "invalid_response"); - return result; + if (!Number.isSafeInteger(value) || (value as number) < 0) throw new AgentCoreError(`Core metrics: ${path} is not a count.`, 0, "invalid_response"); + return value as number; } function text(value: unknown): string | null { @@ -57,6 +56,21 @@ function latency(value: unknown): Latency { return { p50: number(entry.p50), p95: number(entry.p95) }; } +function terminalTurns(value: unknown): TerminalTurns | null { + if (value === undefined || value === null) return null; + const entry = record(value, "execution.terminal_turns"); + return { + total: count(entry.total, "terminal_turns.total"), + completed: count(entry.completed, "terminal_turns.completed"), + failed: count(entry.failed, "terminal_turns.failed"), + cancelled: count(entry.cancelled, "terminal_turns.cancelled"), + failures: list(entry.failures, (failure) => { + if (failure.source !== "turn" || !text(failure.code)) throw new AgentCoreError("Core metrics: invalid failure category.", 0, "invalid_response"); + return { code: String(failure.code), source: "turn" as const, count: count(failure.count, "terminal_turns.failures.count") }; + }, "terminal_turns.failures"), + }; +} + /** Validates the envelope and normalises every missing figure to null. */ export function projectCoreMetrics(value: unknown): CoreMetrics { const body = record(value, "response"); @@ -80,6 +94,7 @@ export function projectCoreMetrics(value: unknown): CoreMetrics { execution_owner: typeof service.execution_owner === "boolean" ? service.execution_owner : null, }, execution: { + terminal_turns: terminalTurns(execution.terminal_turns), slots_in_use: count(execution.slots_in_use, "execution.slots_in_use"), slots_total: count(execution.slots_total, "execution.slots_total"), queued_turns: number(execution.queued_turns), diff --git a/packages/agents-client/src/generated/core-api.ts b/packages/agents-client/src/generated/core-api.ts index 41f6ffd14..9dada9b02 100644 --- a/packages/agents-client/src/generated/core-api.ts +++ b/packages/agents-client/src/generated/core-api.ts @@ -226,7 +226,7 @@ export interface DiagnosticFailure { params: Record; } export const diagnosticFailureFields = ["code", "failed_at", "params"] as const; -export const diagnosticFailureCodeValues = ["harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error", "environment_connection_timeout", "environment_unavailable", "environment_provisioning_failed", "authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy", "connection_failed"] as const; +export const diagnosticFailureCodeValues = ["harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error", "environment_connection_timeout", "environment_unavailable", "environment_provisioning_failed", "authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "provider_billing_unavailable", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy", "connection_failed"] as const; export type DiagnosticFailureCode = (typeof diagnosticFailureCodeValues)[number]; export const diagnosticSourceValues = ["turn", "environment", "environment_input"] as const; export type DiagnosticSource = (typeof diagnosticSourceValues)[number]; @@ -246,10 +246,11 @@ export interface Execution { series: ExecutionBucket[]; slots_in_use: number; slots_total: number; + terminal_turns: TerminalTurns | null; unavailable: number | null; waiting_for_daemon: number | null; } -export const executionFields = ["connected_daemons", "in_progress_turns", "interrupted", "oldest_queued_seconds", "queue_wait_ms", "queued_turns", "series", "slots_in_use", "slots_total", "unavailable", "waiting_for_daemon"] as const; +export const executionFields = ["connected_daemons", "in_progress_turns", "interrupted", "oldest_queued_seconds", "queue_wait_ms", "queued_turns", "series", "slots_in_use", "slots_total", "terminal_turns", "unavailable", "waiting_for_daemon"] as const; export interface ExecutionBucket { in_progress: number | null; queue_wait_p95_ms: number | null; @@ -297,6 +298,13 @@ export interface ExecutorCredentialList { data: ExecutorCredential[]; } export const executorCredentialListFields = ["connection", "data"] as const; +export interface FailureCount { + code?: string; + count?: number; + source?: "turn"; +} +export const failureCountFields = ["code", "count", "source"] as const; +export const failureCountRequired = [] as const; export interface HarnessModelConfiguration { harness: HarnessModelConfigurationHarness; harness_config: Record; @@ -848,7 +856,7 @@ export interface SessionDiagnosticFailure { } export const sessionDiagnosticFailureFields = ["code", "failed_at", "params", "source", "turn_id"] as const; export const sessionDiagnosticFailureRequired = ["code", "failed_at", "params", "source"] as const; -export const sessionDiagnosticFailureCodeValues = ["harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error", "environment_connection_timeout", "environment_unavailable", "environment_provisioning_failed", "authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy", "connection_failed"] as const; +export const sessionDiagnosticFailureCodeValues = ["harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error", "environment_connection_timeout", "environment_unavailable", "environment_provisioning_failed", "authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "provider_billing_unavailable", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy", "connection_failed"] as const; export type SessionDiagnosticFailureCode = (typeof sessionDiagnosticFailureCodeValues)[number]; export interface SessionDiagnostics { failure: SessionDiagnosticFailure | null; @@ -874,6 +882,15 @@ export interface Suspension { retention_seconds: number; } export const suspensionFields = ["idle_seconds", "retention_seconds"] as const; +export interface TerminalTurns { + cancelled?: number; + completed?: number; + failed?: number; + failures?: FailureCount[]; + total?: number; +} +export const terminalTurnsFields = ["cancelled", "completed", "failed", "failures", "total"] as const; +export const terminalTurnsRequired = [] as const; export interface TurnDiagnostics { failure: DiagnosticFailure | null; items: ItemDiagnosticTiming[]; diff --git a/packages/agents-client/src/generated/public-api.ts b/packages/agents-client/src/generated/public-api.ts index 80d6fa6c0..f05688a4c 100644 --- a/packages/agents-client/src/generated/public-api.ts +++ b/packages/agents-client/src/generated/public-api.ts @@ -470,6 +470,12 @@ export interface ErrorResponse2 { error: ErrorBodyResource; } export const errorResponse2Fields = ["error"] as const; +export interface ExecutionDiagnostic { + code: "unknown" | "harness_error" | "model_provider_required" | "runtime_unavailable" | "runtime_disconnected" | "runtime_preparation_failed" | "execution_interrupted" | "delivery_unconfirmed" | "input_rejected" | "executor_protocol_error" | "core_storage_failed" | "environment_connection_timeout" | "environment_unavailable" | "environment_provisioning_failed" | "authentication_error" | "rate_limit_exceeded" | "usage_limit_exceeded" | "provider_billing_unavailable" | "server_overloaded" | "server_error" | "invalid_request" | "resource_not_found" | "request_timeout" | "context_length_exceeded" | "cyber_policy" | "connection_failed"; + failed_at: string | null; + source: "turn" | "environment" | "environment_input"; +} +export const executionDiagnosticFields = ["code", "failed_at", "source"] as const; export interface FileExpirationAfter { anchor: "created_at"; seconds: number; @@ -1109,6 +1115,17 @@ export interface SessionCore { } export const sessionCoreFields = ["installation"] as const; export const sessionCoreRequired = [] as const; +export interface SessionDiagnostics { + diagnostic: ExecutionDiagnostic | null; + input_available?: boolean; + input_unavailable_reason?: "busy" | "requires_action" | "environment_unavailable" | "configuration_required"; + object: "agent.session_diagnostics"; + session_id: string; + status: "idle" | "in_progress" | "requires_action" | "failed"; + turn_id?: string; +} +export const sessionDiagnosticsFields = ["diagnostic", "input_available", "input_unavailable_reason", "object", "session_id", "status", "turn_id"] as const; +export const sessionDiagnosticsRequired = ["diagnostic", "object", "session_id", "status"] as const; export interface SessionEnvironmentErrorResource { type: string; code: string; @@ -1587,6 +1604,14 @@ export interface TokenUsageResource { total_tokens: number; } export const tokenUsageResourceFields = ["input_tokens", "input_tokens_details", "output_tokens", "output_tokens_details", "total_tokens"] as const; +export interface TurnDiagnostics { + diagnostic: ExecutionDiagnostic | null; + object: "agent.turn_diagnostics"; + session_id: string; + status: "queued" | "in_progress" | "waiting" | "completed" | "failed" | "cancelled"; + turn_id: string; +} +export const turnDiagnosticsFields = ["diagnostic", "object", "session_id", "status", "turn_id"] as const; export const turnObjectResourceValues = ["agent.session.turn"] as const; export type TurnObjectResource = (typeof turnObjectResourceValues)[number]; export interface TurnResource { diff --git a/packages/agents-client/src/session-diagnostics.test.ts b/packages/agents-client/src/session-diagnostics.test.ts index d8ef7b405..b37d54c1d 100644 --- a/packages/agents-client/src/session-diagnostics.test.ts +++ b/packages/agents-client/src/session-diagnostics.test.ts @@ -86,3 +86,18 @@ it.each([ ] as const)("rejects arrays in %s", (_field, project) => { expect(project).toThrow(); }); + + it.each([402, null])("retrieves billing diagnostics for Session and Turn with status %s", async (http_status) => { + const f = { ...failure, code: "provider_billing_unavailable", params: { http_status } }; + const sessionValue = { ...session, failure: { ...f, source: "turn", turn_id: turn } }; + const turnValue = { ...snapshot, failure: f }; + const client = new AdminClient({ adminToken: "admin", fetch: (async (url) => new Response(JSON.stringify(String(url).includes("/turns/") ? turnValue : sessionValue), { headers: { "Content-Type": "application/json" } })) as typeof fetch }); + expect(await client.retrieveSessionDiagnostics("project", id)).toEqual(sessionValue); + expect(await client.retrieveTurnDiagnostics("project", id, turn)).toEqual(turnValue); + expect(() => projectSessionDiagnostics({ ...session, failure: { ...f, source: "environment_input" } }, id)).toThrow(); +}); +it.each([{}, { http_status: 400 }, { http_status: 403 }, { http_status: 500 }, { http_status: 401 }, { http_status: 429 }, { http_status: "402" }, { http_status: 402.5 }, { http_status: 402, message: "private-canary" }])("rejects malformed billing parameters on both routes", (params) => { + const f = { ...failure, code: "provider_billing_unavailable", params }; + expect(() => projectSessionDiagnostics({ ...session, failure: { ...f, source: "turn", turn_id: turn } }, id)).toThrow(); + expect(() => projectTurnDiagnostics({ ...snapshot, failure: f }, id, turn)).toThrow(); +}); diff --git a/packages/agents-client/src/session-diagnostics.ts b/packages/agents-client/src/session-diagnostics.ts index 27ed6925d..3bb655ae7 100644 --- a/packages/agents-client/src/session-diagnostics.ts +++ b/packages/agents-client/src/session-diagnostics.ts @@ -20,7 +20,7 @@ export type SessionDiagnostics = Omit & { export type TurnDiagnostics = Omit & { failure: DiagnosticFailure | null }; // The schema's one code enum covers every failure source; each source produces only its subset. -const turnCodes: readonly DiagnosticFailureCode[] = ["authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy", "connection_failed", "harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error"]; +const turnCodes: readonly DiagnosticFailureCode[] = ["authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy", "connection_failed", "provider_billing_unavailable", "harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error"]; const inputCodes: readonly DiagnosticFailureCode[] = ["environment_connection_timeout", "environment_unavailable", "model_provider_required", "internal_error"]; const environmentCodes: readonly DiagnosticFailureCode[] = ["environment_provisioning_failed"]; const fields = (...values: string[]) => new Set(values); @@ -37,9 +37,10 @@ function failure(value: unknown, codes: readonly DiagnosticFailureCode[], sessio (p.index !== null && (!Number.isSafeInteger(p.index) || (p.index as number) < 0 || p.step !== "setup")) || (p.exit_code !== null && (!Number.isSafeInteger(p.exit_code) || (p.exit_code as number) < 1 || (p.exit_code as number) > 255 || !["setup", "python", "npm", "system"].includes(p.step ?? "")))) return invalidAdminResponse(); params = { step: p.step as ProvisioningFailureParams["step"], index: p.index as number | null, exit_code: p.exit_code as number | null }; - } else if (value.code === "connection_failed") { + } else if (value.code === "connection_failed" || value.code === "provider_billing_unavailable") { const p = value.params; if (!exactFields(p, fields("http_status")) || (p.http_status !== null && (!Number.isSafeInteger(p.http_status) || (p.http_status as number) < 100 || (p.http_status as number) > 599))) return invalidAdminResponse(); + if (value.code === "provider_billing_unavailable" && p.http_status !== null && p.http_status !== 402) return invalidAdminResponse(); params = { http_status: p.http_status as number | null }; } else if (Object.keys(value.params).length !== 0) return invalidAdminResponse(); return { code: value.code, params, failed_at: value.failed_at as string | null }; diff --git a/scripts/generate-public-api.py b/scripts/generate-public-api.py index a9621d7f6..e3f75f96a 100644 --- a/scripts/generate-public-api.py +++ b/scripts/generate-public-api.py @@ -277,7 +277,7 @@ def extension_owners(bindings): return owners -def public_document(source, extensions, owners, beta_header): +def public_document(source, extensions, owners, beta_header, extension_paths=None): doc = copy.deepcopy(source) doc['info'] = {'title': 'OpenAgentCore public API', 'version': 'v1', 'description': 'The pinned OpenAI Agents, Files and Skills API with x_agents_core extensions. See the coverage ledger for implementation qualification.'} doc['servers'] = [{'url': '/v1'}] @@ -292,6 +292,12 @@ def public_document(source, extensions, owners, beta_header): operation['parameters'] = operation.get('parameters', []) + [{'name': 'OpenAI-Beta', 'in': 'header', 'required': True, 'schema': {'type': 'string', 'const': beta_header}}] # Extensions are authored in Core's Go types and projected by swag. Official # properties are never rewritten to accommodate a narrower implementation. + bindings = json.loads((CONTRACT / 'go-bindings.json').read_text()) + official_refs = { + '#/definitions/v1.' + name: binding['sources'][0] + for name, binding in bindings.items() + if len(binding.get('sources', [])) == 1 + } def convert(value): if isinstance(value, list): return [convert(v) for v in value] @@ -299,10 +305,29 @@ def convert(value): return value result = {k: convert(v) for k, v in value.items() if k != 'x-nullable'} if '$ref' in result: - result['$ref'] = result['$ref'].replace('#/definitions/', '#/components/schemas/') + result['$ref'] = official_refs.get(result['$ref'], result['$ref'].replace('#/definitions/', '#/components/schemas/')) if value.get('x-nullable'): return {'anyOf': [result, {'type': 'null'}]} return result + for path, item in (extension_paths or {}).items(): + if path in doc['paths']: + raise ValueError(f'extension cannot replace official path: {path}') + projected = convert(copy.deepcopy(item)) + for method, operation in projected.items(): + if method not in HTTP_METHODS: + continue + operation['x-agents-core-extension'] = True + operation.pop('security', None) + operation.pop('produces', None) + operation.pop('consumes', None) + for parameter in operation.get('parameters', []): + if 'schema' not in parameter: + parameter['schema'] = {k: parameter.pop(k) for k in ('type', 'format', 'enum') if k in parameter} + operation['parameters'] = operation.get('parameters', []) + [{'name': 'OpenAI-Beta', 'in': 'header', 'required': True, 'schema': {'type': 'string', 'const': beta_header}}] + for response in operation.get('responses', {}).values(): + if 'schema' in response: + response['content'] = {'application/json': {'schema': response.pop('schema')}} + doc['paths'][path] = projected doc['components']['schemas'].update({k: convert(v) for k, v in extensions.items()}) for name, extension in owners.items(): doc['components']['schemas'][name]['properties']['x_agents_core'] = {'anyOf': [{'$ref': '#/components/schemas/v1.' + extension}, {'type': 'null'}]} @@ -452,12 +477,13 @@ def main(): roots = 'package extensions\n\n' + '\n'.join(f'// @Success 200 {{object}} v1.{name}' for name in sorted(set(owners.values()))) + '\nfunc extensions() {}\n' args.swag_roots.write_text(roots) else: - doc = public_document(source, json.loads(args.extensions.read_text()), owners, pin["beta_header"]) + overlay = json.loads(args.extensions.read_text()) + doc = public_document(source, overlay["definitions"], owners, pin["beta_header"], overlay["paths"]) # JSON is a YAML subset and keeps generation independent of PyYAML. write(CONTRACT / 'openapi.yaml', (json.dumps(doc, indent=2, ensure_ascii=False) + '\n').encode(), args.check) write(CLIENT_TYPES, ts_module(doc['components']['schemas'], 'contracts/agents-api/openapi.yaml'), args.check) write(CORE_CLIENT_TYPES, core_module(json.loads(args.core.read_text()), doc['components']['schemas'], bindings), args.check) - routes = sorted(method.upper() + ' ' + re.sub(r'\{[^}]*\}', '{}', path) for path, item in doc['paths'].items() for method in item if method in HTTP_METHODS) + routes = sorted(method.upper() + ' ' + re.sub(r'\{[^}]*\}', '{}', path) for path, item in doc['paths'].items() for method in item if method in HTTP_METHODS and not item[method].get("x-agents-core-extension")) write(CONTRACT / 'upstream-routes.json', (json.dumps({'openapi_commit': pin['openapi']['commit'], 'generator': 'scripts/generate-public-api.py', 'routes': routes}, indent=2) + '\n').encode(), args.check) diff --git a/scripts/generate-public-api.test.py b/scripts/generate-public-api.test.py index 48ebbed83..d93137546 100644 --- a/scripts/generate-public-api.test.py +++ b/scripts/generate-public-api.test.py @@ -24,10 +24,27 @@ def test_generated_types_are_current(self): def test_public_projection_is_current_and_keeps_source_immutable(self): source = copy.deepcopy(self.source) extensions = {k: v for k, v in self.public['components']['schemas'].items() if k not in source['components']['schemas']} - actual = generator.public_document(source, extensions, generator.extension_owners(self.bindings), self.pin["beta_header"]) + paths = {p: copy.deepcopy(v) for p, v in self.public['paths'].items() if any(op.get('x-agents-core-extension') for op in v.values() if isinstance(op, dict))} + # Feed Swagger-style operations back into the projection. + for item in paths.values(): + for operation in item.values(): + operation['parameters'] = [p for p in operation.get('parameters', []) if p.get('name') != 'OpenAI-Beta'] + actual = generator.public_document(source, extensions, generator.extension_owners(self.bindings), self.pin["beta_header"], paths) self.assertEqual(actual, self.public) self.assertEqual(source, self.source) + def test_diagnostic_extensions_do_not_change_official_routes(self): + extensions = [p for p, item in self.public['paths'].items() if item.get('get', {}).get('x-agents-core-extension')] + self.assertEqual(sorted(extensions), ['/agents/sessions/{session_id}/diagnostics', '/agents/sessions/{session_id}/turns/{turn_id}/diagnostics']) + routes = json.loads((generator.CONTRACT / 'upstream-routes.json').read_text())['routes'] + self.assertEqual(len(routes), 58) + self.assertTrue(all('/diagnostics' not in p for p in routes)) + schemas = self.public['components']['schemas'] + self.assertEqual(set(schemas['v1.ExecutionDiagnostic']['properties']), {'code', 'source', 'failed_at'}) + self.assertEqual(set(schemas['v1.TurnDiagnostics']['properties']['status']['enum']), {'queued', 'in_progress', 'waiting', 'completed', 'failed', 'cancelled'}) + with self.assertRaisesRegex(ValueError, 'cannot replace official path'): + generator.public_document(self.source, {}, {}, self.pin['beta_header'], {'/agents': {}}) + def test_union_and_nullable_fields_keep_the_official_schema(self): schemas = self.public['components']['schemas'] self.assertIn('anyOf', schemas['CreateAgentSessionParams']['properties']['input']) diff --git a/scripts/openapi-split/main.go b/scripts/openapi-split/main.go index 64cf47796..f0f3bc16a 100644 --- a/scripts/openapi-split/main.go +++ b/scripts/openapi-split/main.go @@ -33,8 +33,7 @@ func main() { } // run exports Go-owned definitions for the public extension overlay and the -// Core client types, and emits the two internal contracts. Public operations -// come only from official OpenAPI. +// Core client types, and emits the two internal contracts. Explicit extension operations are exported separately. func run(input, extensionOutput, coreJSONOutput, coreOutput, runtimeOutput string, check bool) error { raw, err := os.ReadFile(input) if err != nil { @@ -44,7 +43,25 @@ func run(input, extensionOutput, coreJSONOutput, coreOutput, runtimeOutput strin if err := yaml.Unmarshal(raw, &source); err != nil { return err } - if err := writeJSON(field(source.Content[0], "definitions"), extensionOutput); err != nil { + var definitions map[string]any + if err := field(source.Content[0], "definitions").Decode(&definitions); err != nil { + return err + } + var paths map[string]any + if err := field(source.Content[0], "paths").Decode(&paths); err != nil { + return err + } + projectPaths := map[string]any{} + for path, item := range paths { + if strings.HasPrefix(path, "/v1/") { + projectPaths[strings.TrimPrefix(path, "/v1")] = item + } + } + extensionJSON, err := json.Marshal(map[string]any{"definitions": definitions, "paths": projectPaths}) + if err != nil { + return err + } + if err := os.WriteFile(extensionOutput, extensionJSON, 0644); err != nil { return err } for _, output := range []struct { diff --git a/scripts/openapi-split/main_test.go b/scripts/openapi-split/main_test.go index 542d47ca5..79aa67e61 100644 --- a/scripts/openapi-split/main_test.go +++ b/scripts/openapi-split/main_test.go @@ -111,7 +111,7 @@ securityDefinitions: t.Fatal(err) } } - if len(extensions) != 4 || len(coreDocument.Definitions) != 2 || coreDocument.Definitions["Node"] == nil { + if len(extensions["definitions"].(map[string]any)) != 4 || len(coreDocument.Definitions) != 2 || coreDocument.Definitions["Node"] == nil { t.Fatalf("definitions: extensions %v, core %v", extensions, coreDocument.Definitions) } // Every internal document keeps only the schemes its operations use. diff --git a/services/core/deploy/codex/README.md b/services/core/deploy/codex/README.md index 645908565..53f81999a 100644 --- a/services/core/deploy/codex/README.md +++ b/services/core/deploy/codex/README.md @@ -55,6 +55,8 @@ The adapter classifies a failed Turn only from the exact root terminal Turn's `c | `httpConnectionFailed`, `responseStreamConnectionFailed`, `responseStreamDisconnected` | `connection_failed`, with the upstream `httpStatusCode` | | `responseTooManyFailedAttempts` | `rate_limit_exceeded` when `httpStatusCode` is 429, otherwise `connection_failed` | +HTTP status 402 normalizes to `provider_billing_unavailable` and is not a retry recommendation. When `codexErrorInfo` is missing, null, or the exact string `other`, the adapter can recognize a single-line native terminal message beginning exactly `unexpected status 402 Payment Required`, followed only by the native colon/comma separator or end of message. It never searches partial assistant text, accumulated outcome text, retry notifications, response-body keywords or unknown structured variants. Known structured variants take precedence. A trailing `, request id: ` UUID is retained as private correlation only; arbitrary identifiers, multiline messages and suffixes are rejected. Terminal logs contain classification, status, bounded correlation and lengths rather than raw native payloads. + Every other variant stays unclassified. ## Runtime image diff --git a/services/core/internal/api/handler.go b/services/core/internal/api/handler.go index dc4af3f1c..ea510f092 100644 --- a/services/core/internal/api/handler.go +++ b/services/core/internal/api/handler.go @@ -101,6 +101,8 @@ func (h *Handler) routes() *chi.Mux { r.Post("/agents/sessions", h.createSession) r.Get("/agents/sessions", h.listSessions) r.Get("/agents/sessions/{session_id}", h.getSession) + r.Get("/agents/sessions/{session_id}/diagnostics", h.getProjectSessionDiagnostics) + r.Get("/agents/sessions/{session_id}/turns/{turn_id}/diagnostics", h.getProjectTurnDiagnostics) r.Post("/agents/sessions/{session_id}", h.updateSession) r.Delete("/agents/sessions/{session_id}", h.deleteSession) r.Post("/agents/sessions/{session_id}/events", h.createEvents) diff --git a/services/core/internal/api/project_diagnostics.go b/services/core/internal/api/project_diagnostics.go new file mode 100644 index 000000000..88c9671e1 --- /dev/null +++ b/services/core/internal/api/project_diagnostics.go @@ -0,0 +1,119 @@ +package api + +import ( + "context" + "net/http" + "time" + + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/go-chi/chi/v5" +) + +// @Summary Retrieve Project Session diagnostics +// @Description OpenAgentCore extension. Project key only. Safe committed failure classification without private execution details. +// @Tags Diagnostics +// @Produce json +// @Security ProjectKey +// @Param session_id path string true "Session ID" +// @Success 200 {object} v1.SessionDiagnostics +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /v1/agents/sessions/{session_id}/diagnostics [get] +func (h *Handler) getProjectSessionDiagnostics(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + session, err := h.SessionsReader.GetSession(ctx, tenantID(r), chi.URLParam(r, "session_id")) + if err != nil { + writeSessionsError(w, r, err) + return + } + public, err := sessionResponse(session, h.Execution.ExecutorURL) + if err != nil { + writeSessionsError(w, r, err) + return + } + snapshot := sessionDiagnosticResponse(session, public.ID, public.Status) + response := v1.SessionDiagnostics{Object: "agent.session_diagnostics", SessionID: public.ID, Status: public.Status} + available, reason := projectInputAvailability(session, public.Status, public.Environment.Type) + response.InputAvailable, response.InputUnavailableReason = &available, reason + if snapshot.Failure != nil { + response.TurnID = snapshot.Failure.TurnID + response.Diagnostic = projectDiagnostic(&snapshot.Failure.DiagnosticFailure, string(snapshot.Failure.Source)) + } + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, http.StatusOK, response) +} + +// A failed Turn is history, not a closed Session. This is a snapshot hint for +// starting a new Turn, never authorization to replay an input or bypass the +// transactional submission gate (including concurrent file writes). +func projectInputAvailability(session sessions.Session, status, environmentType string) (bool, string) { + if session.EnvironmentFailure != nil { + return false, "environment_unavailable" + } + if status == "requires_action" { + return false, "requires_action" + } + if turn := session.LastTurn; turn != nil && (turn.Status == sessions.TurnQueued || turn.Status == sessions.TurnInProgress || turn.Status == sessions.TurnWaiting) { + return false, "busy" + } + if session.PendingInput || status == "in_progress" { + return false, "busy" + } + if environmentType != "none" { + if session.Environment == nil || (session.Environment.Status != "connected" && session.Environment.Status != "ready") { + return false, "environment_unavailable" + } + } + if activity := session.EnvironmentInputActivity; activity != nil && activity.Failure == "model_provider_required" { + return false, "configuration_required" + } + if turn := session.LastTurn; turn != nil && turn.Status == sessions.TurnFailed { + if code, _ := sessions.DiagnosticFailureCode(turn.Outcome); code == "model_provider_required" { + return false, "configuration_required" + } + } + if status != "idle" && status != "failed" { + return false, "configuration_required" + } + return true, "" +} + +// @Summary Retrieve Project Turn diagnostics +// @Description OpenAgentCore extension. Project key only. Safe committed failure classification without private execution details. +// @Tags Diagnostics +// @Produce json +// @Security ProjectKey +// @Param session_id path string true "Session ID" +// @Param turn_id path string true "Root Turn ID" +// @Success 200 {object} v1.TurnDiagnostics +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /v1/agents/sessions/{session_id}/turns/{turn_id}/diagnostics [get] +func (h *Handler) getProjectTurnDiagnostics(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + snapshot, err := h.SessionAdmin.GetTurnDiagnosticsSnapshot(ctx, tenantID(r), chi.URLParam(r, "session_id"), chi.URLParam(r, "turn_id")) + if err != nil { + writeSessionsError(w, r, err) + return + } + public, err := turnResponse(snapshot.Session, snapshot.Turn) + if err != nil { + writeSessionsError(w, r, err) + return + } + response := v1.TurnDiagnostics{Object: "agent.turn_diagnostics", SessionID: public.SessionID, TurnID: public.ID, Status: public.Status, Diagnostic: projectDiagnostic(turnDiagnosticFailure(snapshot.Turn), "turn")} + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, http.StatusOK, response) +} + +func projectDiagnostic(failure *DiagnosticFailure, source string) *v1.ExecutionDiagnostic { + if failure == nil { + return nil + } + code := failure.Code + if code == "internal_error" { + code = "unknown" + } + return &v1.ExecutionDiagnostic{Code: code, Source: source, FailedAt: failure.FailedAt} +} diff --git a/services/core/internal/api/project_diagnostics_test.go b/services/core/internal/api/project_diagnostics_test.go new file mode 100644 index 000000000..67fd800b3 --- /dev/null +++ b/services/core/internal/api/project_diagnostics_test.go @@ -0,0 +1,179 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/google/uuid" +) + +func TestProjectDiagnosticsSafeProjection(t *testing.T) { + id, turn := uuid.NewString(), uuid.NewString() + session := sessions.Session{ID: id, Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`), LastTurn: &sessions.Turn{ID: turn, SessionID: id, Status: sessions.TurnFailed, Outcome: json.RawMessage(`{"error_code":"engine_failed","engine_error_code":"rate_limit_exceeded","error":"secret-canary","engine_http_status":429}`)}} + deps, fakes := testDependencies(t) + key := callerBinding() + fakes.projectsReader.resolveAPIKey = projectKeys(t, key).ResolveAPIKey + serveDiagnostics(diagnosticSnapshotStore{session: session})(&deps, fakes) + h := newTestHandler(t, deps) + for _, suffix := range []string{"/diagnostics", "/turns/" + turn + "/diagnostics"} { + path := "/v1/agents/sessions/" + id + suffix + for _, token := range []string{"", "Bearer admin"} { + if w := diagnosticRequest(h, path, token); w.Code != 401 { + t.Fatalf("credential isolation: %d", w.Code) + } + } + w := diagnosticRequest(h, path, "Bearer caller") + if w.Code != 200 || w.Header().Get("Cache-Control") != "no-store" { + t.Fatal(w.Code, w.Body) + } + body := w.Body.String() + if !strings.Contains(body, `"code":"rate_limit_exceeded"`) || !strings.Contains(body, `"source":"turn"`) || !strings.Contains(body, `"failed_at":null`) { + t.Fatal(body) + } + for _, forbidden := range []string{"canary", "params", "items", "engine_http_status"} { + if strings.Contains(body, forbidden) { + t.Fatal(body) + } + } + } +} + +func TestProjectDiagnosticsReadErrorIsNotExecutionFailure(t *testing.T) { + for _, err := range []error{sessions.ErrNotFound, errors.New("private-secret-canary")} { + deps, fakes := testDependencies(t) + key := callerBinding() + fakes.projectsReader.resolveAPIKey = projectKeys(t, key).ResolveAPIKey + fakes.sessionsReader.getSession = func(_ context.Context, tenant, id string) (sessions.Session, error) { + if tenant != key.TenantID { + t.Fatal("wrong tenant") + } + return sessions.Session{}, err + } + fakes.sessionAdmin.getTurnDiagnosticsSnapshot = func(_ context.Context, tenant, session, turn string) (sessions.TurnDiagnosticsSnapshot, error) { + if tenant != key.TenantID { + t.Fatal("wrong tenant") + } + return sessions.TurnDiagnosticsSnapshot{}, err + } + h := newTestHandler(t, deps) + for _, suffix := range []string{"/diagnostics", "/turns/" + uuid.NewString() + "/diagnostics"} { + w := diagnosticRequest(h, "/v1/agents/sessions/"+uuid.NewString()+suffix, "Bearer caller") + want := 500 + if errors.Is(err, sessions.ErrNotFound) { + want = 404 + } + if w.Code != want || strings.Contains(w.Body.String(), "canary") || strings.Contains(w.Body.String(), `"diagnostic"`) { + t.Fatal(w.Code, w.Body) + } + } + } +} + +func TestProjectDiagnosticUnknownAndNonFailure(t *testing.T) { + if got := projectDiagnostic(nil, "turn"); got != nil { + t.Fatal(got) + } + got := projectDiagnostic(turnDiagnosticFailure(sessions.Turn{Status: sessions.TurnFailed, Outcome: json.RawMessage(`{"error_code":"unrecognized-secret-canary"}`)}), "turn") + if got.Code != "unknown" { + t.Fatal(got) + } +} + +func TestProjectDiagnosticsDatabaseIsolation(t *testing.T) { + service, pool := diagnosticDatabase(t) + key, foreign := callerBinding(), callerBinding() + foreign.TokenSHA256 = runtimedevice.HashCredential("foreign") + deps, fakes := testDependencies(t) + fakes.projectsReader.resolveAPIKey = projectKeys(t, key, foreign).ResolveAPIKey + databaseSessionReads(t, pool)(&deps, fakes) + handler := newTestHandler(t, deps) + created, err := service.CreateSession(t.Context(), key.TenantID, sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "test"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`)}) + if err != nil { + t.Fatal(err) + } + session := created.Session.ID + receipt := submitMessage(t, pool, key.TenantID, session, "message", "private-canary") + paths := []string{"/v1/agents/sessions/" + session + "/diagnostics", "/v1/agents/sessions/" + session + "/turns/" + receipt.TurnID + "/diagnostics"} + for _, path := range paths { + if w := diagnosticRequest(handler, path, "Bearer caller"); w.Code != 200 || !strings.Contains(w.Body.String(), `"diagnostic":null`) { + t.Fatal(w.Code, w.Body) + } + if w := diagnosticRequest(handler, path, "Bearer foreign"); w.Code != 404 { + t.Fatal("foreign resource visible", w.Code, w.Body) + } + } + other, err := service.CreateSession(t.Context(), key.TenantID, sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "test"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`)}) + if err != nil { + t.Fatal(err) + } + wrongParent := "/v1/agents/sessions/" + other.Session.ID + "/turns/" + receipt.TurnID + "/diagnostics" + if w := diagnosticRequest(handler, wrongParent, "Bearer caller"); w.Code != 404 { + t.Fatal("turn visible through another session", w.Code, w.Body) + } + if _, err := pool.Exec(t.Context(), "UPDATE sessions SET deleted_at=clock_timestamp() WHERE id=$1", session); err != nil { + t.Fatal(err) + } + for _, path := range paths { + if w := diagnosticRequest(handler, path, "Bearer caller"); w.Code != 404 { + t.Fatal("deleted visible", w.Code, w.Body) + } + } +} + +func TestProjectTurnDiagnosticsWaiting(t *testing.T) { + id, turn := uuid.NewString(), uuid.NewString() + session := sessions.Session{ID: id, Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`), LastTurn: &sessions.Turn{ID: turn, SessionID: id, Status: sessions.TurnWaiting}} + deps, fakes := testDependencies(t) + fakes.projectsReader.resolveAPIKey = projectKeys(t, callerBinding()).ResolveAPIKey + serveDiagnostics(diagnosticSnapshotStore{session: session})(&deps, fakes) + w := diagnosticRequest(newTestHandler(t, deps), "/v1/agents/sessions/"+id+"/turns/"+turn+"/diagnostics", "Bearer caller") + if w.Code != 200 || !strings.Contains(w.Body.String(), `"status":"waiting"`) || !strings.Contains(w.Body.String(), `"diagnostic":null`) { + t.Fatal(w.Code, w.Body) + } +} + +func TestProjectFailedTurnAllowsExplicitNewTurnPostgres(t *testing.T) { + service, pool := diagnosticDatabase(t) + key := callerBinding() + deps, fakes := testDependencies(t) + fakes.projectsReader.resolveAPIKey = projectKeys(t, key).ResolveAPIKey + databaseSessionReads(t, pool)(&deps, fakes) + h := newTestHandler(t, deps) + created, err := service.CreateSession(t.Context(), key.TenantID, sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "test"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`)}) + if err != nil { + t.Fatal(err) + } + id := created.Session.ID + first := submitMessage(t, pool, key.TenantID, id, "first-explicit-message", "first") + transitionTurn(t, pool, key.TenantID, id, first.TurnID, sessions.TurnTransition{ExpectedStatus: sessions.TurnQueued, Status: sessions.TurnInProgress}) + transitionTurn(t, pool, key.TenantID, id, first.TurnID, sessions.TurnTransition{ExpectedStatus: sessions.TurnInProgress, Status: sessions.TurnFailed, Outcome: json.RawMessage(`{"error_code":"engine_failed","engine_error_code":"provider_billing_unavailable","engine_http_status":402,"engine_request_id":"88a8d808-59c1-4bf3-8950-b34ce57e028b","error":"secret-canary"}`)}) + path := "/v1/agents/sessions/" + id + "/diagnostics" + for i := 0; i < 2; i++ { + w := diagnosticRequest(h, path, "Bearer caller") + if w.Code != 200 || !strings.Contains(w.Body.String(), `"input_available":true`) || !strings.Contains(w.Body.String(), `"code":"provider_billing_unavailable"`) || strings.Contains(w.Body.String(), "secret-canary") || strings.Contains(w.Body.String(), "88a8d808") { + t.Fatal(w.Code, w.Body) + } + } + var count int + if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM turns WHERE session_id=$1", id).Scan(&count); err != nil || count != 1 { + t.Fatal("diagnostic read started work", count, err) + } + next := submitMessage(t, pool, key.TenantID, id, "second-explicit-message", "continue") + if next.TurnID == first.TurnID { + t.Fatal("new message reused failed Turn") + } + w := diagnosticRequest(h, path, "Bearer caller") + if w.Code != 200 || !strings.Contains(w.Body.String(), `"input_available":false`) || !strings.Contains(w.Body.String(), `"input_unavailable_reason":"busy"`) { + t.Fatal(w.Code, w.Body) + } + w = diagnosticRequest(h, "/v1/agents/sessions/"+id+"/turns/"+first.TurnID+"/diagnostics", "Bearer caller") + if w.Code != 200 || !strings.Contains(w.Body.String(), `"code":"provider_billing_unavailable"`) { + t.Fatal("historical failure lost", w.Code, w.Body) + } +} diff --git a/services/core/internal/api/project_input_availability_test.go b/services/core/internal/api/project_input_availability_test.go new file mode 100644 index 000000000..7ad9404e6 --- /dev/null +++ b/services/core/internal/api/project_input_availability_test.go @@ -0,0 +1,61 @@ +package api + +import ( + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "testing" +) + +func TestProjectInputAvailabilitySeparatesTurnFailureFromEnvironment(t *testing.T) { + for _, tc := range []struct { + name, status, envType, envStatus, reason string + pending, environmentFailure bool + activity, coreFailure string + }{ + {name: "failed turn can continue", status: "failed", envType: "openai_hosted", envStatus: "connected"}, + {name: "no environment", status: "failed", envType: "none"}, + {name: "failed environment", status: "failed", envType: "openai_hosted", envStatus: "failed", reason: "environment_unavailable"}, + {name: "expired environment", status: "failed", envType: "self_hosted", envStatus: "expired", reason: "environment_unavailable"}, + {name: "disconnected", status: "failed", envType: "self_hosted", envStatus: "disconnected", reason: "environment_unavailable"}, + {name: "running", status: "in_progress", envType: "none", reason: "busy"}, + {name: "pending reservation", status: "idle", envType: "none", pending: true, reason: "busy"}, + {name: "requires action", status: "requires_action", envType: "none", reason: "requires_action"}, + {name: "missing provider", status: "failed", envType: "none", activity: "model_provider_required", reason: "configuration_required"}, + {name: "turn missing provider", status: "failed", envType: "none", coreFailure: "model_provider_required", reason: "configuration_required"}, + {name: "preparation recoverable", status: "failed", envType: "self_hosted", envStatus: "connected", activity: "runtime_preparation_failed"}, + {name: "environment failure wins", status: "failed", envType: "none", environmentFailure: true, reason: "environment_unavailable"}, + } { + t.Run(tc.name, func(t *testing.T) { + session := sessions.Session{PendingInput: tc.pending, LastTurn: &sessions.Turn{Status: sessions.TurnFailed}} + if tc.envType != "none" { + session.Environment = &sessions.Environment{Status: tc.envStatus} + } + if tc.environmentFailure { + session.EnvironmentFailure = &sessions.EnvironmentFailure{} + } + if tc.activity != "" { + session.EnvironmentInputActivity = &sessions.EnvironmentInputActivity{Status: "failed", Failure: tc.activity} + } + if tc.coreFailure != "" { + session.LastTurn.Outcome = json.RawMessage(`{"error_code":"` + tc.coreFailure + `"}`) + } + available, reason := projectInputAvailability(session, tc.status, tc.envType) + if available != (tc.reason == "") || reason != tc.reason { + t.Fatal(available, reason) + } + }) + } +} + +func TestProjectInputAvailabilityRejectsActiveTurnEvenWithStalePublicStatus(t *testing.T) { + for _, status := range []string{sessions.TurnQueued, sessions.TurnInProgress, sessions.TurnWaiting} { + available, reason := projectInputAvailability(sessions.Session{LastTurn: &sessions.Turn{Status: status}}, "failed", "none") + if available || reason != "busy" { + t.Fatal(status, available, reason) + } + } + available, _ := projectInputAvailability(sessions.Session{}, "unknown", "none") + if available { + t.Fatal("unknown state is not permission") + } +} diff --git a/services/core/internal/api/routing_test.go b/services/core/internal/api/routing_test.go index 910246b45..0afb4c082 100644 --- a/services/core/internal/api/routing_test.go +++ b/services/core/internal/api/routing_test.go @@ -328,7 +328,7 @@ func TestEveryRouteAuthenticatesItsCanonicalPath(t *testing.T) { t.Errorf("self-authenticated route %s is no longer registered", route) } } - // /v1 serves exactly the pinned official method and path set. + // /v1 serves the pinned official method and path set plus explicit Core extensions. var pinned struct { Routes []string `json:"routes"` } @@ -336,6 +336,7 @@ func TestEveryRouteAuthenticatesItsCanonicalPath(t *testing.T) { if err != nil || json.Unmarshal(raw, &pinned) != nil || len(pinned.Routes) == 0 { t.Fatal("read pinned routes", err) } + pinned.Routes = append(pinned.Routes, "GET /agents/sessions/{}/diagnostics", "GET /agents/sessions/{}/turns/{}/diagnostics") served := []string{} for route := range walked { method, path, _ := strings.Cut(route, " ") diff --git a/services/core/internal/api/session_diagnostics.go b/services/core/internal/api/session_diagnostics.go index dd5ad70d3..5db1e3272 100644 --- a/services/core/internal/api/session_diagnostics.go +++ b/services/core/internal/api/session_diagnostics.go @@ -19,7 +19,7 @@ const ( ) type DiagnosticFailure struct { - Code string `json:"code" binding:"required" enums:"harness_error,model_provider_required,runtime_unavailable,runtime_disconnected,runtime_preparation_failed,execution_interrupted,delivery_unconfirmed,input_rejected,executor_protocol_error,core_storage_failed,internal_error,environment_connection_timeout,environment_unavailable,environment_provisioning_failed,authentication_error,rate_limit_exceeded,usage_limit_exceeded,server_overloaded,server_error,invalid_request,resource_not_found,request_timeout,context_length_exceeded,cyber_policy,connection_failed"` + Code string `json:"code" binding:"required" enums:"harness_error,model_provider_required,runtime_unavailable,runtime_disconnected,runtime_preparation_failed,execution_interrupted,delivery_unconfirmed,input_rejected,executor_protocol_error,core_storage_failed,internal_error,environment_connection_timeout,environment_unavailable,environment_provisioning_failed,authentication_error,rate_limit_exceeded,usage_limit_exceeded,provider_billing_unavailable,server_overloaded,server_error,invalid_request,resource_not_found,request_timeout,context_length_exceeded,cyber_policy,connection_failed"` Params CoreErrorDetails `json:"params" swaggertype:"object" binding:"required"` FailedAt *time.Time `json:"failed_at" extensions:"x-nullable" binding:"required"` } @@ -85,34 +85,7 @@ func (h *Handler) getSessionDiagnostics(w http.ResponseWriter, r *http.Request) writeSessionsError(w, r, err) return } - response := SessionDiagnostics{Object: "core.session_diagnostics", SessionID: public.ID, Status: public.Status} - if public.Status == "failed" { - failure := SessionDiagnosticFailure{DiagnosticFailure: DiagnosticFailure{Code: "internal_error", Params: CoreErrorDetails{}}} - switch { - case session.EnvironmentFailure != nil: - failure.Source = DiagnosticEnvironment - failure.Code = "environment_provisioning_failed" - failure.FailedAt = diagnosticTime(session.EnvironmentFailure.FailedAt) - failure.Params = provisioningFailureParams(session.EnvironmentFailure.Detail) - case session.EnvironmentInputActivity != nil: - failure.Source = DiagnosticEnvironmentInput - failure.FailedAt = diagnosticTime(session.EnvironmentInputActivity.LastActiveAt) - switch session.EnvironmentInputActivity.Failure { - case "": - failure.Code = "environment_connection_timeout" - case "environment_unavailable": - failure.Code = "environment_unavailable" - case "runtime_preparation_failed": - failure.Code = "runtime_preparation_failed" - case "model_provider_required": - failure.Code = "model_provider_required" - } - case session.LastTurn != nil: - failure.Source, failure.TurnID = DiagnosticTurn, session.LastTurn.ID - failure.DiagnosticFailure = *turnDiagnosticFailure(*session.LastTurn) - } - response.Failure = &failure - } + response := sessionDiagnosticResponse(session, public.ID, public.Status) w.Header().Set("Cache-Control", "no-store") writeJSON(w, http.StatusOK, response) } @@ -177,3 +150,35 @@ func provisioningFailureParams(detail *sessions.ProvisioningFailureDetail) CoreE } return params } + +func sessionDiagnosticResponse(session sessions.Session, id, status string) SessionDiagnostics { + response := SessionDiagnostics{Object: "core.session_diagnostics", SessionID: id, Status: status} + if status == "failed" { + failure := SessionDiagnosticFailure{DiagnosticFailure: DiagnosticFailure{Code: "internal_error", Params: CoreErrorDetails{}}} + switch { + case session.EnvironmentFailure != nil: + failure.Source = DiagnosticEnvironment + failure.Code = "environment_provisioning_failed" + failure.FailedAt = diagnosticTime(session.EnvironmentFailure.FailedAt) + failure.Params = provisioningFailureParams(session.EnvironmentFailure.Detail) + case session.EnvironmentInputActivity != nil: + failure.Source = DiagnosticEnvironmentInput + failure.FailedAt = diagnosticTime(session.EnvironmentInputActivity.LastActiveAt) + switch session.EnvironmentInputActivity.Failure { + case "": + failure.Code = "environment_connection_timeout" + case "environment_unavailable": + failure.Code = "environment_unavailable" + case "runtime_preparation_failed": + failure.Code = "runtime_preparation_failed" + case "model_provider_required": + failure.Code = "model_provider_required" + } + case session.LastTurn != nil: + failure.Source, failure.TurnID = DiagnosticTurn, session.LastTurn.ID + failure.DiagnosticFailure = *turnDiagnosticFailure(*session.LastTurn) + } + response.Failure = &failure + } + return response +} diff --git a/services/core/internal/api/turn_diagnostic_failure.go b/services/core/internal/api/turn_diagnostic_failure.go index 1f2081511..f288dbb46 100644 --- a/services/core/internal/api/turn_diagnostic_failure.go +++ b/services/core/internal/api/turn_diagnostic_failure.go @@ -1,9 +1,6 @@ package api import ( - "encoding/json" - - "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -11,48 +8,14 @@ func turnDiagnosticFailure(turn sessions.Turn) *DiagnosticFailure { if turn.Status != sessions.TurnFailed { return nil } - var outcome map[string]json.RawMessage - var coreCode string - if json.Unmarshal(turn.Outcome, &outcome) == nil { - _ = json.Unmarshal(outcome["error_code"], &coreCode) - } - code := "internal_error" + code, status := sessions.DiagnosticFailureCode(turn.Outcome) params := CoreErrorDetails{} - switch coreCode { - case "engine_failed": - code = "harness_error" - // Optional malformed metadata cannot hide the authoritative Core error. - var nativeCode string - var nativeStatus *int - _ = json.Unmarshal(outcome["engine_error_code"], &nativeCode) - _ = json.Unmarshal(outcome["engine_http_status"], &nativeStatus) - if classified, status := proto.NormalizeEngineFailure(nativeCode, nativeStatus); classified != "" { - code = classified - if code == "connection_failed" { - params["http_status"] = CoreErrorNull() - if status != nil { - params["http_status"] = CoreErrorNumber(float64(*status)) - } - } + if code == "connection_failed" || code == "provider_billing_unavailable" { + params["http_status"] = CoreErrorNull() + if status != nil { + params["http_status"] = CoreErrorNumber(float64(*status)) } - case "model_provider_required": - code = "model_provider_required" - case "execution_device_unavailable", "execution_unavailable": - code = "runtime_unavailable" - case "device_disconnected", "event_stream_incomplete": - code = "runtime_disconnected" - case "preparation_start_failed", "preparation_interrupted": - code = "runtime_preparation_failed" - case "execution_interrupted": - code = "execution_interrupted" - case "delivery_unknown", "input_outcome_unknown", "cancel_unconfirmed", "cancel_outcome_unavailable", "function_result_unconfirmed": - code = "delivery_unconfirmed" - case "invalid_input", "input_not_applied", "message_input_unsupported", "input_invalid_input", "input_run_inactive", "input_input_conflict", "input_input_limit", "input_unsupported", "input_rejected", "input_not_ready", "input_busy": - code = "input_rejected" - case "invalid_executor_result", "execution_state_unavailable", "execution_state_changed", "function_call_invalid", "function_result_invalid": - code = "executor_protocol_error" - case "event_persistence_failed", "artifact_capture_failed": - code = "core_storage_failed" } + return &DiagnosticFailure{Code: code, Params: params, FailedAt: diagnosticTime(turn.CompletedAt)} } diff --git a/services/core/internal/coremetrics/service.go b/services/core/internal/coremetrics/service.go index 8b2c497e0..c0d02c704 100644 --- a/services/core/internal/coremetrics/service.go +++ b/services/core/internal/coremetrics/service.go @@ -225,6 +225,7 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { if err != nil { view.Service.Status = ServiceDegraded } else { + view.Execution.TerminalTurns = &history.TerminalTurns view.Execution.Interrupted = ptr(history.Interrupted) view.Execution.QueueWaitMS = history.QueueWaitMS for i := range view.Execution.Series { diff --git a/services/core/internal/coremetrics/service_test.go b/services/core/internal/coremetrics/service_test.go index f901a8c62..19f5711dd 100644 --- a/services/core/internal/coremetrics/service_test.go +++ b/services/core/internal/coremetrics/service_test.go @@ -44,7 +44,7 @@ func TestCompleteBucketsAndObservedPercentiles(t *testing.T) { } { s.record(sample) } - source.history = History{Interrupted: 2, QueueWaitMS: Latency{P50: ptr(100.0), P95: ptr(200.0)}, Buckets: map[time.Time]*float64{end.Add(-time.Minute): ptr(200.0)}} + source.history = History{TerminalTurns: TerminalTurns{Total: 5, Completed: 3, Failed: 2, Failures: []FailureCount{{Code: "unknown", Source: "turn", Count: 2}}}, Interrupted: 2, QueueWaitMS: Latency{P50: ptr(100.0), P95: ptr(200.0)}, Buckets: map[time.Time]*float64{end.Add(-time.Minute): ptr(200.0)}} s.now = func() time.Time { return end.Add(-10 * time.Second) } s.RecordUnavailable() s.now = func() time.Time { return now } @@ -56,6 +56,9 @@ func TestCompleteBucketsAndObservedPercentiles(t *testing.T) { if !got.Range.End.Equal(end) || len(got.Execution.Series) != 60 || *got.Execution.QueuedTurns != 99 || *got.Execution.Unavailable != 1 { t.Fatal(got) } + if got.Execution.TerminalTurns == nil || got.Execution.TerminalTurns.Total != 5 || got.Execution.TerminalTurns.Failed != 2 { + t.Fatal(got.Execution.TerminalTurns) + } b := got.Execution.Series[59] d := got.Database.Series[59] if *b.Queued != 3 || *b.InProgress != 4 || *b.QueueWaitP95MS != 200 || *d.PoolInUse != 3 || *d.PingP95MS != 5.8 || *got.Database.PingMS.P50 != 4 { @@ -74,7 +77,7 @@ func TestUnknownAndFailureRemainNull(t *testing.T) { if err != nil { t.Fatal(err) } - if got.Service.Status != "degraded" || got.Execution.Unavailable != nil || got.Execution.Interrupted != nil || got.Execution.QueueWaitMS.P50 != nil || got.Database.SizeBytes != nil { + if got.Service.Status != "degraded" || got.Execution.TerminalTurns != nil || got.Execution.Unavailable != nil || got.Execution.Interrupted != nil || got.Execution.QueueWaitMS.P50 != nil || got.Database.SizeBytes != nil { t.Fatal(got) } raw, _ := json.Marshal(got) diff --git a/services/core/internal/coremetrics/types.go b/services/core/internal/coremetrics/types.go index ca97cce30..a9fdaf993 100644 --- a/services/core/internal/coremetrics/types.go +++ b/services/core/internal/coremetrics/types.go @@ -59,7 +59,20 @@ type Pool struct { Idle *int64 `json:"idle" extensions:"x-nullable" binding:"required"` Max *int64 `json:"max" extensions:"x-nullable" binding:"required"` } +type FailureCount struct { + Code string `json:"code"` + Source string `json:"source" enums:"turn"` + Count int64 `json:"count"` +} +type TerminalTurns struct { + Total int64 `json:"total"` + Completed int64 `json:"completed"` + Failed int64 `json:"failed"` + Cancelled int64 `json:"cancelled"` + Failures []FailureCount `json:"failures"` +} type Execution struct { + TerminalTurns *TerminalTurns `json:"terminal_turns" extensions:"x-nullable" binding:"required"` SlotsInUse int64 `json:"slots_in_use" binding:"required"` SlotsTotal int64 `json:"slots_total" binding:"required"` QueuedTurns *int64 `json:"queued_turns" extensions:"x-nullable" binding:"required"` @@ -135,9 +148,10 @@ type ExecutionSnapshot struct { // History is the root Turn history of a range. Buckets maps each bucket's UTC // start to its queue wait p95 in milliseconds, nil without observations. type History struct { - Interrupted int64 - QueueWaitMS Latency - Buckets map[time.Time]*float64 + TerminalTurns TerminalTurns + Interrupted int64 + QueueWaitMS Latency + Buckets map[time.Time]*float64 } type Source interface { Sample(context.Context) Sample diff --git a/services/core/internal/execution/delivery.go b/services/core/internal/execution/delivery.go index ce62295cc..7f1fbce00 100644 --- a/services/core/internal/execution/delivery.go +++ b/services/core/internal/execution/delivery.go @@ -353,6 +353,7 @@ func (r *Result) mergeObservation(env proto.Envelope) error { return err } r.EngineErrorCode, r.EngineHTTPStatus = proto.NormalizeEngineFailure(failure.Code, failure.HTTPStatus) + r.EngineRequestID = proto.NormalizeProviderRequestID(failure.UpstreamRequestID) case proto.TypeUsage: return env.DecodePayload(&r.Done.Usage) case proto.TypeDone: diff --git a/services/core/internal/execution/dispatcher.go b/services/core/internal/execution/dispatcher.go index 6364a9ae9..9c56c812c 100644 --- a/services/core/internal/execution/dispatcher.go +++ b/services/core/internal/execution/dispatcher.go @@ -66,6 +66,7 @@ type Credentials interface { type Result struct { EngineErrorCode string `json:"engine_error_code,omitempty"` EngineHTTPStatus *int `json:"engine_http_status,omitempty"` + EngineRequestID string `json:"engine_request_id,omitempty"` Done proto.DonePayload `json:"done"` ErrorCode string `json:"error_code,omitempty"` Error string `json:"error,omitempty"` diff --git a/services/core/internal/execution/engine_failure_test.go b/services/core/internal/execution/engine_failure_test.go index 5328e7c79..69373c007 100644 --- a/services/core/internal/execution/engine_failure_test.go +++ b/services/core/internal/execution/engine_failure_test.go @@ -73,3 +73,17 @@ func TestClassifiedErrorCannotOverrideCancellationReceipt(t *testing.T) { } } } + +func TestBillingFailureOutcomePreservesSafeCorrelation(t *testing.T) { + status := 402 + failure, _ := proto.NewEnvelope(proto.TypeError, "run", proto.ErrorPayload{Code: "connection_failed", HTTPStatus: &status, UpstreamRequestID: "88a8d808-59c1-4bf3-8950-b34ce57e028b"}) + result := Result{ErrorCode: "engine_failed"} + if err := result.mergeObservation(failure); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(result) + var saved Result + if json.Unmarshal(raw, &saved) != nil || saved.EngineErrorCode != "provider_billing_unavailable" || saved.EngineRequestID == "" || saved.EngineHTTPStatus == nil || *saved.EngineHTTPStatus != 402 { + t.Fatal(string(raw)) + } +} diff --git a/services/core/internal/modelconfiguration/testdata/observation_cases.json b/services/core/internal/modelconfiguration/testdata/observation_cases.json index 0794fd47d..af85e2372 100644 --- a/services/core/internal/modelconfiguration/testdata/observation_cases.json +++ b/services/core/internal/modelconfiguration/testdata/observation_cases.json @@ -1,5 +1,6 @@ [ {"status": "completed", "observed": true}, + {"status": "failed", "error_code": "engine_failed", "engine_error_code": "provider_billing_unavailable", "observed": false}, {"status": "failed", "error_code": "engine_failed", "engine_error_code": "authentication_error", "observed": true}, {"status": "failed", "error_code": "engine_failed", "engine_error_code": "connection_failed", "observed": true}, {"status": "failed", "error_code": "engine_failed", "engine_error_code": "rate_limit_exceeded", "observed": true}, diff --git a/services/core/internal/persistence/postgres/coremetricspg/store.go b/services/core/internal/persistence/postgres/coremetricspg/store.go index 50ec3fad8..72bea955c 100644 --- a/services/core/internal/persistence/postgres/coremetricspg/store.go +++ b/services/core/internal/persistence/postgres/coremetricspg/store.go @@ -4,6 +4,9 @@ package coremetricspg import ( "context" + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "sort" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" @@ -93,6 +96,49 @@ func (s *Store) ReadExecutionHistory(ctx context.Context, start, end time.Time, result.Buckets[buckets[row.BucketNumber]] = &row.P95Ms } } + terminal, err := tx.Query(ctx, `SELECT status, outcome->>'error_code', outcome->>'engine_error_code', outcome->'engine_http_status', count(*) FROM turns WHERE completed_at >= $1 AND completed_at < $2 AND status IN ('completed','failed','cancelled') GROUP BY 1,2,3,4`, start, end) + if err != nil { + return err + } + defer terminal.Close() + counts := map[string]int64{} + result.TerminalTurns.Failures = []coremetrics.FailureCount{} + for terminal.Next() { + var status string + var coreCode, engineCode *string + var httpStatus json.RawMessage + var count int64 + if err := terminal.Scan(&status, &coreCode, &engineCode, &httpStatus, &count); err != nil { + return err + } + result.TerminalTurns.Total += count + switch status { + case "completed": + result.TerminalTurns.Completed += count + case "cancelled": + result.TerminalTurns.Cancelled += count + case "failed": + result.TerminalTurns.Failed += count + metadata, err := json.Marshal(map[string]any{"error_code": coreCode, "engine_error_code": engineCode, "engine_http_status": httpStatus}) + if err != nil { + return err + } + code, _ := sessions.DiagnosticFailureCode(metadata) + if code == "internal_error" { + code = "unknown" + } + counts[code] += count + } + } + if err := terminal.Err(); err != nil { + return err + } + for code, count := range counts { + result.TerminalTurns.Failures = append(result.TerminalTurns.Failures, coremetrics.FailureCount{Code: code, Source: "turn", Count: count}) + } + sort.Slice(result.TerminalTurns.Failures, func(i, j int) bool { + return result.TerminalTurns.Failures[i].Code < result.TerminalTurns.Failures[j].Code + }) return nil }) if err != nil { diff --git a/services/core/internal/persistence/postgres/coremetricspg/store_test.go b/services/core/internal/persistence/postgres/coremetricspg/store_test.go index db5d8edba..423617800 100644 --- a/services/core/internal/persistence/postgres/coremetricspg/store_test.go +++ b/services/core/internal/persistence/postgres/coremetricspg/store_test.go @@ -179,3 +179,40 @@ func TestCoreMetricsHistoryBounds(t *testing.T) { } } } + +func TestTerminalTurnStatisticsAreSnapshotCounts(t *testing.T) { + pool := pgtest.Open(t) + store := New(pgunit.NewPool(pool)) + start := time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + before, err := store.ReadExecutionHistory(t.Context(), start, end, time.Minute) + if err != nil { + t.Fatal(err) + } + at := start.Add(time.Minute) + session := coreMetricsSession(t, pool, true) + for _, row := range []struct{ status, code string }{{"completed", ""}, {"cancelled", ""}, {"failed", "device_disconnected"}, {"failed", "unrecognized-private-canary"}} { + coreMetricsTurn(t, pool, session, row.status, row.code, start, nil, &at) + } + coreMetricsTurn(t, pool, session, "failed", "engine_failed", start, nil, &end) + for i := 0; i < 2; i++ { + got, err := store.ReadExecutionHistory(t.Context(), start, end, time.Minute) + if err != nil { + t.Fatal(err) + } + stats := got.TerminalTurns + if stats.Total != before.TerminalTurns.Total+4 || stats.Completed != before.TerminalTurns.Completed+1 || stats.Cancelled != before.TerminalTurns.Cancelled+1 || stats.Failed != before.TerminalTurns.Failed+2 { + t.Fatal(stats) + } + counts := map[string]int64{} + for _, f := range stats.Failures { + if f.Source != "turn" || strings.Contains(f.Code, "canary") { + t.Fatal(f) + } + counts[f.Code] = f.Count + } + if counts["runtime_disconnected"] < 1 || counts["unknown"] < 1 { + t.Fatal(stats) + } + } +} diff --git a/services/core/internal/sessions/diagnostic_failure.go b/services/core/internal/sessions/diagnostic_failure.go new file mode 100644 index 000000000..fc59efef3 --- /dev/null +++ b/services/core/internal/sessions/diagnostic_failure.go @@ -0,0 +1,50 @@ +package sessions + +import ( + "encoding/json" + + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" +) + +// DiagnosticFailureCode classifies only typed outcome metadata. Native text is never inspected. +func DiagnosticFailureCode(outcomeJSON json.RawMessage) (string, *int) { + var outcome map[string]json.RawMessage + var coreCode string + if json.Unmarshal(outcomeJSON, &outcome) == nil { + _ = json.Unmarshal(outcome["error_code"], &coreCode) + } + code := "internal_error" + var httpStatus *int + switch coreCode { + case "engine_failed": + code = "harness_error" + // Optional malformed metadata cannot hide the authoritative Core error. + var nativeCode string + var nativeStatus *int + _ = json.Unmarshal(outcome["engine_error_code"], &nativeCode) + _ = json.Unmarshal(outcome["engine_http_status"], &nativeStatus) + if classified, status := proto.NormalizeEngineFailure(nativeCode, nativeStatus); classified != "" { + code = classified + httpStatus = status + } + case "model_provider_required": + code = "model_provider_required" + case "execution_device_unavailable", "execution_unavailable": + code = "runtime_unavailable" + case "device_disconnected", "event_stream_incomplete": + code = "runtime_disconnected" + case "preparation_start_failed", "preparation_interrupted": + code = "runtime_preparation_failed" + case "execution_interrupted": + code = "execution_interrupted" + case "delivery_unknown", "input_outcome_unknown", "cancel_unconfirmed", "cancel_outcome_unavailable", "function_result_unconfirmed": + code = "delivery_unconfirmed" + case "invalid_input", "input_not_applied", "message_input_unsupported", "input_invalid_input", "input_run_inactive", "input_input_conflict", "input_input_limit", "input_unsupported", "input_rejected", "input_not_ready", "input_busy": + code = "input_rejected" + case "invalid_executor_result", "execution_state_unavailable", "execution_state_changed", "function_call_invalid", "function_result_invalid": + code = "executor_protocol_error" + case "event_persistence_failed", "artifact_capture_failed": + code = "core_storage_failed" + } + return code, httpStatus +}