fix(core): ship safe failure diagnostics and manual continuation via beta

apps/daemon/internal/agent/codex/error_classification.go

@@ -2,6 +2,7 @@ package codex
 
 import (
 	"encoding/json"
+	"strings"
 
 	"github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto"
 )
@@ -14,6 +15,18 @@ func classifyTurnError(failure *TurnError) proto.ErrorPayload {
 	}
 	var variant string
 	var code string
+	// Some native HTTP failures arrive as "other" without typed status. Only
+	// the exact native terminal HTTP envelope is recognized, never accumulated
+	// output, notification text or provider response-body keywords.
+	if len(failure.CodexErrorInfo) == 0 || string(failure.CodexErrorInfo) == "null" ||
+		(json.Unmarshal(failure.CodexErrorInfo, &variant) == nil && variant == "other") {
+		const prefix = "unexpected status 402 Payment Required"
+		message := failure.Message
+		if !strings.ContainsAny(message, "\r\n") && (message == prefix || strings.HasPrefix(message, prefix+": ") || strings.HasPrefix(message, prefix+", ")) {
+			status := 402
+			return proto.ErrorPayload{Code: "provider_billing_unavailable", HTTPStatus: &status, UpstreamRequestID: nativeRequestID(message)}
+		}
+	}
 	if json.Unmarshal(failure.CodexErrorInfo, &variant) == nil {
 		switch variant {
 		case "unauthorized":
@@ -53,8 +66,21 @@ func classifyTurnError(failure *TurnError) proto.ErrorPayload {
 				code = "rate_limit_exceeded"
 			}
 			code, status = proto.NormalizeEngineFailure(code, status)
-			return proto.ErrorPayload{Code: code, HTTPStatus: status}
+			return proto.ErrorPayload{Code: code, HTTPStatus: status, UpstreamRequestID: nativeRequestID(failure.Message)}
 		}
 	}
 	return proto.ErrorPayload{}
 }
+
+// The native transport places its request ID last. Ignore arbitrary body text
+// and reject multiline messages or anything other than a UUID suffix.
+func nativeRequestID(message string) string {
+	if !strings.HasPrefix(message, "unexpected status ") || strings.ContainsAny(message, "\r\n") {
+		return ""
+	}
+	_, id, ok := strings.Cut(message, ", request id: ")
+	if !ok {
+		return ""
+	}
+	return proto.NormalizeProviderRequestID(id)
+}

apps/daemon/internal/agent/codex/error_classification_test.go

@@ -93,3 +93,29 @@ func TestClassificationOnlyFromRootTerminalError(t *testing.T) {
 		})
 	}
 }
+
+func TestTerminalProviderBillingEnvelope(t *testing.T) {
+	const id = "88a8d808-59c1-4bf3-8950-b34ce57e028b"
+	message := "unexpected status 402 Payment Required: secret-body-canary, url: https://example.invalid/v1/responses, request id: " + id
+	for _, raw := range []json.RawMessage{nil, json.RawMessage(`null`), json.RawMessage(`"other"`), json.RawMessage(`{"httpConnectionFailed":{"httpStatusCode":402}}`)} {
+		got := classifyTurnError(&TurnError{Message: message, CodexErrorInfo: raw})
+		if got.Code != "provider_billing_unavailable" || got.HTTPStatus == nil || *got.HTTPStatus != 402 || got.UpstreamRequestID != id {
+			t.Fatalf("%s: %+v", raw, got)
+		}
+	}
+	for _, message := range []string{"quoted: " + message, "unexpected status 402 Payment Requiredish", "insufficient credits", "unexpected status 402 Payment Required\nquoted next line"} {
+		got := classifyTurnError(&TurnError{Message: message, CodexErrorInfo: json.RawMessage(`"other"`)})
+		if got.Code != "" || got.UpstreamRequestID != "" {
+			t.Fatal("non-native envelope classified", got)
+		}
+	}
+	for _, raw := range []string{`"unauthorized"`, `"unknownFuture"`, `{"httpConnectionFailed":{"httpStatusCode":"402"}}`, `{"other":true}`} {
+		got := classifyTurnError(&TurnError{Message: message, CodexErrorInfo: json.RawMessage(raw)})
+		if got.Code == "provider_billing_unavailable" {
+			t.Fatal("overrode typed error", got)
+		}
+	}
+	if nativeRequestID(message+" secret-token") != "" || nativeRequestID("quoted: "+message) != "" {
+		t.Fatal("unsafe request ID")
+	}
+}

apps/daemon/internal/agent/codex/session.go

@@ -198,7 +198,7 @@ func (s *Session) onTurnCompleted(raw json.RawMessage) {
 		"status", p.Turn.Status,
 		"final_text_len", len(finalText),
 		"buffered_err_text_len", len(errText),
-		"raw_payload", string(raw))
+		"has_error", p.Turn.Error != nil)
 	if status == "failed" {
 		// Body precedence on failure:
 		//   1. agent's final text (rare on hard failures but exists for
@@ -311,7 +311,7 @@ func (s *Session) onErrorNotif(raw json.RawMessage) {
 	s.cfg.logger.Warn("codex: error notification received",
 		"run_id", s.runID,
 		"thread_id", s.currentThreadID(),
-		"message", p.Message)
+		"message_len", len(p.Message))
 	s.finalTextMu.Lock()
 	s.lastErrText = p.Message
 	s.finalTextMu.Unlock()
@@ -376,15 +376,15 @@ func (s *Session) emitTerminalFailure(message string, asError bool, failure prot
 	}
 	s.stopSteering()
 	s.stopFunctionCalls()
-	// Always log: this is the only place the daemon decides "the prompt is
-	// over, here's what went wrong (if anything)". Without this, post-
-	// mortem requires correlating server-side TypeError frames against
-	// daemon timestamps with no message body anywhere.
+	// Keep safe correlation at terminal settlement; native bodies stay private.
 	if asError {
 		s.cfg.logger.Warn("codex: emitting terminal error",
 			"run_id", s.runID,
 			"thread_id", s.currentThreadID(),
-			"message", message)
+			"error_code", failure.Code,
+			"http_status", failure.HTTPStatus,
+			"upstream_request_id", proto.NormalizeProviderRequestID(failure.UpstreamRequestID),
+			"message_len", len(message))
 	} else {
 		s.cfg.logger.Info("codex: emitting terminal done",
 			"run_id", s.runID,

apps/daemon/internal/agent/codex/session_log_test.go

@@ -12,16 +12,7 @@ import (
 	"github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto"
 )
 
-// TestEmitTerminal_LogsErrorMessage pins the diagnostic invariant:
-// every time the codex session decides "this prompt is over with an
-// error", the error message MUST land in the daemon's structured log.
-//
-// Before this fix, emitTerminal silently sent a TypeError envelope to
-// the upstream channel. If the upstream connector logged only the
-// envelope type (not the body), debugging required guessing what the
-// session had decided to surface. With the fix the daemon log carries
-// the exact message string, the run_id, and the thread_id at the same
-// timestamp the TypeError frame was emitted.
+// Terminal logs retain safe metadata while the private runtime frame retains the native error.
 func TestEmitTerminal_LogsErrorMessage(t *testing.T) {
 	var buf bytes.Buffer
 	logger := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
@@ -42,11 +33,14 @@ func TestEmitTerminal_LogsErrorMessage(t *testing.T) {
 	s.emitTerminal(message, true)
 
 	logs := buf.String()
+	if strings.Contains(logs, message) {
+		t.Fatal("native error body leaked to logs")
+	}
 	for _, want := range []string{
 		`"msg":"codex: emitting terminal error"`,
 		`"run_id":"run-test-123"`,
 		`"thread_id":"thread-abc"`,
-		`"message":"` + message + `"`,
+		`"message_len":`,
 		`"level":"WARN"`,
 	} {
 		if !strings.Contains(logs, want) {
@@ -129,10 +123,13 @@ func TestOnErrorNotif_LogsAndBuffers(t *testing.T) {
 		t.Fatalf("lastErrText not buffered: %q", got)
 	}
 	logs := buf.String()
+	if strings.Contains(logs, "401 from gateway") {
+		t.Fatal("notification body leaked")
+	}
 	for _, want := range []string{
 		`"msg":"codex: error notification received"`,
 		`"thread_id":"thread-y"`,
-		`401 from gateway`,
+		`"message_len":`,
 	} {
 		if !strings.Contains(logs, want) {
 			t.Errorf("log missing %q\n--- log ---\n%s", want, logs)
@@ -254,3 +251,20 @@ func drainEnvelopes(out <-chan proto.Envelope) []proto.Envelope {
 		}
 	}
 }
+
+func TestTerminalBillingLogsOnlySafeCorrelation(t *testing.T) {
+	var buf bytes.Buffer
+	out := make(chan proto.Envelope, 4)
+	s := &Session{runID: "run", cfg: sessionConfig{logger: slog.New(slog.NewJSONHandler(&buf, nil))}, out: out, cancelCtx: context.Background()}
+	message := "unexpected status 402 Payment Required: secret-canary, url: https://private.invalid, request id: 88a8d808-59c1-4bf3-8950-b34ce57e028b"
+	s.emitTerminalFailure(message, true, classifyTurnError(&TurnError{Message: message}))
+	logs := buf.String()
+	for _, want := range []string{`"error_code":"provider_billing_unavailable"`, `"http_status":402`, `"upstream_request_id":"88a8d808-59c1-4bf3-8950-b34ce57e028b"`} {
+		if !strings.Contains(logs, want) {
+			t.Fatal("missing safe correlation", logs)
+		}
+	}
+	if strings.Contains(logs, "secret-canary") || strings.Contains(logs, "private.invalid") {
+		t.Fatal("native body leaked")
+	}
+}

apps/web/src/features/sessions/session-diagnostics.test.tsx

@@ -78,3 +78,12 @@ describe("Core diagnostics in the console", () => {
     client.clear();
   });
 });
+
+it.each(["en", "zh-CN"])("renders billing failure rather than internal error in %s", async (language) => {
+  await i18n.changeLanguage(language);
+  const billing: DiagnosticFailure = { code: "provider_billing_unavailable", params: { http_status: 402 }, failed_at: null };
+  for (const html of [render("session", { ...sessionData, failure: { ...billing, source: "turn", turn_id: turn.id } }), render("turn", { ...turnData, failure: billing })]) {
+    expect(html).toContain(language === "en" ? "billing or usage-limit issue" : "账单或使用额度问题");
+    expect(html).not.toContain(i18n.getFixedT(language, "diagnostics")("failure.internal_error"));
+  }
+});

apps/web/src/i18n/locales/en/diagnostics.ts

@@ -10,6 +10,7 @@ export const diagnostics = {
   providerHelp: "Core records outcomes from hosted Turns using this deployment default. Records are best effort and may lag; missing records do not prove the provider is ready or unused. Replacing the default starts a new observation history.",
   providerStaleHelp: "The latest read failed. These are the last confirmed observations.",
   failure: {
+    provider_billing_unavailable: "The model service rejected this request because of a billing or usage-limit issue. Contact support to check the model service account before trying again.",
     authentication_error: "The model provider rejected authentication. Check its API key.",
     rate_limit_exceeded: "The model provider's rate limit was reached.",
     usage_limit_exceeded: "The model provider's usage limit was reached. Check its quota.",

apps/web/src/i18n/locales/zh-CN/diagnostics.ts

@@ -10,6 +10,7 @@ export const diagnostics = {
   providerHelp: "Core 记录使用此部署默认配置的托管 Turn 结果。记录可能延迟或缺失；暂无记录不代表服务已就绪或从未使用。替换默认配置后会重新记录。",
   providerStaleHelp: "最新读取失败，当前显示上次确认的记录。",
   failure: {
+    provider_billing_unavailable: "模型服务因账单或使用额度问题拒绝了请求。请联系支持人员检查模型服务账户后再尝试。",
     authentication_error: "模型服务认证失败，请检查 API key。",
     rate_limit_exceeded: "已达到模型服务的请求频率上限。",
     usage_limit_exceeded: "已达到模型服务的用量上限，请检查额度。",

contracts/agents-api/core-errors.md

@@ -114,6 +114,7 @@ The [Session and Turn diagnostics reads](./session-diagnostics.md) return these
 | `authentication_error` | Native provider authentication rejected |
 | `rate_limit_exceeded` | Native rate limit classification |
 | `usage_limit_exceeded` | Native billing or usage limit classification |
+| `provider_billing_unavailable` | Model endpoint HTTP 402; billing correction is required, not automatic retry; params contain `http_status` 402 or null |
 | `server_overloaded` | Native overload classification |
 | `server_error` | Native server failure classification |
 | `invalid_request` | Native request rejection |
@@ -138,4 +139,4 @@ The [Session and Turn diagnostics reads](./session-diagnostics.md) return these
 
 A database failure is an error, never an empty or healthy snapshot. Provisioning reasons and native messages are never parsed for categories or parameters.
 
-Native categories apply only to a failed Turn whose outcome has `error_code: engine_failed`. Core accepts only the listed `engine_error_code` values; an unknown, malformed or absent value stays `harness_error`. Only `connection_failed` uses `engine_http_status`. Nested metadata and provider text never classify a failure. Core storage, incomplete-stream and cancellation failures take precedence, and cancelled or completed Turns have no failure. [Native error classification](../../docs/runtime-protocol.md#native-failure-classification) lists which adapters report each category.
+Native categories apply only to a failed Turn whose outcome has `error_code: engine_failed`. Core accepts only the listed `engine_error_code` values; an unknown, malformed or absent value stays `harness_error`. `connection_failed` and `provider_billing_unavailable` use their validated `engine_http_status`. Nested metadata and provider text never classify a failure. Core storage, incomplete-stream and cancellation failures take precedence, and cancelled or completed Turns have no failure. [Native error classification](../../docs/runtime-protocol.md#native-failure-classification) lists which adapters report each category.

contracts/agents-api/core-metrics.md

@@ -63,3 +63,9 @@ Core resolves its own cgroup, including nested and subtree mounts, and reports t
 - The Runtime sampler's `processed` and `failed` count the targets its last sweep observed and failed to observe.
 - A cleanup job's `processed` counts the rows it removed.
 - For the scheduler and the cleanup jobs, a failed pass sets `processed` to null and `failed` to 1; `failed` never estimates lost rows or failed Turns.
+
+## Terminal Turn statistics
+
+`execution.terminal_turns` is a nullable database-backed summary over completed root Turns in the requested `[range.start, range.end)` interval, selected by `completed_at`. It contains `total`, `completed`, `failed`, `cancelled` and `failures` (`code`, `source: turn`, `count`). `total` includes all three terminal statuses; the failure proportion is `failed / total` when total is nonzero. The classification is identical to Project diagnostics, including `unknown`. Empty successful reads contain zeros and an empty failures array. Unavailable history yields null, never measured zero.
+
+These are window aggregates of retained authoritative terminal rows, not process counters. Retried reads, concurrent writers and service restarts cannot add duplicate counts. Deleted Sessions' retained Turns are included. Environment failures without a Turn are excluded from both numerator and denominator. No Project, model, Session or Turn identifiers become dimensions. Aggregation shares the bounded repeatable-read history snapshot and its three-second request budget; no retry or execution mutation occurs.

contracts/agents-api/core.openapi.yaml

@@ -369,6 +369,7 @@ definitions:
           - authentication_error
           - rate_limit_exceeded
           - usage_limit_exceeded
+          - provider_billing_unavailable
           - server_overloaded
           - server_error
           - invalid_request
@@ -750,6 +751,7 @@ definitions:
           - authentication_error
           - rate_limit_exceeded
           - usage_limit_exceeded
+          - provider_billing_unavailable
           - server_overloaded
           - server_error
           - invalid_request
@@ -890,6 +892,10 @@ definitions:
         type: integer
       slots_total:
         type: integer
+      terminal_turns:
+        allOf:
+          - $ref: '#/definitions/coremetrics.TerminalTurns'
+        x-nullable: true
       unavailable:
         type: integer
         x-nullable: true
@@ -906,6 +912,7 @@ definitions:
       - series
       - slots_in_use
       - slots_total
+      - terminal_turns
       - unavailable
       - waiting_for_daemon
     type: object
@@ -928,6 +935,17 @@ definitions:
       - queued
       - start
     type: object
+  coremetrics.FailureCount:
+    properties:
+      code:
+        type: string
+      count:
+        type: integer
+      source:
+        enum:
+          - turn
+        type: string
+    type: object
   coremetrics.Job:
     properties:
       failed:
@@ -1078,6 +1096,21 @@ definitions:
     x-enum-varnames:
       - ServiceRunning
       - ServiceDegraded
+  coremetrics.TerminalTurns:
+    properties:
+      cancelled:
+        type: integer
+      completed:
+        type: integer
+      failed:
+        type: integer
+      failures:
+        items:
+          $ref: '#/definitions/coremetrics.FailureCount'
+        type: array
+      total:
+        type: integer
+    type: object
   coremetrics.View:
     properties:
       database: