fix(otel): alert on Cloud delivery failures only while they are happening

apps/temps-cli/openapi.json

@@ -7747,6 +7747,46 @@
         ],
         "type": "object"
       },
+      "CloudDeliveryGapResponse": {
+        "description": "A stretch of span time whose spans reached this instance but were never\ndelivered to Temps Cloud (their retries were exhausted).\n\nDistinct from [`TelemetryGapWindowResponse`], which records spans refused\nat the door because the queue was full: these were accepted and queued, and\nthe loss happened later, on the way out.",
+        "properties": {
+          "first_span_at": {
+            "description": "When the earliest undelivered span in this stretch reached the instance.",
+            "format": "date-time",
+            "type": "string"
+          },
+          "gave_up_at": {
+            "description": "When delivery of the latest span in this stretch gave up — the same\nfailure `last_error` describes.",
+            "format": "date-time",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "last_error": {
+            "description": "The instance's own bounded failure reason — never span content.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "last_span_at": {
+            "description": "When the latest one did. Traces between the two are incomplete.",
+            "format": "date-time",
+            "type": "string"
+          },
+          "undelivered_spans": {
+            "format": "int64",
+            "type": "integer"
+          }
+        },
+        "required": [
+          "first_span_at",
+          "last_span_at",
+          "undelivered_spans"
+        ],
+        "type": "object"
+      },
       "CloudFeatureSwitchesRequest": {
         "properties": {
           "backups_enabled": {
@@ -35824,6 +35864,49 @@
             "format": "int64",
             "type": "integer"
           },
+          "delivery_failing": {
+            "description": "Whether delivery to Temps Cloud is failing for this project *now*:\nspans that already failed an attempt are still being retried.\n\nThis, not `dead_lettered_spans`, is what decides whether the console\nshows an alert. Dead letters are a permanent record of a past loss and\nstay non-zero after the instance recovers; an alert driven by them tells\na healthy instance it is broken, forever.",
+            "type": "boolean"
+          },
+          "delivery_failing_since": {
+            "description": "When the oldest span still being retried reached this instance.",
+            "format": "date-time",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "delivery_failure_action": {
+            "description": "What the operator has to do to end the failure, when it is something\nthey can fix — today, re-enrolling after Cloud rejected the credential.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "delivery_failure_error": {
+            "description": "Why the most recent attempt failed, while `delivery_failing`.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "delivery_failure_setup_path": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "delivery_gaps": {
+            "description": "Past stretches of span time that were never delivered, newest first.\nRendered as history, not as an alert.",
+            "items": {
+              "$ref": "#/components/schemas/CloudDeliveryGapResponse"
+            },
+            "type": "array"
+          },
+          "delivery_gaps_truncated": {
+            "description": "Whether older delivery gaps exist beyond the ones listed. Their spans\nare still counted in `dead_lettered_spans`, so a client can say how many\nolder undelivered spans the list leaves out rather than presenting it as\ncomplete.",
+            "type": "boolean"
+          },
           "effective_reason": {
             "oneOf": [
               {
@@ -35892,6 +35975,11 @@
               "null"
             ]
           },
+          "retrying_spans": {
+            "description": "Spans that failed at least one attempt and are still being retried.",
+            "format": "int64",
+            "type": "integer"
+          },
           "setup_path": {
             "type": [
               "string",
@@ -35913,6 +36001,10 @@
           "cloud_write_mode_available",
           "queued_spans",
           "dead_lettered_spans",
+          "delivery_failing",
+          "retrying_spans",
+          "delivery_gaps",
+          "delivery_gaps_truncated",
           "gap_windows",
           "intervals"
         ],
@@ -113583,6 +113675,7 @@
             "bearer_auth": []
           }
         ],
+        "summary": "Update application settings",
         "tags": [
           "Settings"
         ]

apps/temps-cli/src/api/sdk.gen.ts

@@ -8929,6 +8929,9 @@ export const getSettings = <ThrowOnError extends boolean = false>(options?: Opti
     ...options
 });
 
+/**
+ * Update application settings
+ */
 export const updateSettings = <ThrowOnError extends boolean = false>(options: Options<UpdateSettingsData, ThrowOnError>): RequestResult<UpdateSettingsResponses, UpdateSettingsErrors, ThrowOnError> => (options.client ?? client).put<UpdateSettingsResponses, UpdateSettingsErrors, ThrowOnError>({
     security: [{ scheme: 'bearer', type: 'http' }],
     url: '/settings',

apps/temps-cli/src/api/types.gen.ts

@@ -3481,6 +3481,35 @@ export type CloudCapability = {
     setup_path: string;
 };
 
+/**
+ * A stretch of span time whose spans reached this instance but were never
+ * delivered to Temps Cloud (their retries were exhausted).
+ *
+ * Distinct from [`TelemetryGapWindowResponse`], which records spans refused
+ * at the door because the queue was full: these were accepted and queued, and
+ * the loss happened later, on the way out.
+ */
+export type CloudDeliveryGapResponse = {
+    /**
+     * When the earliest undelivered span in this stretch reached the instance.
+     */
+    first_span_at: string;
+    /**
+     * When delivery of the latest span in this stretch gave up — the same
+     * failure `last_error` describes.
+     */
+    gave_up_at?: string | null;
+    /**
+     * The instance's own bounded failure reason — never span content.
+     */
+    last_error?: string | null;
+    /**
+     * When the latest one did. Traces between the two are incomplete.
+     */
+    last_span_at: string;
+    undelivered_spans: number;
+};
+
 export type CloudFeatureSwitchesRequest = {
     backups_enabled: boolean;
     notifications_enabled: boolean;
@@ -16351,6 +16380,42 @@ export type ProjectCloudTelemetryResponse = {
      * failures.
      */
     dead_lettered_spans: number;
+    /**
+     * Whether delivery to Temps Cloud is failing for this project *now*:
+     * spans that already failed an attempt are still being retried.
+     *
+     * This, not `dead_lettered_spans`, is what decides whether the console
+     * shows an alert. Dead letters are a permanent record of a past loss and
+     * stay non-zero after the instance recovers; an alert driven by them tells
+     * a healthy instance it is broken, forever.
+     */
+    delivery_failing: boolean;
+    /**
+     * When the oldest span still being retried reached this instance.
+     */
+    delivery_failing_since?: string | null;
+    /**
+     * What the operator has to do to end the failure, when it is something
+     * they can fix — today, re-enrolling after Cloud rejected the credential.
+     */
+    delivery_failure_action?: string | null;
+    /**
+     * Why the most recent attempt failed, while `delivery_failing`.
+     */
+    delivery_failure_error?: string | null;
+    delivery_failure_setup_path?: string | null;
+    /**
+     * Past stretches of span time that were never delivered, newest first.
+     * Rendered as history, not as an alert.
+     */
+    delivery_gaps: Array<CloudDeliveryGapResponse>;
+    /**
+     * Whether older delivery gaps exist beyond the ones listed. Their spans
+     * are still counted in `dead_lettered_spans`, so a client can say how many
+     * older undelivered spans the list leaves out rather than presenting it as
+     * complete.
+     */
+    delivery_gaps_truncated: boolean;
     effective_reason?: null | TelemetryWriteIntervalReason;
     effective_reason_message?: string | null;
     /**
@@ -16386,6 +16451,10 @@ export type ProjectCloudTelemetryResponse = {
      * `cloud_write_mode_available` is false.
      */
     reason?: string | null;
+    /**
+     * Spans that failed at least one attempt and are still being retried.
+     */
+    retrying_spans: number;
     setup_path?: string | null;
     /**
      * The operator's declared intent.

apps/temps-cli/src/commands/cloud/telemetry.test.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
 import { test, expect, describe } from 'bun:test'
-import { formatBytes, formatAge } from './telemetry.js'
+import { describeDelivery, formatAge, formatBytes } from './telemetry.js'
 
 // ---------------------------------------------------------------------------
 // formatBytes
@@ -96,3 +96,79 @@ describe('formatAge', () => {
     expect(formatAge(604800)).toBe('7.0d')
   })
 })
+
+// ---------------------------------------------------------------------------
+// describeDelivery
+// ---------------------------------------------------------------------------
+
+describe('describeDelivery', () => {
+  const recovered = {
+    delivery_failing: false,
+    retrying_spans: 0,
+    dead_lettered_spans: 1234,
+    delivery_gaps: [
+      {
+        first_span_at: '2026-09-24T09:00:00Z',
+        last_span_at: '2026-09-24T15:00:00Z',
+        undelivered_spans: 1234,
+        gave_up_at: '2026-09-24T17:00:00Z',
+        last_error: 'Credential rejected by the backend — re-enroll this instance',
+      },
+    ],
+  }
+
+  test('a recovered instance gets no alert, only dated history', () => {
+    // Dead letters never go away. If they raised the alert, an instance that
+    // recovered days ago would keep telling its operator to re-enroll.
+    const { alert, history } = describeDelivery(recovered)
+    expect(alert).toBeNull()
+    expect(history[0]).toContain('1,234 span(s) never delivered')
+    expect(history[1]).toContain('Credential rejected')
+  })
+
+  test('a failing delivery alerts and names the fix when there is one', () => {
+    const { alert } = describeDelivery({
+      ...recovered,
+      delivery_failing: true,
+      retrying_spans: 42,
+      delivery_failure_error: 'Credential rejected by the backend',
+      delivery_failure_action: 'Re-enroll the instance in Temps Cloud settings.',
+      delivery_failure_setup_path: '/settings/cloud',
+    })
+    expect(alert?.[0]).toContain('failing: 42 span(s) waiting to be delivered')
+    expect(alert).toContain('Re-enroll the instance in Temps Cloud settings.')
+    expect(alert).toContain('Fix it at: /settings/cloud')
+  })
+
+  test('a failure with no operator fix does not invent one', () => {
+    const { alert } = describeDelivery({
+      delivery_failing: true,
+      retrying_spans: 1,
+      dead_lettered_spans: 0,
+      delivery_gaps: [],
+    })
+    expect(alert).toHaveLength(1)
+  })
+
+  test('an older server without dated gaps still reports lost spans', () => {
+    // Such a server sends neither `delivery_failing` nor `delivery_gaps`.
+    // Dropping its dead-letter total would make the loss invisible.
+    const { alert, history } = describeDelivery({
+      dead_lettered_spans: 500,
+      last_dead_letter_error: 'Backend returned 503',
+      last_dead_letter_at: '2026-09-24T17:00:00Z',
+    } as Parameters<typeof describeDelivery>[0])
+    expect(alert).toBeNull()
+    expect(history[0]).toContain('500 span(s) never delivered')
+    expect(history[1]).toContain('Backend returned 503')
+  })
+
+  test('a truncated gap list says how many older spans it leaves out', () => {
+    const { history } = describeDelivery({
+      ...recovered,
+      dead_lettered_spans: 5000,
+      delivery_gaps_truncated: true,
+    })
+    expect(history.at(-1)).toContain('3,766 older undelivered span(s)')
+  })
+})