diagnose incident: cloud-side answer to "why is my project 504ing / down"

README.md

@@ -1143,6 +1143,15 @@ Display EC2 instance metrics (CPU, memory, disk, network).
 npx @insforge/cli diagnose metrics --range 6h
 ```
 
+#### `npx @insforge/cli diagnose incident`
+
+Explain why the project is down or returning gateway timeouts (504). The report is built entirely on the platform side, so the command works even while the instance itself is unreachable. Requires InsForge Platform login (`npx @insforge/cli login`); not available when the project is linked via `--api-key`.
+
+```bash
+npx @insforge/cli diagnose incident
+npx @insforge/cli --json diagnose incident
+```
+
 ---
 
 ### Web Scraper — `npx @insforge/cli webscraper`

src/commands/diagnose/incident.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, it } from 'vitest';
+import { formatIncidentReport, normalizeIncidentReport } from './incident.js';
+
+// The command's human-readable output contract: every evidence field the
+// backend sends must surface next to the verdict.
+
+function baseReport() {
+  return {
+    project_id: 'p1',
+    project_status: 'active',
+    operation_status: null,
+    instance_type: 'nano',
+    reachable: {
+      metrics_last_seen_at: '2026-08-10T14:32:00Z',
+      metrics_reporting: false,
+      database_connect: false,
+    },
+    down_since: '2026-08-10T14:32:00Z',
+    memory_before_down_pct: 96.4,
+    memory_latest_pct: null,
+    postgres_started_at: null,
+    scrape_gaps_24h: 2,
+    recent_platform_operations: [],
+    verdict: 'oom_likely',
+    explanation: 'The instance stopped reporting with memory at 96.4% right before.',
+    recommendation: 'Upgrade to a larger instance.',
+  };
+}
+
+describe('formatIncidentReport', () => {
+  it('renders the OOM verdict with the facts behind it', () => {
+    const text = formatIncidentReport(baseReport()).join('\n');
+    expect(text).toContain('Verdict: Out of memory (likely)');
+    expect(text).toContain('not reporting');
+    expect(text).toContain('Database connection: unreachable');
+    expect(text).toContain('Memory right before: 96.4%');
+    expect(text).toContain('Reporting gaps in the last 24h: 2');
+    expect(text).toContain('Instance type: nano');
+    expect(text).toContain('What to do: Upgrade to a larger instance.');
+  });
+
+  it('surfaces project status and in-flight platform operation as facts', () => {
+    const report = {
+      ...baseReport(),
+      project_status: 'paused',
+      operation_status: 'upgrading_instance_type',
+      verdict: 'platform_operation_in_progress',
+    };
+    const text = formatIncidentReport(report).join('\n');
+    expect(text).toContain('Project status: paused');
+    expect(text).toContain('Platform operation in progress: upgrading_instance_type');
+  });
+
+  it('lists recent platform operations that explain a restart', () => {
+    const report = {
+      ...baseReport(),
+      verdict: 'no_incident_detected',
+      recent_platform_operations: [{ action: 'reset_project', at: '2026-08-10T12:00:00Z' }],
+    };
+    const text = formatIncidentReport(report).join('\n');
+    expect(text).toContain('Recent platform operation: reset_project at');
+  });
+
+  it('falls back to the raw verdict string for unknown verdicts (forward compatibility)', () => {
+    const text = formatIncidentReport({ ...baseReport(), verdict: 'new_verdict' }).join('\n');
+    expect(text).toContain('Verdict: new_verdict');
+  });
+});
+
+describe('normalizeIncidentReport', () => {
+  it('rejects payloads that are not a report at all with a clear error', () => {
+    expect(() => normalizeIncidentReport(null)).toThrow(/not an incident report/);
+    expect(() => normalizeIncidentReport('nope')).toThrow(/not an incident report/);
+    expect(() => normalizeIncidentReport({ error: 'x' })).toThrow(/missing incident report/);
+  });
+
+  it('defends every rendered field against partial payloads', () => {
+    // The exact shape r2d2 called out: reachable present but empty, and
+    // recent_platform_operations null.
+    const report = normalizeIncidentReport({
+      verdict: 'down_unknown',
+      explanation: 'x',
+      reachable: {},
+      recent_platform_operations: null,
+    });
+    expect(report.reachable.metrics_reporting).toBe(false);
+    expect(report.recent_platform_operations).toEqual([]);
+    expect(report.scrape_gaps_24h).toBe(0);
+    expect(report.memory_latest_pct).toBeNull();
+    // and the renderer consumes it without throwing
+    expect(formatIncidentReport(report).join('\n')).toContain('Verdict:');
+  });
+
+  it('drops malformed operation entries and coerces bad numbers', () => {
+    const report = normalizeIncidentReport({
+      verdict: 'no_incident_detected',
+      explanation: 'x',
+      reachable: { metrics_reporting: 'yes', database_connect: true },
+      recent_platform_operations: [{ action: 'reset_project' }, { action: 'ok', at: 't' }, 42],
+      scrape_gaps_24h: 'many',
+      memory_before_down_pct: NaN,
+    });
+    expect(report.reachable.metrics_reporting).toBe(false); // non-boolean → false
+    expect(report.reachable.database_connect).toBe(true);
+    expect(report.recent_platform_operations).toEqual([{ action: 'ok', at: 't' }]);
+    expect(report.scrape_gaps_24h).toBe(0);
+    expect(report.memory_before_down_pct).toBeNull();
+  });
+});

src/commands/diagnose/incident.ts

@@ -0,0 +1,216 @@
+import type { Command } from 'commander';
+import { platformFetch } from '../../lib/api/platform.js';
+import { requireAuth } from '../../lib/credentials.js';
+import { handleError, getRootOpts, CLIError, ProjectNotLinkedError } from '../../lib/errors.js';
+import { getProjectConfig, FAKE_PROJECT_ID } from '../../lib/config.js';
+import { outputJson } from '../../lib/output.js';
+import { trackDiagnose, shutdownAnalytics } from '../../lib/analytics.js';
+
+// Cloud-side incident report. Everything behind this endpoint lives on the
+// platform (Prometheus history, platform records, an outbound DB probe), so
+// the command keeps answering "why is my project 504ing / down?" even while
+// the instance itself is unreachable — exactly when `diagnose logs` dies.
+
+interface IncidentReport {
+  project_id: string;
+  project_status: string;
+  operation_status: string | null;
+  instance_type: string | null;
+  reachable: {
+    metrics_last_seen_at: string | null;
+    metrics_reporting: boolean;
+    database_connect: boolean;
+  };
+  down_since: string | null;
+  memory_before_down_pct: number | null;
+  memory_latest_pct: number | null;
+  postgres_started_at: string | null;
+  scrape_gaps_24h: number;
+  recent_platform_operations: Array<{ action: string; at: string }>;
+  verdict: string;
+  explanation: string;
+  recommendation: string;
+}
+
+const VERDICT_LABELS: Record<string, string> = {
+  paused_or_suspended: 'Project is paused or suspended',
+  platform_operation_in_progress: 'Platform operation in progress',
+  oom_likely: 'Out of memory (likely)',
+  down_unknown: 'Instance down, cause unclear',
+  metrics_stopped: 'Instance stopped reporting metrics',
+  no_incident_detected: 'No incident detected',
+};
+
+function asString(value: unknown, fallback: string): string {
+  return typeof value === 'string' && value.length > 0 ? value : fallback;
+}
+
+function asNullableString(value: unknown): string | null {
+  return typeof value === 'string' && value.length > 0 ? value : null;
+}
+
+function asNullableNumber(value: unknown): number | null {
+  return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+
+/**
+ * Coerce whatever the platform returned into a report the renderer can
+ * safely consume. This command exists precisely for moments when the
+ * backend or instance state is unusual, so every field gets a defensive
+ * default instead of trusting the cast. Throws a CLIError only when the
+ * payload is not a report at all. Exported for tests.
+ */
+export function normalizeIncidentReport(raw: unknown): IncidentReport {
+  if (raw === null || typeof raw !== 'object') {
+    throw new CLIError(
+      'Unexpected response from the platform (not an incident report). Your backend may predate this command.',
+    );
+  }
+  const r = raw as Record<string, unknown>;
+  if (typeof r.verdict !== 'string' || typeof r.explanation !== 'string') {
+    throw new CLIError(
+      'Unexpected response from the platform (missing incident report fields). Your backend may predate this command.',
+    );
+  }
+  const reachable =
+    r.reachable !== null && typeof r.reachable === 'object'
+      ? (r.reachable as Record<string, unknown>)
+      : {};
+  const operations = Array.isArray(r.recent_platform_operations)
+    ? r.recent_platform_operations
+    : [];
+  return {
+    project_id: asString(r.project_id, 'unknown'),
+    project_status: asString(r.project_status, 'unknown'),
+    operation_status: asNullableString(r.operation_status),
+    instance_type: asNullableString(r.instance_type),
+    reachable: {
+      metrics_last_seen_at: asNullableString(reachable.metrics_last_seen_at),
+      metrics_reporting: reachable.metrics_reporting === true,
+      database_connect: reachable.database_connect === true,
+    },
+    down_since: asNullableString(r.down_since),
+    memory_before_down_pct: asNullableNumber(r.memory_before_down_pct),
+    memory_latest_pct: asNullableNumber(r.memory_latest_pct),
+    postgres_started_at: asNullableString(r.postgres_started_at),
+    scrape_gaps_24h: asNullableNumber(r.scrape_gaps_24h) ?? 0,
+    recent_platform_operations: operations
+      .filter(
+        (op): op is { action: string; at: string } =>
+          op !== null &&
+          typeof op === 'object' &&
+          typeof (op as Record<string, unknown>).action === 'string' &&
+          typeof (op as Record<string, unknown>).at === 'string',
+      )
+      .map((op) => ({ action: op.action, at: op.at })),
+    verdict: r.verdict,
+    explanation: r.explanation,
+    recommendation: asString(r.recommendation, ''),
+  };
+}
+
+function formatWhen(iso: string | null): string {
+  if (!iso) return 'unknown';
+  return new Date(iso).toLocaleString();
+}
+
+/**
+ * Render the report as printable lines. Pure and exported for tests — this
+ * is the command's output contract. Every evidence field the backend sends
+ * shows up here so the verdict is always accompanied by the facts behind it.
+ */
+export function formatIncidentReport(report: IncidentReport): string[] {
+  const lines: string[] = [];
+  lines.push(`Verdict: ${VERDICT_LABELS[report.verdict] ?? report.verdict}`);
+  lines.push('');
+  lines.push(report.explanation);
+  lines.push('');
+  const facts: string[] = [];
+  if (report.project_status !== 'active') {
+    facts.push(`Project status: ${report.project_status}`);
+  }
+  if (report.operation_status) {
+    facts.push(`Platform operation in progress: ${report.operation_status}`);
+  }
+  facts.push(
+    `Instance metrics: ${
+      report.reachable.metrics_reporting
+        ? 'reporting'
+        : `not reporting (last seen ${formatWhen(report.reachable.metrics_last_seen_at)})`
+    }`,
+  );
+  facts.push(
+    `Database connection: ${report.reachable.database_connect ? 'accepting connections' : 'unreachable'}`,
+  );
+  if (report.down_since) {
+    facts.push(`Down since: ${formatWhen(report.down_since)}`);
+  }
+  if (report.memory_before_down_pct !== null) {
+    facts.push(`Memory right before: ${report.memory_before_down_pct}%`);
+  } else if (report.memory_latest_pct !== null) {
+    facts.push(`Memory latest: ${report.memory_latest_pct}%`);
+  }
+  if (report.postgres_started_at) {
+    facts.push(`Postgres started: ${formatWhen(report.postgres_started_at)}`);
+  }
+  if (report.scrape_gaps_24h > 0) {
+    facts.push(`Reporting gaps in the last 24h: ${report.scrape_gaps_24h}`);
+  }
+  if (report.instance_type) {
+    facts.push(`Instance type: ${report.instance_type}`);
+  }
+  for (const op of report.recent_platform_operations) {
+    facts.push(`Recent platform operation: ${op.action} at ${formatWhen(op.at)}`);
+  }
+  for (const fact of facts) {
+    lines.push(`  - ${fact}`);
+  }
+  lines.push('');
+  lines.push(`What to do: ${report.recommendation}`);
+  return lines;
+}
+
+export function registerDiagnoseIncidentCommand(diagnoseCmd: Command): void {
+  diagnoseCmd
+    .command('incident')
+    .description(
+      'Explain why the project is down or returning gateway timeouts (504) — works even while the instance is unreachable',
+    )
+    .action(async (_opts, cmd) => {
+      const { json, apiUrl } = getRootOpts(cmd);
+      try {
+        await requireAuth(apiUrl);
+        const config = getProjectConfig();
+        if (!config) throw new ProjectNotLinkedError();
+        if (config.project_id === FAKE_PROJECT_ID) {
+          throw new CLIError(
+            'Incident report requires InsForge Platform login. Not available when linked via --api-key.',
+          );
+        }
+        trackDiagnose('incident', config);
+
+        const res = await platformFetch(
+          `/projects/v1/${config.project_id}/diagnose/incident`,
+          {},
+          apiUrl,
+        );
+        const raw = await res.json();
+
+        if (json) {
+          // Raw passthrough: machine consumers get the exact server payload,
+          // including fields this CLI version does not know about yet.
+          outputJson(raw);
+        } else {
+          const report = normalizeIncidentReport(raw);
+          for (const line of formatIncidentReport(report)) {
+            console.log(line);
+          }
+        }
+      } catch (err) {
+        await shutdownAnalytics();
+        handleError(err, json);
+      } finally {
+        await shutdownAnalytics();
+      }
+    });
+}

src/commands/diagnose/index.ts

@@ -14,6 +14,7 @@ import { fetchMetricsSummary, registerDiagnoseMetricsCommand } from './metrics.j
 import { fetchAdvisorSummary, registerDiagnoseAdvisorCommand } from './advisor.js';
 import { runDbChecks, registerDiagnoseDbCommand } from './db.js';
 import { fetchLogsSummary, registerDiagnoseLogsCommand } from './logs.js';
+import { registerDiagnoseIncidentCommand } from './incident.js';
 
 function sectionHeader(title: string): string {
   return `── ${title} ${'─'.repeat(Math.max(0, 44 - title.length))}`;
@@ -379,6 +380,7 @@ export function registerDiagnoseCommands(diagnoseCmd: Command): void {
   registerDiagnoseAdvisorCommand(diagnoseCmd);
   registerDiagnoseDbCommand(diagnoseCmd);
   registerDiagnoseLogsCommand(diagnoseCmd);
+  registerDiagnoseIncidentCommand(diagnoseCmd);
 }
 
 function formatBytesCompact(bytes: number): string {