fix(checkin): detect unusable X discovery read budgets

checks/_helpers/collectors.mjs

@@ -91,7 +91,11 @@ function readDailyBudget(root) {
   try {
     const src = readFileSync(join(root, 'shared/server/x-ingest.ts'), 'utf8')
     const match = src.match(/DAILY_DISCOVERY_READ_BUDGET\s*=\s*(\d+)/)
-    return match ? Number(match[1]) : null
+    const minimum = src.match(/MIN_SEARCH_PAGE_SIZE\s*=\s*(\d+)/)
+    return {
+      target: match ? Number(match[1]) : null,
+      minimum: minimum ? Number(minimum[1]) : null,
+    }
   }
   catch (error) {
     throw new Error('X discovery budget source is unavailable.', { cause: error })
@@ -171,8 +175,10 @@ export function collectD1(context) {
       has('jobs') ? `(SELECT COUNT(*) FROM jobs WHERE reserved_at IS NOT NULL AND reserved_at < ${Math.floor(now.getTime() / 1000) - 900} AND completed_at IS NULL AND failed_at IS NULL) AS stale_reserved_jobs` : 'NULL AS stale_reserved_jobs',
     ]
     /**
-     * Turn raw X read counts into the number that matters: dollars, and whether
-     * the month is on track. Reads are $0.005 each with no included allowance.
+     * Estimate Post-only spend from returned reads, not an invoice.
+     * X usually charges repeated resources once per UTC day. The request
+     * ceiling includes repeats, while stored observations show distinct posts.
+     * User expansions and billing receipts are not recorded here.
      *
      * The budget is read out of shared/server/x-ingest.ts rather than copied.
      * This script is plain node and cannot import the app's module graph, and a
@@ -181,38 +187,41 @@ export function collectD1(context) {
      */
     function withXSpend(costRow) {
       const USD_PER_READ = 0.005
-      const budgetTarget = readDailyBudget(root)
+      const budget = readDailyBudget(root)
+      const budgetTarget = budget.minimum === null ? null : budget.target
       const row = costRow ?? {}
       const discoveryToday = Number(row.x_discovery_reads_today ?? 0)
       const hot = Number(row.x_hot_posts ?? 0)
-      // Refresh only pays for a hot post again when its window crosses midnight.
-      // Treating every hot post as one more charge is the pessimistic bound.
+      // Treat every returned post and remaining hot post as another charge.
+      // This conservative post-only projection does not prove actual spend.
       const projectedDaily = discoveryToday + hot
       return {
         ...row,
         x_budget_target: budgetTarget,
         x_budget_used_pct: budgetTarget > 0 ? Math.round((discoveryToday / budgetTarget) * 100) : null,
         x_projected_daily_reads: projectedDaily,
         x_projected_monthly_usd: Math.round(projectedDaily * 30 * USD_PER_READ * 100) / 100,
-        x_over_budget: discoveryToday > budgetTarget,
+        x_over_budget: budgetTarget === null ? null : discoveryToday > budgetTarget,
         // The value question: reads bought per skill actually verified. Rising
         // means the query is getting less precise or the stream is noisier.
         x_reads_per_verified_skill: Number(row.x_verified_skills ?? 0) > 0
           ? Math.round(Number(row.x_discovery_reads_total ?? 0) / Number(row.x_verified_skills))
           : null,
-        // Budget spent without finishing the window means the stream is
-        // outrunning the ingest and the backlog grows daily.
-        x_falling_behind: discoveryToday >= budgetTarget,
+        // Exhaustion can leave unread results. This historical field is a
+        // warning heuristic; returned reads alone cannot prove a backlog.
+        x_falling_behind: budgetTarget === null ? null : budgetTarget - discoveryToday < budget.minimum,
       }
     }
     const costParts = [
       has('ai_batch_costs') ? `(SELECT COALESCE(SUM(est_cost_usd), 0) FROM ai_batch_costs WHERE submitted_at >= ${sinceSec}) AS ai_cost_usd` : 'NULL AS ai_cost_usd',
-      // X is pay-per-use with no included allowance: every post read is a real
-      // $0.005 invoice line, so it belongs in the health report rather than only
-      // in task logs. `budget_spent` counts today only when `budget_day` matches;
-      // a stale day reads as 0, matching how the ingest resets it.
+      // Returned reads consume the local ceiling. They can include repeat IDs
+      // that X normally deduplicates for billing within the UTC day.
+      // A stale budget day reads as 0, matching the ingest's reset.
       has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(CASE WHEN budget_day = '${utcDay}' THEN budget_spent ELSE 0 END), 0) FROM x_ingest_cursor) AS x_discovery_reads_today` : 'NULL AS x_discovery_reads_today',
       has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(posts_read_total), 0) FROM x_ingest_cursor) AS x_discovery_reads_total` : 'NULL AS x_discovery_reads_total',
+      // Discovery and refresh share snapshots. Count their distinct observed
+      // post IDs separately from returned reads. This is not billed usage.
+      has('x_post_metrics') && has('x_posts') ? `(SELECT COUNT(DISTINCT m.post_id) FROM x_post_metrics m JOIN x_posts p ON p.post_id = m.post_id WHERE p.platform = 'x' AND m.observed_at >= strftime('%s', '${utcDay}') AND m.observed_at <= ${Math.floor(now.getTime() / 1000)}) AS x_observed_posts_today` : 'NULL AS x_observed_posts_today',
       // Hot posts are refresh's remaining exposure: each can cost at most one
       // more read, when its window crosses midnight UTC.
       has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE refresh_tier = 'hot') AS x_hot_posts` : 'NULL AS x_hot_posts',

shared/checkin-external.ts

@@ -22,16 +22,19 @@ Interpret this site evidence:
     },
     {
       id: 'skilld.spend',
-      prompt: `- X API is pay-per-use at $0.005 per post read, with no included allowance and a $40/month accepted line. Gate on \`d1.cost\`:
-     - \`x_over_budget\` true is AMBER. The daily discovery read ceiling failed to hold. Investigate before inferring billed spend.
-     - \`x_projected_monthly_usd\` above 40 is AMBER, above 60 is RED, even when \`x_over_budget\` is false. Harlan raised the accepted line from $30 to $40 on 2026-09-07 (issue #160). Report the figure every run, not only when it breaches.
-     - \`x_budget_used_pct\` at 100 on consecutive days means discovery is truncating. Check cursor progress and repeated pages before calling it steady state.
-     - \`x_discovery_reads_today\` counts returned posts, including repeats. X normally charges once per resource per UTC day. The monthly projection extrapolates today's returns plus hot posts for 30 days. Report it as an upper-bound estimate, never billed spend. A catch-up burst does not establish the steady rate.
-     - \`x_posts_24h\` counts newly stored posts, including posts without repo references. A large gap from returned reads can indicate replay. Check continuation and cursor progress before proposing query tuning.`,
+      prompt: `- X API lists $0.005 per Post and $0.010 per User. The accepted monthly line is $40, recorded in #160.
+     - X usually deduplicates resource charges within each UTC day. This guarantee has exceptions: https://docs.x.com/x-api/getting-started/pricing.
+     - \`x_over_budget\` true is AMBER: discovery exceeded its returned-read ceiling. Null means the ceiling evidence is unavailable.
+     - \`x_projected_monthly_usd\` above 40 is AMBER; above 60 is RED. Report it as a conservative Post-only estimate. Actual billing and User expansion charges are unavailable.
+     - \`x_falling_behind\` warns that fewer reads remain than X's minimum request size. Exhaustion alone does not prove unread results remain.
+     - If \`x_budget_used_pct\` reaches 100 on consecutive days, check continuation, cursor progress, and repeated pages. A catch-up burst does not establish the steady rate.
+     - \`x_discovery_reads_today\` counts returned Posts, including repeated IDs. It is not billed usage.
+     - \`x_observed_posts_today\` counts distinct X Posts observed by discovery or refresh within the current UTC day. It is not billed usage.
+     - \`x_posts_24h\` counts newly stored Posts over a rolling 24 hours, including Posts without Repository links. These windows differ. Never infer discarded Posts or invoice charges from their gap. Check continuation and cursor progress before proposing query tuning.`,
     },
     {
       id: 'skilld.pulse',
-      prompt: `Include a Pulse section with users, skills, repo changes, command copies split as run and install, digests, known AI cost, and X as "$X.XX/mo upper-bound estimate (N/<x_budget_target> returned reads today)". Read these values from collected evidence. Include deploy and CI changes in Overnight. Include migration, schema, deploy SHA, stale task, and observability gaps in Drift.`,
+      prompt: `Include a Pulse section with users, skills, repo changes, command copies split as run and install, digests, known AI cost, and X spend as "$X.XX/mo Post-only estimate (N/<x_budget_target> returned reads today)". State that actual billing is unavailable. Read these values from collected evidence. Include deploy and CI changes in Overnight. Include migration, schema, deploy SHA, stale task, and observability gaps in Drift.`,
     },
     {
       id: 'skilld.seo-recovery',

test/unit/daily-checkin-data.test.ts

@@ -37,7 +37,7 @@ it('archives the identities behind the skill sync failure count', async () => {
   roots.push(root)
   await mkdir(join(root, 'shared/server'), { recursive: true })
   await mkdir(join(root, 'migrations'))
-  await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400')
+  await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400\nconst MIN_SEARCH_PAGE_SIZE = 10')
   await writeFile(join(root, 'migrations/001.sql'), '')
   const skills = [
     { owner: 'anthu', repo: 'vite', name: 'env', sync_status: 'error', last_synced_at: Math.floor(since.getTime() / 1000) + 3600 },

test/unit/external-checkin-collectors.test.ts

@@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { runExternalChecks } from '@harlan-zw/nuxt-checkin/external'
+import Database from 'better-sqlite3'
 import { afterEach, expect, it, vi } from 'vitest'
 import analyticsCheck from '../../checks/external/analytics'
 import ciCheck from '../../checks/external/ci'
@@ -23,13 +24,19 @@ afterEach(async () => {
   vi.unstubAllGlobals()
 })
 
-it('shares deployment and database evidence without blocking nested collectors', async () => {
+it.each([
+  [10, false, true],
+  [390, false, true],
+  [391, true, true],
+  [400, true, true],
+  [10, null, false],
+])('reports discovery exhaustion after %i returned reads', async (readsToday, exhausted, hasMinimum) => {
   const now = new Date('2026-09-14T00:00:00Z')
   const root = await mkdtemp(join(tmpdir(), 'skilld-checkin-'))
   roots.push(root)
   await mkdir(join(root, 'shared/server'), { recursive: true })
   await mkdir(join(root, 'migrations'))
-  await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400')
+  await writeFile(join(root, 'shared/server/x-ingest.ts'), `export const DAILY_DISCOVERY_READ_BUDGET = 400${hasMinimum ? '\nconst MIN_SEARCH_PAGE_SIZE = 10' : ''}`)
   await writeFile(join(root, 'migrations/001.sql'), '')
   boundary.command.mockImplementation(async (_context, command: string, args: string[]) => {
     let output: unknown = ''
@@ -56,19 +63,53 @@ it('shares deployment and database evidence without blocking nested collectors',
       else if (sql.includes('AS newly_broken_repos_total'))
         rows = [{ newly_broken_repos_total: 0, stale_reserved_jobs: 0 }]
       else if (sql.includes('AS ai_cost_usd'))
-        rows = [{ x_discovery_reads_today: 10, x_hot_posts: 0 }]
+        rows = [{ x_discovery_reads_today: readsToday, x_hot_posts: 0 }]
       output = [{ success: true, results: rows }]
     }
     return { _tag: 'Ok', stdout: JSON.stringify(output), stderr: '' }
   })
   const checks = [gitCheck, deployCheck, databaseCheck]
   const result = await runExternalChecks(checks, { required: checks.map(check => check.id), timeoutMs: 1000, totalTimeoutMs: 1500 }, { rootDir: root, env: {}, clock: () => now })
-  expect(result.report).toMatchObject({ severity: 'pass', coverage: 'complete' })
+  expect(result.report.coverage).toBe(hasMinimum ? 'complete' : 'incomplete')
   expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { inventory: { skills: 23 } } })
+  expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { cost: { x_falling_behind: exhausted } } })
+  if (!hasMinimum)
+    expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { cost: { x_over_budget: null, x_budget_target: null } } })
   expect(boundary.command.mock.calls.filter(([, command, args]) => command === 'git' && args[0] === 'fetch')).toHaveLength(1)
   expect(boundary.command.mock.calls.filter(([, , args]) => args.includes('SELECT name FROM sqlite_master WHERE type = \'table\' ORDER BY name'))).toHaveLength(1)
 })
 
+it('counts only X Posts observed within the UTC day at the check time', async () => {
+  const now = new Date('2026-10-05T10:00:00Z')
+  const root = await mkdtemp(join(tmpdir(), 'skilld-checkin-x-observed-'))
+  roots.push(root)
+  await mkdir(join(root, 'shared/server'), { recursive: true })
+  await mkdir(join(root, 'migrations'))
+  await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400\nconst MIN_SEARCH_PAGE_SIZE = 10')
+  const db = new Database(':memory:')
+  db.exec(`
+    CREATE TABLE x_posts (post_id TEXT, platform TEXT, refresh_tier TEXT, first_seen_at INTEGER, posted_at INTEGER);
+    CREATE TABLE x_post_metrics (post_id TEXT, observed_at INTEGER);
+    INSERT INTO x_posts VALUES ('x-one', 'x', 'frozen', 0, 0), ('bsky-one', 'bsky', 'frozen', 0, 0), ('x-future', 'x', 'frozen', 0, 0), ('x-old', 'x', 'frozen', 0, 0);
+    INSERT INTO x_post_metrics VALUES
+      ('x-one', unixepoch('2026-10-05T08:00:00Z')),
+      ('x-one', unixepoch('2026-10-05T09:00:00Z')),
+      ('bsky-one', unixepoch('2026-10-05T09:00:00Z')),
+      ('x-future', unixepoch('2026-10-05T11:00:00Z')),
+      ('x-old', unixepoch('2026-10-04T23:59:59Z'));
+  `)
+  boundary.command.mockImplementation(async (_context, command: string, args: string[]) => {
+    if (command === 'git')
+      return { _tag: 'Ok', stdout: args[0] === 'rev-parse' ? 'abc' : '', stderr: '' }
+    const sql = args[args.indexOf('--command') + 1]!
+    const rows = db.prepare(sql).all()
+    return { _tag: 'Ok', stdout: JSON.stringify([{ success: true, results: rows }]), stderr: '' }
+  })
+  const { report } = await runExternalChecks([databaseCheck], { required: [databaseCheck.id] }, { rootDir: root, env: {}, clock: () => now })
+  db.close()
+  expect(report.results[0]?.result).toMatchObject({ evidence: { cost: { x_observed_posts_today: 1 } } })
+})
+
 it('rejects oversized Worker analytics evidence', async () => {
   vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json({ data: { viewer: { accounts: [{ workersInvocationsAdaptive: [] }] } }, padding: 'x'.repeat(2_097_153) })))
   const { report } = await runExternalChecks([workersCheck], { required: [workersCheck.id] }, { env: { CLOUDFLARE_USAGE_TOKEN: 'test-token' } })