From 7fa798e3706d44f5fe318512f4c686aa14957984 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Mon, 5 Oct 2026 17:07:45 +1100 Subject: [PATCH 1/2] fix(checkin): detect unusable X discovery read budgets --- checks/_helpers/collectors.mjs | 35 ++++++++++++------- test/unit/daily-checkin-data.test.ts | 2 +- test/unit/external-checkin-collectors.test.ts | 14 +++++--- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs index 6ae98b3a..e543d4d4 100644 --- a/checks/_helpers/collectors.mjs +++ b/checks/_helpers/collectors.mjs @@ -91,7 +91,11 @@ function readDailyBudget(root) { try { const src = readFileSync(join(root, 'shared/server/x-ingest.ts'), 'utf8') const match = src.match(/DAILY_DISCOVERY_READ_BUDGET\s*=\s*(\d+)/) - return match ? Number(match[1]) : null + const minimum = src.match(/MIN_SEARCH_PAGE_SIZE\s*=\s*(\d+)/) + return { + target: match ? Number(match[1]) : null, + minimum: minimum ? Number(minimum[1]) : null, + } } catch (error) { throw new Error('X discovery budget source is unavailable.', { cause: error }) @@ -171,8 +175,10 @@ export function collectD1(context) { has('jobs') ? `(SELECT COUNT(*) FROM jobs WHERE reserved_at IS NOT NULL AND reserved_at < ${Math.floor(now.getTime() / 1000) - 900} AND completed_at IS NULL AND failed_at IS NULL) AS stale_reserved_jobs` : 'NULL AS stale_reserved_jobs', ] /** - * Turn raw X read counts into the number that matters: dollars, and whether - * the month is on track. Reads are $0.005 each with no included allowance. + * Project an upper bound from returned post reads, not an invoice. + * X usually charges repeated resources once per UTC day. The request + * ceiling includes repeats, while stored observations show distinct posts. + * User expansions and billing receipts are not recorded here. * * The budget is read out of shared/server/x-ingest.ts rather than copied. * This script is plain node and cannot import the app's module graph, and a @@ -181,12 +187,13 @@ export function collectD1(context) { */ function withXSpend(costRow) { const USD_PER_READ = 0.005 - const budgetTarget = readDailyBudget(root) + const budget = readDailyBudget(root) + const budgetTarget = budget.minimum === null ? null : budget.target const row = costRow ?? {} const discoveryToday = Number(row.x_discovery_reads_today ?? 0) const hot = Number(row.x_hot_posts ?? 0) - // Refresh only pays for a hot post again when its window crosses midnight. - // Treating every hot post as one more charge is the pessimistic bound. + // Treat every returned post and remaining hot post as another charge. + // This conservative post-only projection does not prove actual spend. const projectedDaily = discoveryToday + hot return { ...row, @@ -200,19 +207,21 @@ export function collectD1(context) { x_reads_per_verified_skill: Number(row.x_verified_skills ?? 0) > 0 ? Math.round(Number(row.x_discovery_reads_total ?? 0) / Number(row.x_verified_skills)) : null, - // Budget spent without finishing the window means the stream is - // outrunning the ingest and the backlog grows daily. - x_falling_behind: discoveryToday >= budgetTarget, + // Exhaustion can leave unread results. This historical field is a + // warning heuristic; returned reads alone cannot prove a backlog. + x_falling_behind: budgetTarget === null ? null : budgetTarget - discoveryToday < budget.minimum, } } const costParts = [ has('ai_batch_costs') ? `(SELECT COALESCE(SUM(est_cost_usd), 0) FROM ai_batch_costs WHERE submitted_at >= ${sinceSec}) AS ai_cost_usd` : 'NULL AS ai_cost_usd', - // X is pay-per-use with no included allowance: every post read is a real - // $0.005 invoice line, so it belongs in the health report rather than only - // in task logs. `budget_spent` counts today only when `budget_day` matches; - // a stale day reads as 0, matching how the ingest resets it. + // Returned reads consume the local ceiling. They can include repeat IDs + // that X normally deduplicates for billing within the UTC day. + // A stale budget day reads as 0, matching the ingest's reset. has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(CASE WHEN budget_day = '${utcDay}' THEN budget_spent ELSE 0 END), 0) FROM x_ingest_cursor) AS x_discovery_reads_today` : 'NULL AS x_discovery_reads_today', has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(posts_read_total), 0) FROM x_ingest_cursor) AS x_discovery_reads_total` : 'NULL AS x_discovery_reads_total', + // Discovery and refresh share snapshots. Count their distinct observed + // post IDs separately from returned reads. This is not billed usage. + has('x_post_metrics') ? `(SELECT COUNT(DISTINCT post_id) FROM x_post_metrics WHERE observed_at >= strftime('%s', '${utcDay}')) AS x_observed_posts_today` : 'NULL AS x_observed_posts_today', // Hot posts are refresh's remaining exposure: each can cost at most one // more read, when its window crosses midnight UTC. has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE refresh_tier = 'hot') AS x_hot_posts` : 'NULL AS x_hot_posts', diff --git a/test/unit/daily-checkin-data.test.ts b/test/unit/daily-checkin-data.test.ts index 032bfdf2..6d2b4e15 100644 --- a/test/unit/daily-checkin-data.test.ts +++ b/test/unit/daily-checkin-data.test.ts @@ -37,7 +37,7 @@ it('archives the identities behind the skill sync failure count', async () => { roots.push(root) await mkdir(join(root, 'shared/server'), { recursive: true }) await mkdir(join(root, 'migrations')) - await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400') + await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400\nconst MIN_SEARCH_PAGE_SIZE = 10') await writeFile(join(root, 'migrations/001.sql'), '') const skills = [ { owner: 'anthu', repo: 'vite', name: 'env', sync_status: 'error', last_synced_at: Math.floor(since.getTime() / 1000) + 3600 }, diff --git a/test/unit/external-checkin-collectors.test.ts b/test/unit/external-checkin-collectors.test.ts index a0fe3152..55a5c157 100644 --- a/test/unit/external-checkin-collectors.test.ts +++ b/test/unit/external-checkin-collectors.test.ts @@ -23,13 +23,18 @@ afterEach(async () => { vi.unstubAllGlobals() }) -it('shares deployment and database evidence without blocking nested collectors', async () => { +it.each([ + [10, false], + [390, false], + [391, true], + [400, true], +])('reports discovery exhaustion after %i returned reads', async (readsToday, exhausted) => { const now = new Date('2026-09-14T00:00:00Z') const root = await mkdtemp(join(tmpdir(), 'skilld-checkin-')) roots.push(root) await mkdir(join(root, 'shared/server'), { recursive: true }) await mkdir(join(root, 'migrations')) - await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400') + await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400\nconst MIN_SEARCH_PAGE_SIZE = 10') await writeFile(join(root, 'migrations/001.sql'), '') boundary.command.mockImplementation(async (_context, command: string, args: string[]) => { let output: unknown = '' @@ -56,15 +61,16 @@ it('shares deployment and database evidence without blocking nested collectors', else if (sql.includes('AS newly_broken_repos_total')) rows = [{ newly_broken_repos_total: 0, stale_reserved_jobs: 0 }] else if (sql.includes('AS ai_cost_usd')) - rows = [{ x_discovery_reads_today: 10, x_hot_posts: 0 }] + rows = [{ x_discovery_reads_today: readsToday, x_hot_posts: 0 }] output = [{ success: true, results: rows }] } return { _tag: 'Ok', stdout: JSON.stringify(output), stderr: '' } }) const checks = [gitCheck, deployCheck, databaseCheck] const result = await runExternalChecks(checks, { required: checks.map(check => check.id), timeoutMs: 1000, totalTimeoutMs: 1500 }, { rootDir: root, env: {}, clock: () => now }) - expect(result.report).toMatchObject({ severity: 'pass', coverage: 'complete' }) + expect(result.report.coverage).toBe('complete') expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { inventory: { skills: 23 } } }) + expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { cost: { x_falling_behind: exhausted } } }) expect(boundary.command.mock.calls.filter(([, command, args]) => command === 'git' && args[0] === 'fetch')).toHaveLength(1) expect(boundary.command.mock.calls.filter(([, , args]) => args.includes('SELECT name FROM sqlite_master WHERE type = \'table\' ORDER BY name'))).toHaveLength(1) }) From 6d91dda3c1b302fa82d5d760f6540a02c35b03e6 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Mon, 5 Oct 2026 17:14:38 +1100 Subject: [PATCH 2/2] fix(checkin): distinguish X observations from billed usage --- checks/_helpers/collectors.mjs | 6 +-- shared/checkin-external.ts | 15 +++--- test/unit/external-checkin-collectors.test.ts | 49 ++++++++++++++++--- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/checks/_helpers/collectors.mjs b/checks/_helpers/collectors.mjs index e543d4d4..b6702243 100644 --- a/checks/_helpers/collectors.mjs +++ b/checks/_helpers/collectors.mjs @@ -175,7 +175,7 @@ export function collectD1(context) { has('jobs') ? `(SELECT COUNT(*) FROM jobs WHERE reserved_at IS NOT NULL AND reserved_at < ${Math.floor(now.getTime() / 1000) - 900} AND completed_at IS NULL AND failed_at IS NULL) AS stale_reserved_jobs` : 'NULL AS stale_reserved_jobs', ] /** - * Project an upper bound from returned post reads, not an invoice. + * Estimate Post-only spend from returned reads, not an invoice. * X usually charges repeated resources once per UTC day. The request * ceiling includes repeats, while stored observations show distinct posts. * User expansions and billing receipts are not recorded here. @@ -201,7 +201,7 @@ export function collectD1(context) { x_budget_used_pct: budgetTarget > 0 ? Math.round((discoveryToday / budgetTarget) * 100) : null, x_projected_daily_reads: projectedDaily, x_projected_monthly_usd: Math.round(projectedDaily * 30 * USD_PER_READ * 100) / 100, - x_over_budget: discoveryToday > budgetTarget, + x_over_budget: budgetTarget === null ? null : discoveryToday > budgetTarget, // The value question: reads bought per skill actually verified. Rising // means the query is getting less precise or the stream is noisier. x_reads_per_verified_skill: Number(row.x_verified_skills ?? 0) > 0 @@ -221,7 +221,7 @@ export function collectD1(context) { has('x_ingest_cursor') ? `(SELECT COALESCE(SUM(posts_read_total), 0) FROM x_ingest_cursor) AS x_discovery_reads_total` : 'NULL AS x_discovery_reads_total', // Discovery and refresh share snapshots. Count their distinct observed // post IDs separately from returned reads. This is not billed usage. - has('x_post_metrics') ? `(SELECT COUNT(DISTINCT post_id) FROM x_post_metrics WHERE observed_at >= strftime('%s', '${utcDay}')) AS x_observed_posts_today` : 'NULL AS x_observed_posts_today', + has('x_post_metrics') && has('x_posts') ? `(SELECT COUNT(DISTINCT m.post_id) FROM x_post_metrics m JOIN x_posts p ON p.post_id = m.post_id WHERE p.platform = 'x' AND m.observed_at >= strftime('%s', '${utcDay}') AND m.observed_at <= ${Math.floor(now.getTime() / 1000)}) AS x_observed_posts_today` : 'NULL AS x_observed_posts_today', // Hot posts are refresh's remaining exposure: each can cost at most one // more read, when its window crosses midnight UTC. has('x_posts') ? `(SELECT COUNT(*) FROM x_posts WHERE refresh_tier = 'hot') AS x_hot_posts` : 'NULL AS x_hot_posts', diff --git a/shared/checkin-external.ts b/shared/checkin-external.ts index 0fec5d3d..3fad96b8 100644 --- a/shared/checkin-external.ts +++ b/shared/checkin-external.ts @@ -22,15 +22,18 @@ Interpret this site evidence: }, { id: 'skilld.spend', - prompt: `- X API is pay-per-use at $0.005 per post read, with no included allowance and a $20/month target. Accepted 2026-08-31 at a projected $18.15/month, which raised the target from $10. Gate on \`d1.cost\`: - - \`x_over_budget\` true is AMBER. The daily discovery budget failed to hold and the month will overrun. - - \`x_projected_monthly_usd\` above 40 is AMBER, above 60 is RED, even when \`x_over_budget\` is false. Harlan raised the accepted line from $30 to $40 on 2026-09-07 (issue #160). Report the figure every run, not only when it breaches. - - \`x_budget_used_pct\` at 100 on consecutive days means discovery is truncating. That is the designed steady state, not a fault. Say so rather than reporting it as a failure. - - \`x_discovery_reads_today\` is the billed number. \`x_posts_24h\` counts only posts that survived repo extraction, so a large gap between them means the search query is paying for posts it discards. Flag a gap above 50% as a query-tuning action.`, + prompt: `- X API lists $0.005 per Post and $0.010 per User. The accepted monthly line is $40, recorded in #160. + - X usually deduplicates resource charges within each UTC day. This guarantee has exceptions: https://docs.x.com/x-api/getting-started/pricing. + - \`x_over_budget\` true is AMBER: discovery exceeded its returned-read ceiling. Null means the ceiling evidence is unavailable. + - \`x_projected_monthly_usd\` above 40 is AMBER; above 60 is RED. Report it as a conservative Post-only estimate. Actual billing and User expansion charges are unavailable. + - \`x_falling_behind\` warns that fewer reads remain than X's minimum request size. Exhaustion alone does not prove unread results remain. + - \`x_discovery_reads_today\` counts returned Posts, including repeated IDs. It is not billed usage. + - \`x_observed_posts_today\` counts distinct X Posts observed by discovery or refresh within the current UTC day. It is not billed usage. + - \`x_posts_24h\` counts newly stored Posts over a rolling 24 hours, including Posts without Repository links. These windows differ. Never infer discarded Posts or invoice charges from their gap.`, }, { id: 'skilld.pulse', - prompt: `Include a Pulse section with users, skills, repo changes, command copies split as run and install, digests, known AI cost, and X spend as "$X.XX/mo projected (N/ reads today)". Read these values from collected evidence. Include deploy and CI changes in Overnight. Include migration, schema, deploy SHA, stale task, and observability gaps in Drift.`, + prompt: `Include a Pulse section with users, skills, repo changes, command copies split as run and install, digests, known AI cost, and X spend as "$X.XX/mo Post-only estimate (N/ returned reads today)". State that actual billing is unavailable. Read these values from collected evidence. Include deploy and CI changes in Overnight. Include migration, schema, deploy SHA, stale task, and observability gaps in Drift.`, }, { id: 'skilld.seo-recovery', diff --git a/test/unit/external-checkin-collectors.test.ts b/test/unit/external-checkin-collectors.test.ts index 55a5c157..bd590e7c 100644 --- a/test/unit/external-checkin-collectors.test.ts +++ b/test/unit/external-checkin-collectors.test.ts @@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { runExternalChecks } from '@harlan-zw/nuxt-checkin/external' +import Database from 'better-sqlite3' import { afterEach, expect, it, vi } from 'vitest' import analyticsCheck from '../../checks/external/analytics' import ciCheck from '../../checks/external/ci' @@ -24,17 +25,18 @@ afterEach(async () => { }) it.each([ - [10, false], - [390, false], - [391, true], - [400, true], -])('reports discovery exhaustion after %i returned reads', async (readsToday, exhausted) => { + [10, false, true], + [390, false, true], + [391, true, true], + [400, true, true], + [10, null, false], +])('reports discovery exhaustion after %i returned reads', async (readsToday, exhausted, hasMinimum) => { const now = new Date('2026-09-14T00:00:00Z') const root = await mkdtemp(join(tmpdir(), 'skilld-checkin-')) roots.push(root) await mkdir(join(root, 'shared/server'), { recursive: true }) await mkdir(join(root, 'migrations')) - await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400\nconst MIN_SEARCH_PAGE_SIZE = 10') + await writeFile(join(root, 'shared/server/x-ingest.ts'), `export const DAILY_DISCOVERY_READ_BUDGET = 400${hasMinimum ? '\nconst MIN_SEARCH_PAGE_SIZE = 10' : ''}`) await writeFile(join(root, 'migrations/001.sql'), '') boundary.command.mockImplementation(async (_context, command: string, args: string[]) => { let output: unknown = '' @@ -68,13 +70,46 @@ it.each([ }) const checks = [gitCheck, deployCheck, databaseCheck] const result = await runExternalChecks(checks, { required: checks.map(check => check.id), timeoutMs: 1000, totalTimeoutMs: 1500 }, { rootDir: root, env: {}, clock: () => now }) - expect(result.report.coverage).toBe('complete') + expect(result.report.coverage).toBe(hasMinimum ? 'complete' : 'incomplete') expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { inventory: { skills: 23 } } }) expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { cost: { x_falling_behind: exhausted } } }) + if (!hasMinimum) + expect(result.report.results.find(check => check.id === 'skilld.database')?.result).toMatchObject({ evidence: { cost: { x_over_budget: null, x_budget_target: null } } }) expect(boundary.command.mock.calls.filter(([, command, args]) => command === 'git' && args[0] === 'fetch')).toHaveLength(1) expect(boundary.command.mock.calls.filter(([, , args]) => args.includes('SELECT name FROM sqlite_master WHERE type = \'table\' ORDER BY name'))).toHaveLength(1) }) +it('counts only X Posts observed within the UTC day at the check time', async () => { + const now = new Date('2026-10-05T10:00:00Z') + const root = await mkdtemp(join(tmpdir(), 'skilld-checkin-x-observed-')) + roots.push(root) + await mkdir(join(root, 'shared/server'), { recursive: true }) + await mkdir(join(root, 'migrations')) + await writeFile(join(root, 'shared/server/x-ingest.ts'), 'export const DAILY_DISCOVERY_READ_BUDGET = 400\nconst MIN_SEARCH_PAGE_SIZE = 10') + const db = new Database(':memory:') + db.exec(` + CREATE TABLE x_posts (post_id TEXT, platform TEXT, refresh_tier TEXT, first_seen_at INTEGER, posted_at INTEGER); + CREATE TABLE x_post_metrics (post_id TEXT, observed_at INTEGER); + INSERT INTO x_posts VALUES ('x-one', 'x', 'frozen', 0, 0), ('bsky-one', 'bsky', 'frozen', 0, 0), ('x-future', 'x', 'frozen', 0, 0), ('x-old', 'x', 'frozen', 0, 0); + INSERT INTO x_post_metrics VALUES + ('x-one', unixepoch('2026-10-05T08:00:00Z')), + ('x-one', unixepoch('2026-10-05T09:00:00Z')), + ('bsky-one', unixepoch('2026-10-05T09:00:00Z')), + ('x-future', unixepoch('2026-10-05T11:00:00Z')), + ('x-old', unixepoch('2026-10-04T23:59:59Z')); + `) + boundary.command.mockImplementation(async (_context, command: string, args: string[]) => { + if (command === 'git') + return { _tag: 'Ok', stdout: args[0] === 'rev-parse' ? 'abc' : '', stderr: '' } + const sql = args[args.indexOf('--command') + 1]! + const rows = db.prepare(sql).all() + return { _tag: 'Ok', stdout: JSON.stringify([{ success: true, results: rows }]), stderr: '' } + }) + const { report } = await runExternalChecks([databaseCheck], { required: [databaseCheck.id] }, { rootDir: root, env: {}, clock: () => now }) + db.close() + expect(report.results[0]?.result).toMatchObject({ evidence: { cost: { x_observed_posts_today: 1 } } }) +}) + it('rejects oversized Worker analytics evidence', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json({ data: { viewer: { accounts: [{ workersInvocationsAdaptive: [] }] } }, padding: 'x'.repeat(2_097_153) }))) const { report } = await runExternalChecks([workersCheck], { required: [workersCheck.id] }, { env: { CLOUDFLARE_USAGE_TOKEN: 'test-token' } })