From 283dbf6d6988abf42c8f9be339d9c99772d6dc6b Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 01:20:34 +1100 Subject: [PATCH 1/2] fix(artifact-delivery): keep a known sweep failure through a spent quota A retryable failure replaced the recorded tag, so the next check that saw the known failure again alarmed as new. --- .../server/utils/run-sweep.ts | 19 +++++++++++++++++-- test/unit/skill-run-sweep.test.ts | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/layers/artifact-delivery/server/utils/run-sweep.ts b/layers/artifact-delivery/server/utils/run-sweep.ts index 79a7815d..3ae3af2e 100644 --- a/layers/artifact-delivery/server/utils/run-sweep.ts +++ b/layers/artifact-delivery/server/utils/run-sweep.ts @@ -57,6 +57,7 @@ const checkRowSchema = z.object({ name: z.string(), outcome: z.enum(['ready', 'failing']).nullable(), tag: z.string().nullable(), + detail: z.string().nullable(), retryable: z.union([z.literal(0), z.literal(1)]), failing_since: z.number().int().nullable(), settled_at: z.number().int().nullable(), @@ -108,7 +109,7 @@ export async function runSkillRunSweep(dependencies: SkillRunSweepDependencies): if (row.outcome === 'failing' && row.retryable === 0) report.recovered.push(skill) await writeSettled(dependencies.db, row, { outcome: 'ready', tag: null, detail: null, retryable: false, failingSince: null }, now) - rows.set(skillKey(skill), { ...row, outcome: 'ready', tag: null, retryable: 0, pending_resolution_id: null, settled_at: now }) + rows.set(skillKey(skill), { ...row, outcome: 'ready', tag: null, detail: null, retryable: 0, pending_resolution_id: null, settled_at: now }) continue } const failure = { ...skill, tag: settled.tag, detail: settled.detail } @@ -117,6 +118,19 @@ export async function runSkillRunSweep(dependencies: SkillRunSweepDependencies): report.transientFailures.push(failure) else if (!sameFailure) report.newFailures.push(failure) + // A retryable failure says nothing about the Skill. Over a known failure it + // keeps that failure, so the next check that sees it again is not new. + if (settled.retryable && row.outcome === 'failing' && row.retryable === 0) { + await writeSettled(dependencies.db, row, { + outcome: 'failing', + tag: row.tag, + detail: row.detail, + retryable: false, + failingSince: row.failing_since, + }, now) + rows.set(skillKey(skill), { ...row, pending_resolution_id: null, settled_at: now }) + continue + } const failingSince = sameFailure ? row.failing_since ?? now : now await writeSettled(dependencies.db, row, { outcome: 'failing', @@ -129,6 +143,7 @@ export async function runSkillRunSweep(dependencies: SkillRunSweepDependencies): ...row, outcome: 'failing', tag: settled.tag, + detail: settled.detail, retryable: settled.retryable ? 1 : 0, failing_since: failingSince, pending_resolution_id: null, @@ -204,7 +219,7 @@ export function settleResolution(resolution: ResolutionRow | null, pendingSecond async function loadCheckRows(db: D1Database): Promise> { const result = await db.prepare( - `SELECT owner, repository, name, outcome, tag, retryable, failing_since, + `SELECT owner, repository, name, outcome, tag, detail, retryable, failing_since, settled_at, pending_resolution_id, requested_at FROM artifact_run_checks`, ).all>() diff --git a/test/unit/skill-run-sweep.test.ts b/test/unit/skill-run-sweep.test.ts index bd99c8a8..9d426ced 100644 --- a/test/unit/skill-run-sweep.test.ts +++ b/test/unit/skill-run-sweep.test.ts @@ -61,6 +61,24 @@ describe('skilld run sweep', () => { harness.close() }) + it('does not alarm again when a spent quota interrupts a known failure', async () => { + const harness = createHarness([archify]) + const blocked: SourceRejection = { _tag: 'rejected', code: 'INVALID_SOURCE', summary: 'Too large.', findings: [] } + + await runSkillRunSweep(harness.dependencies) + await harness.buildEach({ archify: blocked }) + await runSkillRunSweep(harness.dependencies) + await harness.buildEach({ archify: { _tag: 'rejected', code: 'RATE_LIMITED', summary: 'Spent.', findings: [] } }) + const interrupted = await runSkillRunSweep(harness.dependencies) + await harness.buildEach({ archify: blocked }) + const resumed = await runSkillRunSweep(harness.dependencies) + + expect(interrupted.transientFailures).toHaveLength(1) + expect(interrupted.failing).toBe(1) + expect(resumed.newFailures).toEqual([]) + harness.close() + }) + it('checks the Skill checked longest ago first', async () => { const harness = createHarness([archify, busy], 1) From e3b8765cb75a427241d4c3ebecaf00cdee114e54 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 01:21:26 +1100 Subject: [PATCH 2/2] fix(artifact-delivery): size the sweep batch to the indexed Skill count The registry indexes 5,949 Skills, not about 2,100, so 25 checks per run took two and a half days per cycle. --- .../artifact-delivery/server/tasks/sweep-skill-runs.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts b/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts index 87876138..7fe8c2dd 100644 --- a/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts +++ b/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts @@ -11,11 +11,13 @@ import { runSkillRunSweep } from '../utils/run-sweep' const CRON = '7,22,37,52 * * * *' /** - * 25 checks every 15 minutes is 2,400 a day, so about 2,100 indexed Skills - * cycle in under a day. A ready build of the same commit is reused without a - * GitHub read, so a repeat check of an unchanged Skill costs D1 rows only. + * 40 checks every 15 minutes is 3,840 a day. The registry indexed 5,949 Skills + * on 2026-10-07, so one cycle takes about 37 hours. A ready build of the same + * commit is reused without a GitHub read, so a repeat check of an unchanged + * Skill costs D1 rows only. A first check loads from GitHub with about 8 REST + * requests, so a cycle of first checks spends about 1,300 an hour. */ -const BATCH_SIZE = 25 +const BATCH_SIZE = 40 /** * Scheduled task: check that every indexed Skill still runs.