From 148bc95d4fe8e7b6de0007fe93fe6e46ec0838b6 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Fri, 9 Oct 2026 16:07:34 +1100 Subject: [PATCH 1/2] fix(research): retain reviews when source snapshots change --- docs/ops/description-reviews.md | 6 ++++-- scripts/lib/description-review.ts | 5 ++++- test/unit/description-review.test.ts | 10 +++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/ops/description-reviews.md b/docs/ops/description-reviews.md index 92860597..4d9c66e2 100644 --- a/docs/ops/description-reviews.md +++ b/docs/ops/description-reviews.md @@ -62,8 +62,10 @@ pnpm exec wrangler d1 execute DB --remote --config wrangler.jsonc --file OUTPUT. Findings live in `skill_generated`, using `description-review::` kinds. The existing JSON payload supports these fields. No schema migration is required. Each rubric and source version retains a separate row. Replaying SQL preserves the earlier finding. -Writes require matching blob SHA, raw SHA-256, identity, and successful render status. -Changed sources are skipped. Re-export them for a later run. +Writes require matching identity, blob SHA, raw SHA-256, and the exact author description. +Evidence can match the successful current render or a retained source snapshot in `skill_description_history`. +Historical findings keep their original blob SHA. Current-source queries exclude them after a source change. +Unknown source versions are skipped. Re-export changed sources to review their new descriptions. The payload preserves description, source hashes, source commit, questions, probabilities, labels, model, and usage. `reviewKey` identifies the shared request for duplicate descriptions. diff --git a/scripts/lib/description-review.ts b/scripts/lib/description-review.ts index 93d33bbf..0f8ec325 100644 --- a/scripts/lib/description-review.ts +++ b/scripts/lib/description-review.ts @@ -91,7 +91,10 @@ export function descriptionReviewStatement(source: z.infer= 0.8) diff --git a/test/unit/description-review.test.ts b/test/unit/description-review.test.ts index 94a66e46..852ce246 100644 --- a/test/unit/description-review.test.ts +++ b/test/unit/description-review.test.ts @@ -37,7 +37,7 @@ describe('description reviews', () => { if (!parsed.success) throw parsed.error try { - await fixture.db.prepare(`INSERT INTO skills(owner,repo,name,display_name,slug,current_sha,rendered_raw_sha256,rendered_status) VALUES ('author','skills','review','Review','author/review',?,?,'ok')`).bind(source.sourceBlobSha, source.rawSha256).run() + await fixture.db.prepare(`INSERT INTO skills(owner,repo,name,display_name,slug,current_sha,rendered_raw_sha256,rendered_status,rendered_frontmatter) VALUES ('author','skills','review','Review','author/review',?,?,'ok',?)`).bind(source.sourceBlobSha, source.rawSha256, JSON.stringify({ description: source.description })).run() const statement = descriptionReviewStatement(source, parsed.data, '2026-10-09T00:00:00Z') await fixture.db.prepare(statement).run() await fixture.db.prepare(statement).run() @@ -46,6 +46,14 @@ describe('description reviews', () => { await fixture.db.prepare(`UPDATE skills SET rendered_raw_sha256=?`).bind('d'.repeat(64)).run() await fixture.db.prepare(descriptionReviewStatement({ ...source, rawSha256: 'd'.repeat(64) }, parsed.data, '2026-10-09T00:00:01Z')).run() expect((await fixture.db.prepare(`SELECT COUNT(*) AS count FROM skill_generated WHERE kind LIKE 'description-review:%'`).first())).toEqual({ count: 2 }) + await fixture.db.prepare(`DELETE FROM skill_generated WHERE json_extract(payload,'$.rawSha256')=?`).bind(source.rawSha256).run() + await fixture.db.prepare(`INSERT INTO skill_description_history(owner,repo,name,raw_sha256,source_blob_sha,source_commit,frontmatter,first_observed_at,last_observed_at) VALUES ('author','skills','review',?,?,?,?,0,0)`).bind(source.rawSha256, source.sourceBlobSha, source.sourceCommit, JSON.stringify({ description: source.description })).run() + await fixture.db.prepare(statement).run() + expect((await fixture.db.prepare(`SELECT COUNT(*) AS count FROM skill_generated WHERE kind LIKE 'description-review:%'`).first())).toEqual({ count: 2 }) + await fixture.db.prepare(`DELETE FROM skill_generated`).run() + await fixture.db.prepare(descriptionReviewStatement({ ...source, description: 'An altered description.' }, parsed.data, '2026-10-09T00:00:02Z')).run() + await fixture.db.prepare(descriptionReviewStatement({ ...source, rawSha256: 'd'.repeat(64), description: 'An altered description.' }, parsed.data, '2026-10-09T00:00:02Z')).run() + expect((await fixture.db.prepare(`SELECT COUNT(*) AS count FROM skill_generated`).first())).toEqual({ count: 0 }) } finally { fixture.close() From f082b014be8ddbf7e63b00d6a20b7ec63b7cd6fb Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Fri, 9 Oct 2026 16:18:06 +1100 Subject: [PATCH 2/2] fix(research): replace review checkpoints atomically --- docs/ops/description-reviews.md | 5 +++-- scripts/review-skill-descriptions.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/ops/description-reviews.md b/docs/ops/description-reviews.md index 4d9c66e2..b037c929 100644 --- a/docs/ops/description-reviews.md +++ b/docs/ops/description-reviews.md @@ -48,6 +48,7 @@ Confirm [current model pricing](https://developers.cloudflare.com/ai/models/type The key includes the exact description, model version, rubric version, and questions. Identical descriptions reuse the same assessment. Changed descriptions require new assessments. The tool checkpoints SQL after each batch and at completion. +Each checkpoint replaces the SQL file atomically. A failed write preserves the earlier complete checkpoint. Failed requests appear in `OUTPUT.sql.failures.jsonl`; the process exits with code 1. Keep these files outside the repository. @@ -59,8 +60,8 @@ Apply a completed or copied checkpoint through the existing D1 command: pnpm exec wrangler d1 execute DB --remote --config wrangler.jsonc --file OUTPUT.sql ``` -Findings live in `skill_generated`, using `description-review::` kinds. -The existing JSON payload supports these fields. No schema migration is required. +Findings live in `skill_generated`, using `description-review:` kinds with the rubric version and raw source hash. +The existing JSON payload stores these fields. Each rubric and source version retains a separate row. Replaying SQL preserves the earlier finding. Writes require matching identity, blob SHA, raw SHA-256, and the exact author description. Evidence can match the successful current render or a retained source snapshot in `skill_description_history`. diff --git a/scripts/review-skill-descriptions.ts b/scripts/review-skill-descriptions.ts index 6aa1c81d..2950b616 100644 --- a/scripts/review-skill-descriptions.ts +++ b/scripts/review-skill-descriptions.ts @@ -1,4 +1,4 @@ -import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs' +import { appendFileSync, existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs' import process from 'node:process' import { setTimeout as pause } from 'node:timers/promises' import { parseArgs } from 'node:util' @@ -103,7 +103,8 @@ function writeSnapshot() { throw new Error(`Invalid review: ${key}`) return descriptionReviewStatement(row, parsed.data, record.reviewedAt) }) - writeFileSync(output, `${statements.join('\n')}\n`) + writeFileSync(`${output}.tmp`, `${statements.join('\n')}\n`) + renameSync(`${output}.tmp`, output) } const unique = [...new Set(sources.map(row => descriptionReviewKey(row.description)))].filter(key => completed.has(key)).map(key => parseDescriptionReview(completed.get(key)!.response)).filter(result => result.success) console.log(JSON.stringify({ sources: sources.length, uniqueReviews: unique.length, failed, inputTokens: unique.reduce((sum, result) => sum + result.data.usage.input_tokens, 0), inputCostUSD: unique.reduce((sum, result) => sum + result.data.usage.input_tokens * 0.042 / 1_000_000, 0), elapsedSeconds: (Date.now() - started) / 1000, sql: output }))