fix(research): preserve source-version review checkpoints (#599)

Author: harlan-zw

docs/ops/description-reviews.md

@@ -48,6 +48,7 @@ Confirm [current model pricing](https://developers.cloudflare.com/ai/models/type
 The key includes the exact description, model version, rubric version, and questions.
 Identical descriptions reuse the same assessment. Changed descriptions require new assessments.
 The tool checkpoints SQL after each batch and at completion.
+Each checkpoint replaces the SQL file atomically. A failed write preserves the earlier complete checkpoint.
 Failed requests appear in `OUTPUT.sql.failures.jsonl`; the process exits with code 1.
 Keep these files outside the repository.
 
@@ -59,11 +60,13 @@ Apply a completed or copied checkpoint through the existing D1 command:
 pnpm exec wrangler d1 execute DB --remote --config wrangler.jsonc --file OUTPUT.sql
 ```
 
-Findings live in `skill_generated`, using `description-review:<rubric-version>:<raw-sha256>` kinds.
-The existing JSON payload supports these fields. No schema migration is required.
+Findings live in `skill_generated`, using `description-review:` kinds with the rubric version and raw source hash.
+The existing JSON payload stores these fields.
 Each rubric and source version retains a separate row. Replaying SQL preserves the earlier finding.
-Writes require matching blob SHA, raw SHA-256, identity, and successful render status.
-Changed sources are skipped. Re-export them for a later run.
+Writes require matching identity, blob SHA, raw SHA-256, and the exact author description.
+Evidence can match the successful current render or a retained source snapshot in `skill_description_history`.
+Historical findings keep their original blob SHA. Current-source queries exclude them after a source change.
+Unknown source versions are skipped. Re-export changed sources to review their new descriptions.
 
 The payload preserves description, source hashes, source commit, questions, probabilities, labels, model, and usage.
 `reviewKey` identifies the shared request for duplicate descriptions.

scripts/lib/description-review.ts

@@ -91,7 +91,10 @@ export function descriptionReviewStatement(source: z.infer<typeof descriptionRev
     answer: review,
   }
   const kind = `description-review:${DESCRIPTION_REVIEW_VERSION}:${source.rawSha256}`
-  return `INSERT INTO skill_generated(owner,repo,name,kind,sha,payload,generated_at) SELECT owner,repo,name,${quote(kind)},current_sha,${quote(JSON.stringify(payload))},${quote(reviewedAt)} FROM skills WHERE owner=${quote(source.owner)} AND repo=${quote(source.repo)} AND name=${quote(source.name)} AND current_sha=${quote(source.sourceBlobSha)} AND rendered_raw_sha256=${quote(source.rawSha256)} AND rendered_status='ok' ON CONFLICT(owner,repo,name,kind) DO NOTHING;`
+  const identity = `owner=${quote(source.owner)} AND repo=${quote(source.repo)} AND name=${quote(source.name)}`
+  const current = `current_sha=${quote(source.sourceBlobSha)} AND rendered_raw_sha256=${quote(source.rawSha256)} AND rendered_status='ok' AND json_valid(rendered_frontmatter) AND json_extract(rendered_frontmatter,'$.description')=${quote(source.description)}`
+  const historical = `EXISTS(SELECT 1 FROM skill_description_history WHERE ${identity} AND raw_sha256=${quote(source.rawSha256)} AND source_blob_sha=${quote(source.sourceBlobSha)} AND json_extract(frontmatter,'$.description')=${quote(source.description)})`
+  return `INSERT INTO skill_generated(owner,repo,name,kind,sha,payload,generated_at) SELECT owner,repo,name,${quote(kind)},${quote(source.sourceBlobSha)},${quote(JSON.stringify(payload))},${quote(reviewedAt)} FROM skills WHERE ${identity} AND ((${current}) OR ${historical}) ON CONFLICT(owner,repo,name,kind) DO NOTHING;`
 }
 export function reviewLabel(evidence: { task: number, activation: number, scope: number, redundant: number }): 'clear' | 'needs-work' | 'uncertain' {
   if (evidence.task <= 0.2 || evidence.activation <= 0.2 || evidence.scope <= 0.2 || evidence.redundant >= 0.8)

scripts/review-skill-descriptions.ts

@@ -1,4 +1,4 @@
-import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { appendFileSync, existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
 import process from 'node:process'
 import { setTimeout as pause } from 'node:timers/promises'
 import { parseArgs } from 'node:util'
@@ -103,7 +103,8 @@ function writeSnapshot() {
       throw new Error(`Invalid review: ${key}`)
     return descriptionReviewStatement(row, parsed.data, record.reviewedAt)
   })
-  writeFileSync(output, `${statements.join('\n')}\n`)
+  writeFileSync(`${output}.tmp`, `${statements.join('\n')}\n`)
+  renameSync(`${output}.tmp`, output)
 }
 const unique = [...new Set(sources.map(row => descriptionReviewKey(row.description)))].filter(key => completed.has(key)).map(key => parseDescriptionReview(completed.get(key)!.response)).filter(result => result.success)
 console.log(JSON.stringify({ sources: sources.length, uniqueReviews: unique.length, failed, inputTokens: unique.reduce((sum, result) => sum + result.data.usage.input_tokens, 0), inputCostUSD: unique.reduce((sum, result) => sum + result.data.usage.input_tokens * 0.042 / 1_000_000, 0), elapsedSeconds: (Date.now() - started) / 1000, sql: output }))

test/unit/description-review.test.ts

@@ -37,7 +37,7 @@ describe('description reviews', () => {
     if (!parsed.success)
       throw parsed.error
     try {
-      await fixture.db.prepare(`INSERT INTO skills(owner,repo,name,display_name,slug,current_sha,rendered_raw_sha256,rendered_status) VALUES ('author','skills','review','Review','author/review',?,?,'ok')`).bind(source.sourceBlobSha, source.rawSha256).run()
+      await fixture.db.prepare(`INSERT INTO skills(owner,repo,name,display_name,slug,current_sha,rendered_raw_sha256,rendered_status,rendered_frontmatter) VALUES ('author','skills','review','Review','author/review',?,?,'ok',?)`).bind(source.sourceBlobSha, source.rawSha256, JSON.stringify({ description: source.description })).run()
       const statement = descriptionReviewStatement(source, parsed.data, '2026-10-09T00:00:00Z')
       await fixture.db.prepare(statement).run()
       await fixture.db.prepare(statement).run()
@@ -46,6 +46,14 @@ describe('description reviews', () => {
       await fixture.db.prepare(`UPDATE skills SET rendered_raw_sha256=?`).bind('d'.repeat(64)).run()
       await fixture.db.prepare(descriptionReviewStatement({ ...source, rawSha256: 'd'.repeat(64) }, parsed.data, '2026-10-09T00:00:01Z')).run()
       expect((await fixture.db.prepare(`SELECT COUNT(*) AS count FROM skill_generated WHERE kind LIKE 'description-review:%'`).first())).toEqual({ count: 2 })
+      await fixture.db.prepare(`DELETE FROM skill_generated WHERE json_extract(payload,'$.rawSha256')=?`).bind(source.rawSha256).run()
+      await fixture.db.prepare(`INSERT INTO skill_description_history(owner,repo,name,raw_sha256,source_blob_sha,source_commit,frontmatter,first_observed_at,last_observed_at) VALUES ('author','skills','review',?,?,?,?,0,0)`).bind(source.rawSha256, source.sourceBlobSha, source.sourceCommit, JSON.stringify({ description: source.description })).run()
+      await fixture.db.prepare(statement).run()
+      expect((await fixture.db.prepare(`SELECT COUNT(*) AS count FROM skill_generated WHERE kind LIKE 'description-review:%'`).first())).toEqual({ count: 2 })
+      await fixture.db.prepare(`DELETE FROM skill_generated`).run()
+      await fixture.db.prepare(descriptionReviewStatement({ ...source, description: 'An altered description.' }, parsed.data, '2026-10-09T00:00:02Z')).run()
+      await fixture.db.prepare(descriptionReviewStatement({ ...source, rawSha256: 'd'.repeat(64), description: 'An altered description.' }, parsed.data, '2026-10-09T00:00:02Z')).run()
+      expect((await fixture.db.prepare(`SELECT COUNT(*) AS count FROM skill_generated`).first())).toEqual({ count: 0 })
     }
     finally {
       fixture.close()