diff --git a/docs/ops/skill-description-analysis.md b/docs/ops/skill-description-analysis.md new file mode 100644 index 00000000..bfd3b6df --- /dev/null +++ b/docs/ops/skill-description-analysis.md @@ -0,0 +1,73 @@ +# Description evidence + +Use author frontmatter for description research. `skills.description` can contain repository fallback text. +The current snapshot is `skills.rendered_frontmatter`. Its source identity includes the content hash and source commit. + +`skill_description_history` preserves complete snapshots across refreshes. +Each content hash has one row per Skill identity, with first and last observation times. +The stored frontmatter includes missing descriptions and invocation settings. +Repository moves carry the history to the new identity. Removing a source retains its observations. + +The migration backfills complete current snapshots. It cannot recover overwritten versions. +The production deployment workflow applies the migration before deploying code. +Database triggers capture later snapshots inside the existing write transaction. +Incomplete snapshots and malformed frontmatter never enter the history. +Repeated content extends observation times without adding another row. + +## Current source sample + +```sql +SELECT s.owner, s.repo, s.name, s.rendered_frontmatter, + s.current_sha, s.rendered_commit_sha, s.rendered_raw_sha256, + s.rendered_skill_path, s.last_synced_at +FROM skills s +WHERE s.source_resolved=1 AND s.rendered_status='ok' + AND json_valid(s.rendered_frontmatter); +``` + +Count missing, non-string, and blank descriptions before excluding them. +Measure decoded Unicode characters and UTF-8 bytes separately. +Record the export time and checksum. Keep source identities beside every result. + +## Description changes + +```sql +SELECT owner, repo, name, raw_sha256, source_blob_sha, source_commit, + source_path, first_observed_at, last_observed_at, + json_type(frontmatter, '$.description') AS description_type, + json_extract(frontmatter, '$.description') AS description +FROM skill_description_history +ORDER BY owner, repo, name, first_observed_at; +``` + +A content version can change without changing its description. Compare decoded descriptions before counting description edits. +Source commits may be absent in legacy snapshots. Exclude those rows when requiring exact commit attribution. + +## Classification controls + +The older `skill_repo_focus` screen covers individual maintainers only. +Organization rows with model `owner-kind` were excluded by ownership, without a Jev purpose judgment. +Use `model LIKE 'jev%' AND probability>=0.8` for its positive group. + +The newer `repository_purpose` classifier distinguishes Skill publishing, software, directories, mirrors, and uncertainty. +Report its missing and uncertain rows separately. Do not treat either as a negative finding. +Require `repository_purpose.source_commit=skills.rendered_commit_sha` for same-commit comparisons. + +Skill classifications live in `skill_generated` with kind `abstractness`. +Require `skill_generated.sha=skills.current_sha` before using their payload. +Its `abstract` value means general-purpose. Its `package-specific` value identifies package knowledge. +These classifications are machine evidence. They do not establish quality or human approval. + +## Interpretation limits + +Report Skill-weighted results and the median of repository medians. +Repeat results after removing duplicate descriptions. Show collection coverage against its source inventory. +Our registry admission rules create a selected sample. It is not the skills.sh top 10,000. + +Description length measures context cost. It does not establish routing quality. +Phrase matching measures visible markers, not whether a model understands a trigger. +Synthetic catalogs must name their renderer assumptions, budget, sampling seed, and missing policy metadata. +Agent version, invocation policy, enabled Skills, and actual prompt captures require separate runtime evidence. + +The skills.sh crawler records dated ranks and install estimates. +Check observation dates before using those values. `skills.installs` is not a fresh ranking snapshot. diff --git a/layers/registry/server/utils/repository-move.ts b/layers/registry/server/utils/repository-move.ts index 83b3d9d5..b6d3b12f 100644 --- a/layers/registry/server/utils/repository-move.ts +++ b/layers/registry/server/utils/repository-move.ts @@ -124,6 +124,7 @@ const skillTable = (table: string, options: Omit = {}): Key const SKILL_TABLES: KeyedTable[] = [ skillTable('skill_likes'), skillTable('skill_revisions'), + skillTable('skill_description_history'), skillTable('skill_generated', { keep: `kind = 'embedding'` }), skillTable('skill_dirty'), skillTable('skill_trending_admissions'), diff --git a/migrations/0150_skill_description_history.sql b/migrations/0150_skill_description_history.sql new file mode 100644 index 00000000..b184967a --- /dev/null +++ b/migrations/0150_skill_description_history.sql @@ -0,0 +1,60 @@ +-- Keep author frontmatter across refreshes. Listing descriptions may contain +-- repository fallback text, so they cannot measure author descriptions. +-- No foreign key: removing a source must not erase its observed history. +CREATE TABLE skill_description_history ( + owner TEXT NOT NULL, + repo TEXT NOT NULL, + name TEXT NOT NULL, + raw_sha256 TEXT NOT NULL CHECK (length(raw_sha256) = 64), + source_blob_sha TEXT NOT NULL, + source_commit TEXT, + source_path TEXT, + frontmatter TEXT NOT NULL CHECK (json_valid(frontmatter)), + first_observed_at INTEGER NOT NULL, + last_observed_at INTEGER NOT NULL, + PRIMARY KEY (owner, repo, name, raw_sha256), + CHECK (first_observed_at <= last_observed_at) +); + +INSERT INTO skill_description_history + (owner,repo,name,raw_sha256,source_blob_sha,source_commit,source_path, + frontmatter,first_observed_at,last_observed_at) +SELECT owner,repo,name,rendered_raw_sha256,current_sha,rendered_commit_sha, + rendered_skill_path,rendered_frontmatter,rendered_at,rendered_at +FROM skills +WHERE rendered_status='ok' AND length(rendered_raw_sha256)=64 + AND current_sha IS NOT NULL AND rendered_at IS NOT NULL + AND json_valid(rendered_frontmatter); + +CREATE TRIGGER skill_description_history_insert AFTER INSERT ON skills +WHEN NEW.rendered_status='ok' AND length(NEW.rendered_raw_sha256)=64 + AND NEW.current_sha IS NOT NULL AND NEW.rendered_at IS NOT NULL + AND json_valid(NEW.rendered_frontmatter) +BEGIN + INSERT INTO skill_description_history + (owner,repo,name,raw_sha256,source_blob_sha,source_commit,source_path, + frontmatter,first_observed_at,last_observed_at) + VALUES (NEW.owner,NEW.repo,NEW.name,NEW.rendered_raw_sha256,NEW.current_sha, + NEW.rendered_commit_sha,NEW.rendered_skill_path,NEW.rendered_frontmatter, + NEW.rendered_at,NEW.rendered_at) + ON CONFLICT(owner,repo,name,raw_sha256) DO UPDATE SET + first_observed_at=MIN(first_observed_at,excluded.first_observed_at), + last_observed_at=MAX(last_observed_at,excluded.last_observed_at); +END; + +CREATE TRIGGER skill_description_history_update +AFTER UPDATE OF rendered_raw_sha256,rendered_frontmatter,rendered_at ON skills +WHEN NEW.rendered_status='ok' AND length(NEW.rendered_raw_sha256)=64 + AND NEW.current_sha IS NOT NULL AND NEW.rendered_at IS NOT NULL + AND json_valid(NEW.rendered_frontmatter) +BEGIN + INSERT INTO skill_description_history + (owner,repo,name,raw_sha256,source_blob_sha,source_commit,source_path, + frontmatter,first_observed_at,last_observed_at) + VALUES (NEW.owner,NEW.repo,NEW.name,NEW.rendered_raw_sha256,NEW.current_sha, + NEW.rendered_commit_sha,NEW.rendered_skill_path,NEW.rendered_frontmatter, + NEW.rendered_at,NEW.rendered_at) + ON CONFLICT(owner,repo,name,raw_sha256) DO UPDATE SET + first_observed_at=MIN(first_observed_at,excluded.first_observed_at), + last_observed_at=MAX(last_observed_at,excluded.last_observed_at); +END; diff --git a/test/unit/skill-description-history.test.ts b/test/unit/skill-description-history.test.ts new file mode 100644 index 00000000..3978b185 --- /dev/null +++ b/test/unit/skill-description-history.test.ts @@ -0,0 +1,86 @@ +import type { SqliteD1 } from './helpers/d1-sqlite' +import { readFileSync } from 'node:fs' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { moveRepository } from '../../layers/registry/server/utils/repository-move' +import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' + +const migration = 'migrations/0150_skill_description_history.sql' +let fixture: SqliteD1 + +beforeEach(() => { + fixture = createSqliteD1(allMigrations()) +}) +afterEach(() => fixture.close()) + +async function snapshot(frontmatter: Record, hash = 'a'.repeat(64), observedAt = 100) { + await fixture.db.prepare(` + INSERT INTO skills(owner,repo,name,display_name,slug,description,current_sha, + rendered_status,rendered_raw_sha256,rendered_frontmatter,rendered_at, + rendered_skill_path,rendered_commit_sha) + VALUES ('author','skills','review','Review','author/review','Repository fallback', + 'blob','ok',?,?,?,'skills/review/SKILL.md',?) + ON CONFLICT(owner,repo,name) DO UPDATE SET + rendered_raw_sha256=excluded.rendered_raw_sha256, + rendered_frontmatter=excluded.rendered_frontmatter, + rendered_at=excluded.rendered_at, + rendered_commit_sha=excluded.rendered_commit_sha + `).bind(hash, JSON.stringify(frontmatter), observedAt, 'c'.repeat(40)).run() +} + +function history() { + return fixture.db.prepare(`SELECT frontmatter,first_observed_at,last_observed_at + FROM skill_description_history ORDER BY first_observed_at`).all() +} + +describe('description history at the snapshot boundary', () => { + it('preserves the author description when a later snapshot replaces it', async () => { + await snapshot({ 'description': 'Review changes. Use when reviewing.', 'disable-model-invocation': true }) + await snapshot({ description: 'Review code.' }, 'b'.repeat(64), 200) + await snapshot({ description: 'Review code.' }, 'b'.repeat(64), 300) + expect((await history()).results).toEqual([ + { frontmatter: JSON.stringify({ 'description': 'Review changes. Use when reviewing.', 'disable-model-invocation': true }), first_observed_at: 100, last_observed_at: 100 }, + { frontmatter: JSON.stringify({ description: 'Review code.' }), first_observed_at: 200, last_observed_at: 300 }, + ]) + }) + + it('records missing descriptions without substituting repository text', async () => { + await snapshot({ name: 'review' }) + expect((await history()).results).toEqual([ + { frontmatter: '{"name":"review"}', first_observed_at: 100, last_observed_at: 100 }, + ]) + }) + + it('ignores incomplete snapshots and preserves history after source removal', async () => { + await snapshot({ description: 'Original' }) + await fixture.db.prepare(`UPDATE skills SET rendered_status='unavailable', + rendered_raw_sha256=?, rendered_frontmatter=?, rendered_at=200`).bind('b'.repeat(64), '{"description":"Partial"}').run() + await fixture.db.prepare('DELETE FROM skills').run() + expect((await history()).results).toEqual([ + { frontmatter: '{"description":"Original"}', first_observed_at: 100, last_observed_at: 100 }, + ]) + }) + + it('moves historical descriptions with the repository identity', async () => { + await fixture.db.prepare(`INSERT INTO repos(owner,repo,repository_id) VALUES ('author','skills',123)`).run() + await snapshot({ description: 'Original' }) + await moveRepository(fixture.db, { + from: { owner: 'author', repo: 'skills' }, + to: { owner: 'author', repo: 'renamed' }, + source: { owner: 'author', repo: 'renamed' }, + repositoryId: 123, + movedAt: 200, + }) + expect((await fixture.db.prepare('SELECT owner,repo,name FROM skill_description_history').all()).results) + .toEqual([{ owner: 'author', repo: 'renamed', name: 'review' }]) + }) + + it('backfills only complete stored snapshots', async () => { + fixture.close() + fixture = createSqliteD1(allMigrations().filter(path => path !== migration)) + await snapshot({ description: 'Stored original' }) + fixture.raw.exec(readFileSync(migration, 'utf8')) + expect((await history()).results).toEqual([ + { frontmatter: '{"description":"Stored original"}', first_observed_at: 100, last_observed_at: 100 }, + ]) + }) +}) diff --git a/test/unit/skill-snapshot-migration.test.ts b/test/unit/skill-snapshot-migration.test.ts index af15077f..3939e037 100644 --- a/test/unit/skill-snapshot-migration.test.ts +++ b/test/unit/skill-snapshot-migration.test.ts @@ -10,7 +10,7 @@ const migration = 'migrations/0148_skill_snapshot_commit.sql' describe('skill snapshot migration', () => { it.each([true, false])('refreshes dormant nested Skills, recorded revision: %s', async (hasRevision) => { - const fixture = createSqliteD1(allMigrations().filter(path => path !== migration)) + const fixture = createSqliteD1(allMigrations().filter(path => path < migration)) const skill = { owner: 'acme', repo: 'skills', name: 'setup' } fixture.raw.exec(` INSERT INTO repos (owner, repo, default_branch, last_tree_sha, pushed_at, repo_meta_synced_at)