From f22f16356d3a46eab9e4e7206b38fe7af84e7832 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 02:19:46 +1100 Subject: [PATCH 1/2] fix(registry): spend less GitHub quota on the sync and page views GITHUB_TOKEN is a personal token, so the registry shares its hourly quota with every other tool on that account. Measured 2026-10-06: about 1,650 of 1,900 daily sync jobs only advanced the freshness cursor, about 630 /gh views a day re-asked GitHub for a repository it had already said was missing, and the owner and commit reads went out anonymous and failed. The hourly task now reads candidate summaries in GraphQL batches of 100 and records unchanged repositories in place. Sync and page tree reads go by tree SHA. The repository page caches 404s and tree scans at the edge, answers 404 for a private repository, and falls back to ungh.cc metadata when GitHub refuses. Owner and commit reads authenticate through the ETag client, so a repeat read answers 304 for free. Sync pauses name the bucket. --- .../registry/server/api/orgs/[owner].get.ts | 52 ++-- .../server/api/repos/[owner]/[repo].get.ts | 129 +++++++--- .../server/api/skill-related/[...slug].get.ts | 35 +-- .../server/api/skills/featured.get.ts | 48 ++-- .../server/tasks/sync-github-skills.ts | 45 +++- layers/registry/server/utils/github-client.ts | 237 +++++++++++++----- .../server/utils/github-sync-control.ts | 18 +- layers/registry/server/utils/sync-prefetch.ts | 144 +++++++++++ layers/registry/server/utils/sync-repo.ts | 116 +++++---- layers/registry/server/utils/ungh-repo.ts | 72 ++++++ server/jobs/registry/repo-maintenance.ts | 2 + test/unit/github-conditional-read.test.ts | 43 ++++ test/unit/github-repo-summary-batch.test.ts | 99 ++++++++ test/unit/github-sync-control.test.ts | 16 ++ test/unit/repo-source-profile-quota.test.ts | 109 ++++++++ test/unit/repo-source-profile-timeout.test.ts | 3 + test/unit/skill-detail-d1-cost.test.ts | 6 +- test/unit/sync-prefetch.test.ts | 113 +++++++++ test/unit/sync-repo-correctness.test.ts | 2 +- test/unit/sync-repo-freshness.test.ts | 1 + 20 files changed, 1067 insertions(+), 223 deletions(-) create mode 100644 layers/registry/server/utils/sync-prefetch.ts create mode 100644 layers/registry/server/utils/ungh-repo.ts create mode 100644 test/unit/github-conditional-read.test.ts create mode 100644 test/unit/github-repo-summary-batch.test.ts create mode 100644 test/unit/repo-source-profile-quota.test.ts create mode 100644 test/unit/sync-prefetch.test.ts diff --git a/layers/registry/server/api/orgs/[owner].get.ts b/layers/registry/server/api/orgs/[owner].get.ts index 751aaf13..c9b26af2 100644 --- a/layers/registry/server/api/orgs/[owner].get.ts +++ b/layers/registry/server/api/orgs/[owner].get.ts @@ -1,11 +1,12 @@ import type { H3Event } from 'h3' import type { TagPayload } from '../../jobs/generate-tags' +import type { GithubBindings } from '../../utils/github-client' import type { RegistrySkill } from '../../utils/skills-registry' import { getDB } from '#server/utils/db' import { cached, readCache, writeCache } from '#shared/server/cache' import { officialRepos } from '../../data/official-repos' import { TAG_BY_SLUG } from '../../jobs/taxonomy' -import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client' +import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client' import { resolveRepoSourceIdentitiesForOwner } from '../../utils/repo-source-identity' import { getGeneratedBatch } from '../../utils/skill-generated' import { querySkills } from '../../utils/skills-registry' @@ -68,19 +69,27 @@ const ORG_PROFILE_CACHE_STALE_TTL = 60 * 5 const officialOwners = new Set(officialRepos.map(r => r.owner)) const kindByOwner = new Map(officialRepos.map(r => [r.owner, r.kind])) -async function fetchAndStoreOwner(owner: string, db: D1Database): Promise { - const res = await fetch(`https://api.github.com/users/${owner}`, { - signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS), - headers: { - 'User-Agent': 'skilld.dev', - 'Accept': 'application/vnd.github+json', - }, - }).catch(() => { - emitOperationalEvent(createWideEvent({ operation: 'org-profile-fetch', outcome: 'failed' })) - return null - }) +interface GithubOwnerResponse { + name?: string + bio?: string + blog?: string + location?: string + followers?: number + public_repos?: number + type?: string +} + +async function fetchAndStoreOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise { + // Authenticated and conditional: an unchanged profile answers 304 from + // the ETag cache for free. Anonymous, this read shared the Worker IP's 60 + // requests an hour with every other Cloudflare tenant, and failed. + const res = await getGithubJson(`/users/${encodeURIComponent(owner)}`, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) + .catch(() => { + emitOperationalEvent(createWideEvent({ operation: 'org-profile-fetch', outcome: 'failed' })) + return null + }) - if (!res || !res.ok) { + if (!res?.data) { if (res?.status === 404) { await db .prepare( @@ -93,16 +102,7 @@ async function fetchAndStoreOwner(owner: string, db: D1Database): Promise { +async function loadOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise { const cached = await db .prepare( `SELECT kind, name, bio, blog, location, followers, last_synced_at, sync_status @@ -145,7 +145,7 @@ async function loadOwner(owner: string, db: D1Database): Promise { emitOperationalEvent(createWideEvent({ operation, @@ -31,6 +48,44 @@ function githubReadFailed(operation: string, error: unknown): FetchOutcome }): SkillFileScan { + return { + status: tree.truncated ? 'truncated' : 'ok', + skillFiles: tree.tree + .filter(entry => entry.type === 'blob' && isRegistrySkillPath(entry.path)) + .map(entry => entry.path) + .sort(), + } +} + +function profileFrom(meta: RepoMeta, scan: SkillFileScan | null): RepoSourceProfile { + const skillFiles = scan?.skillFiles ?? [] + return { + owner: meta.owner.login, + repo: meta.name, + description: meta.description, + githubUrl: meta.html_url || `https://github.com/${meta.owner.login}/${meta.name}`, + defaultBranch: meta.default_branch, + stars: meta.stargazers_count, + forks: meta.forks_count, + pushedAt: meta.pushed_at, + createdAt: meta.created_at, + archived: Boolean(meta.archived), + fork: Boolean(meta.fork), + skillFileScanStatus: scan?.status ?? 'unavailable', + skillFileCount: skillFiles.length, + skillFiles, + } +} + export default defineCachedEventHandler(async (event) => { const ownerParam = getRouterParam(event, 'owner') const repoParam = getRouterParam(event, 'repo') @@ -39,55 +94,61 @@ export default defineCachedEventHandler(async (event) => { const owner = ownerParam.toLowerCase() const repo = repoParam.toLowerCase() + const edgeCache = useStorage('edge-cache') + if (await readCache(edgeCache, missingRepoKey(owner, repo))) + throw createError({ statusCode: 404, message: 'Repository not found' }) + const db = getDB(event) - const source = await resolveRepoSourceIdentity(db, { owner, repo }) + const row = await db + .prepare(`SELECT source_owner, source_repo, last_tree_sha FROM repos WHERE owner = ? AND repo = ?`) + .bind(owner, repo) + .first<{ source_owner: string | null, source_repo: string | null, last_tree_sha: string | null }>() + const source = resolveRepoSourceIdentityFromRow({ owner, repo }, row) const bindings = resolveGithubBindings(event.context.platform.env) // A read that timed out or dropped is an outage, not a missing repository. const repoRes = await getRepo(source.owner, source.repo, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) .catch((error: unknown) => githubReadFailed('repo-source-profile-repo', error)) - if (repoRes.status === 404) + // The token can read private repositories, and this page is public, so a + // private one answers exactly like one that does not exist. + if (repoRes.status === 404 || repoRes.data?.private) { + await writeCache(edgeCache, missingRepoKey(owner, repo), true, { ttl: MISSING_REPO_TTL_SECONDS }) throw createError({ statusCode: 404, message: 'Repository not found' }) + } if (!repoRes.data) { emitOperationalEvent(createWideEvent({ 'operation': 'repo-source-profile', 'outcome': 'unavailable', 'upstream.status': repoRes.status || null, })) + // Metadata from ungh.cc keeps the page whole while the shared quota is + // spent. The Skill file list needs the tree, which only GitHub serves. + if (githubRefused(repoRes)) { + const fallback = await readUnghRepoMeta(source.owner, source.repo) + if (fallback._tag === 'found') + return profileFrom(fallback.meta, null) + emitOperationalEvent(createWideEvent({ operation: 'repo-source-profile-ungh', outcome: 'unavailable', reason: fallback.reason })) + } return buildUnavailableRepoSourceProfile(owner, repo) satisfies RepoSourceProfile } const meta = repoRes.data - const repoOwner = meta.owner.login - const repoName = meta.name - const treeRes = await getTree(repoOwner, repoName, meta.default_branch, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) - .catch((error: unknown) => githubReadFailed('repo-source-profile-tree', error)) - const skillFiles = (treeRes.data?.tree ?? []) - .filter(entry => entry.type === 'blob' && isRegistrySkillPath(entry.path)) - .map(entry => entry.path) - .sort() - const skillFileScanStatus = !treeRes.data - ? 'unavailable' - : treeRes.data.truncated - ? 'truncated' - : 'ok' + // The sync records the tree it last read. Read at that SHA, the listing + // never changes, so one scan serves every later view. A repository the + // registry has not synced is read at its default branch. + const treeSha = row?.last_tree_sha ?? null + const cachedScan = treeSha + ? await readCache(edgeCache, treeScanKey(meta.owner.login, meta.name, treeSha)) + : null + if (cachedScan) + return profileFrom(meta, cachedScan) satisfies RepoSourceProfile - return { - owner: repoOwner, - repo: repoName, - description: meta.description, - githubUrl: meta.html_url || `https://github.com/${repoOwner}/${repoName}`, - defaultBranch: meta.default_branch, - stars: meta.stargazers_count, - forks: meta.forks_count, - pushedAt: meta.pushed_at, - createdAt: meta.created_at, - archived: Boolean(meta.archived), - fork: Boolean(meta.fork), - skillFileScanStatus, - skillFileCount: skillFiles.length, - skillFiles, - } satisfies RepoSourceProfile + const treeRes = await getTree(meta.owner.login, meta.name, treeSha ?? meta.default_branch, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) + .catch((error: unknown) => githubReadFailed('repo-source-profile-tree', error)) + const scan = treeRes.data ? scanTree(treeRes.data) : null + if (scan && treeSha) + await writeCache(edgeCache, treeScanKey(meta.owner.login, meta.name, treeSha), scan, { ttl: TREE_SCAN_TTL_SECONDS }) + return profileFrom(meta, scan) satisfies RepoSourceProfile }, { maxAge: 60 * 15, swr: true, diff --git a/layers/registry/server/api/skill-related/[...slug].get.ts b/layers/registry/server/api/skill-related/[...slug].get.ts index 4125397b..63163716 100644 --- a/layers/registry/server/api/skill-related/[...slug].get.ts +++ b/layers/registry/server/api/skill-related/[...slug].get.ts @@ -1,12 +1,13 @@ import type { H3Event } from 'h3' import type { EmbeddingNeighbor } from '../../jobs/generate-embeddings' +import type { GithubBindings } from '../../utils/github-client' import type { CoOccurrenceNeighbor } from '../../utils/skill-co-occurrence' import type { SkillCommitSourceRow } from '../../utils/skill-commit-source' import type { CachedRelated } from '../../utils/skill-related' import { readCache, readThroughCache, writeCache } from '#shared/server/cache' import { defineApiHandler } from '#shared/server/handler' import { getEmbeddingNeighbors } from '../../jobs/generate-embeddings' -import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client' +import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client' import { getCoOccurrenceNeighbors } from '../../utils/skill-co-occurrence' import { skillCommitSourceFromRow } from '../../utils/skill-commit-source' import { @@ -76,7 +77,7 @@ export default defineApiHandler({ const [commits, related, coOccurrenceNeighbors, embeddingNeighbors] = await Promise.all([ source - ? getSkillCommits(source.owner, source.repo, source.path) + ? getSkillCommits(source.owner, source.repo, source.path, resolveGithubBindings(platform.env)) : Promise.resolve([]), findRelatedSkills(event, { owner: skill.owner, repo: skill.repo, excludeName: skill.name, limit: 6 }), getCoOccurrenceNeighbors(platform.db, skill.name), @@ -166,26 +167,32 @@ interface GhCommitResponse { author: { login: string, avatar_url: string } | null } -async function getSkillCommits(owner: string, repo: string, path: string): Promise { +async function getSkillCommits(owner: string, repo: string, path: string, bindings: GithubBindings): Promise { const cacheKey = `skills:commits:v2:${owner}/${repo}:${path}` const cached = await readCache(useStorage('edge-cache'), cacheKey) if (cached) return cached - const data = await $fetch(`https://api.github.com/repos/${owner}/${repo}/commits`, { - query: { path, per_page: 5 }, - headers: { 'Accept': 'application/vnd.github+json', 'User-Agent': 'skilld.dev' }, - // ofetch retries a GET once on its own. The commit list is optional - // decoration, so a slow GitHub costs the list, never the page. - retry: 0, - timeout: GITHUB_PAGE_READ_TIMEOUT_MS, - }).catch(() => { - emitOperationalEvent(createWideEvent({ operation: 'skill-related-commits-fetch', outcome: 'failed' })) + // Authenticated and conditional: an unchanged history answers 304 from the + // ETag cache for free. Anonymous, this read shared the Worker IP's 60 + // requests an hour with every other Cloudflare tenant, and about 330 a day + // failed (2026-10-06). The commit list is optional decoration, so a slow + // GitHub costs the list, never the page. + const query = new URLSearchParams({ path, per_page: '5' }) + const response = await getGithubJson( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits?${query.toString()}`, + bindings, + { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }, + ).catch((error: unknown) => { + emitOperationalEvent(createWideEvent({ operation: 'skill-related-commits-fetch', outcome: 'failed', reason: error instanceof Error ? error.message : String(error) })) return null }) - - if (!Array.isArray(data)) + const data = response?.data + if (!Array.isArray(data)) { + if (response) + emitOperationalEvent(createWideEvent({ 'operation': 'skill-related-commits-fetch', 'outcome': 'failed', 'upstream.status': response.status })) return [] + } const commits: SkillCommit[] = data.map(c => ({ sha: c.sha, diff --git a/layers/registry/server/api/skills/featured.get.ts b/layers/registry/server/api/skills/featured.get.ts index 9c914068..433e6c6c 100644 --- a/layers/registry/server/api/skills/featured.get.ts +++ b/layers/registry/server/api/skills/featured.get.ts @@ -1,8 +1,9 @@ import type { TagPayload } from '../../jobs/generate-tags' +import type { GithubBindings } from '../../utils/github-client' import { defineApiHandler } from '#shared/server/handler' import { officialRepos } from '../../data/official-repos' import { FeaturedSkillsQuery } from '../../schemas/featured-query' -import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client' +import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client' import { getGeneratedBatch } from '../../utils/skill-generated' import { getFeaturedOfficialSections, getTopReposByCount, getTopReposByStars } from '../../utils/skills-registry' @@ -15,27 +16,30 @@ interface OwnerProfileRow { sync_status: string | null } -async function fetchAndStoreOwner(owner: string, db: D1Database): Promise { - const res = await fetch(`https://api.github.com/users/${owner}`, { - signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS), - headers: { 'User-Agent': 'skilld.dev', 'Accept': 'application/vnd.github+json' }, - }).catch(() => { - emitOperationalEvent(createWideEvent({ operation: 'featured-owner-fetch', outcome: 'failed' })) - return null - }) +interface GithubOwnerResponse { + name?: string + bio?: string + blog?: string + location?: string + followers?: number + public_repos?: number + type?: string +} - if (!res?.ok) +async function fetchAndStoreOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise { + // Authenticated and conditional: an unchanged profile answers 304 from + // the ETag cache for free. Anonymous, this read shared the Worker IP's 60 + // requests an hour with every other Cloudflare tenant, and failed. + const res = await getGithubJson(`/users/${encodeURIComponent(owner)}`, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) + .catch(() => { + emitOperationalEvent(createWideEvent({ operation: 'featured-owner-fetch', outcome: 'failed' })) + return null + }) + + if (!res?.data) return null - const data = await res.json() as { - name?: string - bio?: string - blog?: string - location?: string - followers?: number - public_repos?: number - type?: string - } + const data = res.data const kind = data.type === 'Organization' ? 'org' : 'user' await db.prepare( @@ -66,7 +70,7 @@ async function fetchAndStoreOwner(owner: string, db: D1Database): Promise> { +async function loadOwnerProfiles(owners: string[], db: D1Database, bindings: GithubBindings): Promise> { const profiles = new Map() const now = Math.floor(Date.now() / 1000) @@ -84,7 +88,7 @@ async function loadOwnerProfiles(owners: string[], db: D1Database): Promise s.owner), platform.db) + const profileMap = await loadOwnerProfiles(devSections.map(s => s.owner), platform.db, resolveGithubBindings(platform.env)) const enriched = sections.map(enrichSkills) const enrichedDevs = devSections.map((section) => { const profile = profileMap.get(section.owner) diff --git a/layers/registry/server/tasks/sync-github-skills.ts b/layers/registry/server/tasks/sync-github-skills.ts index 403f0717..48c6d06a 100644 --- a/layers/registry/server/tasks/sync-github-skills.ts +++ b/layers/registry/server/tasks/sync-github-skills.ts @@ -8,6 +8,8 @@ import { SUBSCRIBED_REPO_STALE_SECONDS, } from '~~/server/utils/sync-thresholds' import { observedSchedulePolicy } from '#shared/schedule-policy' +import { resolveGithubBindings } from '../utils/github-client' +import { githubSyncPermit } from '../utils/github-sync-control' import { appendRepoReverificationCandidates, DISCOVERY_SYNC_CANDIDATES_SQL, @@ -18,6 +20,7 @@ import { STAGE_HISTORICAL_DISCOVERY_CANDIDATES_SQL, SUBSCRIBED_SYNC_CANDIDATES_SQL, } from '../utils/sync-candidates' +import { prefetchUnchangedRepos } from '../utils/sync-prefetch' const MAX_REPOS_PER_RUN = 250 const MAX_HISTORICAL_CANDIDATES_PER_RUN = 250 @@ -106,26 +109,47 @@ export default defineScheduledTask({ { limit: MAX_REPOS_PER_RUN, maxReverified: MAX_REVERIFIED_REPOS_PER_RUN }, ) - if (queue.length === 0) { + const reverified = queue.length - ordered.length + const candidates = queue.map(({ owner, repo, ownerVerified }) => ({ + owner, + repo, + ownerVerified, + claimDiscovery: discoveryKeys.has(`${owner}/${repo}`), + })) + // While the sync is paused, every job would release itself unread, so + // a prefetch would only spend the quota the pause is saving. + const permit = await githubSyncPermit(db, now) + const prefetch = permit._tag === 'paused' + ? { _tag: 'unread' as const, queue: candidates, reason: 'sync-paused' } + : await prefetchUnchangedRepos({ db, bindings: resolveGithubBindings(env), now }, candidates) + .catch((error: unknown) => { + // The jobs read every repository themselves, as before the + // prefetch existed, so the run still completes. + emitOperationalEvent(createWideEvent({ + operation: 'sync-github-skills-prefetch', + outcome: 'failed', + reason: error instanceof Error ? error.message : String(error), + })) + return { _tag: 'unread' as const, queue: candidates, reason: 'prefetch-threw' } + }) + const prefetched = prefetch._tag === 'prefetched' + ? { unchanged: prefetch.unchanged, graphqlRequests: prefetch.requests } + : { unread: prefetch.reason } + + if (prefetch.queue.length === 0) { await reportJobRun(db, 'sync-github-skills', { cron: CRON, status: 'ok', durationMs: Date.now() - startedAt, }) - return { result: { queued: 0, deferred, stagedHistorical, reverified: 0 } } + return { result: { queued: 0, deferred, stagedHistorical, reverified, prefetched } } } const batch = await createRegistryJobBatch( env as Cloudflare.Env & Record, { name: `registry-sync:${now}`, - jobs: queue.map(({ owner, repo, ownerVerified }) => ({ - operation: 'sync', - owner, - repo, - ownerVerified, - claimDiscovery: discoveryKeys.has(`${owner}/${repo}`), - })), + jobs: prefetch.queue.map(candidate => ({ operation: 'sync' as const, ...candidate })), }, ) const dispatchFailed = batch.dispatched.filter(result => result.status !== 'sent') @@ -145,7 +169,8 @@ export default defineScheduledTask({ queued: batch.jobIds.length, deferred, stagedHistorical, - reverified: queue.length - ordered.length, + reverified, + prefetched, dispatchFailed: dispatchFailed.length, }, } diff --git a/layers/registry/server/utils/github-client.ts b/layers/registry/server/utils/github-client.ts index 565dcc78..851ee513 100644 --- a/layers/registry/server/utils/github-client.ts +++ b/layers/registry/server/utils/github-client.ts @@ -101,6 +101,12 @@ export interface RateLimitInfo { remaining: number limit: number reset: number + /** + * The bucket GitHub counted the request against: `core` for REST and + * `graphql` for GraphQL. The two are separate quotas. A pause reason that + * said only "163 requests remaining" could not tell which one ran out. + */ + resource: string | null } export interface FetchOutcome { @@ -149,6 +155,7 @@ function parseRateLimit(headers: Headers): RateLimitInfo | null { remaining: Number(remaining), limit: Number(headers.get('x-ratelimit-limit') ?? 0), reset: Number(headers.get('x-ratelimit-reset') ?? 0), + resource: headers.get('x-ratelimit-resource'), } } @@ -197,6 +204,26 @@ async function ghRequest( return { status: res.status, data: body, rateLimit, notModified: false } } +/** + * One authenticated, conditional GET of an API path, such as + * `/users/nuxt`. A 304 answers from the KV ETag cache and costs no quota. + */ +export async function getGithubJson( + path: `/${string}`, + bindings: GithubBindings, + options?: GithubReadOptions, +): Promise> { + return ghRequest(`${API_BASE}${path}`, bindings, readInit(options)) +} + +/** + * Whether GitHub refused a read for want of quota or capacity, as opposed to + * answering it. A timeout or dropped connection reads as status 0. + */ +export function githubRefused(outcome: Pick, 'status'>): boolean { + return outcome.status === 0 || outcome.status === 403 || outcome.status === 429 || outcome.status >= 500 +} + export async function getRepo( owner: string, repo: string, @@ -206,31 +233,52 @@ export async function getRepo( return ghRequest(`${API_BASE}/repos/${owner}/${repo}`, bindings, readInit(options)) } -interface RepoSummaryGqlResponse { - repository: { +interface RepoSummaryGql { + name: string + nameWithOwner: string + url: string + owner: { login: string } + description: string | null + stargazerCount: number + forkCount: number + pushedAt: string + createdAt: string + isArchived: boolean + isFork: boolean + defaultBranchRef: { name: string - nameWithOwner: string - url: string - owner: { login: string } - description: string | null - stargazerCount: number - forkCount: number - pushedAt: string - createdAt: string - isArchived: boolean - isFork: boolean - defaultBranchRef: { - name: string - target: { oid: string, tree: { oid: string } } | null - } | null + target: { oid: string, tree: { oid: string } } | null } | null } +const REPO_SUMMARY_FIELDS = `name nameWithOwner url owner{login} + description stargazerCount forkCount pushedAt createdAt isArchived isFork + defaultBranchRef{name target{... on Commit{oid tree{oid}}}}` + export interface RepoSummary { meta: RepoMeta headTreeSha: string | null } +function repoSummaryFromGql(r: RepoSummaryGql): RepoSummary { + const branch = r.defaultBranchRef?.name || 'main' + const meta: RepoMeta = { + name: r.name, + full_name: r.nameWithOwner, + html_url: r.url, + owner: { login: r.owner.login }, + default_branch: branch, + description: r.description, + stargazers_count: r.stargazerCount, + forks_count: r.forkCount, + pushed_at: r.pushedAt, + created_at: r.createdAt, + archived: r.isArchived, + fork: r.isFork, + } + return { meta, headTreeSha: r.defaultBranchRef?.target?.tree.oid ?? null } +} + /** * GraphQL-backed combo of getRepo + head-tree-SHA. One subrequest replaces * the REST getRepo (1) and lets sync-repo skip the REST getTree call (1) @@ -248,59 +296,92 @@ export async function getRepoSummary( ): Promise> { const query = `query($owner:String!,$repo:String!){ repository(owner:$owner,name:$repo){ - name nameWithOwner url owner{login} - description stargazerCount forkCount pushedAt createdAt isArchived isFork - defaultBranchRef{name target{... on Commit{oid tree{oid}}}} + ${REPO_SUMMARY_FIELDS} } }` - const headers = new Headers() - headers.set('Accept', 'application/vnd.github+json') - headers.set('Content-Type', 'application/json') - headers.set('User-Agent', 'skilld.dev') - if (bindings.GITHUB_TOKEN) - headers.set('Authorization', `Bearer ${bindings.GITHUB_TOKEN}`) - - const res = await fetch(GRAPHQL_URL, { - method: 'POST', - headers, - body: JSON.stringify({ query, variables: { owner, repo } }), - }) - const rateLimit = parseRateLimit(res.headers) - if (!res.ok) - return { status: res.status, data: null, rateLimit, notModified: false } - - const body = await res.json() as { data?: RepoSummaryGqlResponse, errors?: Array<{ type?: string, message?: string }> } + const out = await gqlRequest<{ repository: RepoSummaryGql | null }>(query, { owner, repo }, bindings) + if (out._tag === 'failed') + return { status: out.status, data: null, rateLimit: out.rateLimit, notModified: false } // GraphQL surfaces NOT_FOUND in errors with HTTP 200. Map it to 404 so // callers preserve their existing rate-limit / broken-repo branching. - if (body.errors?.length) { - const notFound = body.errors.some(e => e.type === 'NOT_FOUND') - return { status: notFound ? 404 : 502, data: null, rateLimit, notModified: false } + if (out.errors.length) { + const notFound = out.errors.some(e => e.type === 'NOT_FOUND') + return { status: notFound ? 404 : 502, data: null, rateLimit: out.rateLimit, notModified: false } } - const r = body.data?.repository + const r = out.data?.repository if (!r) - return { status: 404, data: null, rateLimit, notModified: false } + return { status: 404, data: null, rateLimit: out.rateLimit, notModified: false } + return { status: 200, data: repoSummaryFromGql(r), rateLimit: out.rateLimit, notModified: false } +} - const branch = r.defaultBranchRef?.name || 'main' - const meta: RepoMeta = { - name: r.name, - full_name: r.nameWithOwner, - html_url: r.url, - owner: { login: r.owner.login }, - default_branch: branch, - description: r.description, - stargazers_count: r.stargazerCount, - forks_count: r.forkCount, - pushed_at: r.pushedAt, - created_at: r.createdAt, - archived: r.isArchived, - fork: r.isFork, +/** + * Repositories one batched summary query reads. GitHub prices a query by the + * connections it asks for, and a summary asks for none, so one query of 100 + * costs the same single point as one query of 1. 100 is GitHub's ceiling for + * nodes per connection, and keeps the query under 15 KB. + */ +export const REPO_SUMMARY_BATCH_SIZE = 100 + +export interface RepoSummaryRequest { + owner: string + repo: string +} + +export type RepoSummaryBatchOutcome + = | { + _tag: 'read' + /** + * One entry per request, in request order. Null means GitHub gave no + * summary for that alias, such as NOT_FOUND. The caller leaves those to + * the per-repository sync, which owns the missing and renamed verdicts. + */ + summaries: Array + rateLimit: RateLimitInfo | null + /** GraphQL requests sent. */ + requests: number } - return { - status: 200, - data: { meta, headTreeSha: r.defaultBranchRef?.target?.tree.oid ?? null }, - rateLimit, - notModified: false, + | { _tag: 'failed', status: number, rateLimit: RateLimitInfo | null, requests: number } + +/** + * The summary of many repositories in aliased GraphQL queries of up to + * {@link REPO_SUMMARY_BATCH_SIZE}. + * + * The hourly sync used to send one summary query per repository from its own + * queue job. About 87% of those found the tree unchanged (1,649 of 1,900 jobs + * on 2026-10-06), so the job and its query did nothing but advance the + * freshness cursor. + */ +export async function getRepoSummariesBatch( + requests: RepoSummaryRequest[], + bindings: GithubBindings, +): Promise { + const summaries: Array = [] + let rateLimit: RateLimitInfo | null = null + let sent = 0 + for (const batch of chunk(requests, REPO_SUMMARY_BATCH_SIZE)) { + const varDecls = batch.flatMap((_, i) => [`$o${i}:String!`, `$n${i}:String!`]) + const aliases = batch.map((_, i) => `r${i}:repository(owner:$o${i},name:$n${i}){${REPO_SUMMARY_FIELDS}}`).join(' ') + const variables: Record = {} + batch.forEach((request, i) => { + variables[`o${i}`] = request.owner + variables[`n${i}`] = request.repo + }) + sent++ + const out = await gqlRequest>(`query(${varDecls.join(',')}){${aliases}}`, variables, bindings) + rateLimit = out.rateLimit ?? rateLimit + if (out._tag === 'failed') + return { _tag: 'failed', status: out.status, rateLimit, requests: sent } + // A NOT_FOUND names one alias and leaves the others whole. Any other + // error type, such as RATE_LIMITED, says nothing trustworthy about the + // batch, so it fails as a whole and the per-repository sync takes over. + if (out.errors.some(error => error.type !== 'NOT_FOUND')) + return { _tag: 'failed', status: 502, rateLimit, requests: sent } + for (let i = 0; i < batch.length; i++) { + const r = out.data?.[`r${i}`] + summaries.push(r ? repoSummaryFromGql(r) : null) + } } + return { _tag: 'read', summaries, rateLimit, requests: sent } } export async function getTree( @@ -341,11 +422,18 @@ export async function getCommits( ) } -async function gqlPost( +interface GqlError { type?: string, message?: string } + +type GqlRequestOutcome + = | { _tag: 'answered', data: T | null, errors: GqlError[], rateLimit: RateLimitInfo | null } + | { _tag: 'failed', status: number, rateLimit: RateLimitInfo | null } + +/** One GraphQL POST. Partial data and per-alias errors come back together. */ +async function gqlRequest( query: string, variables: Record, bindings: GithubBindings, -): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null }> { +): Promise> { const headers = new Headers() headers.set('Accept', 'application/vnd.github+json') headers.set('Content-Type', 'application/json') @@ -359,21 +447,32 @@ async function gqlPost( }) const rateLimit = parseRateLimit(res.headers) if (!res.ok) - return { status: res.status, data: null, rateLimit } + return { _tag: 'failed', status: res.status, rateLimit } // A gateway can answer 200 with a truncated or non-JSON body. Parsing that // eagerly threw `Unexpected end of JSON input` out of the client and reached // the sync summary as an opaque reason with no status attached. const body = await res.json().catch(() => { emitOperationalEvent(createWideEvent({ operation: 'github-graphql-parse', outcome: 'invalid-response' })) return null - }) as { data?: T, errors?: Array<{ type?: string, message?: string }> } | null + }) as { data?: T, errors?: GqlError[] } | null if (!body) - return { status: 502, data: null, rateLimit } - if (body.errors?.length) { - const notFound = body.errors.some(e => e.type === 'NOT_FOUND') - return { status: notFound ? 404 : 502, data: null, rateLimit } + return { _tag: 'failed', status: 502, rateLimit } + return { _tag: 'answered', data: body.data ?? null, errors: body.errors ?? [], rateLimit } +} + +async function gqlPost( + query: string, + variables: Record, + bindings: GithubBindings, +): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null }> { + const out = await gqlRequest(query, variables, bindings) + if (out._tag === 'failed') + return { status: out.status, data: null, rateLimit: out.rateLimit } + if (out.errors.length) { + const notFound = out.errors.some(e => e.type === 'NOT_FOUND') + return { status: notFound ? 404 : 502, data: null, rateLimit: out.rateLimit } } - return { status: 200, data: body.data ?? null, rateLimit } + return { status: 200, data: out.data, rateLimit: out.rateLimit } } export interface BlobBatchOutcome extends FetchOutcome> { diff --git a/layers/registry/server/utils/github-sync-control.ts b/layers/registry/server/utils/github-sync-control.ts index 8e28ff28..3ee37942 100644 --- a/layers/registry/server/utils/github-sync-control.ts +++ b/layers/registry/server/utils/github-sync-control.ts @@ -57,11 +57,24 @@ export function githubRatePauseUntil( return Math.min(now + 12 * 60 * 60, Math.max(fallback, (resetAt ?? 0) + 5)) } +/** + * Requests the sync leaves unspent in a GitHub bucket before it pauses. + * + * `GITHUB_TOKEN` is a personal token, and GitHub counts its quota per + * account, so every tool on that account draws on the same bucket. The sync + * spends about 10 REST requests and 130 GraphQL points an hour, and page + * views about 50 REST requests at most. 200 covers page views for the rest + * of any hour once the sync stops. + */ +export const GITHUB_SYNC_RESERVE = 200 + export function githubSyncPauseDecision(input: { owner: string repo: string now: number remaining?: number + /** The bucket `remaining` counts, from `x-ratelimit-resource`. */ + resource?: string resetAt?: number rateLimited: boolean unauthorized: boolean @@ -73,11 +86,12 @@ export function githubSyncPauseDecision(input: { reason: `${input.owner}/${input.repo}: GitHub credential rejected`, } } - if (!input.rateLimited && (input.remaining == null || input.remaining >= 200)) + if (!input.rateLimited && (input.remaining == null || input.remaining >= GITHUB_SYNC_RESERVE)) return { _tag: 'continue' } + const bucket = input.resource ? `${input.resource} ` : '' return { _tag: 'pause', pauseUntil: githubRatePauseUntil(input.now, input.resetAt), - reason: `${input.owner}/${input.repo}: ${input.rateLimited ? 'rate limited' : `${input.remaining} requests remaining`}`, + reason: `${input.owner}/${input.repo}: ${input.rateLimited ? 'rate limited' : `${input.remaining} ${bucket}requests remaining`}`, } } diff --git a/layers/registry/server/utils/sync-prefetch.ts b/layers/registry/server/utils/sync-prefetch.ts new file mode 100644 index 00000000..feaf7981 --- /dev/null +++ b/layers/registry/server/utils/sync-prefetch.ts @@ -0,0 +1,144 @@ +/// + +import type { GithubBindings, RepoSummary } from './github-client' +import type { ExistingRepo } from './sync-repo' +import { getRepoSummariesBatch } from './github-client' +import { resolveRepoSourceIdentityFromRow } from './repo-source-identity' +import { repoUnchangedStatements, unchangedRepoStatus } from './sync-repo' + +/** One repository the hourly sync is about to queue. */ +export interface SyncCandidate { + owner: string + repo: string + ownerVerified: boolean + claimDiscovery: boolean +} + +export type SyncPrefetchResult + = | { + _tag: 'prefetched' + /** Candidates that still need a sync job. */ + queue: SyncCandidate[] + /** Repositories recorded as unchanged without a job. */ + unchanged: number + /** GraphQL requests the prefetch sent. */ + requests: number + } + | { + _tag: 'unread' + /** Every candidate: the per-repository sync reads each one itself. */ + queue: SyncCandidate[] + reason: string + } + +export interface SyncPrefetchDependencies { + db: D1Database + bindings: GithubBindings + /** Unix seconds. */ + now: number +} + +/** A sync job that touched its progress row this recently may still hold it. */ +const PROGRESS_ACTIVE_SECONDS = 30 * 60 + +/** Repositories whose unchanged writes share one D1 batch. */ +const WRITE_BATCH_REPOSITORIES = 25 + +interface PrefetchRow extends ExistingRepo { + owner: string + repo: string + has_skills: number + in_progress: number +} + +/** + * Read every eligible candidate's summary in batched GraphQL queries, record + * the unchanged ones in place, and return the candidates that still need a job. + * + * Each candidate used to get a queue job whose first act was one GraphQL + * summary read. 1,649 of 1,900 such jobs on 2026-10-06 found the tree + * unchanged and only advanced the freshness cursor. The cursor writes here + * are the same statements `syncRepo` runs on that path. + * + * A discovery claim, a job still holding the repository's progress row, a + * Skill-less repository and any repository GitHub gave no summary for keep + * their job: `syncRepo` owns those verdicts. + */ +export async function prefetchUnchangedRepos( + dependencies: SyncPrefetchDependencies, + candidates: SyncCandidate[], +): Promise { + const eligible = candidates.filter(candidate => !candidate.claimDiscovery) + if (eligible.length === 0) + return { _tag: 'prefetched', queue: candidates, unchanged: 0, requests: 0 } + + const rows = await loadPrefetchRows(dependencies.db, eligible, dependencies.now) + const requests = eligible.map(candidate => resolveRepoSourceIdentityFromRow(candidate, rows.get(repoKey(candidate)))) + const batch = await getRepoSummariesBatch(requests, dependencies.bindings) + if (batch._tag === 'failed') + return { _tag: 'unread', queue: candidates, reason: `graphql-${batch.status}` } + + const unchanged = new Set() + const writes: D1PreparedStatement[][] = [] + eligible.forEach((candidate, index) => { + const row = rows.get(repoKey(candidate)) + const summary = batch.summaries[index] + if (!row || !summary || row.in_progress === 1 || !isUnchanged(row, summary)) + return + unchanged.add(repoKey(candidate)) + writes.push(repoUnchangedStatements(dependencies.db, { + owner: candidate.owner, + repo: candidate.repo, + meta: summary.meta, + checkedAt: dependencies.now, + ownerVerified: candidate.ownerVerified, + })) + }) + for (let offset = 0; offset < writes.length; offset += WRITE_BATCH_REPOSITORIES) + await dependencies.db.batch(writes.slice(offset, offset + WRITE_BATCH_REPOSITORIES).flat()) + + return { + _tag: 'prefetched', + queue: candidates.filter(candidate => !unchanged.has(repoKey(candidate))), + unchanged: unchanged.size, + requests: batch.requests, + } +} + +function isUnchanged(row: PrefetchRow, summary: RepoSummary): boolean { + const pushedAt = Date.parse(summary.meta.pushed_at) + return unchangedRepoStatus({ + existing: row, + hasAdmittedSkills: row.has_skills === 1, + headTreeSha: summary.headTreeSha, + repoPushedAt: Number.isFinite(pushedAt) ? Math.floor(pushedAt / 1000) : null, + }) !== null +} + +function repoKey(identity: { owner: string, repo: string }): string { + return `${identity.owner}/${identity.repo}` +} + +/** One D1 read for every candidate: a JSON array binds as a single parameter. */ +async function loadPrefetchRows( + db: D1Database, + candidates: SyncCandidate[], + now: number, +): Promise> { + const pairs = JSON.stringify(candidates.map(candidate => [candidate.owner, candidate.repo])) + const result = await db.prepare( + `WITH candidates(owner, repo) AS ( + SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') + FROM json_each(?1) + ) + SELECT r.owner, r.repo, r.last_tree_sha, r.pushed_at, r.source_owner, r.source_repo, + EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) AS has_skills, + EXISTS ( + SELECT 1 FROM repo_sync_progress p + WHERE p.owner = r.owner AND p.repo = r.repo AND p.updated_at > ?2 + ) AS in_progress + FROM candidates c + JOIN repos r ON r.owner = c.owner AND r.repo = c.repo`, + ).bind(pairs, now - PROGRESS_ACTIVE_SECONDS).all() + return new Map((result.results ?? []).map(row => [repoKey(row), row])) +} diff --git a/layers/registry/server/utils/sync-repo.ts b/layers/registry/server/utils/sync-repo.ts index f7844afa..72df3bfa 100644 --- a/layers/registry/server/utils/sync-repo.ts +++ b/layers/registry/server/utils/sync-repo.ts @@ -25,6 +25,8 @@ export interface SyncRepoStats { revisionsInserted: number activityEmitted: number rateLimitRemaining?: number + /** The GitHub bucket `rateLimitRemaining` counts: `core` or `graphql`. */ + rateLimitResource?: string rateLimitResetAt?: number continuation?: SyncRepoContinuation } @@ -52,7 +54,7 @@ interface ExistingSkillAssets { assets: string } -interface ExistingRepo { +export interface ExistingRepo { last_tree_sha: string | null pushed_at: number | null source_owner: string | null @@ -358,7 +360,7 @@ export async function refreshRepoAssets( SET source_owner = ?, source_repo = ? WHERE owner = ? AND repo = ? `).bind(sourceOwner, sourceRepo, owner, repo).run() - const treeRes = await getTree(sourceOwner, sourceRepo, branch, bindings) + const treeRes = await getTree(sourceOwner, sourceRepo, repoRes.data.headTreeSha ?? branch, bindings) logRateLimit(`asset-backfill tree ${owner}/${repo}`, treeRes.rateLimit) const rateLimitRemaining = Math.min( rate.rateLimitRemaining ?? Number.POSITIVE_INFINITY, @@ -516,18 +518,62 @@ async function repoHasAdmittedSkills(db: D1Database, owner: string, repo: string return row?.admitted === 1 } -async function markUnchangedOwnerVerified( +/** + * Why a repository needs no content sync this hour, or null when it does. + * + * Only a repository with admitted Skills qualifies: a Skill-less candidate is + * still being discovered, so its tree must be read whatever the cursor says. + */ +export function unchangedRepoStatus(input: { + existing: ExistingRepo | null + hasAdmittedSkills: boolean + headTreeSha: string | null + repoPushedAt: number | null +}): 'skipped-tree-sha' | 'skipped-pushed-at' | null { + const { existing, hasAdmittedSkills, headTreeSha, repoPushedAt } = input + if (!hasAdmittedSkills || !existing?.last_tree_sha) + return null + if (headTreeSha && existing.last_tree_sha === headTreeSha) + return 'skipped-tree-sha' + if (existing.pushed_at != null && repoPushedAt != null && existing.pushed_at >= repoPushedAt) + return 'skipped-pushed-at' + return null +} + +/** + * The writes that record an unchanged repository: fresh metadata, the + * freshness cursor, the day's star count and, for an owner-verified repository, + * the verified flag on every Skill. + */ +export function repoUnchangedStatements( db: D1Database, - owner: string, - repo: string, - meta: RepoMeta, - pushedAt: number | null, - checkedAt: number, -): Promise { - await db.batch([ + input: { + owner: string + repo: string + meta: RepoMeta + checkedAt: number + ownerVerified: boolean + }, +): D1PreparedStatement[] { + const { owner, repo, meta, checkedAt } = input + const pushedAt = epoch(meta.pushed_at) + const statements = [ markRepoSummaryCheckedStatement(db, owner, repo, meta, pushedAt, checkedAt), clearRepoMissingSkillsStatement(db, owner, repo, checkedAt), ...repoStarObservationStatements(db, owner, repo, meta.stargazers_count ?? 0, checkedAt), + ] + return input.ownerVerified + ? [...statements, ...ownerVerifiedStatements(db, owner, repo, checkedAt)] + : statements +} + +function ownerVerifiedStatements( + db: D1Database, + owner: string, + repo: string, + checkedAt: number, +): D1PreparedStatement[] { + return [ db.prepare( `UPDATE skills SET owner_verified = 1 @@ -542,7 +588,7 @@ async function markUnchangedOwnerVerified( queued_at = excluded.queued_at, attempts = 0`, ).bind(checkedAt, owner, repo), - ]) + ] } /** @@ -726,12 +772,13 @@ export async function syncRepo( revisionsInserted: 0, activityEmitted: 0, } - const trackRateLimit = (rateLimit: { remaining: number, reset: number } | null): void => { + const trackRateLimit = (rateLimit: { remaining: number, reset: number, resource: string | null } | null): void => { if (!rateLimit) return - stats.rateLimitRemaining = stats.rateLimitRemaining == null - ? rateLimit.remaining - : Math.min(stats.rateLimitRemaining, rateLimit.remaining) + if (stats.rateLimitRemaining == null || rateLimit.remaining < stats.rateLimitRemaining) { + stats.rateLimitRemaining = rateLimit.remaining + stats.rateLimitResource = rateLimit.resource ?? undefined + } stats.rateLimitResetAt = Math.max(stats.rateLimitResetAt ?? 0, rateLimit.reset) } @@ -775,39 +822,24 @@ export async function syncRepo( const hasAdmittedSkills = await repoHasAdmittedSkills(db, owner, repo) const markUnchanged = async (status: 'skipped-tree-sha' | 'skipped-pushed-at'): Promise => { - if (opts.ownerVerified) { - await markUnchangedOwnerVerified(db, owner, repo, meta, repoPushedAt, checkedAt) - stats.status = 'verified-only' - return stats - } - await db.batch([ - markRepoSummaryCheckedStatement(db, owner, repo, meta, repoPushedAt, checkedAt), - clearRepoMissingSkillsStatement(db, owner, repo, checkedAt), - ...repoStarObservationStatements(db, owner, repo, meta.stargazers_count ?? 0, checkedAt), - ]) - stats.status = status + await db.batch(repoUnchangedStatements(db, { owner, repo, meta, checkedAt, ownerVerified: opts.ownerVerified === true })) + stats.status = opts.ownerVerified ? 'verified-only' : status return stats } // GraphQL gave us the head tree SHA in the same request. If it matches // our cached value, the repo is unchanged and we skip the REST getTree // call entirely. Skill-less candidates deliberately bypass this cursor. - if (!opts.continuation && !opts.forceContent && hasAdmittedSkills && existingRepo?.last_tree_sha && headTreeSha && existingRepo.last_tree_sha === headTreeSha) - return markUnchanged('skipped-tree-sha') - - if ( - !opts.continuation - && !opts.forceContent - && hasAdmittedSkills - && existingRepo?.pushed_at != null - && existingRepo.last_tree_sha != null - && repoPushedAt != null - && existingRepo.pushed_at >= repoPushedAt - ) { - return markUnchanged('skipped-pushed-at') - } - - const treeRes = await getTree(sourceOwner, sourceRepo, branch, bindings) + const unchanged = opts.continuation || opts.forceContent + ? null + : unchangedRepoStatus({ existing: existingRepo, hasAdmittedSkills, headTreeSha, repoPushedAt }) + if (unchanged) + return markUnchanged(unchanged) + + // The head tree SHA names the exact tree the summary saw. A branch name + // can move between the two reads, and a SHA URL is immutable, so its ETag + // cache entry answers 304 for as long as the tree stands. + const treeRes = await getTree(sourceOwner, sourceRepo, headTreeSha ?? branch, bindings) logRateLimit(`tree ${owner}/${repo}`, treeRes.rateLimit) trackRateLimit(treeRes.rateLimit) diff --git a/layers/registry/server/utils/ungh-repo.ts b/layers/registry/server/utils/ungh-repo.ts new file mode 100644 index 00000000..2673261f --- /dev/null +++ b/layers/registry/server/utils/ungh-repo.ts @@ -0,0 +1,72 @@ +import type { RepoMeta } from './github-client' +import { z } from 'zod' +import { GITHUB_PAGE_READ_TIMEOUT_MS } from './github-client' + +const unghRepoSchema = z.object({ + repo: z.object({ + name: z.string().min(1), + repo: z.string().regex(/^[^/]+\/[^/]+$/), + description: z.string().nullable().optional(), + createdAt: z.string(), + pushedAt: z.string(), + stars: z.number().int().nonnegative(), + forks: z.number().int().nonnegative(), + defaultBranch: z.string().min(1), + }), +}) + +export type UnghRepoResult + = | { _tag: 'found', meta: RepoMeta } + | { _tag: 'unavailable', reason: string } + +/** + * Public repository metadata from ungh.cc, the unjs GitHub proxy that caches + * anonymous reads. + * + * A page-view fallback for when `GITHUB_TOKEN` is spent or GitHub is down. It + * answers metadata only. It never serves a tree or file content, and nothing + * built or signed reads it. ungh.cc knows no archived or fork flag, so both + * read false. Its 404 is no verdict about GitHub, so it reads as unavailable. + */ +export async function readUnghRepoMeta( + owner: string, + repo: string, + fetcher: typeof fetch = fetch, +): Promise { + const response = await fetcher(`https://ungh.cc/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, { + headers: { 'User-Agent': 'skilld.dev', 'Accept': 'application/json' }, + signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS), + }).catch((error: unknown) => ({ _tag: 'threw' as const, reason: error instanceof Error ? error.message : String(error) })) + if ('_tag' in response) + return { _tag: 'unavailable', reason: response.reason } + if (!response.ok) { + await response.body?.cancel() + return { _tag: 'unavailable', reason: `ungh ${response.status}` } + } + const parsed = unghRepoSchema.safeParse(await response.json().catch(() => { + // A body that is not JSON fails the schema below, which reports it. + return null + })) + if (!parsed.success) + return { _tag: 'unavailable', reason: 'ungh invalid response' } + const { repo: found } = parsed.data + const [login] = found.repo.split('/') as [string, string] + return { + _tag: 'found', + meta: { + name: found.name, + full_name: found.repo, + html_url: `https://github.com/${found.repo}`, + owner: { login }, + default_branch: found.defaultBranch, + description: found.description ?? null, + stargazers_count: found.stars, + forks_count: found.forks, + pushed_at: found.pushedAt, + created_at: found.createdAt, + archived: false, + fork: false, + private: false, + }, + } +} diff --git a/server/jobs/registry/repo-maintenance.ts b/server/jobs/registry/repo-maintenance.ts index dc8d16b9..1559f297 100644 --- a/server/jobs/registry/repo-maintenance.ts +++ b/server/jobs/registry/repo-maintenance.ts @@ -190,6 +190,7 @@ async function applyRateGuard( repo: string now: number remaining?: number + resource?: string resetAt?: number rateLimited: boolean unauthorized: boolean @@ -333,6 +334,7 @@ export async function handleRegistryRepoJob( repo: payload.repo, now, remaining: stats.rateLimitRemaining, + resource: stats.rateLimitResource, resetAt: stats.rateLimitResetAt, rateLimited: stats.status === 'rate-limited', unauthorized: stats.status === 'unauthorized', diff --git a/test/unit/github-conditional-read.test.ts b/test/unit/github-conditional-read.test.ts new file mode 100644 index 00000000..1734cab7 --- /dev/null +++ b/test/unit/github-conditional-read.test.ts @@ -0,0 +1,43 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getGithubJson } from '../../layers/registry/server/utils/github-client' + +/** A KV namespace over a Map: the ETag cache the client writes and reads. */ +function memoryKv(): KVNamespace { + const values = new Map() + return { + get: async (key: string, type?: string) => { + const value = values.get(key) + return value === undefined ? null : type === 'json' ? JSON.parse(value) : value + }, + put: async (key: string, value: string) => { + values.set(key, value) + }, + } as unknown as KVNamespace +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('getGithubJson', () => { + it('asks again with the stored ETag and answers a 304 from the cache', async () => { + const seen: Array<{ authorization: string | null, ifNoneMatch: string | null }> = [] + vi.stubGlobal('fetch', vi.fn(async (_url: string, init: RequestInit) => { + const headers = new Headers(init.headers) + seen.push({ authorization: headers.get('authorization'), ifNoneMatch: headers.get('if-none-match') }) + return headers.get('if-none-match') === '"v1"' + ? new Response(null, { status: 304 }) + : new Response(JSON.stringify({ login: 'nuxt', type: 'Organization' }), { status: 200, headers: { etag: '"v1"' } }) + })) + const bindings = { GITHUB_TOKEN: 'registry-token', KV_CACHE: memoryKv() } + + await getGithubJson('/users/nuxt', bindings) + const repeat = await getGithubJson<{ type: string }>('/users/nuxt', bindings) + + expect(repeat).toMatchObject({ status: 304, notModified: true, data: { type: 'Organization' } }) + expect(seen).toEqual([ + { authorization: 'Bearer registry-token', ifNoneMatch: null }, + { authorization: 'Bearer registry-token', ifNoneMatch: '"v1"' }, + ]) + }) +}) diff --git a/test/unit/github-repo-summary-batch.test.ts b/test/unit/github-repo-summary-batch.test.ts new file mode 100644 index 00000000..e300de3b --- /dev/null +++ b/test/unit/github-repo-summary-batch.test.ts @@ -0,0 +1,99 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getRepoSummariesBatch } from '../../layers/registry/server/utils/github-client' + +const bindings = { GITHUB_TOKEN: 'test-token' } + +interface GraphqlBody { + query: string + variables: Record +} + +function repository(owner: string, name: string, tree: string) { + return { + name, + nameWithOwner: `${owner}/${name}`, + url: `https://github.com/${owner}/${name}`, + owner: { login: owner }, + description: null, + stargazerCount: 3, + forkCount: 1, + pushedAt: '2026-10-01T00:00:00Z', + createdAt: '2025-01-01T00:00:00Z', + isArchived: false, + isFork: false, + defaultBranchRef: { name: 'main', target: { oid: 'c'.repeat(40), tree: { oid: tree } } }, + } +} + +/** Answers every alias, except owners named `gone`, which GitHub reports NOT_FOUND. */ +function graphqlAnswer(_input: unknown, init?: RequestInit): Response { + const body = JSON.parse(String(init?.body)) as GraphqlBody + const data: Record = {} + const errors: unknown[] = [] + for (const key of Object.keys(body.variables).filter(name => name.startsWith('o'))) { + const index = key.slice(1) + const owner = body.variables[key]! + const name = body.variables[`n${index}`]! + if (owner === 'gone') { + data[`r${index}`] = null + errors.push({ type: 'NOT_FOUND', path: [`r${index}`], message: 'Could not resolve' }) + } + else { + data[`r${index}`] = repository(owner, name, `tree-${owner}-${name}`) + } + } + return new Response(JSON.stringify({ data, errors: errors.length ? errors : undefined }), { + status: 200, + headers: { 'x-ratelimit-remaining': '4990', 'x-ratelimit-resource': 'graphql' }, + }) +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('getRepoSummariesBatch', () => { + it('reads 230 repositories in three queries, in request order', async () => { + const fetchMock = vi.fn(graphqlAnswer) + vi.stubGlobal('fetch', fetchMock) + const requests = Array.from({ length: 230 }, (_, i) => ({ owner: 'acme', repo: `repo-${i}` })) + + const result = await getRepoSummariesBatch(requests, bindings) + + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(result._tag).toBe('read') + if (result._tag !== 'read') + return + expect(result.summaries).toHaveLength(230) + expect(result.summaries[229]?.headTreeSha).toBe('tree-acme-repo-229') + expect(result.summaries[0]?.meta.full_name).toBe('acme/repo-0') + expect(result.rateLimit?.resource).toBe('graphql') + }) + + it('leaves a repository GitHub cannot find empty and keeps the rest', async () => { + vi.stubGlobal('fetch', vi.fn(graphqlAnswer)) + + const result = await getRepoSummariesBatch([ + { owner: 'acme', repo: 'one' }, + { owner: 'gone', repo: 'two' }, + { owner: 'acme', repo: 'three' }, + ], bindings) + + expect(result._tag === 'read' && result.summaries.map(summary => summary?.headTreeSha ?? null)).toEqual([ + 'tree-acme-one', + null, + 'tree-acme-three', + ]) + }) + + it('fails the whole batch on an error that is not NOT_FOUND', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ + data: null, + errors: [{ type: 'RATE_LIMITED', message: 'API rate limit exceeded' }], + }), { status: 200 }))) + + const result = await getRepoSummariesBatch([{ owner: 'acme', repo: 'one' }], bindings) + + expect(result).toMatchObject({ _tag: 'failed', status: 502 }) + }) +}) diff --git a/test/unit/github-sync-control.test.ts b/test/unit/github-sync-control.test.ts index 42cc26f3..114ecded 100644 --- a/test/unit/github-sync-control.test.ts +++ b/test/unit/github-sync-control.test.ts @@ -41,4 +41,20 @@ describe('github sync pause decision', () => { remaining: 200, })).toEqual({ _tag: 'continue' }) }) + + it('names the GitHub bucket that ran low', () => { + expect(githubSyncPauseDecision({ + now: 1_000, + owner: 'nuxt', + repo: 'ui', + unauthorized: false, + rateLimited: false, + remaining: 163, + resource: 'graphql', + resetAt: 2_000, + })).toMatchObject({ + _tag: 'pause', + reason: 'nuxt/ui: 163 graphql requests remaining', + }) + }) }) diff --git a/test/unit/repo-source-profile-quota.test.ts b/test/unit/repo-source-profile-quota.test.ts new file mode 100644 index 00000000..c1597be9 --- /dev/null +++ b/test/unit/repo-source-profile-quota.test.ts @@ -0,0 +1,109 @@ +import type { H3Event } from 'h3' +import type { SqliteD1 } from './helpers/d1-sqlite' +import { createStorage } from 'unstorage' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' + +const route = { owner: 'acme', repo: 'skills' } +let edgeCache = createStorage() +let harness: SqliteD1 + +function stubRuntime(): void { + vi.stubGlobal('defineCachedEventHandler', (handler: unknown) => handler) + vi.stubGlobal('createError', (input: Record) => Object.assign(new Error(String(input.message)), input)) + vi.stubGlobal('getRouterParam', (_event: unknown, key: 'owner' | 'repo') => route[key]) + vi.stubGlobal('useStorage', () => edgeCache) +} + +const TREE_SHA = 'a'.repeat(40) + +function repoMeta(overrides: Record = {}) { + return { + name: 'skills', + full_name: 'acme/skills', + html_url: 'https://github.com/acme/skills', + owner: { login: 'acme' }, + default_branch: 'main', + description: 'Skills', + stargazers_count: 12, + forks_count: 3, + pushed_at: '2026-10-01T00:00:00Z', + created_at: '2025-01-01T00:00:00Z', + private: false, + ...overrides, + } +} + +function json(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { status: 200, ...init }) +} + +async function profile(): Promise> { + const handler = (await import('../../layers/registry/server/api/repos/[owner]/[repo].get')).default as (event: H3Event) => Promise> + return handler({ context: { platform: { db: harness.db, env: { GITHUB_TOKEN: 'token' } } } } as unknown as H3Event) +} + +function calls(fetchMock: ReturnType, fragment: string): number { + return fetchMock.mock.calls.filter(([input]) => String(input).includes(fragment)).length +} + +beforeEach(() => { + edgeCache = createStorage() + harness = createSqliteD1(allMigrations()) + route.repo = 'skills' + stubRuntime() +}) + +// The global wide event stubs come from the setup file, so this file never +// calls `vi.unstubAllGlobals()`: each test replaces the stubs it needs. +afterEach(() => { + harness.close() +}) + +describe('repository source profile GitHub spend', () => { + it('answers 404 for a private repository and never lists its tree', async () => { + const fetchMock = vi.fn(async () => json(repoMeta({ private: true }))) + vi.stubGlobal('fetch', fetchMock) + + await expect(profile()).rejects.toMatchObject({ statusCode: 404 }) + expect(calls(fetchMock, '/git/trees/')).toBe(0) + }) + + it('remembers a repository GitHub cannot find, so a repeat view reads nothing', async () => { + route.repo = 'gone' + const fetchMock = vi.fn(async () => json({ message: 'Not Found' }, { status: 404 })) + vi.stubGlobal('fetch', fetchMock) + + await expect(profile()).rejects.toMatchObject({ statusCode: 404 }) + await expect(profile()).rejects.toMatchObject({ statusCode: 404 }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('reads repository metadata from ungh.cc when GitHub refuses with a spent quota', async () => { + const fetchMock = vi.fn(async (input: unknown) => String(input).startsWith('https://ungh.cc/') + ? json({ repo: { id: 1, name: 'skills', repo: 'acme/skills', description: 'From ungh', createdAt: '2025-01-01T00:00:00Z', pushedAt: '2026-10-01T00:00:00Z', stars: 99, forks: 4, defaultBranch: 'trunk' } }) + : json({ message: 'API rate limit exceeded' }, { status: 403, headers: { 'x-ratelimit-remaining': '0' } })) + vi.stubGlobal('fetch', fetchMock) + + const body = await profile() + + expect(body).toMatchObject({ owner: 'acme', repo: 'skills', stars: 99, description: 'From ungh', defaultBranch: 'trunk', skillFileScanStatus: 'unavailable' }) + expect(calls(fetchMock, '/git/trees/')).toBe(0) + }) + + it('lists a registry repository at its synced tree, once per tree', async () => { + harness.raw.prepare(`INSERT INTO repos (owner, repo, last_tree_sha) VALUES ('acme', 'skills', ?)`).run(TREE_SHA) + const fetchMock = vi.fn(async (input: unknown) => String(input).includes('/git/trees/') + ? json({ sha: TREE_SHA, tree: [{ path: 'skills/one/SKILL.md', type: 'blob', sha: 'b'.repeat(40) }] }) + : json(repoMeta())) + vi.stubGlobal('fetch', fetchMock) + + const first = await profile() + const second = await profile() + + expect(first).toMatchObject({ skillFileScanStatus: 'ok', skillFiles: ['skills/one/SKILL.md'] }) + expect(second).toMatchObject({ skillFiles: ['skills/one/SKILL.md'] }) + expect(fetchMock.mock.calls.filter(([input]) => String(input).includes(`/git/trees/${TREE_SHA}`))).toHaveLength(1) + }) +}) diff --git a/test/unit/repo-source-profile-timeout.test.ts b/test/unit/repo-source-profile-timeout.test.ts index 1e6f4baf..4458ecf6 100644 --- a/test/unit/repo-source-profile-timeout.test.ts +++ b/test/unit/repo-source-profile-timeout.test.ts @@ -1,16 +1,19 @@ import type { H3Event } from 'h3' +import { createStorage } from 'unstorage' import { afterEach, describe, expect, it, vi } from 'vitest' import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' vi.stubGlobal('defineCachedEventHandler', (handler: unknown) => handler) vi.stubGlobal('createError', (input: Record) => Object.assign(new Error(String(input.message)), input)) vi.stubGlobal('getRouterParam', (_event: unknown, key: string) => (key === 'owner' ? 'skilld-dev' : 'skills')) +vi.stubGlobal('useStorage', () => createStorage()) afterEach(() => { vi.unstubAllGlobals() vi.stubGlobal('defineCachedEventHandler', (handler: unknown) => handler) vi.stubGlobal('createError', (input: Record) => Object.assign(new Error(String(input.message)), input)) vi.stubGlobal('getRouterParam', (_event: unknown, key: string) => (key === 'owner' ? 'skilld-dev' : 'skills')) + vi.stubGlobal('useStorage', () => createStorage()) }) async function profile() { diff --git a/test/unit/skill-detail-d1-cost.test.ts b/test/unit/skill-detail-d1-cost.test.ts index 152ef08e..eee0d577 100644 --- a/test/unit/skill-detail-d1-cost.test.ts +++ b/test/unit/skill-detail-d1-cost.test.ts @@ -236,17 +236,17 @@ describe('findSkill', () => { describe('uncached skill related render', () => { it('reads D1 at most three times and keeps the commit source', async () => { seed({ maximumQueries: 3 }) - upstreamFetch.mockImplementationOnce(async () => [{ + rawFetch.mockImplementationOnce(async () => new Response(JSON.stringify([{ sha: 'abcdef1234567', html_url: 'https://github.com/c/abcdef1', commit: { message: 'edit alpha', author: { name: 'Eric', date: '2026-09-01T00:00:00Z' } }, author: null, - }] as never) + }]))) const handler = (await import('../../layers/registry/server/api/skill-related/[...slug].get')).default as (event: H3Event) => Promise> const body = await handler({ context: { platform: { db: harness.db, env: {} } }, node: { req: { headers: {} } } } as unknown as H3Event) - expect(upstreamFetch).toHaveBeenCalledWith(`https://api.github.com/repos/${OWNER}/${REPO}/commits`, expect.objectContaining({ query: { path: 'skills/alpha/SKILL.md', per_page: 5 } })) + expect(String(rawFetch.mock.calls[0]?.[0])).toBe(`https://api.github.com/repos/${OWNER}/${REPO}/commits?path=skills%2Falpha%2FSKILL.md&per_page=5`) expect(body.commits.map((commit: { shortSha: string }) => commit.shortSha)).toEqual(['abcdef1']) expect(body.relatedRepoSkills.map((skill: { name: string }) => skill.name)).toEqual(['beta']) }) diff --git a/test/unit/sync-prefetch.test.ts b/test/unit/sync-prefetch.test.ts new file mode 100644 index 00000000..a8d0f640 --- /dev/null +++ b/test/unit/sync-prefetch.test.ts @@ -0,0 +1,113 @@ +import type { SqliteD1 } from './helpers/d1-sqlite' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { prefetchUnchangedRepos } from '../../layers/registry/server/utils/sync-prefetch' +import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' + +const NOW = 1_791_300_000 +const bindings = { GITHUB_TOKEN: 'test-token' } + +/** The head tree GitHub reports for each repository. */ +const heads: Record = { + 'acme/unchanged': 'tree-same', + 'acme/changed': 'tree-new', + 'acme/fresh': 'tree-fresh', + 'acme/busy': 'tree-busy', + 'acme/discovered': 'tree-discovered', +} + +function graphqlAnswer(_input: unknown, init?: RequestInit): Response { + const body = JSON.parse(String(init?.body)) as { variables: Record } + const data: Record = {} + const errors: unknown[] = [] + for (const key of Object.keys(body.variables).filter(name => name.startsWith('o'))) { + const index = key.slice(1) + const fullName = `${body.variables[key]}/${body.variables[`n${index}`]}` + const tree = heads[fullName] + if (!tree) { + data[`r${index}`] = null + errors.push({ type: 'NOT_FOUND', path: [`r${index}`] }) + continue + } + const [owner, name] = fullName.split('/') as [string, string] + data[`r${index}`] = { + name, + nameWithOwner: fullName, + url: `https://github.com/${fullName}`, + owner: { login: owner }, + description: 'Skills', + stargazerCount: 77, + forkCount: 2, + pushedAt: '2026-10-01T00:00:00Z', + createdAt: '2025-01-01T00:00:00Z', + isArchived: false, + isFork: false, + defaultBranchRef: { name: 'main', target: { oid: 'c'.repeat(40), tree: { oid: tree } } }, + } + } + return new Response(JSON.stringify({ data, errors: errors.length ? errors : undefined }), { status: 200 }) +} + +function candidate(repo: string, claimDiscovery = false, owner = 'acme') { + return { owner, repo, ownerVerified: false, claimDiscovery } +} + +describe('prefetchUnchangedRepos', () => { + let harness: SqliteD1 + + beforeEach(() => { + harness = createSqliteD1(allMigrations()) + for (const [repo, tree] of [['unchanged', 'tree-same'], ['changed', 'tree-old'], ['fresh', null], ['busy', 'tree-busy'], ['discovered', 'tree-discovered']] as const) { + harness.raw.prepare(`INSERT INTO repos (owner, repo, last_tree_sha, pushed_at, stars) VALUES ('acme', ?, ?, 100, 1)`).run(repo, tree) + if (repo !== 'fresh') + harness.raw.prepare(`INSERT INTO skills (owner, repo, name, display_name, slug) VALUES ('acme', ?, 'one', 'One', 'one')`).run(repo) + } + harness.raw.prepare(`INSERT INTO repo_sync_progress (owner, repo, job_id, tree_sha, checked_at, next_offset, updated_at) VALUES ('acme', 'busy', 'job-1', 'tree-busy', ?, 250, ?)`).run(NOW - 60, NOW - 60) + }) + + afterEach(() => { + harness.close() + vi.unstubAllGlobals() + }) + + it('records an unchanged repository in place and queues only the ones that need a job', async () => { + const fetchMock = vi.fn(graphqlAnswer) + vi.stubGlobal('fetch', fetchMock) + + const result = await prefetchUnchangedRepos({ db: harness.db, bindings, now: NOW }, [ + candidate('unchanged'), + candidate('changed'), + candidate('fresh'), + candidate('busy'), + candidate('discovered', true), + candidate('missing', false, 'gone'), + ]) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(result.queue.map(item => `${item.owner}/${item.repo}`)).toEqual([ + 'acme/changed', + 'acme/fresh', + 'acme/busy', + 'acme/discovered', + 'gone/missing', + ]) + expect(result).toMatchObject({ _tag: 'prefetched', unchanged: 1 }) + expect(harness.raw.prepare(`SELECT stars, repo_meta_synced_at FROM repos WHERE owner = 'acme' AND repo = 'unchanged'`).get()) + .toEqual({ stars: 77, repo_meta_synced_at: NOW }) + expect(harness.raw.prepare(`SELECT repo_meta_synced_at FROM repos WHERE owner = 'acme' AND repo = 'changed'`).get()) + .toEqual({ repo_meta_synced_at: null }) + }) + + it('queues every candidate when GitHub does not answer the batch', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('bad gateway', { status: 502 }))) + + const result = await prefetchUnchangedRepos({ db: harness.db, bindings, now: NOW }, [ + candidate('unchanged'), + candidate('changed'), + ]) + + expect(result).toMatchObject({ _tag: 'unread', reason: 'graphql-502' }) + expect(result.queue).toHaveLength(2) + expect(harness.raw.prepare(`SELECT repo_meta_synced_at FROM repos WHERE owner = 'acme' AND repo = 'unchanged'`).get()) + .toEqual({ repo_meta_synced_at: null }) + }) +}) diff --git a/test/unit/sync-repo-correctness.test.ts b/test/unit/sync-repo-correctness.test.ts index cb5edc7d..42aa1138 100644 --- a/test/unit/sync-repo-correctness.test.ts +++ b/test/unit/sync-repo-correctness.test.ts @@ -136,7 +136,7 @@ describe('syncRepo content acknowledgement', () => { const result = await syncRepo('steipete', 'clawdis', {}, db, { ownerVerified: true }) expect(result).toMatchObject({ owner: 'steipete', repo: 'clawdis', status: 'indexed' }) - expect(github.getTree).toHaveBeenCalledWith('openclaw', 'openclaw', 'main', {}) + expect(github.getTree).toHaveBeenCalledWith('openclaw', 'openclaw', 'new-tree', {}) expect(github.getBlobsBatch).toHaveBeenCalledWith( 'openclaw', 'openclaw', diff --git a/test/unit/sync-repo-freshness.test.ts b/test/unit/sync-repo-freshness.test.ts index 966c06e2..8771d110 100644 --- a/test/unit/sync-repo-freshness.test.ts +++ b/test/unit/sync-repo-freshness.test.ts @@ -215,6 +215,7 @@ describe('syncRepo freshness cursor', () => { FROM repos WHERE owner = 'acme' AND repo = 'skills' `).get() + expect(github.getTree).toHaveBeenCalledWith('acme', 'skills', 'moved-tree', {}) expect(result.status).toBe('failed') expect(result.reason).toBe('tree_truncated') expect(row).toEqual({ From 450a61198deab98a6ab2d0d9459a8e46911dc883 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 02:21:09 +1100 Subject: [PATCH 2/2] docs(registry): state what the sync job count measured --- layers/registry/server/utils/github-client.ts | 5 ++--- layers/registry/server/utils/sync-prefetch.ts | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/layers/registry/server/utils/github-client.ts b/layers/registry/server/utils/github-client.ts index 851ee513..16b5a8a4 100644 --- a/layers/registry/server/utils/github-client.ts +++ b/layers/registry/server/utils/github-client.ts @@ -347,9 +347,8 @@ export type RepoSummaryBatchOutcome * {@link REPO_SUMMARY_BATCH_SIZE}. * * The hourly sync used to send one summary query per repository from its own - * queue job. About 87% of those found the tree unchanged (1,649 of 1,900 jobs - * on 2026-10-06), so the job and its query did nothing but advance the - * freshness cursor. + * queue job. On 2026-10-06, 1,649 of 1,900 such jobs wrote no Skill row, and + * a job that finds its tree unchanged only advances the freshness cursor. */ export async function getRepoSummariesBatch( requests: RepoSummaryRequest[], diff --git a/layers/registry/server/utils/sync-prefetch.ts b/layers/registry/server/utils/sync-prefetch.ts index feaf7981..ec59cf2a 100644 --- a/layers/registry/server/utils/sync-prefetch.ts +++ b/layers/registry/server/utils/sync-prefetch.ts @@ -56,9 +56,9 @@ interface PrefetchRow extends ExistingRepo { * the unchanged ones in place, and return the candidates that still need a job. * * Each candidate used to get a queue job whose first act was one GraphQL - * summary read. 1,649 of 1,900 such jobs on 2026-10-06 found the tree - * unchanged and only advanced the freshness cursor. The cursor writes here - * are the same statements `syncRepo` runs on that path. + * summary read. On 2026-10-06, 1,649 of 1,900 such jobs wrote no Skill row. + * A job that finds its tree unchanged only advances the freshness cursor, and + * the writes here are the same statements `syncRepo` runs on that path. * * A discovery claim, a job still holding the repository's progress row, a * Skill-less repository and any repository GitHub gave no summary for keep