diff --git a/layers/registry/server/api/orgs/[owner].get.ts b/layers/registry/server/api/orgs/[owner].get.ts index 751aaf13..c9b26af2 100644 --- a/layers/registry/server/api/orgs/[owner].get.ts +++ b/layers/registry/server/api/orgs/[owner].get.ts @@ -1,11 +1,12 @@ import type { H3Event } from 'h3' import type { TagPayload } from '../../jobs/generate-tags' +import type { GithubBindings } from '../../utils/github-client' import type { RegistrySkill } from '../../utils/skills-registry' import { getDB } from '#server/utils/db' import { cached, readCache, writeCache } from '#shared/server/cache' import { officialRepos } from '../../data/official-repos' import { TAG_BY_SLUG } from '../../jobs/taxonomy' -import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client' +import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client' import { resolveRepoSourceIdentitiesForOwner } from '../../utils/repo-source-identity' import { getGeneratedBatch } from '../../utils/skill-generated' import { querySkills } from '../../utils/skills-registry' @@ -68,19 +69,27 @@ const ORG_PROFILE_CACHE_STALE_TTL = 60 * 5 const officialOwners = new Set(officialRepos.map(r => r.owner)) const kindByOwner = new Map(officialRepos.map(r => [r.owner, r.kind])) -async function fetchAndStoreOwner(owner: string, db: D1Database): Promise { - const res = await fetch(`https://api.github.com/users/${owner}`, { - signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS), - headers: { - 'User-Agent': 'skilld.dev', - 'Accept': 'application/vnd.github+json', - }, - }).catch(() => { - emitOperationalEvent(createWideEvent({ operation: 'org-profile-fetch', outcome: 'failed' })) - return null - }) +interface GithubOwnerResponse { + name?: string + bio?: string + blog?: string + location?: string + followers?: number + public_repos?: number + type?: string +} + +async function fetchAndStoreOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise { + // Authenticated and conditional: an unchanged profile answers 304 from + // the ETag cache for free. Anonymous, this read shared the Worker IP's 60 + // requests an hour with every other Cloudflare tenant, and failed. + const res = await getGithubJson(`/users/${encodeURIComponent(owner)}`, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) + .catch(() => { + emitOperationalEvent(createWideEvent({ operation: 'org-profile-fetch', outcome: 'failed' })) + return null + }) - if (!res || !res.ok) { + if (!res?.data) { if (res?.status === 404) { await db .prepare( @@ -93,16 +102,7 @@ async function fetchAndStoreOwner(owner: string, db: D1Database): Promise { +async function loadOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise { const cached = await db .prepare( `SELECT kind, name, bio, blog, location, followers, last_synced_at, sync_status @@ -145,7 +145,7 @@ async function loadOwner(owner: string, db: D1Database): Promise { emitOperationalEvent(createWideEvent({ operation, @@ -31,6 +48,44 @@ function githubReadFailed(operation: string, error: unknown): FetchOutcome }): SkillFileScan { + return { + status: tree.truncated ? 'truncated' : 'ok', + skillFiles: tree.tree + .filter(entry => entry.type === 'blob' && isRegistrySkillPath(entry.path)) + .map(entry => entry.path) + .sort(), + } +} + +function profileFrom(meta: RepoMeta, scan: SkillFileScan | null): RepoSourceProfile { + const skillFiles = scan?.skillFiles ?? [] + return { + owner: meta.owner.login, + repo: meta.name, + description: meta.description, + githubUrl: meta.html_url || `https://github.com/${meta.owner.login}/${meta.name}`, + defaultBranch: meta.default_branch, + stars: meta.stargazers_count, + forks: meta.forks_count, + pushedAt: meta.pushed_at, + createdAt: meta.created_at, + archived: Boolean(meta.archived), + fork: Boolean(meta.fork), + skillFileScanStatus: scan?.status ?? 'unavailable', + skillFileCount: skillFiles.length, + skillFiles, + } +} + export default defineCachedEventHandler(async (event) => { const ownerParam = getRouterParam(event, 'owner') const repoParam = getRouterParam(event, 'repo') @@ -39,55 +94,61 @@ export default defineCachedEventHandler(async (event) => { const owner = ownerParam.toLowerCase() const repo = repoParam.toLowerCase() + const edgeCache = useStorage('edge-cache') + if (await readCache(edgeCache, missingRepoKey(owner, repo))) + throw createError({ statusCode: 404, message: 'Repository not found' }) + const db = getDB(event) - const source = await resolveRepoSourceIdentity(db, { owner, repo }) + const row = await db + .prepare(`SELECT source_owner, source_repo, last_tree_sha FROM repos WHERE owner = ? AND repo = ?`) + .bind(owner, repo) + .first<{ source_owner: string | null, source_repo: string | null, last_tree_sha: string | null }>() + const source = resolveRepoSourceIdentityFromRow({ owner, repo }, row) const bindings = resolveGithubBindings(event.context.platform.env) // A read that timed out or dropped is an outage, not a missing repository. const repoRes = await getRepo(source.owner, source.repo, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) .catch((error: unknown) => githubReadFailed('repo-source-profile-repo', error)) - if (repoRes.status === 404) + // The token can read private repositories, and this page is public, so a + // private one answers exactly like one that does not exist. + if (repoRes.status === 404 || repoRes.data?.private) { + await writeCache(edgeCache, missingRepoKey(owner, repo), true, { ttl: MISSING_REPO_TTL_SECONDS }) throw createError({ statusCode: 404, message: 'Repository not found' }) + } if (!repoRes.data) { emitOperationalEvent(createWideEvent({ 'operation': 'repo-source-profile', 'outcome': 'unavailable', 'upstream.status': repoRes.status || null, })) + // Metadata from ungh.cc keeps the page whole while the shared quota is + // spent. The Skill file list needs the tree, which only GitHub serves. + if (githubRefused(repoRes)) { + const fallback = await readUnghRepoMeta(source.owner, source.repo) + if (fallback._tag === 'found') + return profileFrom(fallback.meta, null) + emitOperationalEvent(createWideEvent({ operation: 'repo-source-profile-ungh', outcome: 'unavailable', reason: fallback.reason })) + } return buildUnavailableRepoSourceProfile(owner, repo) satisfies RepoSourceProfile } const meta = repoRes.data - const repoOwner = meta.owner.login - const repoName = meta.name - const treeRes = await getTree(repoOwner, repoName, meta.default_branch, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) - .catch((error: unknown) => githubReadFailed('repo-source-profile-tree', error)) - const skillFiles = (treeRes.data?.tree ?? []) - .filter(entry => entry.type === 'blob' && isRegistrySkillPath(entry.path)) - .map(entry => entry.path) - .sort() - const skillFileScanStatus = !treeRes.data - ? 'unavailable' - : treeRes.data.truncated - ? 'truncated' - : 'ok' + // The sync records the tree it last read. Read at that SHA, the listing + // never changes, so one scan serves every later view. A repository the + // registry has not synced is read at its default branch. + const treeSha = row?.last_tree_sha ?? null + const cachedScan = treeSha + ? await readCache(edgeCache, treeScanKey(meta.owner.login, meta.name, treeSha)) + : null + if (cachedScan) + return profileFrom(meta, cachedScan) satisfies RepoSourceProfile - return { - owner: repoOwner, - repo: repoName, - description: meta.description, - githubUrl: meta.html_url || `https://github.com/${repoOwner}/${repoName}`, - defaultBranch: meta.default_branch, - stars: meta.stargazers_count, - forks: meta.forks_count, - pushedAt: meta.pushed_at, - createdAt: meta.created_at, - archived: Boolean(meta.archived), - fork: Boolean(meta.fork), - skillFileScanStatus, - skillFileCount: skillFiles.length, - skillFiles, - } satisfies RepoSourceProfile + const treeRes = await getTree(meta.owner.login, meta.name, treeSha ?? meta.default_branch, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) + .catch((error: unknown) => githubReadFailed('repo-source-profile-tree', error)) + const scan = treeRes.data ? scanTree(treeRes.data) : null + if (scan && treeSha) + await writeCache(edgeCache, treeScanKey(meta.owner.login, meta.name, treeSha), scan, { ttl: TREE_SCAN_TTL_SECONDS }) + return profileFrom(meta, scan) satisfies RepoSourceProfile }, { maxAge: 60 * 15, swr: true, diff --git a/layers/registry/server/api/skill-related/[...slug].get.ts b/layers/registry/server/api/skill-related/[...slug].get.ts index 4125397b..63163716 100644 --- a/layers/registry/server/api/skill-related/[...slug].get.ts +++ b/layers/registry/server/api/skill-related/[...slug].get.ts @@ -1,12 +1,13 @@ import type { H3Event } from 'h3' import type { EmbeddingNeighbor } from '../../jobs/generate-embeddings' +import type { GithubBindings } from '../../utils/github-client' import type { CoOccurrenceNeighbor } from '../../utils/skill-co-occurrence' import type { SkillCommitSourceRow } from '../../utils/skill-commit-source' import type { CachedRelated } from '../../utils/skill-related' import { readCache, readThroughCache, writeCache } from '#shared/server/cache' import { defineApiHandler } from '#shared/server/handler' import { getEmbeddingNeighbors } from '../../jobs/generate-embeddings' -import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client' +import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client' import { getCoOccurrenceNeighbors } from '../../utils/skill-co-occurrence' import { skillCommitSourceFromRow } from '../../utils/skill-commit-source' import { @@ -76,7 +77,7 @@ export default defineApiHandler({ const [commits, related, coOccurrenceNeighbors, embeddingNeighbors] = await Promise.all([ source - ? getSkillCommits(source.owner, source.repo, source.path) + ? getSkillCommits(source.owner, source.repo, source.path, resolveGithubBindings(platform.env)) : Promise.resolve([]), findRelatedSkills(event, { owner: skill.owner, repo: skill.repo, excludeName: skill.name, limit: 6 }), getCoOccurrenceNeighbors(platform.db, skill.name), @@ -166,26 +167,32 @@ interface GhCommitResponse { author: { login: string, avatar_url: string } | null } -async function getSkillCommits(owner: string, repo: string, path: string): Promise { +async function getSkillCommits(owner: string, repo: string, path: string, bindings: GithubBindings): Promise { const cacheKey = `skills:commits:v2:${owner}/${repo}:${path}` const cached = await readCache(useStorage('edge-cache'), cacheKey) if (cached) return cached - const data = await $fetch(`https://api.github.com/repos/${owner}/${repo}/commits`, { - query: { path, per_page: 5 }, - headers: { 'Accept': 'application/vnd.github+json', 'User-Agent': 'skilld.dev' }, - // ofetch retries a GET once on its own. The commit list is optional - // decoration, so a slow GitHub costs the list, never the page. - retry: 0, - timeout: GITHUB_PAGE_READ_TIMEOUT_MS, - }).catch(() => { - emitOperationalEvent(createWideEvent({ operation: 'skill-related-commits-fetch', outcome: 'failed' })) + // Authenticated and conditional: an unchanged history answers 304 from the + // ETag cache for free. Anonymous, this read shared the Worker IP's 60 + // requests an hour with every other Cloudflare tenant, and about 330 a day + // failed (2026-10-06). The commit list is optional decoration, so a slow + // GitHub costs the list, never the page. + const query = new URLSearchParams({ path, per_page: '5' }) + const response = await getGithubJson( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits?${query.toString()}`, + bindings, + { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }, + ).catch((error: unknown) => { + emitOperationalEvent(createWideEvent({ operation: 'skill-related-commits-fetch', outcome: 'failed', reason: error instanceof Error ? error.message : String(error) })) return null }) - - if (!Array.isArray(data)) + const data = response?.data + if (!Array.isArray(data)) { + if (response) + emitOperationalEvent(createWideEvent({ 'operation': 'skill-related-commits-fetch', 'outcome': 'failed', 'upstream.status': response.status })) return [] + } const commits: SkillCommit[] = data.map(c => ({ sha: c.sha, diff --git a/layers/registry/server/api/skills/featured.get.ts b/layers/registry/server/api/skills/featured.get.ts index 9c914068..433e6c6c 100644 --- a/layers/registry/server/api/skills/featured.get.ts +++ b/layers/registry/server/api/skills/featured.get.ts @@ -1,8 +1,9 @@ import type { TagPayload } from '../../jobs/generate-tags' +import type { GithubBindings } from '../../utils/github-client' import { defineApiHandler } from '#shared/server/handler' import { officialRepos } from '../../data/official-repos' import { FeaturedSkillsQuery } from '../../schemas/featured-query' -import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client' +import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client' import { getGeneratedBatch } from '../../utils/skill-generated' import { getFeaturedOfficialSections, getTopReposByCount, getTopReposByStars } from '../../utils/skills-registry' @@ -15,27 +16,30 @@ interface OwnerProfileRow { sync_status: string | null } -async function fetchAndStoreOwner(owner: string, db: D1Database): Promise { - const res = await fetch(`https://api.github.com/users/${owner}`, { - signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS), - headers: { 'User-Agent': 'skilld.dev', 'Accept': 'application/vnd.github+json' }, - }).catch(() => { - emitOperationalEvent(createWideEvent({ operation: 'featured-owner-fetch', outcome: 'failed' })) - return null - }) +interface GithubOwnerResponse { + name?: string + bio?: string + blog?: string + location?: string + followers?: number + public_repos?: number + type?: string +} - if (!res?.ok) +async function fetchAndStoreOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise { + // Authenticated and conditional: an unchanged profile answers 304 from + // the ETag cache for free. Anonymous, this read shared the Worker IP's 60 + // requests an hour with every other Cloudflare tenant, and failed. + const res = await getGithubJson(`/users/${encodeURIComponent(owner)}`, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS }) + .catch(() => { + emitOperationalEvent(createWideEvent({ operation: 'featured-owner-fetch', outcome: 'failed' })) + return null + }) + + if (!res?.data) return null - const data = await res.json() as { - name?: string - bio?: string - blog?: string - location?: string - followers?: number - public_repos?: number - type?: string - } + const data = res.data const kind = data.type === 'Organization' ? 'org' : 'user' await db.prepare( @@ -66,7 +70,7 @@ async function fetchAndStoreOwner(owner: string, db: D1Database): Promise> { +async function loadOwnerProfiles(owners: string[], db: D1Database, bindings: GithubBindings): Promise> { const profiles = new Map() const now = Math.floor(Date.now() / 1000) @@ -84,7 +88,7 @@ async function loadOwnerProfiles(owners: string[], db: D1Database): Promise s.owner), platform.db) + const profileMap = await loadOwnerProfiles(devSections.map(s => s.owner), platform.db, resolveGithubBindings(platform.env)) const enriched = sections.map(enrichSkills) const enrichedDevs = devSections.map((section) => { const profile = profileMap.get(section.owner) diff --git a/layers/registry/server/tasks/sync-github-skills.ts b/layers/registry/server/tasks/sync-github-skills.ts index 403f0717..48c6d06a 100644 --- a/layers/registry/server/tasks/sync-github-skills.ts +++ b/layers/registry/server/tasks/sync-github-skills.ts @@ -8,6 +8,8 @@ import { SUBSCRIBED_REPO_STALE_SECONDS, } from '~~/server/utils/sync-thresholds' import { observedSchedulePolicy } from '#shared/schedule-policy' +import { resolveGithubBindings } from '../utils/github-client' +import { githubSyncPermit } from '../utils/github-sync-control' import { appendRepoReverificationCandidates, DISCOVERY_SYNC_CANDIDATES_SQL, @@ -18,6 +20,7 @@ import { STAGE_HISTORICAL_DISCOVERY_CANDIDATES_SQL, SUBSCRIBED_SYNC_CANDIDATES_SQL, } from '../utils/sync-candidates' +import { prefetchUnchangedRepos } from '../utils/sync-prefetch' const MAX_REPOS_PER_RUN = 250 const MAX_HISTORICAL_CANDIDATES_PER_RUN = 250 @@ -106,26 +109,47 @@ export default defineScheduledTask({ { limit: MAX_REPOS_PER_RUN, maxReverified: MAX_REVERIFIED_REPOS_PER_RUN }, ) - if (queue.length === 0) { + const reverified = queue.length - ordered.length + const candidates = queue.map(({ owner, repo, ownerVerified }) => ({ + owner, + repo, + ownerVerified, + claimDiscovery: discoveryKeys.has(`${owner}/${repo}`), + })) + // While the sync is paused, every job would release itself unread, so + // a prefetch would only spend the quota the pause is saving. + const permit = await githubSyncPermit(db, now) + const prefetch = permit._tag === 'paused' + ? { _tag: 'unread' as const, queue: candidates, reason: 'sync-paused' } + : await prefetchUnchangedRepos({ db, bindings: resolveGithubBindings(env), now }, candidates) + .catch((error: unknown) => { + // The jobs read every repository themselves, as before the + // prefetch existed, so the run still completes. + emitOperationalEvent(createWideEvent({ + operation: 'sync-github-skills-prefetch', + outcome: 'failed', + reason: error instanceof Error ? error.message : String(error), + })) + return { _tag: 'unread' as const, queue: candidates, reason: 'prefetch-threw' } + }) + const prefetched = prefetch._tag === 'prefetched' + ? { unchanged: prefetch.unchanged, graphqlRequests: prefetch.requests } + : { unread: prefetch.reason } + + if (prefetch.queue.length === 0) { await reportJobRun(db, 'sync-github-skills', { cron: CRON, status: 'ok', durationMs: Date.now() - startedAt, }) - return { result: { queued: 0, deferred, stagedHistorical, reverified: 0 } } + return { result: { queued: 0, deferred, stagedHistorical, reverified, prefetched } } } const batch = await createRegistryJobBatch( env as Cloudflare.Env & Record, { name: `registry-sync:${now}`, - jobs: queue.map(({ owner, repo, ownerVerified }) => ({ - operation: 'sync', - owner, - repo, - ownerVerified, - claimDiscovery: discoveryKeys.has(`${owner}/${repo}`), - })), + jobs: prefetch.queue.map(candidate => ({ operation: 'sync' as const, ...candidate })), }, ) const dispatchFailed = batch.dispatched.filter(result => result.status !== 'sent') @@ -145,7 +169,8 @@ export default defineScheduledTask({ queued: batch.jobIds.length, deferred, stagedHistorical, - reverified: queue.length - ordered.length, + reverified, + prefetched, dispatchFailed: dispatchFailed.length, }, } diff --git a/layers/registry/server/utils/github-client.ts b/layers/registry/server/utils/github-client.ts index 565dcc78..16b5a8a4 100644 --- a/layers/registry/server/utils/github-client.ts +++ b/layers/registry/server/utils/github-client.ts @@ -101,6 +101,12 @@ export interface RateLimitInfo { remaining: number limit: number reset: number + /** + * The bucket GitHub counted the request against: `core` for REST and + * `graphql` for GraphQL. The two are separate quotas. A pause reason that + * said only "163 requests remaining" could not tell which one ran out. + */ + resource: string | null } export interface FetchOutcome { @@ -149,6 +155,7 @@ function parseRateLimit(headers: Headers): RateLimitInfo | null { remaining: Number(remaining), limit: Number(headers.get('x-ratelimit-limit') ?? 0), reset: Number(headers.get('x-ratelimit-reset') ?? 0), + resource: headers.get('x-ratelimit-resource'), } } @@ -197,6 +204,26 @@ async function ghRequest( return { status: res.status, data: body, rateLimit, notModified: false } } +/** + * One authenticated, conditional GET of an API path, such as + * `/users/nuxt`. A 304 answers from the KV ETag cache and costs no quota. + */ +export async function getGithubJson( + path: `/${string}`, + bindings: GithubBindings, + options?: GithubReadOptions, +): Promise> { + return ghRequest(`${API_BASE}${path}`, bindings, readInit(options)) +} + +/** + * Whether GitHub refused a read for want of quota or capacity, as opposed to + * answering it. A timeout or dropped connection reads as status 0. + */ +export function githubRefused(outcome: Pick, 'status'>): boolean { + return outcome.status === 0 || outcome.status === 403 || outcome.status === 429 || outcome.status >= 500 +} + export async function getRepo( owner: string, repo: string, @@ -206,31 +233,52 @@ export async function getRepo( return ghRequest(`${API_BASE}/repos/${owner}/${repo}`, bindings, readInit(options)) } -interface RepoSummaryGqlResponse { - repository: { +interface RepoSummaryGql { + name: string + nameWithOwner: string + url: string + owner: { login: string } + description: string | null + stargazerCount: number + forkCount: number + pushedAt: string + createdAt: string + isArchived: boolean + isFork: boolean + defaultBranchRef: { name: string - nameWithOwner: string - url: string - owner: { login: string } - description: string | null - stargazerCount: number - forkCount: number - pushedAt: string - createdAt: string - isArchived: boolean - isFork: boolean - defaultBranchRef: { - name: string - target: { oid: string, tree: { oid: string } } | null - } | null + target: { oid: string, tree: { oid: string } } | null } | null } +const REPO_SUMMARY_FIELDS = `name nameWithOwner url owner{login} + description stargazerCount forkCount pushedAt createdAt isArchived isFork + defaultBranchRef{name target{... on Commit{oid tree{oid}}}}` + export interface RepoSummary { meta: RepoMeta headTreeSha: string | null } +function repoSummaryFromGql(r: RepoSummaryGql): RepoSummary { + const branch = r.defaultBranchRef?.name || 'main' + const meta: RepoMeta = { + name: r.name, + full_name: r.nameWithOwner, + html_url: r.url, + owner: { login: r.owner.login }, + default_branch: branch, + description: r.description, + stargazers_count: r.stargazerCount, + forks_count: r.forkCount, + pushed_at: r.pushedAt, + created_at: r.createdAt, + archived: r.isArchived, + fork: r.isFork, + } + return { meta, headTreeSha: r.defaultBranchRef?.target?.tree.oid ?? null } +} + /** * GraphQL-backed combo of getRepo + head-tree-SHA. One subrequest replaces * the REST getRepo (1) and lets sync-repo skip the REST getTree call (1) @@ -248,59 +296,91 @@ export async function getRepoSummary( ): Promise> { const query = `query($owner:String!,$repo:String!){ repository(owner:$owner,name:$repo){ - name nameWithOwner url owner{login} - description stargazerCount forkCount pushedAt createdAt isArchived isFork - defaultBranchRef{name target{... on Commit{oid tree{oid}}}} + ${REPO_SUMMARY_FIELDS} } }` - const headers = new Headers() - headers.set('Accept', 'application/vnd.github+json') - headers.set('Content-Type', 'application/json') - headers.set('User-Agent', 'skilld.dev') - if (bindings.GITHUB_TOKEN) - headers.set('Authorization', `Bearer ${bindings.GITHUB_TOKEN}`) - - const res = await fetch(GRAPHQL_URL, { - method: 'POST', - headers, - body: JSON.stringify({ query, variables: { owner, repo } }), - }) - const rateLimit = parseRateLimit(res.headers) - if (!res.ok) - return { status: res.status, data: null, rateLimit, notModified: false } - - const body = await res.json() as { data?: RepoSummaryGqlResponse, errors?: Array<{ type?: string, message?: string }> } + const out = await gqlRequest<{ repository: RepoSummaryGql | null }>(query, { owner, repo }, bindings) + if (out._tag === 'failed') + return { status: out.status, data: null, rateLimit: out.rateLimit, notModified: false } // GraphQL surfaces NOT_FOUND in errors with HTTP 200. Map it to 404 so // callers preserve their existing rate-limit / broken-repo branching. - if (body.errors?.length) { - const notFound = body.errors.some(e => e.type === 'NOT_FOUND') - return { status: notFound ? 404 : 502, data: null, rateLimit, notModified: false } + if (out.errors.length) { + const notFound = out.errors.some(e => e.type === 'NOT_FOUND') + return { status: notFound ? 404 : 502, data: null, rateLimit: out.rateLimit, notModified: false } } - const r = body.data?.repository + const r = out.data?.repository if (!r) - return { status: 404, data: null, rateLimit, notModified: false } + return { status: 404, data: null, rateLimit: out.rateLimit, notModified: false } + return { status: 200, data: repoSummaryFromGql(r), rateLimit: out.rateLimit, notModified: false } +} - const branch = r.defaultBranchRef?.name || 'main' - const meta: RepoMeta = { - name: r.name, - full_name: r.nameWithOwner, - html_url: r.url, - owner: { login: r.owner.login }, - default_branch: branch, - description: r.description, - stargazers_count: r.stargazerCount, - forks_count: r.forkCount, - pushed_at: r.pushedAt, - created_at: r.createdAt, - archived: r.isArchived, - fork: r.isFork, +/** + * Repositories one batched summary query reads. GitHub prices a query by the + * connections it asks for, and a summary asks for none, so one query of 100 + * costs the same single point as one query of 1. 100 is GitHub's ceiling for + * nodes per connection, and keeps the query under 15 KB. + */ +export const REPO_SUMMARY_BATCH_SIZE = 100 + +export interface RepoSummaryRequest { + owner: string + repo: string +} + +export type RepoSummaryBatchOutcome + = | { + _tag: 'read' + /** + * One entry per request, in request order. Null means GitHub gave no + * summary for that alias, such as NOT_FOUND. The caller leaves those to + * the per-repository sync, which owns the missing and renamed verdicts. + */ + summaries: Array + rateLimit: RateLimitInfo | null + /** GraphQL requests sent. */ + requests: number } - return { - status: 200, - data: { meta, headTreeSha: r.defaultBranchRef?.target?.tree.oid ?? null }, - rateLimit, - notModified: false, + | { _tag: 'failed', status: number, rateLimit: RateLimitInfo | null, requests: number } + +/** + * The summary of many repositories in aliased GraphQL queries of up to + * {@link REPO_SUMMARY_BATCH_SIZE}. + * + * The hourly sync used to send one summary query per repository from its own + * queue job. On 2026-10-06, 1,649 of 1,900 such jobs wrote no Skill row, and + * a job that finds its tree unchanged only advances the freshness cursor. + */ +export async function getRepoSummariesBatch( + requests: RepoSummaryRequest[], + bindings: GithubBindings, +): Promise { + const summaries: Array = [] + let rateLimit: RateLimitInfo | null = null + let sent = 0 + for (const batch of chunk(requests, REPO_SUMMARY_BATCH_SIZE)) { + const varDecls = batch.flatMap((_, i) => [`$o${i}:String!`, `$n${i}:String!`]) + const aliases = batch.map((_, i) => `r${i}:repository(owner:$o${i},name:$n${i}){${REPO_SUMMARY_FIELDS}}`).join(' ') + const variables: Record = {} + batch.forEach((request, i) => { + variables[`o${i}`] = request.owner + variables[`n${i}`] = request.repo + }) + sent++ + const out = await gqlRequest>(`query(${varDecls.join(',')}){${aliases}}`, variables, bindings) + rateLimit = out.rateLimit ?? rateLimit + if (out._tag === 'failed') + return { _tag: 'failed', status: out.status, rateLimit, requests: sent } + // A NOT_FOUND names one alias and leaves the others whole. Any other + // error type, such as RATE_LIMITED, says nothing trustworthy about the + // batch, so it fails as a whole and the per-repository sync takes over. + if (out.errors.some(error => error.type !== 'NOT_FOUND')) + return { _tag: 'failed', status: 502, rateLimit, requests: sent } + for (let i = 0; i < batch.length; i++) { + const r = out.data?.[`r${i}`] + summaries.push(r ? repoSummaryFromGql(r) : null) + } } + return { _tag: 'read', summaries, rateLimit, requests: sent } } export async function getTree( @@ -341,11 +421,18 @@ export async function getCommits( ) } -async function gqlPost( +interface GqlError { type?: string, message?: string } + +type GqlRequestOutcome + = | { _tag: 'answered', data: T | null, errors: GqlError[], rateLimit: RateLimitInfo | null } + | { _tag: 'failed', status: number, rateLimit: RateLimitInfo | null } + +/** One GraphQL POST. Partial data and per-alias errors come back together. */ +async function gqlRequest( query: string, variables: Record, bindings: GithubBindings, -): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null }> { +): Promise> { const headers = new Headers() headers.set('Accept', 'application/vnd.github+json') headers.set('Content-Type', 'application/json') @@ -359,21 +446,32 @@ async function gqlPost( }) const rateLimit = parseRateLimit(res.headers) if (!res.ok) - return { status: res.status, data: null, rateLimit } + return { _tag: 'failed', status: res.status, rateLimit } // A gateway can answer 200 with a truncated or non-JSON body. Parsing that // eagerly threw `Unexpected end of JSON input` out of the client and reached // the sync summary as an opaque reason with no status attached. const body = await res.json().catch(() => { emitOperationalEvent(createWideEvent({ operation: 'github-graphql-parse', outcome: 'invalid-response' })) return null - }) as { data?: T, errors?: Array<{ type?: string, message?: string }> } | null + }) as { data?: T, errors?: GqlError[] } | null if (!body) - return { status: 502, data: null, rateLimit } - if (body.errors?.length) { - const notFound = body.errors.some(e => e.type === 'NOT_FOUND') - return { status: notFound ? 404 : 502, data: null, rateLimit } + return { _tag: 'failed', status: 502, rateLimit } + return { _tag: 'answered', data: body.data ?? null, errors: body.errors ?? [], rateLimit } +} + +async function gqlPost( + query: string, + variables: Record, + bindings: GithubBindings, +): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null }> { + const out = await gqlRequest(query, variables, bindings) + if (out._tag === 'failed') + return { status: out.status, data: null, rateLimit: out.rateLimit } + if (out.errors.length) { + const notFound = out.errors.some(e => e.type === 'NOT_FOUND') + return { status: notFound ? 404 : 502, data: null, rateLimit: out.rateLimit } } - return { status: 200, data: body.data ?? null, rateLimit } + return { status: 200, data: out.data, rateLimit: out.rateLimit } } export interface BlobBatchOutcome extends FetchOutcome> { diff --git a/layers/registry/server/utils/github-sync-control.ts b/layers/registry/server/utils/github-sync-control.ts index 8e28ff28..3ee37942 100644 --- a/layers/registry/server/utils/github-sync-control.ts +++ b/layers/registry/server/utils/github-sync-control.ts @@ -57,11 +57,24 @@ export function githubRatePauseUntil( return Math.min(now + 12 * 60 * 60, Math.max(fallback, (resetAt ?? 0) + 5)) } +/** + * Requests the sync leaves unspent in a GitHub bucket before it pauses. + * + * `GITHUB_TOKEN` is a personal token, and GitHub counts its quota per + * account, so every tool on that account draws on the same bucket. The sync + * spends about 10 REST requests and 130 GraphQL points an hour, and page + * views about 50 REST requests at most. 200 covers page views for the rest + * of any hour once the sync stops. + */ +export const GITHUB_SYNC_RESERVE = 200 + export function githubSyncPauseDecision(input: { owner: string repo: string now: number remaining?: number + /** The bucket `remaining` counts, from `x-ratelimit-resource`. */ + resource?: string resetAt?: number rateLimited: boolean unauthorized: boolean @@ -73,11 +86,12 @@ export function githubSyncPauseDecision(input: { reason: `${input.owner}/${input.repo}: GitHub credential rejected`, } } - if (!input.rateLimited && (input.remaining == null || input.remaining >= 200)) + if (!input.rateLimited && (input.remaining == null || input.remaining >= GITHUB_SYNC_RESERVE)) return { _tag: 'continue' } + const bucket = input.resource ? `${input.resource} ` : '' return { _tag: 'pause', pauseUntil: githubRatePauseUntil(input.now, input.resetAt), - reason: `${input.owner}/${input.repo}: ${input.rateLimited ? 'rate limited' : `${input.remaining} requests remaining`}`, + reason: `${input.owner}/${input.repo}: ${input.rateLimited ? 'rate limited' : `${input.remaining} ${bucket}requests remaining`}`, } } diff --git a/layers/registry/server/utils/sync-prefetch.ts b/layers/registry/server/utils/sync-prefetch.ts new file mode 100644 index 00000000..ec59cf2a --- /dev/null +++ b/layers/registry/server/utils/sync-prefetch.ts @@ -0,0 +1,144 @@ +/// + +import type { GithubBindings, RepoSummary } from './github-client' +import type { ExistingRepo } from './sync-repo' +import { getRepoSummariesBatch } from './github-client' +import { resolveRepoSourceIdentityFromRow } from './repo-source-identity' +import { repoUnchangedStatements, unchangedRepoStatus } from './sync-repo' + +/** One repository the hourly sync is about to queue. */ +export interface SyncCandidate { + owner: string + repo: string + ownerVerified: boolean + claimDiscovery: boolean +} + +export type SyncPrefetchResult + = | { + _tag: 'prefetched' + /** Candidates that still need a sync job. */ + queue: SyncCandidate[] + /** Repositories recorded as unchanged without a job. */ + unchanged: number + /** GraphQL requests the prefetch sent. */ + requests: number + } + | { + _tag: 'unread' + /** Every candidate: the per-repository sync reads each one itself. */ + queue: SyncCandidate[] + reason: string + } + +export interface SyncPrefetchDependencies { + db: D1Database + bindings: GithubBindings + /** Unix seconds. */ + now: number +} + +/** A sync job that touched its progress row this recently may still hold it. */ +const PROGRESS_ACTIVE_SECONDS = 30 * 60 + +/** Repositories whose unchanged writes share one D1 batch. */ +const WRITE_BATCH_REPOSITORIES = 25 + +interface PrefetchRow extends ExistingRepo { + owner: string + repo: string + has_skills: number + in_progress: number +} + +/** + * Read every eligible candidate's summary in batched GraphQL queries, record + * the unchanged ones in place, and return the candidates that still need a job. + * + * Each candidate used to get a queue job whose first act was one GraphQL + * summary read. On 2026-10-06, 1,649 of 1,900 such jobs wrote no Skill row. + * A job that finds its tree unchanged only advances the freshness cursor, and + * the writes here are the same statements `syncRepo` runs on that path. + * + * A discovery claim, a job still holding the repository's progress row, a + * Skill-less repository and any repository GitHub gave no summary for keep + * their job: `syncRepo` owns those verdicts. + */ +export async function prefetchUnchangedRepos( + dependencies: SyncPrefetchDependencies, + candidates: SyncCandidate[], +): Promise { + const eligible = candidates.filter(candidate => !candidate.claimDiscovery) + if (eligible.length === 0) + return { _tag: 'prefetched', queue: candidates, unchanged: 0, requests: 0 } + + const rows = await loadPrefetchRows(dependencies.db, eligible, dependencies.now) + const requests = eligible.map(candidate => resolveRepoSourceIdentityFromRow(candidate, rows.get(repoKey(candidate)))) + const batch = await getRepoSummariesBatch(requests, dependencies.bindings) + if (batch._tag === 'failed') + return { _tag: 'unread', queue: candidates, reason: `graphql-${batch.status}` } + + const unchanged = new Set() + const writes: D1PreparedStatement[][] = [] + eligible.forEach((candidate, index) => { + const row = rows.get(repoKey(candidate)) + const summary = batch.summaries[index] + if (!row || !summary || row.in_progress === 1 || !isUnchanged(row, summary)) + return + unchanged.add(repoKey(candidate)) + writes.push(repoUnchangedStatements(dependencies.db, { + owner: candidate.owner, + repo: candidate.repo, + meta: summary.meta, + checkedAt: dependencies.now, + ownerVerified: candidate.ownerVerified, + })) + }) + for (let offset = 0; offset < writes.length; offset += WRITE_BATCH_REPOSITORIES) + await dependencies.db.batch(writes.slice(offset, offset + WRITE_BATCH_REPOSITORIES).flat()) + + return { + _tag: 'prefetched', + queue: candidates.filter(candidate => !unchanged.has(repoKey(candidate))), + unchanged: unchanged.size, + requests: batch.requests, + } +} + +function isUnchanged(row: PrefetchRow, summary: RepoSummary): boolean { + const pushedAt = Date.parse(summary.meta.pushed_at) + return unchangedRepoStatus({ + existing: row, + hasAdmittedSkills: row.has_skills === 1, + headTreeSha: summary.headTreeSha, + repoPushedAt: Number.isFinite(pushedAt) ? Math.floor(pushedAt / 1000) : null, + }) !== null +} + +function repoKey(identity: { owner: string, repo: string }): string { + return `${identity.owner}/${identity.repo}` +} + +/** One D1 read for every candidate: a JSON array binds as a single parameter. */ +async function loadPrefetchRows( + db: D1Database, + candidates: SyncCandidate[], + now: number, +): Promise> { + const pairs = JSON.stringify(candidates.map(candidate => [candidate.owner, candidate.repo])) + const result = await db.prepare( + `WITH candidates(owner, repo) AS ( + SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') + FROM json_each(?1) + ) + SELECT r.owner, r.repo, r.last_tree_sha, r.pushed_at, r.source_owner, r.source_repo, + EXISTS (SELECT 1 FROM skills s WHERE s.owner = r.owner AND s.repo = r.repo) AS has_skills, + EXISTS ( + SELECT 1 FROM repo_sync_progress p + WHERE p.owner = r.owner AND p.repo = r.repo AND p.updated_at > ?2 + ) AS in_progress + FROM candidates c + JOIN repos r ON r.owner = c.owner AND r.repo = c.repo`, + ).bind(pairs, now - PROGRESS_ACTIVE_SECONDS).all() + return new Map((result.results ?? []).map(row => [repoKey(row), row])) +} diff --git a/layers/registry/server/utils/sync-repo.ts b/layers/registry/server/utils/sync-repo.ts index f7844afa..72df3bfa 100644 --- a/layers/registry/server/utils/sync-repo.ts +++ b/layers/registry/server/utils/sync-repo.ts @@ -25,6 +25,8 @@ export interface SyncRepoStats { revisionsInserted: number activityEmitted: number rateLimitRemaining?: number + /** The GitHub bucket `rateLimitRemaining` counts: `core` or `graphql`. */ + rateLimitResource?: string rateLimitResetAt?: number continuation?: SyncRepoContinuation } @@ -52,7 +54,7 @@ interface ExistingSkillAssets { assets: string } -interface ExistingRepo { +export interface ExistingRepo { last_tree_sha: string | null pushed_at: number | null source_owner: string | null @@ -358,7 +360,7 @@ export async function refreshRepoAssets( SET source_owner = ?, source_repo = ? WHERE owner = ? AND repo = ? `).bind(sourceOwner, sourceRepo, owner, repo).run() - const treeRes = await getTree(sourceOwner, sourceRepo, branch, bindings) + const treeRes = await getTree(sourceOwner, sourceRepo, repoRes.data.headTreeSha ?? branch, bindings) logRateLimit(`asset-backfill tree ${owner}/${repo}`, treeRes.rateLimit) const rateLimitRemaining = Math.min( rate.rateLimitRemaining ?? Number.POSITIVE_INFINITY, @@ -516,18 +518,62 @@ async function repoHasAdmittedSkills(db: D1Database, owner: string, repo: string return row?.admitted === 1 } -async function markUnchangedOwnerVerified( +/** + * Why a repository needs no content sync this hour, or null when it does. + * + * Only a repository with admitted Skills qualifies: a Skill-less candidate is + * still being discovered, so its tree must be read whatever the cursor says. + */ +export function unchangedRepoStatus(input: { + existing: ExistingRepo | null + hasAdmittedSkills: boolean + headTreeSha: string | null + repoPushedAt: number | null +}): 'skipped-tree-sha' | 'skipped-pushed-at' | null { + const { existing, hasAdmittedSkills, headTreeSha, repoPushedAt } = input + if (!hasAdmittedSkills || !existing?.last_tree_sha) + return null + if (headTreeSha && existing.last_tree_sha === headTreeSha) + return 'skipped-tree-sha' + if (existing.pushed_at != null && repoPushedAt != null && existing.pushed_at >= repoPushedAt) + return 'skipped-pushed-at' + return null +} + +/** + * The writes that record an unchanged repository: fresh metadata, the + * freshness cursor, the day's star count and, for an owner-verified repository, + * the verified flag on every Skill. + */ +export function repoUnchangedStatements( db: D1Database, - owner: string, - repo: string, - meta: RepoMeta, - pushedAt: number | null, - checkedAt: number, -): Promise { - await db.batch([ + input: { + owner: string + repo: string + meta: RepoMeta + checkedAt: number + ownerVerified: boolean + }, +): D1PreparedStatement[] { + const { owner, repo, meta, checkedAt } = input + const pushedAt = epoch(meta.pushed_at) + const statements = [ markRepoSummaryCheckedStatement(db, owner, repo, meta, pushedAt, checkedAt), clearRepoMissingSkillsStatement(db, owner, repo, checkedAt), ...repoStarObservationStatements(db, owner, repo, meta.stargazers_count ?? 0, checkedAt), + ] + return input.ownerVerified + ? [...statements, ...ownerVerifiedStatements(db, owner, repo, checkedAt)] + : statements +} + +function ownerVerifiedStatements( + db: D1Database, + owner: string, + repo: string, + checkedAt: number, +): D1PreparedStatement[] { + return [ db.prepare( `UPDATE skills SET owner_verified = 1 @@ -542,7 +588,7 @@ async function markUnchangedOwnerVerified( queued_at = excluded.queued_at, attempts = 0`, ).bind(checkedAt, owner, repo), - ]) + ] } /** @@ -726,12 +772,13 @@ export async function syncRepo( revisionsInserted: 0, activityEmitted: 0, } - const trackRateLimit = (rateLimit: { remaining: number, reset: number } | null): void => { + const trackRateLimit = (rateLimit: { remaining: number, reset: number, resource: string | null } | null): void => { if (!rateLimit) return - stats.rateLimitRemaining = stats.rateLimitRemaining == null - ? rateLimit.remaining - : Math.min(stats.rateLimitRemaining, rateLimit.remaining) + if (stats.rateLimitRemaining == null || rateLimit.remaining < stats.rateLimitRemaining) { + stats.rateLimitRemaining = rateLimit.remaining + stats.rateLimitResource = rateLimit.resource ?? undefined + } stats.rateLimitResetAt = Math.max(stats.rateLimitResetAt ?? 0, rateLimit.reset) } @@ -775,39 +822,24 @@ export async function syncRepo( const hasAdmittedSkills = await repoHasAdmittedSkills(db, owner, repo) const markUnchanged = async (status: 'skipped-tree-sha' | 'skipped-pushed-at'): Promise => { - if (opts.ownerVerified) { - await markUnchangedOwnerVerified(db, owner, repo, meta, repoPushedAt, checkedAt) - stats.status = 'verified-only' - return stats - } - await db.batch([ - markRepoSummaryCheckedStatement(db, owner, repo, meta, repoPushedAt, checkedAt), - clearRepoMissingSkillsStatement(db, owner, repo, checkedAt), - ...repoStarObservationStatements(db, owner, repo, meta.stargazers_count ?? 0, checkedAt), - ]) - stats.status = status + await db.batch(repoUnchangedStatements(db, { owner, repo, meta, checkedAt, ownerVerified: opts.ownerVerified === true })) + stats.status = opts.ownerVerified ? 'verified-only' : status return stats } // GraphQL gave us the head tree SHA in the same request. If it matches // our cached value, the repo is unchanged and we skip the REST getTree // call entirely. Skill-less candidates deliberately bypass this cursor. - if (!opts.continuation && !opts.forceContent && hasAdmittedSkills && existingRepo?.last_tree_sha && headTreeSha && existingRepo.last_tree_sha === headTreeSha) - return markUnchanged('skipped-tree-sha') - - if ( - !opts.continuation - && !opts.forceContent - && hasAdmittedSkills - && existingRepo?.pushed_at != null - && existingRepo.last_tree_sha != null - && repoPushedAt != null - && existingRepo.pushed_at >= repoPushedAt - ) { - return markUnchanged('skipped-pushed-at') - } - - const treeRes = await getTree(sourceOwner, sourceRepo, branch, bindings) + const unchanged = opts.continuation || opts.forceContent + ? null + : unchangedRepoStatus({ existing: existingRepo, hasAdmittedSkills, headTreeSha, repoPushedAt }) + if (unchanged) + return markUnchanged(unchanged) + + // The head tree SHA names the exact tree the summary saw. A branch name + // can move between the two reads, and a SHA URL is immutable, so its ETag + // cache entry answers 304 for as long as the tree stands. + const treeRes = await getTree(sourceOwner, sourceRepo, headTreeSha ?? branch, bindings) logRateLimit(`tree ${owner}/${repo}`, treeRes.rateLimit) trackRateLimit(treeRes.rateLimit) diff --git a/layers/registry/server/utils/ungh-repo.ts b/layers/registry/server/utils/ungh-repo.ts new file mode 100644 index 00000000..2673261f --- /dev/null +++ b/layers/registry/server/utils/ungh-repo.ts @@ -0,0 +1,72 @@ +import type { RepoMeta } from './github-client' +import { z } from 'zod' +import { GITHUB_PAGE_READ_TIMEOUT_MS } from './github-client' + +const unghRepoSchema = z.object({ + repo: z.object({ + name: z.string().min(1), + repo: z.string().regex(/^[^/]+\/[^/]+$/), + description: z.string().nullable().optional(), + createdAt: z.string(), + pushedAt: z.string(), + stars: z.number().int().nonnegative(), + forks: z.number().int().nonnegative(), + defaultBranch: z.string().min(1), + }), +}) + +export type UnghRepoResult + = | { _tag: 'found', meta: RepoMeta } + | { _tag: 'unavailable', reason: string } + +/** + * Public repository metadata from ungh.cc, the unjs GitHub proxy that caches + * anonymous reads. + * + * A page-view fallback for when `GITHUB_TOKEN` is spent or GitHub is down. It + * answers metadata only. It never serves a tree or file content, and nothing + * built or signed reads it. ungh.cc knows no archived or fork flag, so both + * read false. Its 404 is no verdict about GitHub, so it reads as unavailable. + */ +export async function readUnghRepoMeta( + owner: string, + repo: string, + fetcher: typeof fetch = fetch, +): Promise { + const response = await fetcher(`https://ungh.cc/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, { + headers: { 'User-Agent': 'skilld.dev', 'Accept': 'application/json' }, + signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS), + }).catch((error: unknown) => ({ _tag: 'threw' as const, reason: error instanceof Error ? error.message : String(error) })) + if ('_tag' in response) + return { _tag: 'unavailable', reason: response.reason } + if (!response.ok) { + await response.body?.cancel() + return { _tag: 'unavailable', reason: `ungh ${response.status}` } + } + const parsed = unghRepoSchema.safeParse(await response.json().catch(() => { + // A body that is not JSON fails the schema below, which reports it. + return null + })) + if (!parsed.success) + return { _tag: 'unavailable', reason: 'ungh invalid response' } + const { repo: found } = parsed.data + const [login] = found.repo.split('/') as [string, string] + return { + _tag: 'found', + meta: { + name: found.name, + full_name: found.repo, + html_url: `https://github.com/${found.repo}`, + owner: { login }, + default_branch: found.defaultBranch, + description: found.description ?? null, + stargazers_count: found.stars, + forks_count: found.forks, + pushed_at: found.pushedAt, + created_at: found.createdAt, + archived: false, + fork: false, + private: false, + }, + } +} diff --git a/server/jobs/registry/repo-maintenance.ts b/server/jobs/registry/repo-maintenance.ts index dc8d16b9..1559f297 100644 --- a/server/jobs/registry/repo-maintenance.ts +++ b/server/jobs/registry/repo-maintenance.ts @@ -190,6 +190,7 @@ async function applyRateGuard( repo: string now: number remaining?: number + resource?: string resetAt?: number rateLimited: boolean unauthorized: boolean @@ -333,6 +334,7 @@ export async function handleRegistryRepoJob( repo: payload.repo, now, remaining: stats.rateLimitRemaining, + resource: stats.rateLimitResource, resetAt: stats.rateLimitResetAt, rateLimited: stats.status === 'rate-limited', unauthorized: stats.status === 'unauthorized', diff --git a/test/unit/github-conditional-read.test.ts b/test/unit/github-conditional-read.test.ts new file mode 100644 index 00000000..1734cab7 --- /dev/null +++ b/test/unit/github-conditional-read.test.ts @@ -0,0 +1,43 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getGithubJson } from '../../layers/registry/server/utils/github-client' + +/** A KV namespace over a Map: the ETag cache the client writes and reads. */ +function memoryKv(): KVNamespace { + const values = new Map() + return { + get: async (key: string, type?: string) => { + const value = values.get(key) + return value === undefined ? null : type === 'json' ? JSON.parse(value) : value + }, + put: async (key: string, value: string) => { + values.set(key, value) + }, + } as unknown as KVNamespace +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('getGithubJson', () => { + it('asks again with the stored ETag and answers a 304 from the cache', async () => { + const seen: Array<{ authorization: string | null, ifNoneMatch: string | null }> = [] + vi.stubGlobal('fetch', vi.fn(async (_url: string, init: RequestInit) => { + const headers = new Headers(init.headers) + seen.push({ authorization: headers.get('authorization'), ifNoneMatch: headers.get('if-none-match') }) + return headers.get('if-none-match') === '"v1"' + ? new Response(null, { status: 304 }) + : new Response(JSON.stringify({ login: 'nuxt', type: 'Organization' }), { status: 200, headers: { etag: '"v1"' } }) + })) + const bindings = { GITHUB_TOKEN: 'registry-token', KV_CACHE: memoryKv() } + + await getGithubJson('/users/nuxt', bindings) + const repeat = await getGithubJson<{ type: string }>('/users/nuxt', bindings) + + expect(repeat).toMatchObject({ status: 304, notModified: true, data: { type: 'Organization' } }) + expect(seen).toEqual([ + { authorization: 'Bearer registry-token', ifNoneMatch: null }, + { authorization: 'Bearer registry-token', ifNoneMatch: '"v1"' }, + ]) + }) +}) diff --git a/test/unit/github-repo-summary-batch.test.ts b/test/unit/github-repo-summary-batch.test.ts new file mode 100644 index 00000000..e300de3b --- /dev/null +++ b/test/unit/github-repo-summary-batch.test.ts @@ -0,0 +1,99 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getRepoSummariesBatch } from '../../layers/registry/server/utils/github-client' + +const bindings = { GITHUB_TOKEN: 'test-token' } + +interface GraphqlBody { + query: string + variables: Record +} + +function repository(owner: string, name: string, tree: string) { + return { + name, + nameWithOwner: `${owner}/${name}`, + url: `https://github.com/${owner}/${name}`, + owner: { login: owner }, + description: null, + stargazerCount: 3, + forkCount: 1, + pushedAt: '2026-10-01T00:00:00Z', + createdAt: '2025-01-01T00:00:00Z', + isArchived: false, + isFork: false, + defaultBranchRef: { name: 'main', target: { oid: 'c'.repeat(40), tree: { oid: tree } } }, + } +} + +/** Answers every alias, except owners named `gone`, which GitHub reports NOT_FOUND. */ +function graphqlAnswer(_input: unknown, init?: RequestInit): Response { + const body = JSON.parse(String(init?.body)) as GraphqlBody + const data: Record = {} + const errors: unknown[] = [] + for (const key of Object.keys(body.variables).filter(name => name.startsWith('o'))) { + const index = key.slice(1) + const owner = body.variables[key]! + const name = body.variables[`n${index}`]! + if (owner === 'gone') { + data[`r${index}`] = null + errors.push({ type: 'NOT_FOUND', path: [`r${index}`], message: 'Could not resolve' }) + } + else { + data[`r${index}`] = repository(owner, name, `tree-${owner}-${name}`) + } + } + return new Response(JSON.stringify({ data, errors: errors.length ? errors : undefined }), { + status: 200, + headers: { 'x-ratelimit-remaining': '4990', 'x-ratelimit-resource': 'graphql' }, + }) +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('getRepoSummariesBatch', () => { + it('reads 230 repositories in three queries, in request order', async () => { + const fetchMock = vi.fn(graphqlAnswer) + vi.stubGlobal('fetch', fetchMock) + const requests = Array.from({ length: 230 }, (_, i) => ({ owner: 'acme', repo: `repo-${i}` })) + + const result = await getRepoSummariesBatch(requests, bindings) + + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(result._tag).toBe('read') + if (result._tag !== 'read') + return + expect(result.summaries).toHaveLength(230) + expect(result.summaries[229]?.headTreeSha).toBe('tree-acme-repo-229') + expect(result.summaries[0]?.meta.full_name).toBe('acme/repo-0') + expect(result.rateLimit?.resource).toBe('graphql') + }) + + it('leaves a repository GitHub cannot find empty and keeps the rest', async () => { + vi.stubGlobal('fetch', vi.fn(graphqlAnswer)) + + const result = await getRepoSummariesBatch([ + { owner: 'acme', repo: 'one' }, + { owner: 'gone', repo: 'two' }, + { owner: 'acme', repo: 'three' }, + ], bindings) + + expect(result._tag === 'read' && result.summaries.map(summary => summary?.headTreeSha ?? null)).toEqual([ + 'tree-acme-one', + null, + 'tree-acme-three', + ]) + }) + + it('fails the whole batch on an error that is not NOT_FOUND', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ + data: null, + errors: [{ type: 'RATE_LIMITED', message: 'API rate limit exceeded' }], + }), { status: 200 }))) + + const result = await getRepoSummariesBatch([{ owner: 'acme', repo: 'one' }], bindings) + + expect(result).toMatchObject({ _tag: 'failed', status: 502 }) + }) +}) diff --git a/test/unit/github-sync-control.test.ts b/test/unit/github-sync-control.test.ts index 42cc26f3..114ecded 100644 --- a/test/unit/github-sync-control.test.ts +++ b/test/unit/github-sync-control.test.ts @@ -41,4 +41,20 @@ describe('github sync pause decision', () => { remaining: 200, })).toEqual({ _tag: 'continue' }) }) + + it('names the GitHub bucket that ran low', () => { + expect(githubSyncPauseDecision({ + now: 1_000, + owner: 'nuxt', + repo: 'ui', + unauthorized: false, + rateLimited: false, + remaining: 163, + resource: 'graphql', + resetAt: 2_000, + })).toMatchObject({ + _tag: 'pause', + reason: 'nuxt/ui: 163 graphql requests remaining', + }) + }) }) diff --git a/test/unit/repo-source-profile-quota.test.ts b/test/unit/repo-source-profile-quota.test.ts new file mode 100644 index 00000000..c1597be9 --- /dev/null +++ b/test/unit/repo-source-profile-quota.test.ts @@ -0,0 +1,109 @@ +import type { H3Event } from 'h3' +import type { SqliteD1 } from './helpers/d1-sqlite' +import { createStorage } from 'unstorage' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' + +const route = { owner: 'acme', repo: 'skills' } +let edgeCache = createStorage() +let harness: SqliteD1 + +function stubRuntime(): void { + vi.stubGlobal('defineCachedEventHandler', (handler: unknown) => handler) + vi.stubGlobal('createError', (input: Record) => Object.assign(new Error(String(input.message)), input)) + vi.stubGlobal('getRouterParam', (_event: unknown, key: 'owner' | 'repo') => route[key]) + vi.stubGlobal('useStorage', () => edgeCache) +} + +const TREE_SHA = 'a'.repeat(40) + +function repoMeta(overrides: Record = {}) { + return { + name: 'skills', + full_name: 'acme/skills', + html_url: 'https://github.com/acme/skills', + owner: { login: 'acme' }, + default_branch: 'main', + description: 'Skills', + stargazers_count: 12, + forks_count: 3, + pushed_at: '2026-10-01T00:00:00Z', + created_at: '2025-01-01T00:00:00Z', + private: false, + ...overrides, + } +} + +function json(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { status: 200, ...init }) +} + +async function profile(): Promise> { + const handler = (await import('../../layers/registry/server/api/repos/[owner]/[repo].get')).default as (event: H3Event) => Promise> + return handler({ context: { platform: { db: harness.db, env: { GITHUB_TOKEN: 'token' } } } } as unknown as H3Event) +} + +function calls(fetchMock: ReturnType, fragment: string): number { + return fetchMock.mock.calls.filter(([input]) => String(input).includes(fragment)).length +} + +beforeEach(() => { + edgeCache = createStorage() + harness = createSqliteD1(allMigrations()) + route.repo = 'skills' + stubRuntime() +}) + +// The global wide event stubs come from the setup file, so this file never +// calls `vi.unstubAllGlobals()`: each test replaces the stubs it needs. +afterEach(() => { + harness.close() +}) + +describe('repository source profile GitHub spend', () => { + it('answers 404 for a private repository and never lists its tree', async () => { + const fetchMock = vi.fn(async () => json(repoMeta({ private: true }))) + vi.stubGlobal('fetch', fetchMock) + + await expect(profile()).rejects.toMatchObject({ statusCode: 404 }) + expect(calls(fetchMock, '/git/trees/')).toBe(0) + }) + + it('remembers a repository GitHub cannot find, so a repeat view reads nothing', async () => { + route.repo = 'gone' + const fetchMock = vi.fn(async () => json({ message: 'Not Found' }, { status: 404 })) + vi.stubGlobal('fetch', fetchMock) + + await expect(profile()).rejects.toMatchObject({ statusCode: 404 }) + await expect(profile()).rejects.toMatchObject({ statusCode: 404 }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('reads repository metadata from ungh.cc when GitHub refuses with a spent quota', async () => { + const fetchMock = vi.fn(async (input: unknown) => String(input).startsWith('https://ungh.cc/') + ? json({ repo: { id: 1, name: 'skills', repo: 'acme/skills', description: 'From ungh', createdAt: '2025-01-01T00:00:00Z', pushedAt: '2026-10-01T00:00:00Z', stars: 99, forks: 4, defaultBranch: 'trunk' } }) + : json({ message: 'API rate limit exceeded' }, { status: 403, headers: { 'x-ratelimit-remaining': '0' } })) + vi.stubGlobal('fetch', fetchMock) + + const body = await profile() + + expect(body).toMatchObject({ owner: 'acme', repo: 'skills', stars: 99, description: 'From ungh', defaultBranch: 'trunk', skillFileScanStatus: 'unavailable' }) + expect(calls(fetchMock, '/git/trees/')).toBe(0) + }) + + it('lists a registry repository at its synced tree, once per tree', async () => { + harness.raw.prepare(`INSERT INTO repos (owner, repo, last_tree_sha) VALUES ('acme', 'skills', ?)`).run(TREE_SHA) + const fetchMock = vi.fn(async (input: unknown) => String(input).includes('/git/trees/') + ? json({ sha: TREE_SHA, tree: [{ path: 'skills/one/SKILL.md', type: 'blob', sha: 'b'.repeat(40) }] }) + : json(repoMeta())) + vi.stubGlobal('fetch', fetchMock) + + const first = await profile() + const second = await profile() + + expect(first).toMatchObject({ skillFileScanStatus: 'ok', skillFiles: ['skills/one/SKILL.md'] }) + expect(second).toMatchObject({ skillFiles: ['skills/one/SKILL.md'] }) + expect(fetchMock.mock.calls.filter(([input]) => String(input).includes(`/git/trees/${TREE_SHA}`))).toHaveLength(1) + }) +}) diff --git a/test/unit/repo-source-profile-timeout.test.ts b/test/unit/repo-source-profile-timeout.test.ts index 1e6f4baf..4458ecf6 100644 --- a/test/unit/repo-source-profile-timeout.test.ts +++ b/test/unit/repo-source-profile-timeout.test.ts @@ -1,16 +1,19 @@ import type { H3Event } from 'h3' +import { createStorage } from 'unstorage' import { afterEach, describe, expect, it, vi } from 'vitest' import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' vi.stubGlobal('defineCachedEventHandler', (handler: unknown) => handler) vi.stubGlobal('createError', (input: Record) => Object.assign(new Error(String(input.message)), input)) vi.stubGlobal('getRouterParam', (_event: unknown, key: string) => (key === 'owner' ? 'skilld-dev' : 'skills')) +vi.stubGlobal('useStorage', () => createStorage()) afterEach(() => { vi.unstubAllGlobals() vi.stubGlobal('defineCachedEventHandler', (handler: unknown) => handler) vi.stubGlobal('createError', (input: Record) => Object.assign(new Error(String(input.message)), input)) vi.stubGlobal('getRouterParam', (_event: unknown, key: string) => (key === 'owner' ? 'skilld-dev' : 'skills')) + vi.stubGlobal('useStorage', () => createStorage()) }) async function profile() { diff --git a/test/unit/skill-detail-d1-cost.test.ts b/test/unit/skill-detail-d1-cost.test.ts index 152ef08e..eee0d577 100644 --- a/test/unit/skill-detail-d1-cost.test.ts +++ b/test/unit/skill-detail-d1-cost.test.ts @@ -236,17 +236,17 @@ describe('findSkill', () => { describe('uncached skill related render', () => { it('reads D1 at most three times and keeps the commit source', async () => { seed({ maximumQueries: 3 }) - upstreamFetch.mockImplementationOnce(async () => [{ + rawFetch.mockImplementationOnce(async () => new Response(JSON.stringify([{ sha: 'abcdef1234567', html_url: 'https://github.com/c/abcdef1', commit: { message: 'edit alpha', author: { name: 'Eric', date: '2026-09-01T00:00:00Z' } }, author: null, - }] as never) + }]))) const handler = (await import('../../layers/registry/server/api/skill-related/[...slug].get')).default as (event: H3Event) => Promise> const body = await handler({ context: { platform: { db: harness.db, env: {} } }, node: { req: { headers: {} } } } as unknown as H3Event) - expect(upstreamFetch).toHaveBeenCalledWith(`https://api.github.com/repos/${OWNER}/${REPO}/commits`, expect.objectContaining({ query: { path: 'skills/alpha/SKILL.md', per_page: 5 } })) + expect(String(rawFetch.mock.calls[0]?.[0])).toBe(`https://api.github.com/repos/${OWNER}/${REPO}/commits?path=skills%2Falpha%2FSKILL.md&per_page=5`) expect(body.commits.map((commit: { shortSha: string }) => commit.shortSha)).toEqual(['abcdef1']) expect(body.relatedRepoSkills.map((skill: { name: string }) => skill.name)).toEqual(['beta']) }) diff --git a/test/unit/sync-prefetch.test.ts b/test/unit/sync-prefetch.test.ts new file mode 100644 index 00000000..a8d0f640 --- /dev/null +++ b/test/unit/sync-prefetch.test.ts @@ -0,0 +1,113 @@ +import type { SqliteD1 } from './helpers/d1-sqlite' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { prefetchUnchangedRepos } from '../../layers/registry/server/utils/sync-prefetch' +import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' + +const NOW = 1_791_300_000 +const bindings = { GITHUB_TOKEN: 'test-token' } + +/** The head tree GitHub reports for each repository. */ +const heads: Record = { + 'acme/unchanged': 'tree-same', + 'acme/changed': 'tree-new', + 'acme/fresh': 'tree-fresh', + 'acme/busy': 'tree-busy', + 'acme/discovered': 'tree-discovered', +} + +function graphqlAnswer(_input: unknown, init?: RequestInit): Response { + const body = JSON.parse(String(init?.body)) as { variables: Record } + const data: Record = {} + const errors: unknown[] = [] + for (const key of Object.keys(body.variables).filter(name => name.startsWith('o'))) { + const index = key.slice(1) + const fullName = `${body.variables[key]}/${body.variables[`n${index}`]}` + const tree = heads[fullName] + if (!tree) { + data[`r${index}`] = null + errors.push({ type: 'NOT_FOUND', path: [`r${index}`] }) + continue + } + const [owner, name] = fullName.split('/') as [string, string] + data[`r${index}`] = { + name, + nameWithOwner: fullName, + url: `https://github.com/${fullName}`, + owner: { login: owner }, + description: 'Skills', + stargazerCount: 77, + forkCount: 2, + pushedAt: '2026-10-01T00:00:00Z', + createdAt: '2025-01-01T00:00:00Z', + isArchived: false, + isFork: false, + defaultBranchRef: { name: 'main', target: { oid: 'c'.repeat(40), tree: { oid: tree } } }, + } + } + return new Response(JSON.stringify({ data, errors: errors.length ? errors : undefined }), { status: 200 }) +} + +function candidate(repo: string, claimDiscovery = false, owner = 'acme') { + return { owner, repo, ownerVerified: false, claimDiscovery } +} + +describe('prefetchUnchangedRepos', () => { + let harness: SqliteD1 + + beforeEach(() => { + harness = createSqliteD1(allMigrations()) + for (const [repo, tree] of [['unchanged', 'tree-same'], ['changed', 'tree-old'], ['fresh', null], ['busy', 'tree-busy'], ['discovered', 'tree-discovered']] as const) { + harness.raw.prepare(`INSERT INTO repos (owner, repo, last_tree_sha, pushed_at, stars) VALUES ('acme', ?, ?, 100, 1)`).run(repo, tree) + if (repo !== 'fresh') + harness.raw.prepare(`INSERT INTO skills (owner, repo, name, display_name, slug) VALUES ('acme', ?, 'one', 'One', 'one')`).run(repo) + } + harness.raw.prepare(`INSERT INTO repo_sync_progress (owner, repo, job_id, tree_sha, checked_at, next_offset, updated_at) VALUES ('acme', 'busy', 'job-1', 'tree-busy', ?, 250, ?)`).run(NOW - 60, NOW - 60) + }) + + afterEach(() => { + harness.close() + vi.unstubAllGlobals() + }) + + it('records an unchanged repository in place and queues only the ones that need a job', async () => { + const fetchMock = vi.fn(graphqlAnswer) + vi.stubGlobal('fetch', fetchMock) + + const result = await prefetchUnchangedRepos({ db: harness.db, bindings, now: NOW }, [ + candidate('unchanged'), + candidate('changed'), + candidate('fresh'), + candidate('busy'), + candidate('discovered', true), + candidate('missing', false, 'gone'), + ]) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(result.queue.map(item => `${item.owner}/${item.repo}`)).toEqual([ + 'acme/changed', + 'acme/fresh', + 'acme/busy', + 'acme/discovered', + 'gone/missing', + ]) + expect(result).toMatchObject({ _tag: 'prefetched', unchanged: 1 }) + expect(harness.raw.prepare(`SELECT stars, repo_meta_synced_at FROM repos WHERE owner = 'acme' AND repo = 'unchanged'`).get()) + .toEqual({ stars: 77, repo_meta_synced_at: NOW }) + expect(harness.raw.prepare(`SELECT repo_meta_synced_at FROM repos WHERE owner = 'acme' AND repo = 'changed'`).get()) + .toEqual({ repo_meta_synced_at: null }) + }) + + it('queues every candidate when GitHub does not answer the batch', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('bad gateway', { status: 502 }))) + + const result = await prefetchUnchangedRepos({ db: harness.db, bindings, now: NOW }, [ + candidate('unchanged'), + candidate('changed'), + ]) + + expect(result).toMatchObject({ _tag: 'unread', reason: 'graphql-502' }) + expect(result.queue).toHaveLength(2) + expect(harness.raw.prepare(`SELECT repo_meta_synced_at FROM repos WHERE owner = 'acme' AND repo = 'unchanged'`).get()) + .toEqual({ repo_meta_synced_at: null }) + }) +}) diff --git a/test/unit/sync-repo-correctness.test.ts b/test/unit/sync-repo-correctness.test.ts index cb5edc7d..42aa1138 100644 --- a/test/unit/sync-repo-correctness.test.ts +++ b/test/unit/sync-repo-correctness.test.ts @@ -136,7 +136,7 @@ describe('syncRepo content acknowledgement', () => { const result = await syncRepo('steipete', 'clawdis', {}, db, { ownerVerified: true }) expect(result).toMatchObject({ owner: 'steipete', repo: 'clawdis', status: 'indexed' }) - expect(github.getTree).toHaveBeenCalledWith('openclaw', 'openclaw', 'main', {}) + expect(github.getTree).toHaveBeenCalledWith('openclaw', 'openclaw', 'new-tree', {}) expect(github.getBlobsBatch).toHaveBeenCalledWith( 'openclaw', 'openclaw', diff --git a/test/unit/sync-repo-freshness.test.ts b/test/unit/sync-repo-freshness.test.ts index 966c06e2..8771d110 100644 --- a/test/unit/sync-repo-freshness.test.ts +++ b/test/unit/sync-repo-freshness.test.ts @@ -215,6 +215,7 @@ describe('syncRepo freshness cursor', () => { FROM repos WHERE owner = 'acme' AND repo = 'skills' `).get() + expect(github.getTree).toHaveBeenCalledWith('acme', 'skills', 'moved-tree', {}) expect(result.status).toBe('failed') expect(result.reason).toBe('tree_truncated') expect(row).toEqual({