fix(registry): spend less GitHub quota on the sync and page views

layers/registry/server/api/orgs/[owner].get.ts

@@ -1,11 +1,12 @@
 import type { H3Event } from 'h3'
 import type { TagPayload } from '../../jobs/generate-tags'
+import type { GithubBindings } from '../../utils/github-client'
 import type { RegistrySkill } from '../../utils/skills-registry'
 import { getDB } from '#server/utils/db'
 import { cached, readCache, writeCache } from '#shared/server/cache'
 import { officialRepos } from '../../data/official-repos'
 import { TAG_BY_SLUG } from '../../jobs/taxonomy'
-import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client'
+import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client'
 import { resolveRepoSourceIdentitiesForOwner } from '../../utils/repo-source-identity'
 import { getGeneratedBatch } from '../../utils/skill-generated'
 import { querySkills } from '../../utils/skills-registry'
@@ -68,19 +69,27 @@ const ORG_PROFILE_CACHE_STALE_TTL = 60 * 5
 const officialOwners = new Set(officialRepos.map(r => r.owner))
 const kindByOwner = new Map(officialRepos.map(r => [r.owner, r.kind]))
 
-async function fetchAndStoreOwner(owner: string, db: D1Database): Promise<OwnerRow | null> {
-  const res = await fetch(`https://api.github.com/users/${owner}`, {
-    signal: AbortSignal.timeout(GITHUB_PAGE_READ_TIMEOUT_MS),
-    headers: {
-      'User-Agent': 'skilld.dev',
-      'Accept': 'application/vnd.github+json',
-    },
-  }).catch(() => {
-    emitOperationalEvent(createWideEvent({ operation: 'org-profile-fetch', outcome: 'failed' }))
-    return null
-  })
+interface GithubOwnerResponse {
+  name?: string
+  bio?: string
+  blog?: string
+  location?: string
+  followers?: number
+  public_repos?: number
+  type?: string
+}
+
+async function fetchAndStoreOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise<OwnerRow | null> {
+  // Authenticated and conditional: an unchanged profile answers 304 from
+  // the ETag cache for free. Anonymous, this read shared the Worker IP's 60
+  // requests an hour with every other Cloudflare tenant, and failed.
+  const res = await getGithubJson<GithubOwnerResponse>(`/users/${encodeURIComponent(owner)}`, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS })
+    .catch(() => {
+      emitOperationalEvent(createWideEvent({ operation: 'org-profile-fetch', outcome: 'failed' }))
+      return null
+    })
 
-  if (!res || !res.ok) {
+  if (!res?.data) {
     if (res?.status === 404) {
       await db
         .prepare(
@@ -93,16 +102,7 @@ async function fetchAndStoreOwner(owner: string, db: D1Database): Promise<OwnerR
     return null
   }
 
-  const data = await res.json() as {
-    name?: string
-    bio?: string
-    blog?: string
-    location?: string
-    followers?: number
-    public_repos?: number
-    type?: string
-  }
-
+  const data = res.data
   const kind: 'user' | 'org' = data.type === 'Organization' ? 'org' : 'user'
   const row: OwnerRow = {
     kind,
@@ -130,7 +130,7 @@ async function fetchAndStoreOwner(owner: string, db: D1Database): Promise<OwnerR
   return row
 }
 
-async function loadOwner(owner: string, db: D1Database): Promise<OwnerRow | null> {
+async function loadOwner(owner: string, db: D1Database, bindings: GithubBindings): Promise<OwnerRow | null> {
   const cached = await db
     .prepare(
       `SELECT kind, name, bio, blog, location, followers, last_synced_at, sync_status
@@ -145,7 +145,7 @@ async function loadOwner(owner: string, db: D1Database): Promise<OwnerRow | null
   if (cached && fresh)
     return cached.sync_status === '404' ? null : cached
 
-  const fetched = await fetchAndStoreOwner(owner, db)
+  const fetched = await fetchAndStoreOwner(owner, db, bindings)
   return fetched ?? cached ?? null
 }
 
@@ -185,7 +185,7 @@ async function loadOrgProfile(event: H3Event, owner: string): Promise<OrgProfile
     throw createError({ statusCode: 404, message: `No skills found for @${owner}` })
   }
 
-  const ownerRow = await loadOwner(owner, db)
+  const ownerRow = await loadOwner(owner, db, resolveGithubBindings(event.context.platform.env))
 
   const manifestKind = kindByOwner.get(owner)
   const kind: OrgKind = manifestKind

layers/registry/server/api/repos/[owner]/[repo].get.ts

@@ -1,9 +1,11 @@
-import type { FetchOutcome } from '../../../utils/github-client'
+import type { FetchOutcome, RepoMeta } from '../../../utils/github-client'
 import { getDB } from '#server/utils/db'
+import { readCache, writeCache } from '#shared/server/cache'
 import { isRegistrySkillPath } from '#shared/skill-path'
-import { getRepo, getTree, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../../utils/github-client'
-import { resolveRepoSourceIdentity } from '../../../utils/repo-source-identity'
+import { getRepo, getTree, GITHUB_PAGE_READ_TIMEOUT_MS, githubRefused, resolveGithubBindings } from '../../../utils/github-client'
+import { resolveRepoSourceIdentityFromRow } from '../../../utils/repo-source-identity'
 import { buildUnavailableRepoSourceProfile } from '../../../utils/repo-source-profile'
+import { readUnghRepoMeta } from '../../../utils/ungh-repo'
 
 export interface RepoSourceProfile {
   owner: string
@@ -22,6 +24,21 @@ export interface RepoSourceProfile {
   skillFiles: string[]
 }
 
+interface SkillFileScan {
+  status: 'ok' | 'truncated'
+  skillFiles: string[]
+}
+
+/**
+ * How long a repository GitHub could not find answers 404 without asking
+ * again. Crawlers retry dead `/gh` links: about 630 such reads a day each
+ * spent one request of the shared `GITHUB_TOKEN` quota (2026-10-06).
+ */
+const MISSING_REPO_TTL_SECONDS = 60 * 60
+
+/** A tree SHA names immutable content, so its scan can live as long as the cache keeps it. */
+const TREE_SCAN_TTL_SECONDS = 7 * 24 * 60 * 60
+
 function githubReadFailed(operation: string, error: unknown): FetchOutcome<never> {
   emitOperationalEvent(createWideEvent({
     operation,
@@ -31,6 +48,44 @@ function githubReadFailed(operation: string, error: unknown): FetchOutcome<never
   return { status: 0, data: null, rateLimit: null, notModified: false }
 }
 
+function missingRepoKey(owner: string, repo: string): string {
+  return `repo-source-missing:v1:${owner}/${repo}`
+}
+
+function treeScanKey(owner: string, repo: string, treeSha: string): string {
+  return `repo-source-tree:v1:${owner}/${repo}/${treeSha}`
+}
+
+function scanTree(tree: { truncated?: boolean, tree: Array<{ path: string, type: string }> }): SkillFileScan {
+  return {
+    status: tree.truncated ? 'truncated' : 'ok',
+    skillFiles: tree.tree
+      .filter(entry => entry.type === 'blob' && isRegistrySkillPath(entry.path))
+      .map(entry => entry.path)
+      .sort(),
+  }
+}
+
+function profileFrom(meta: RepoMeta, scan: SkillFileScan | null): RepoSourceProfile {
+  const skillFiles = scan?.skillFiles ?? []
+  return {
+    owner: meta.owner.login,
+    repo: meta.name,
+    description: meta.description,
+    githubUrl: meta.html_url || `https://github.com/${meta.owner.login}/${meta.name}`,
+    defaultBranch: meta.default_branch,
+    stars: meta.stargazers_count,
+    forks: meta.forks_count,
+    pushedAt: meta.pushed_at,
+    createdAt: meta.created_at,
+    archived: Boolean(meta.archived),
+    fork: Boolean(meta.fork),
+    skillFileScanStatus: scan?.status ?? 'unavailable',
+    skillFileCount: skillFiles.length,
+    skillFiles,
+  }
+}
+
 export default defineCachedEventHandler(async (event) => {
   const ownerParam = getRouterParam(event, 'owner')
   const repoParam = getRouterParam(event, 'repo')
@@ -39,55 +94,61 @@ export default defineCachedEventHandler(async (event) => {
 
   const owner = ownerParam.toLowerCase()
   const repo = repoParam.toLowerCase()
+  const edgeCache = useStorage('edge-cache')
+  if (await readCache<true>(edgeCache, missingRepoKey(owner, repo)))
+    throw createError({ statusCode: 404, message: 'Repository not found' })
+
   const db = getDB(event)
-  const source = await resolveRepoSourceIdentity(db, { owner, repo })
+  const row = await db
+    .prepare(`SELECT source_owner, source_repo, last_tree_sha FROM repos WHERE owner = ? AND repo = ?`)
+    .bind(owner, repo)
+    .first<{ source_owner: string | null, source_repo: string | null, last_tree_sha: string | null }>()
+  const source = resolveRepoSourceIdentityFromRow({ owner, repo }, row)
   const bindings = resolveGithubBindings(event.context.platform.env)
 
   // A read that timed out or dropped is an outage, not a missing repository.
   const repoRes = await getRepo(source.owner, source.repo, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS })
     .catch((error: unknown) => githubReadFailed('repo-source-profile-repo', error))
-  if (repoRes.status === 404)
+  // The token can read private repositories, and this page is public, so a
+  // private one answers exactly like one that does not exist.
+  if (repoRes.status === 404 || repoRes.data?.private) {
+    await writeCache(edgeCache, missingRepoKey(owner, repo), true, { ttl: MISSING_REPO_TTL_SECONDS })
     throw createError({ statusCode: 404, message: 'Repository not found' })
+  }
   if (!repoRes.data) {
     emitOperationalEvent(createWideEvent({
       'operation': 'repo-source-profile',
       'outcome': 'unavailable',
       'upstream.status': repoRes.status || null,
     }))
+    // Metadata from ungh.cc keeps the page whole while the shared quota is
+    // spent. The Skill file list needs the tree, which only GitHub serves.
+    if (githubRefused(repoRes)) {
+      const fallback = await readUnghRepoMeta(source.owner, source.repo)
+      if (fallback._tag === 'found')
+        return profileFrom(fallback.meta, null)
+      emitOperationalEvent(createWideEvent({ operation: 'repo-source-profile-ungh', outcome: 'unavailable', reason: fallback.reason }))
+    }
     return buildUnavailableRepoSourceProfile(owner, repo) satisfies RepoSourceProfile
   }
 
   const meta = repoRes.data
-  const repoOwner = meta.owner.login
-  const repoName = meta.name
-  const treeRes = await getTree(repoOwner, repoName, meta.default_branch, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS })
-    .catch((error: unknown) => githubReadFailed('repo-source-profile-tree', error))
-  const skillFiles = (treeRes.data?.tree ?? [])
-    .filter(entry => entry.type === 'blob' && isRegistrySkillPath(entry.path))
-    .map(entry => entry.path)
-    .sort()
-  const skillFileScanStatus = !treeRes.data
-    ? 'unavailable'
-    : treeRes.data.truncated
-      ? 'truncated'
-      : 'ok'
+  // The sync records the tree it last read. Read at that SHA, the listing
+  // never changes, so one scan serves every later view. A repository the
+  // registry has not synced is read at its default branch.
+  const treeSha = row?.last_tree_sha ?? null
+  const cachedScan = treeSha
+    ? await readCache<SkillFileScan>(edgeCache, treeScanKey(meta.owner.login, meta.name, treeSha))
+    : null
+  if (cachedScan)
+    return profileFrom(meta, cachedScan) satisfies RepoSourceProfile
 
-  return {
-    owner: repoOwner,
-    repo: repoName,
-    description: meta.description,
-    githubUrl: meta.html_url || `https://github.com/${repoOwner}/${repoName}`,
-    defaultBranch: meta.default_branch,
-    stars: meta.stargazers_count,
-    forks: meta.forks_count,
-    pushedAt: meta.pushed_at,
-    createdAt: meta.created_at,
-    archived: Boolean(meta.archived),
-    fork: Boolean(meta.fork),
-    skillFileScanStatus,
-    skillFileCount: skillFiles.length,
-    skillFiles,
-  } satisfies RepoSourceProfile
+  const treeRes = await getTree(meta.owner.login, meta.name, treeSha ?? meta.default_branch, bindings, { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS })
+    .catch((error: unknown) => githubReadFailed('repo-source-profile-tree', error))
+  const scan = treeRes.data ? scanTree(treeRes.data) : null
+  if (scan && treeSha)
+    await writeCache(edgeCache, treeScanKey(meta.owner.login, meta.name, treeSha), scan, { ttl: TREE_SCAN_TTL_SECONDS })
+  return profileFrom(meta, scan) satisfies RepoSourceProfile
 }, {
   maxAge: 60 * 15,
   swr: true,

layers/registry/server/api/skill-related/[...slug].get.ts

@@ -1,12 +1,13 @@
 import type { H3Event } from 'h3'
 import type { EmbeddingNeighbor } from '../../jobs/generate-embeddings'
+import type { GithubBindings } from '../../utils/github-client'
 import type { CoOccurrenceNeighbor } from '../../utils/skill-co-occurrence'
 import type { SkillCommitSourceRow } from '../../utils/skill-commit-source'
 import type { CachedRelated } from '../../utils/skill-related'
 import { readCache, readThroughCache, writeCache } from '#shared/server/cache'
 import { defineApiHandler } from '#shared/server/handler'
 import { getEmbeddingNeighbors } from '../../jobs/generate-embeddings'
-import { GITHUB_PAGE_READ_TIMEOUT_MS } from '../../utils/github-client'
+import { getGithubJson, GITHUB_PAGE_READ_TIMEOUT_MS, resolveGithubBindings } from '../../utils/github-client'
 import { getCoOccurrenceNeighbors } from '../../utils/skill-co-occurrence'
 import { skillCommitSourceFromRow } from '../../utils/skill-commit-source'
 import {
@@ -76,7 +77,7 @@ export default defineApiHandler({
 
         const [commits, related, coOccurrenceNeighbors, embeddingNeighbors] = await Promise.all([
           source
-            ? getSkillCommits(source.owner, source.repo, source.path)
+            ? getSkillCommits(source.owner, source.repo, source.path, resolveGithubBindings(platform.env))
             : Promise.resolve([]),
           findRelatedSkills(event, { owner: skill.owner, repo: skill.repo, excludeName: skill.name, limit: 6 }),
           getCoOccurrenceNeighbors(platform.db, skill.name),
@@ -166,26 +167,32 @@ interface GhCommitResponse {
   author: { login: string, avatar_url: string } | null
 }
 
-async function getSkillCommits(owner: string, repo: string, path: string): Promise<SkillCommit[]> {
+async function getSkillCommits(owner: string, repo: string, path: string, bindings: GithubBindings): Promise<SkillCommit[]> {
   const cacheKey = `skills:commits:v2:${owner}/${repo}:${path}`
   const cached = await readCache<SkillCommit[]>(useStorage('edge-cache'), cacheKey)
   if (cached)
     return cached
 
-  const data = await $fetch<GhCommitResponse[]>(`https://api.github.com/repos/${owner}/${repo}/commits`, {
-    query: { path, per_page: 5 },
-    headers: { 'Accept': 'application/vnd.github+json', 'User-Agent': 'skilld.dev' },
-    // ofetch retries a GET once on its own. The commit list is optional
-    // decoration, so a slow GitHub costs the list, never the page.
-    retry: 0,
-    timeout: GITHUB_PAGE_READ_TIMEOUT_MS,
-  }).catch(() => {
-    emitOperationalEvent(createWideEvent({ operation: 'skill-related-commits-fetch', outcome: 'failed' }))
+  // Authenticated and conditional: an unchanged history answers 304 from the
+  // ETag cache for free. Anonymous, this read shared the Worker IP's 60
+  // requests an hour with every other Cloudflare tenant, and about 330 a day
+  // failed (2026-10-06). The commit list is optional decoration, so a slow
+  // GitHub costs the list, never the page.
+  const query = new URLSearchParams({ path, per_page: '5' })
+  const response = await getGithubJson<GhCommitResponse[]>(
+    `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits?${query.toString()}`,
+    bindings,
+    { timeoutMs: GITHUB_PAGE_READ_TIMEOUT_MS },
+  ).catch((error: unknown) => {
+    emitOperationalEvent(createWideEvent({ operation: 'skill-related-commits-fetch', outcome: 'failed', reason: error instanceof Error ? error.message : String(error) }))
     return null
   })
-
-  if (!Array.isArray(data))
+  const data = response?.data
+  if (!Array.isArray(data)) {
+    if (response)
+      emitOperationalEvent(createWideEvent({ 'operation': 'skill-related-commits-fetch', 'outcome': 'failed', 'upstream.status': response.status }))
     return []
+  }
 
   const commits: SkillCommit[] = data.map(c => ({
     sha: c.sha,