From bb03880e9962cd9ecc79867d5ec8b21f389e6f6c Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 03:18:19 +1100 Subject: [PATCH 1/3] feat(github): read GitHub as the App everywhere, tokens as fallback GITHUB_TOKEN is a personal token, so the registry sync, page views and mention checks spent the same bucket as its owner's tools and CI. Every Worker GitHub read now uses the read App installation token, shared with builds through one isolate cache. GITHUB_TOKEN is read only for a read GitHub denied the App, and each use is reported. Without it, a denied read fails for that Repository only: the sync no longer counts a denial as a spent quota, and a build answers SOURCE_ACCESS_DENIED unless the signed-in requester's token can read it. --- .dev.vars.example | 14 +- docs/ops/artifact-delivery-rollout.md | 14 +- .../server/utils/github-app.ts | 86 +--- .../server/utils/github-read-credential.ts | 235 +---------- .../artifact-delivery/server/utils/queue.ts | 34 +- .../server/utils/requester-github.ts | 17 +- .../server/utils/daily-health-check.ts | 4 +- layers/registry/server/utils/github-client.ts | 179 ++++++-- .../server/utils/github-rate-limited.ts | 10 + layers/registry/server/utils/sync-repo.ts | 19 +- shared/server/github-app-credential.ts | 393 ++++++++++++++++++ test/unit/artifact-github-read-app.test.ts | 22 +- test/unit/artifact-requester-github.test.ts | 11 + .../registry-github-app-credential.test.ts | 196 +++++++++ 14 files changed, 845 insertions(+), 389 deletions(-) create mode 100644 layers/registry/server/utils/github-rate-limited.ts create mode 100644 shared/server/github-app-credential.ts create mode 100644 test/unit/registry-github-app-credential.test.ts diff --git a/.dev.vars.example b/.dev.vars.example index d5a4b14a..953dad2b 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -10,13 +10,17 @@ NUXT_OG_IMAGE_SECRET= # Signs same-origin image proxy addresses. Keep it separate from NUXT_TOKEN_KEY. NUXT_IMAGE_PROXY_KEY= NUXT_AI_READY_RUNTIME_SYNC_SECRET= +# Every Worker GitHub read uses the read App (SKILLD_READ_APP_*): metadata read +# only, installed on skilld-dev, with a quota nothing else spends. The key is a +# PKCS #8 PEM. Without the App secrets, as in local development, reads use +# GITHUB_TOKEN. +# +# With the App set, GITHUB_TOKEN is a fallback only: it repeats a read GitHub +# denied the App, such as an organization that restricts Apps. Builds try +# ARTIFACT_GITHUB_TOKEN first. Each fallback read is logged as an app-denied event. +# Both may stay empty. GITHUB_TOKEN= -# Public Artifact builds read GitHub with this token when it is set. Use a token -# from another GitHub account, so runs keep a quota the registry sync cannot spend. ARTIFACT_GITHUB_TOKEN= -# Public Artifact builds read GitHub as the read App: metadata read only, -# installed on skilld-dev. Its installation has a quota nothing else spends. -# The key is a PKCS #8 PEM. Without these, builds use the tokens above. SKILLD_READ_APP_ID= SKILLD_READ_APP_INSTALLATION_ID= SKILLD_READ_APP_PRIVATE_KEY_PKCS8= diff --git a/docs/ops/artifact-delivery-rollout.md b/docs/ops/artifact-delivery-rollout.md index 16e2ad09..eddf17dc 100644 --- a/docs/ops/artifact-delivery-rollout.md +++ b/docs/ops/artifact-delivery-rollout.md @@ -111,17 +111,17 @@ Preserve these existing values: - `NUXT_OAUTH_GITHUB_CLIENT_ID` - `NUXT_OAUTH_GITHUB_CLIENT_SECRET` - `NUXT_TOKEN_KEY` -- `GITHUB_TOKEN` -The existing `GITHUB_TOKEN` reads public [GitHub](https://github.com) sources for the registry sync and `/gh` pages. +Every Worker GitHub read uses the read App `skilld-dev-registry-reads`: the registry sync, repository maintenance, mention checks, `/gh` pages and public Artifact builds. Set `SKILLD_READ_APP_ID`, `SKILLD_READ_APP_INSTALLATION_ID` and `SKILLD_READ_APP_PRIVATE_KEY_PKCS8` (a PKCS #8 PEM). The App has metadata read permission only, and its installation has its own quota. -Public Artifact builds read GitHub with a credential of their own, in this order: +Personal tokens are fallbacks only. They repeat a read GitHub denied the App, such as an organization that restricts Apps: -1. The read App `skilld-dev-registry-reads`: `SKILLD_READ_APP_ID`, `SKILLD_READ_APP_INSTALLATION_ID` and `SKILLD_READ_APP_PRIVATE_KEY_PKCS8` (a PKCS #8 PEM). It has metadata read permission only, and its installation has its own quota. -2. `ARTIFACT_GITHUB_TOKEN`. -3. `GITHUB_TOKEN`. +1. Builds try `ARTIFACT_GITHUB_TOKEN`, then `GITHUB_TOKEN`, then the token of the signed-in account that asked for the run. +2. Every other caller tries `GITHUB_TOKEN`. -If the App secrets are set but unusable, builds read with the next credential and emit an `artifact-github-credential` event with the reason. +Each denial emits an `app-denied` event: `github-credential` for registry reads and `artifact-github-credential` for builds. The `github.credential` field names the fallback, or `none`. Without a fallback, the read fails for that Repository only: a build answers `SOURCE_ACCESS_DENIED`, and the sync records a failure without pausing. + +If the App secrets are set but unusable, reads use the fallback and emit an event with the reason. Keep the read App apart from the `GITHUB_APP_*` secrets below. Those belong to private delivery. Changing `NUXT_TOKEN_KEY` breaks stored GitHub credentials. diff --git a/layers/artifact-delivery/server/utils/github-app.ts b/layers/artifact-delivery/server/utils/github-app.ts index f6a7d541..b8b7d695 100644 --- a/layers/artifact-delivery/server/utils/github-app.ts +++ b/layers/artifact-delivery/server/utils/github-app.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import { decryptToken, encryptToken } from '#layers/identity/server/utils/crypto' +import { requestInstallationToken, signGithubAppJwt } from '#shared/server/github-app-credential' import { base64ToBytes, bytesToBase64Url } from './encoding' import { fetchNoRedirect } from './fetch-no-redirect' @@ -48,11 +49,6 @@ const githubUserSchema = z.object({ login: z.string().min(1).max(100), }).passthrough() -const installationTokenSchema = z.object({ - token: z.string().min(1).max(2048), - expires_at: z.string().datetime(), -}).passthrough() - const refreshedUserTokenSchema = z.object({ access_token: z.string().min(1).max(2048), expires_in: z.number().int().positive(), @@ -541,50 +537,6 @@ export async function verifyGithubWebhookSignature( return different === 0 } -export type InstallationTokenResult - = | { _tag: 'created', token: string, expiresAt: string } - /** GitHub answered 401, 403 or 404: the App, its key or the installation is gone. */ - | { _tag: 'refused', status: number } - -/** - * Exchange an App JWT for an installation access token. - * - * A refusal is a value. Any other failure, such as a 5xx or an unreadable - * answer, throws: it says nothing about the App. - */ -export async function requestInstallationToken(input: { - fetch: typeof globalThis.fetch - jwt: string - installationId: number - body: { repository_ids?: number[], permissions: Record } -}): Promise { - const fetched = await fetchNoRedirect(input.fetch, `${GITHUB_API}/app/installations/${input.installationId}/access_tokens`, { - method: 'POST', - headers: { - 'Accept': 'application/vnd.github+json', - 'Authorization': `Bearer ${input.jwt}`, - 'Content-Type': 'application/json', - 'User-Agent': 'skilld.dev', - 'X-GitHub-Api-Version': GITHUB_API_VERSION, - }, - body: JSON.stringify(input.body), - signal: AbortSignal.timeout(GITHUB_REQUEST_TIMEOUT_MS), - }) - if (fetched._tag === 'unexpected-redirect') - throw new Error(`GitHub App request redirected ${fetched.status}`) - const response = fetched.response - if (response.status === 401 || response.status === 403 || response.status === 404) { - await response.body?.cancel() - return { _tag: 'refused', status: response.status } - } - if (!response.ok) - throw new Error(`GitHub App request returned ${response.status}`) - const value = installationTokenSchema.safeParse(await readBoundedJson(response, MAX_GITHUB_TOKEN_RESPONSE_BYTES)) - if (!value.success) - throw new Error('GitHub App returned an invalid response') - return { _tag: 'created', token: value.data.token, expiresAt: value.data.expires_at } -} - async function createGithubAppJwt(config: GithubAppConfig): Promise { const privateKey = decodeCanonicalBase64Url(config.privateKeyPkcs8) if (!privateKey) @@ -592,42 +544,6 @@ async function createGithubAppJwt(config: GithubAppConfig): Promise { return await signGithubAppJwt({ issuer: config.clientId, privateKeyPkcs8: privateKey, now: config.now() }) } -/** - * An RS256 GitHub App JWT. `issuer` is the App's client ID or its App ID; - * GitHub accepts either. - * - * It is valid for ten minutes from a minute in the past, which absorbs clock - * drift between the Worker and GitHub. Ten minutes is GitHub's maximum. - */ -export async function signGithubAppJwt(input: { - issuer: string - privateKeyPkcs8: Uint8Array - /** Unix seconds. */ - now: number -}): Promise { - const issuedAt = input.now - 60 - const header = bytesToBase64Url(new TextEncoder().encode(JSON.stringify({ alg: 'RS256', typ: 'JWT' }))) - const payload = bytesToBase64Url(new TextEncoder().encode(JSON.stringify({ - iat: issuedAt, - exp: issuedAt + 600, - iss: input.issuer, - }))) - const unsigned = `${header}.${payload}` - const imported = await crypto.subtle.importKey( - 'pkcs8', - toArrayBuffer(input.privateKeyPkcs8), - { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' }, - false, - ['sign'], - ) - const signature = await crypto.subtle.sign( - 'RSASSA-PKCS1-v1_5', - imported, - new TextEncoder().encode(unsigned), - ) - return `${unsigned}.${bytesToBase64Url(new Uint8Array(signature))}` -} - async function readBoundedJson(response: Response, maximumBytes: number): Promise { const declared = Number(response.headers.get('content-length')) if (Number.isFinite(declared) && declared > maximumBytes) diff --git a/layers/artifact-delivery/server/utils/github-read-credential.ts b/layers/artifact-delivery/server/utils/github-read-credential.ts index 4c836c5b..151ae7c1 100644 --- a/layers/artifact-delivery/server/utils/github-read-credential.ts +++ b/layers/artifact-delivery/server/utils/github-read-credential.ts @@ -1,161 +1,13 @@ +import type { GithubCredential, GithubCredentialConfig, GithubCredentialEnv, GithubCredentialReport } from '#shared/server/github-app-credential' import type { LoadSourceResult, PublicGithubSourceClient, ResolveSourceResult } from './github-source' -import { requestInstallationToken, signGithubAppJwt } from './github-app' +import { parseGithubCredentialConfig } from '#shared/server/github-app-credential' /** - * Renew an installation token this long before GitHub says it expires, so a - * build that starts with it never sees it lapse part way through. - */ -export const INSTALLATION_TOKEN_RENEW_SECONDS = 5 * 60 - -/** The read App: metadata read only, installed on skilld-dev. */ -export interface ReadAppConfig { - appId: number - installationId: number - privateKeyPkcs8: Uint8Array -} - -/** The token a build reads with when the App gives none, by its secret name. */ -export interface FallbackToken { - name: 'ARTIFACT_GITHUB_TOKEN' | 'GITHUB_TOKEN' - token: string -} - -/** - * Where public Artifact builds get their GitHub credential, parsed once from - * the Worker env. - * - * GitHub counts a personal token's quota per account, so every token of one - * account shares one bucket. An App installation has a bucket of its own. - */ -export type ArtifactGithubCredentialConfig - = | { _tag: 'app', app: ReadAppConfig, fallback: FallbackToken | null } - /** App secrets are set but unusable. Builds read with the fallback and report why. */ - | { _tag: 'app-misconfigured', reason: string, fallback: FallbackToken | null } - | { _tag: 'token', token: FallbackToken } - | { _tag: 'anonymous' } - -export interface ArtifactGithubCredentialEnv { - SKILLD_READ_APP_ID?: string - SKILLD_READ_APP_INSTALLATION_ID?: string - SKILLD_READ_APP_PRIVATE_KEY_PKCS8?: string - ARTIFACT_GITHUB_TOKEN?: string - GITHUB_TOKEN?: string -} - -/** Why a build read with a fallback token. It names secrets, never a value. */ -export interface ArtifactGithubCredentialReport { - outcome: 'app-misconfigured' | 'app-token-unavailable' | 'app-denied' - reason: string - fallback: FallbackToken['name'] | 'anonymous' -} - -interface CachedInstallationToken { - token: string - /** Unix seconds. */ - expiresAt: number -} - -/** - * Installation tokens kept between builds in one isolate. A token lives an - * hour; minting one per build would add a request and a signature to each. - */ -export interface InstallationTokenCache { - tokens: Map - pending: Map> -} - -export function createInstallationTokenCache(): InstallationTokenCache { - return { tokens: new Map(), pending: new Map() } -} - -export interface ArtifactGithubCredentialRuntime { - fetch: typeof globalThis.fetch - /** Unix seconds. */ - now: () => number - tokenCache: InstallationTokenCache - report: (event: ArtifactGithubCredentialReport) => void -} - -/** The credential for one GitHub read. An undefined token reads anonymously. */ -export interface GithubReadCredential { - token: string | undefined - /** The read App's installation token, which a Repository can deny. */ - isApp: boolean -} - -export interface ArtifactGithubCredential { - current: () => Promise - /** The token after the read App, or null. */ - fallback: FallbackToken | null -} - -export function parseArtifactGithubCredentialConfig(env: ArtifactGithubCredentialEnv): ArtifactGithubCredentialConfig { - const fallback = fallbackToken(env) - const appSecrets = [env.SKILLD_READ_APP_ID, env.SKILLD_READ_APP_INSTALLATION_ID, env.SKILLD_READ_APP_PRIVATE_KEY_PKCS8] - if (appSecrets.every(value => !value?.trim())) - return fallback ? { _tag: 'token', token: fallback } : { _tag: 'anonymous' } - const misconfigured = (reason: string): ArtifactGithubCredentialConfig => ({ _tag: 'app-misconfigured', reason, fallback }) - const appId = positiveInteger(env.SKILLD_READ_APP_ID) - if (appId === null) - return misconfigured('SKILLD_READ_APP_ID is not a GitHub App ID') - const installationId = positiveInteger(env.SKILLD_READ_APP_INSTALLATION_ID) - if (installationId === null) - return misconfigured('SKILLD_READ_APP_INSTALLATION_ID is not an installation ID') - const privateKeyPkcs8 = pkcs8PemBytes(env.SKILLD_READ_APP_PRIVATE_KEY_PKCS8 ?? '') - if (!privateKeyPkcs8) - return misconfigured('SKILLD_READ_APP_PRIVATE_KEY_PKCS8 is not a PKCS #8 PEM private key') - return { _tag: 'app', app: { appId, installationId, privateKeyPkcs8 }, fallback } -} - -/** - * The GitHub credential public Artifact builds read with: the read App's + * Where public Artifact builds get their GitHub credential: the read App's * installation token, then `ARTIFACT_GITHUB_TOKEN`, then `GITHUB_TOKEN`. - * - * The registry sync and `/gh` page views keep `GITHUB_TOKEN`. When the App - * cannot give a token, the build still runs on the fallback and the reason is - * reported, so a broken key shows up as an event instead of as a spent quota. */ -export function createArtifactGithubCredential( - config: ArtifactGithubCredentialConfig, - runtime: ArtifactGithubCredentialRuntime, -): ArtifactGithubCredential { - if (config._tag === 'anonymous') - return { current: async () => ({ token: undefined, isApp: false }), fallback: null } - if (config._tag === 'token') - return { current: async () => ({ token: config.token.token, isApp: false }), fallback: null } - const fallbackName = config.fallback?.name ?? 'anonymous' - const fallback = { token: config.fallback?.token, isApp: false } - if (config._tag === 'app-misconfigured') { - return { - current: async () => { - runtime.report({ outcome: 'app-misconfigured', reason: config.reason, fallback: fallbackName }) - return fallback - }, - fallback: null, - } - } - const app = config.app - const key = `${app.appId}:${app.installationId}` - return { - current: async () => { - const cached = runtime.tokenCache.tokens.get(key) - if (cached && cached.expiresAt - INSTALLATION_TOKEN_RENEW_SECONDS > runtime.now()) - return { token: cached.token, isApp: true } - let pending = runtime.tokenCache.pending.get(key) - if (!pending) { - pending = mintInstallationToken(app, runtime).finally(() => runtime.tokenCache.pending.delete(key)) - runtime.tokenCache.pending.set(key, pending) - } - const minted = await pending - if ('token' in minted) { - runtime.tokenCache.tokens.set(key, minted) - return { token: minted.token, isApp: true } - } - runtime.report({ outcome: 'app-token-unavailable', reason: minted.reason, fallback: fallbackName }) - return fallback - }, - fallback: config.fallback, - } +export function parseArtifactGithubCredentialConfig(env: GithubCredentialEnv): GithubCredentialConfig { + return parseGithubCredentialConfig(env, ['ARTIFACT_GITHUB_TOKEN', 'GITHUB_TOKEN']) } /** @@ -165,12 +17,12 @@ export function createArtifactGithubCredential( * Some organizations deny the read App on a public Repository that a personal * token reads: `neondatabase/agent-skills` answered SOURCE_ACCESS_DENIED to * every run on 2026-10-06. A read the App was denied repeats once with the - * fallback token, and the denial is reported. + * fallback token. Every denial is reported, with or without a fallback. */ export function withGithubCredential( - credential: ArtifactGithubCredential, + credential: GithubCredential, create: (token: string | undefined) => PublicGithubSourceClient, - report: (event: ArtifactGithubCredentialReport) => void, + report: (event: GithubCredentialReport) => void, ): PublicGithubSourceClient { const read = async ( repository: { owner: string, repository: string }, @@ -178,76 +30,23 @@ export function withGithubCredential( ): Promise => { const current = await credential.current() const first = await run(create(current.token)) - const fallback = credential.fallback - if (!current.isApp || !fallback || first._tag !== 'rejected' || first.code !== 'SOURCE_ACCESS_DENIED') + if (!current.isApp || first._tag !== 'rejected' || first.code !== 'SOURCE_ACCESS_DENIED') return first + const fallback = credential.fallback report({ outcome: 'app-denied', reason: `GitHub denied the read App for ${repository.owner}/${repository.repository}`, - fallback: fallback.name, + fallback: fallback?.name ?? 'none', }) - return await run(create(fallback.token)) + if (fallback) + return await run(create(fallback.token)) + return { + ...first, + summary: 'GitHub denies the skilld.dev read App access to this Repository, and no fallback token is set.', + } } return { resolve: async request => await read(request, client => client.resolve(request)), load: async (source, options) => await read(source, client => client.load(source, options)), } } - -async function mintInstallationToken( - app: ReadAppConfig, - runtime: Pick, -): Promise { - const minted = await signGithubAppJwt({ issuer: String(app.appId), privateKeyPkcs8: app.privateKeyPkcs8, now: runtime.now() }) - .then(jwt => requestInstallationToken({ - fetch: runtime.fetch, - jwt, - installationId: app.installationId, - body: { permissions: { metadata: 'read' } }, - })) - // A network fault, a 5xx or a key WebCrypto rejects: the build reads with - // the fallback token, and the reason reaches the report. - .catch((error: unknown) => ({ _tag: 'failed' as const, reason: error instanceof Error ? error.message : String(error) })) - if (minted._tag === 'failed') - return { _tag: 'unavailable', reason: minted.reason.slice(0, 200) } - if (minted._tag === 'refused') - return { _tag: 'unavailable', reason: `GitHub refused the installation token (${minted.status})` } - const expiresAt = Math.floor(Date.parse(minted.expiresAt) / 1000) - return { token: minted.token, expiresAt } -} - -function fallbackToken(env: ArtifactGithubCredentialEnv): FallbackToken | null { - if (env.ARTIFACT_GITHUB_TOKEN?.trim()) - return { name: 'ARTIFACT_GITHUB_TOKEN', token: env.ARTIFACT_GITHUB_TOKEN.trim() } - if (env.GITHUB_TOKEN?.trim()) - return { name: 'GITHUB_TOKEN', token: env.GITHUB_TOKEN.trim() } - return null -} - -function positiveInteger(value: string | undefined): number | null { - const trimmed = value?.trim() ?? '' - if (!/^[1-9]\d*$/.test(trimmed)) - return null - const parsed = Number(trimmed) - return Number.isSafeInteger(parsed) ? parsed : null -} - -/** - * The DER bytes of a PKCS #8 PEM private key, or null. - * - * A secret pasted through a dashboard or a JSON file can hold its line breaks - * as the two characters `\n`, so both forms parse. A PKCS #1 key - * (`BEGIN RSA PRIVATE KEY`, as GitHub downloads it) does not: WebCrypto - * imports PKCS #8 only. Convert it with `openssl pkcs8 -topk8 -nocrypt`. - */ -function pkcs8PemBytes(value: string): Uint8Array | null { - const text = value.replaceAll('\\n', '\n').trim() - const match = /^-----BEGIN PRIVATE KEY-----([A-Z0-9+/=\s]+)-----END PRIVATE KEY-----$/i.exec(text) - if (!match) - return null - const base64 = match[1]!.replace(/\s+/g, '') - if (!/^[A-Z0-9+/]+={0,2}$/i.test(base64) || base64.length % 4 !== 0) - return null - const binary = atob(base64) - return Uint8Array.from(binary, character => character.charCodeAt(0)) -} diff --git a/layers/artifact-delivery/server/utils/queue.ts b/layers/artifact-delivery/server/utils/queue.ts index 074f3e9c..55fd516e 100644 --- a/layers/artifact-delivery/server/utils/queue.ts +++ b/layers/artifact-delivery/server/utils/queue.ts @@ -1,14 +1,11 @@ import type { QueueBatch } from '#cf-jobs/server' +import type { GithubCredentialEnv, GithubCredentialReport, GithubCredentialRuntime } from '#shared/server/github-app-credential' import type { ArtifactBuildDependencies } from './build' -import type { - ArtifactGithubCredentialEnv, - ArtifactGithubCredentialReport, - ArtifactGithubCredentialRuntime, -} from './github-read-credential' import type { GithubObjectCache, GithubReadFailure, PublicGithubSourceClient } from './github-source' import { createWideEvent } from '@harlan-zw/nuxt-wide-events/standalone' import { z } from 'zod' import { emitOperationalEvent } from '#server/utils/operational-event' +import { createGithubCredential, isolateInstallationTokenCache } from '#shared/server/github-app-credential' import { createArtifactSigner } from './attestation' import { failResolution, processArtifactBuild } from './build' import { @@ -16,12 +13,7 @@ import { githubAppUserTokenDependenciesFromEnv, loadAccountGithubAppUserToken, } from './github-app' -import { - createArtifactGithubCredential, - createInstallationTokenCache, - parseArtifactGithubCredentialConfig, - withGithubCredential, -} from './github-read-credential' +import { parseArtifactGithubCredentialConfig, withGithubCredential } from './github-read-credential' import { createGithubSourceClient, createPublicGithubSourceClient } from './github-source' import { createD1PrivateArtifactKeyProvider, privateArtifactWrappingKeysFromEnv } from './private-crypto' import { privateArtifactAccessEnabled } from './private-feature' @@ -165,12 +157,6 @@ export function createKvGithubObjectCache(kv: KVNamespace): GithubObjectCache { } } -/** - * One installation token cache per isolate. A token lives an hour, so it - * outlives the queue batch that minted it. - */ -const installationTokens = createInstallationTokenCache() - /** * The GitHub source public Artifact builds read with. * @@ -181,11 +167,11 @@ const installationTokens = createInstallationTokenCache() * else spends, then `ARTIFACT_GITHUB_TOKEN`, then `GITHUB_TOKEN`. */ export function createArtifactGithubSource( - env: ArtifactGithubCredentialEnv, - runtime: ArtifactGithubCredentialRuntime = defaultGithubSourceRuntime(), + env: GithubCredentialEnv, + runtime: GithubCredentialRuntime = defaultGithubSourceRuntime(), cache?: GithubObjectCache, ): PublicGithubSourceClient { - const credential = createArtifactGithubCredential(parseArtifactGithubCredentialConfig(env), runtime) + const credential = createGithubCredential(parseArtifactGithubCredentialConfig(env), runtime) return withGithubCredential( credential, token => createPublicGithubSourceClient({ fetch: runtime.fetch, token, onReadFailure: reportGithubReadFailure, cache }), @@ -193,16 +179,18 @@ export function createArtifactGithubSource( ) } -function defaultGithubSourceRuntime(): ArtifactGithubCredentialRuntime { +function defaultGithubSourceRuntime(): GithubCredentialRuntime { return { fetch: globalThis.fetch.bind(globalThis), now: () => Math.floor(Date.now() / 1000), - tokenCache: installationTokens, + // A token lives an hour, so it outlives the queue batch that minted it, + // and the registry reads in this isolate share it. + tokenCache: isolateInstallationTokenCache(), report: reportGithubCredential, } } -function reportGithubCredential(event: ArtifactGithubCredentialReport): void { +function reportGithubCredential(event: GithubCredentialReport): void { emitOperationalEvent(createWideEvent({ 'operation': 'artifact-github-credential', 'outcome': event.outcome, diff --git a/layers/artifact-delivery/server/utils/requester-github.ts b/layers/artifact-delivery/server/utils/requester-github.ts index 43a46ce7..5c85eef5 100644 --- a/layers/artifact-delivery/server/utils/requester-github.ts +++ b/layers/artifact-delivery/server/utils/requester-github.ts @@ -1,4 +1,4 @@ -import type { GithubObjectCache, GithubReadFailure, PublicGithubSourceClient } from './github-source' +import type { GithubObjectCache, GithubReadFailure, LoadSourceResult, PublicGithubSourceClient, ResolveSourceResult } from './github-source' import type { ResolutionRow } from './state' import { decryptToken } from '#layers/identity/server/utils/crypto' import { createPublicGithubSourceClient } from './github-source' @@ -86,8 +86,10 @@ export function createRequesterGithubSource( } /** - * A client that reads with the shared credential, and repeats a read the - * shared quota refused with the requester's own client. + * A client that reads with the shared credential, and repeats with the + * requester's own client a read the shared credentials could not make: a spent + * quota, or a Repository whose organization denies the read App when no + * fallback token is set. * * Every other answer, success or rejection, stays the shared one, so a * requester's token is spent only when nothing else could read. @@ -104,17 +106,22 @@ export function withRequesterFallback( return { async resolve(request) { const first = await shared.resolve(request) - if (first._tag !== 'rejected' || first.code !== 'RATE_LIMITED') + if (!sharedCouldNotRead(first)) return first const client = await ownClient() return client ? await client.resolve(request) : first }, async load(source, options) { const first = await shared.load(source, options) - if (first._tag !== 'rejected' || first.code !== 'RATE_LIMITED') + if (!sharedCouldNotRead(first)) return first const client = await ownClient() return client ? await client.load(source, options) : first }, } } + +/** A spent quota, or a denial no shared credential could get past. */ +function sharedCouldNotRead(result: ResolveSourceResult | LoadSourceResult): boolean { + return result._tag === 'rejected' && (result.code === 'RATE_LIMITED' || result.code === 'SOURCE_ACCESS_DENIED') +} diff --git a/layers/identity/server/utils/daily-health-check.ts b/layers/identity/server/utils/daily-health-check.ts index 45434f85..c393969c 100644 --- a/layers/identity/server/utils/daily-health-check.ts +++ b/layers/identity/server/utils/daily-health-check.ts @@ -605,9 +605,9 @@ export function evaluateDailyHealthStatus( // while rotating is still routine rather than an incident. const token = summary.credentials.githubToken if (token._tag === 'expired') - red.push(`GITHUB_TOKEN expired ${plural(Math.abs(token.daysRemaining), 'day')} ago. Rotate it: sync is down until you do.`) + red.push(`GITHUB_TOKEN expired ${plural(Math.abs(token.daysRemaining), 'day')} ago. Repositories that deny the read App cannot sync or build until you rotate or delete it.`) else if (token._tag === 'expiring') - amber.push(`GITHUB_TOKEN expires in ${plural(token.daysRemaining, 'day')}. Rotate it before sync starts failing.`) + amber.push(`GITHUB_TOKEN expires in ${plural(token.daysRemaining, 'day')}. Rotate or delete it: it reads only Repositories that deny the read App.`) // Report every finding, ordered by severity, rather than only the winning // tier. Returning `red` alone meant one loud red hid every amber underneath diff --git a/layers/registry/server/utils/github-client.ts b/layers/registry/server/utils/github-client.ts index 40845d2f..64bdb98e 100644 --- a/layers/registry/server/utils/github-client.ts +++ b/layers/registry/server/utils/github-client.ts @@ -1,5 +1,18 @@ /// +import type { + GithubCredential, + GithubCredentialReport, + GithubReadCredential, + InstallationTokenCache, +} from '#shared/server/github-app-credential' +import { + createGithubCredential, + githubDeniedApp, + isolateInstallationTokenCache, + parseGithubCredentialConfig, +} from '#shared/server/github-app-credential' + const API_BASE = 'https://api.github.com' const GRAPHQL_URL = 'https://api.github.com/graphql' @@ -39,10 +52,29 @@ function readInit(options: GithubReadOptions | undefined): RequestInit | undefin export interface GithubBindings { KV_CACHE?: KVNamespace + /** + * The credential every Worker read asks for: the read App's installation + * token, with `GITHUB_TOKEN` only for a read GitHub denied the App. + */ + credential?: GithubCredential + /** A static token, for scripts and tests that pass no `credential`. */ GITHUB_TOKEN?: string + /** Receives each read GitHub denied the App. Defaults to a wide event. */ + reportCredential?: (event: GithubCredentialReport) => void } -type GithubBindingSource = Partial> +type GithubBindingSource = Partial> + +/** What a test or a caller with its own clock passes in place of the isolate defaults. */ +export interface GithubBindingRuntime { + tokenCache: InstallationTokenCache + /** Unix seconds. */ + now: () => number + report: (event: GithubCredentialReport) => void +} /** * Resolve GitHub bindings from a Cloudflare env (Worker runtime) or process.env (local dev). @@ -50,14 +82,62 @@ type GithubBindingSource = Partial = {}, +): GithubBindings { const env = cloudflareEnv ?? {} + const report = runtime.report ?? reportGithubCredential + const config = parseGithubCredentialConfig({ ...env, GITHUB_TOKEN: env.GITHUB_TOKEN ?? process.env.GITHUB_TOKEN }, ['GITHUB_TOKEN']) return { KV_CACHE: env.KV_CACHE, - GITHUB_TOKEN: env.GITHUB_TOKEN ?? process.env.GITHUB_TOKEN, + credential: createGithubCredential(config, { + // Late-bound, so a test that stubs the global fetch reaches the mint too. + fetch: (input, init) => fetch(input, init), + now: runtime.now ?? (() => Math.floor(Date.now() / 1000)), + tokenCache: runtime.tokenCache ?? isolateInstallationTokenCache(), + report, + }), + reportCredential: report, } } +function reportGithubCredential(event: GithubCredentialReport): void { + emitOperationalEvent(createWideEvent({ + 'operation': 'github-credential', + 'outcome': event.outcome, + 'reason': event.reason, + 'github.credential': event.fallback, + })) +} + +async function readCredential(bindings: GithubBindings): Promise { + return bindings.credential ? await bindings.credential.current() : { token: bindings.GITHUB_TOKEN, isApp: false } +} + +/** + * Report a read GitHub denied the App, and answer the token to repeat it + * with, or null when there is none. Every use of the fallback is reported. + */ +function fallbackAfterDenial(bindings: GithubBindings, label: string): string | null { + const fallback = bindings.credential?.fallback ?? null + ;(bindings.reportCredential ?? reportGithubCredential)({ + outcome: 'app-denied', + reason: `GitHub denied the read App for ${label}`, + fallback: fallback?.name ?? 'none', + }) + return fallback?.token ?? null +} + +function withAuthorization(headers: Headers, token: string | undefined): Headers { + const authorized = new Headers(headers) + if (token) + authorized.set('Authorization', `Bearer ${token}`) + else + authorized.delete('Authorization') + return authorized +} + export interface RepoMeta { name: string full_name: string @@ -114,6 +194,11 @@ export interface FetchOutcome { data: T | null rateLimit: RateLimitInfo | null notModified: boolean + /** + * GitHub denied the read App this read, and no fallback token could repeat + * it. The status is 403, but no quota is spent. + */ + denied?: true } /** @@ -168,8 +253,6 @@ async function ghRequest( headers.set('Accept', 'application/vnd.github+json') headers.set('User-Agent', 'skilld.dev') headers.set('X-GitHub-Api-Version', '2022-11-28') - if (bindings.GITHUB_TOKEN) - headers.set('Authorization', `Bearer ${bindings.GITHUB_TOKEN}`) const cacheKey = etagKey(url) let cached: CachedEntry | null = null @@ -182,7 +265,15 @@ async function ghRequest( headers.set('If-None-Match', cached.etag) } - const res = await fetch(url, { ...init, headers }) + const send = (token: string | undefined) => fetch(url, { ...init, headers: withAuthorization(headers, token) }) + const credential = await readCredential(bindings) + let res = await send(credential.token) + if (credential.isApp && await githubDeniedApp(res)) { + const fallback = fallbackAfterDenial(bindings, new URL(url).pathname) + if (fallback === null) + return { status: 403, data: null, rateLimit: parseRateLimit(res.headers), notModified: false, denied: true } + res = await send(fallback) + } const rateLimit = parseRateLimit(res.headers) if (res.status === 304 && cached) { @@ -312,9 +403,12 @@ export async function getRepoSummary( ${REPO_SUMMARY_FIELDS} } }` - const out = await gqlRequest<{ repository: RepoSummaryGql | null }>(query, { owner, repo }, bindings) - if (out._tag === 'failed') - return { status: out.status, data: null, rateLimit: out.rateLimit, notModified: false } + const out = await gqlRequest<{ repository: RepoSummaryGql | null }>(query, { owner, repo }, bindings, { label: `${owner}/${repo}`, retryForbidden: true }) + if (out._tag === 'failed') { + return out.denied + ? { status: out.status, data: null, rateLimit: out.rateLimit, notModified: false, denied: true } + : { status: out.status, data: null, rateLimit: out.rateLimit, notModified: false } + } // GraphQL surfaces NOT_FOUND in errors with HTTP 200. Map it to 404 so // callers preserve their existing rate-limit / broken-repo branching. if (out.errors.length) { @@ -383,14 +477,16 @@ export async function getRepoSummariesBatch( variables[`n${i}`] = request.repo }) sent++ - const out = await gqlRequest>(`query(${varDecls.join(',')}){${aliases}}`, variables, bindings) + const out = await gqlRequest>(`query(${varDecls.join(',')}){${aliases}}`, variables, bindings, { label: 'a summary batch', retryForbidden: false }) rateLimit = out.rateLimit ?? rateLimit if (out._tag === 'failed') return { _tag: 'failed', status: out.status, rateLimit, requests: sent } // A NOT_FOUND names one alias and leaves the others whole. Any other // error type, such as RATE_LIMITED, says nothing trustworthy about the // batch, so it fails as a whole and the per-repository sync takes over. - if (out.errors.some(error => error.type !== 'NOT_FOUND')) + // A FORBIDDEN alias is a Repository GitHub denies the read App. It reads + // as null here, and the per-repository sync repeats it with the fallback. + if (out.errors.some(error => error.type !== 'NOT_FOUND' && error.type !== 'FORBIDDEN')) return { _tag: 'failed', status: 502, rateLimit, requests: sent } for (let i = 0; i < batch.length; i++) { const r = out.data?.[`r${i}`] @@ -444,28 +540,57 @@ interface GqlError { type?: string, message?: string } type GqlRequestOutcome = | { _tag: 'answered', data: T | null, errors: GqlError[], rateLimit: RateLimitInfo | null } - | { _tag: 'failed', status: number, rateLimit: RateLimitInfo | null } + | { _tag: 'failed', status: number, rateLimit: RateLimitInfo | null, denied?: true } -/** One GraphQL POST. Partial data and per-alias errors come back together. */ +/** + * One GraphQL POST. Partial data and per-alias errors come back together. + * + * A query about one Repository that GitHub denied the read App, with a 403 + * or a FORBIDDEN error, repeats once with the fallback token. A batch over + * many Repositories passes `retryForbidden: false`: one denied alias says + * nothing about the others, and the per-repository sync repeats it. + */ async function gqlRequest( query: string, variables: Record, bindings: GithubBindings, + options: { label: string, retryForbidden: boolean }, +): Promise> { + const credential = await readCredential(bindings) + const first = await gqlSend(query, variables, credential.token, credential.isApp) + const deniedApp = first._tag === 'failed' + ? first.denied === true + : options.retryForbidden && first.errors.some(error => error.type === 'FORBIDDEN') + if (!credential.isApp || !deniedApp) + return first + const fallback = fallbackAfterDenial(bindings, options.label) + if (fallback === null) + return { _tag: 'failed', status: 403, rateLimit: first.rateLimit, denied: true } + return await gqlSend(query, variables, fallback, false) +} + +async function gqlSend( + query: string, + variables: Record, + token: string | undefined, + asApp: boolean, ): Promise> { const headers = new Headers() headers.set('Accept', 'application/vnd.github+json') headers.set('Content-Type', 'application/json') headers.set('User-Agent', 'skilld.dev') - if (bindings.GITHUB_TOKEN) - headers.set('Authorization', `Bearer ${bindings.GITHUB_TOKEN}`) + if (token) + headers.set('Authorization', `Bearer ${token}`) const res = await fetch(GRAPHQL_URL, { method: 'POST', headers, body: JSON.stringify({ query, variables }), }) const rateLimit = parseRateLimit(res.headers) - if (!res.ok) - return { _tag: 'failed', status: res.status, rateLimit } + if (!res.ok) { + const denied = asApp && await githubDeniedApp(res) + return denied ? { _tag: 'failed', status: res.status, rateLimit, denied: true } : { _tag: 'failed', status: res.status, rateLimit } + } // A gateway can answer 200 with a truncated or non-JSON body. Parsing that // eagerly threw `Unexpected end of JSON input` out of the client and reached // the sync summary as an opaque reason with no status attached. @@ -482,10 +607,11 @@ async function gqlPost( query: string, variables: Record, bindings: GithubBindings, -): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null }> { - const out = await gqlRequest(query, variables, bindings) + label: string, +): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null, denied?: true }> { + const out = await gqlRequest(query, variables, bindings, { label, retryForbidden: true }) if (out._tag === 'failed') - return { status: out.status, data: null, rateLimit: out.rateLimit } + return out.denied ? { status: out.status, data: null, rateLimit: out.rateLimit, denied: true } : { status: out.status, data: null, rateLimit: out.rateLimit } if (out.errors.length) { const notFound = out.errors.some(e => e.type === 'NOT_FOUND') return { status: notFound ? 404 : 502, data: null, rateLimit: out.rateLimit } @@ -537,12 +663,12 @@ export async function getBlobsBatch( variables[`expr${i}`] = `${branch}:${p}` }) - const out = await gqlPost<{ repository: Record | null }>(query, variables, bindings) + const out = await gqlPost<{ repository: Record | null }>(query, variables, bindings, `${owner}/${repo}`) rateLimit = out.rateLimit ?? rateLimit // A partial map would look like a repo that lost files, and the caller // would delete skills it simply failed to read. Fail the whole batch. if (!out.data?.repository) - return { status: out.status, data: null, unreadable, rateLimit, notModified: false } + return { status: out.status, data: null, unreadable, rateLimit, notModified: false, ...(out.denied ? { denied: true as const } : {}) } batch.forEach((p, i) => { const blob = out.data!.repository![`b${i}`] if (typeof blob?.text === 'string') @@ -590,11 +716,11 @@ export async function getCommitsBatch( variables[`path${i}`] = p }) - const out = await gqlPost(query, variables, bindings) + const out = await gqlPost(query, variables, bindings, `${owner}/${repo}`) rateLimit = out.rateLimit ?? rateLimit const target = out.data?.repository?.defaultBranchRef?.target if (!target) - return { status: out.status, data: null, rateLimit, notModified: false } + return { status: out.status, data: null, rateLimit, notModified: false, ...(out.denied ? { denied: true as const } : {}) } batch.forEach((p, i) => { const nodes = target[`h${i}`]?.nodes ?? [] map.set(p, nodes.map(c => ({ @@ -624,8 +750,9 @@ export async function getRawFile( ): Promise { const headers = new Headers() headers.set('User-Agent', 'skilld.dev') - if (bindings.GITHUB_TOKEN) - headers.set('Authorization', `Bearer ${bindings.GITHUB_TOKEN}`) + const credential = await readCredential(bindings) + if (credential.token) + headers.set('Authorization', `Bearer ${credential.token}`) const res = await fetch(`${RAW_BASE}/${owner}/${repo}/${ref}/${path}`, { headers }) if (!res.ok) return null diff --git a/layers/registry/server/utils/github-rate-limited.ts b/layers/registry/server/utils/github-rate-limited.ts new file mode 100644 index 00000000..8439a62b --- /dev/null +++ b/layers/registry/server/utils/github-rate-limited.ts @@ -0,0 +1,10 @@ +/** + * Whether GitHub refused a read for want of quota. + * + * A read GitHub denied the read App also answers 403, but it is a fact about + * one Repository: `denied` marks it, and it never pauses the sync. It lives + * apart from the GitHub client, which sync tests replace whole. + */ +export function githubRateLimited(outcome: { status: number, denied?: true }): boolean { + return (outcome.status === 403 || outcome.status === 429) && !outcome.denied +} diff --git a/layers/registry/server/utils/sync-repo.ts b/layers/registry/server/utils/sync-repo.ts index b72d581f..b24a31bb 100644 --- a/layers/registry/server/utils/sync-repo.ts +++ b/layers/registry/server/utils/sync-repo.ts @@ -6,6 +6,7 @@ import type { SkillTrustTier } from './skill-trust' import { canonicalSkillPaths, isRegistrySkillPath, isSkilldCacheSkill } from '#shared/skill-path' import { isCategoryPinned } from '../data/clusters' import { getBlobsBatch, getCommitsBatch, getRepoSummary, getTree, logRateLimit } from './github-client' +import { githubRateLimited } from './github-rate-limited' import { repoStarObservationStatements } from './repo-history' import { resolveRepoSourceIdentityFromRow } from './repo-source-identity' import { findHeldRepositoryName, moveRepository, planRepositoryMove, sameRepositoryName } from './repository-move' @@ -352,8 +353,8 @@ export async function refreshRepoAssets( owner, repo, reason: `repo fetch ${repoRes.status}`, - retryable: repoRes.status === 403 || repoRes.status === 429 || repoRes.status >= 500, - rateLimited: repoRes.status === 403 || repoRes.status === 429, + retryable: githubRateLimited(repoRes) || repoRes.status >= 500, + rateLimited: githubRateLimited(repoRes), unauthorized: repoRes.status === 401, ...rate, } @@ -384,8 +385,8 @@ export async function refreshRepoAssets( owner, repo, reason: `tree fetch ${treeRes.status}`, - retryable: treeRes.status === 403 || treeRes.status === 429 || treeRes.status >= 500, - rateLimited: treeRes.status === 403 || treeRes.status === 429, + retryable: githubRateLimited(treeRes) || treeRes.status >= 500, + rateLimited: githubRateLimited(treeRes), unauthorized: treeRes.status === 401, ...finalRate, } @@ -827,7 +828,7 @@ export async function syncRepo( return stats } - if (repoRes.status === 403 || repoRes.status === 429) { + if (githubRateLimited(repoRes)) { stats.status = 'rate-limited' stats.reason = `rate-limited (${repoRes.rateLimit?.remaining ?? '?'} remaining)` return stats @@ -837,7 +838,7 @@ export async function syncRepo( if (repoRes.status === 404 || repoRes.status === 410) await markRepoMissing(db, requestedOwner, requestedRepo, nowSec()) stats.status = 'failed' - stats.reason = `repo fetch ${repoRes.status}` + stats.reason = repoRes.denied ? 'github denied the read App (403)' : `repo fetch ${repoRes.status}` return stats } @@ -893,7 +894,7 @@ export async function syncRepo( trackRateLimit(treeRes.rateLimit) if (!treeRes.data) { - stats.status = treeRes.status === 403 || treeRes.status === 429 ? 'rate-limited' : 'failed' + stats.status = githubRateLimited(treeRes) ? 'rate-limited' : 'failed' stats.reason = `tree_fetch_failed:${treeRes.status}` return stats } @@ -1161,7 +1162,7 @@ export async function syncRepo( logRateLimit(`blobs ${owner}/${repo}`, blobsRes.rateLimit) trackRateLimit(blobsRes.rateLimit) if (!blobsRes.data) { - stats.status = blobsRes.status === 403 || blobsRes.status === 429 ? 'rate-limited' : 'failed' + stats.status = githubRateLimited(blobsRes) ? 'rate-limited' : 'failed' stats.reason = `blob_batch_failed:${blobsRes.status}` return stats } @@ -1303,7 +1304,7 @@ export async function syncRepo( logRateLimit(`commits-batch ${owner}/${repo}`, commitsRes.rateLimit) trackRateLimit(commitsRes.rateLimit) if (!commitsRes.data) { - stats.status = commitsRes.status === 403 || commitsRes.status === 429 ? 'rate-limited' : 'failed' + stats.status = githubRateLimited(commitsRes) ? 'rate-limited' : 'failed' stats.reason = `commit_batch_failed:${commitsRes.status}` return stats } diff --git a/shared/server/github-app-credential.ts b/shared/server/github-app-credential.ts new file mode 100644 index 00000000..03dd0973 --- /dev/null +++ b/shared/server/github-app-credential.ts @@ -0,0 +1,393 @@ +import { z } from 'zod' + +/** + * The read App's GitHub credential, shared by every Worker caller. + * + * GitHub counts a personal token's quota per account, so every token of one + * account shares one bucket, and the account owner's own tools spend it too. + * The read App (`skilld-dev-registry-reads`, metadata read only) has an + * installation with a bucket of its own. A personal token remains only as the + * fallback for a Repository whose organization denies the App. + */ + +const GITHUB_API = 'https://api.github.com' +const GITHUB_API_VERSION = '2026-03-10' +const GITHUB_REQUEST_TIMEOUT_MS = 15_000 +const MAX_TOKEN_RESPONSE_BYTES = 64 * 1024 + +/** + * Renew an installation token this long before GitHub says it expires, so a + * read that starts with it never sees it lapse part way through. + */ +export const INSTALLATION_TOKEN_RENEW_SECONDS = 5 * 60 + +/** The read App: metadata read only, installed on skilld-dev. */ +export interface ReadAppConfig { + appId: number + installationId: number + privateKeyPkcs8: Uint8Array +} + +/** A personal token a caller may fall back to, by its secret name. */ +export interface FallbackToken { + name: 'ARTIFACT_GITHUB_TOKEN' | 'GITHUB_TOKEN' + token: string +} + +/** Where a caller's GitHub credential comes from, parsed once from the Worker env. */ +export type GithubCredentialConfig + = | { _tag: 'app', app: ReadAppConfig, fallback: FallbackToken | null } + /** App secrets are set but unusable. Reads use the fallback and report why. */ + | { _tag: 'app-misconfigured', reason: string, fallback: FallbackToken | null } + /** No App secret is set, as in local development and tests. */ + | { _tag: 'token', token: FallbackToken } + | { _tag: 'anonymous' } + +export interface GithubCredentialEnv { + SKILLD_READ_APP_ID?: string + SKILLD_READ_APP_INSTALLATION_ID?: string + SKILLD_READ_APP_PRIVATE_KEY_PKCS8?: string + ARTIFACT_GITHUB_TOKEN?: string + GITHUB_TOKEN?: string +} + +/** + * Why a read used a token other than the App's, or found none. It names + * secrets and paths, never a token value. + */ +export interface GithubCredentialReport { + outcome: 'app-misconfigured' | 'app-token-unavailable' | 'app-denied' + reason: string + fallback: FallbackToken['name'] | 'anonymous' | 'none' +} + +interface CachedInstallationToken { + token: string + /** Unix seconds. */ + expiresAt: number +} + +/** + * Installation tokens kept between reads in one isolate. A token lives an + * hour; minting one per read would add a request and a signature to each. + */ +export interface InstallationTokenCache { + tokens: Map + pending: Map> +} + +export function createInstallationTokenCache(): InstallationTokenCache { + return { tokens: new Map(), pending: new Map() } +} + +let isolateTokenCache: InstallationTokenCache | null = null + +/** + * The one installation token cache of this isolate, so the registry and the + * build queue share a token instead of minting one each. + */ +export function isolateInstallationTokenCache(): InstallationTokenCache { + isolateTokenCache ??= createInstallationTokenCache() + return isolateTokenCache +} + +export interface GithubCredentialRuntime { + fetch: typeof globalThis.fetch + /** Unix seconds. */ + now: () => number + tokenCache: InstallationTokenCache + report: (event: GithubCredentialReport) => void +} + +/** The credential for one GitHub read. An undefined token reads anonymously. */ +export interface GithubReadCredential { + token: string | undefined + /** The read App's installation token, which a Repository can deny. */ + isApp: boolean +} + +export interface GithubCredential { + current: () => Promise + /** The token for a read GitHub denied the App, or null. */ + fallback: FallbackToken | null +} + +/** + * Parse the read App secrets and the caller's fallback tokens, in order. + * + * With no App secret at all, the first fallback token becomes the primary + * credential, which is what local development runs on. + */ +export function parseGithubCredentialConfig( + env: GithubCredentialEnv, + fallbackNames: ReadonlyArray, +): GithubCredentialConfig { + const fallback = firstToken(env, fallbackNames) + const appSecrets = [env.SKILLD_READ_APP_ID, env.SKILLD_READ_APP_INSTALLATION_ID, env.SKILLD_READ_APP_PRIVATE_KEY_PKCS8] + if (appSecrets.every(value => !value?.trim())) + return fallback ? { _tag: 'token', token: fallback } : { _tag: 'anonymous' } + const misconfigured = (reason: string): GithubCredentialConfig => ({ _tag: 'app-misconfigured', reason, fallback }) + const appId = positiveInteger(env.SKILLD_READ_APP_ID) + if (appId === null) + return misconfigured('SKILLD_READ_APP_ID is not a GitHub App ID') + const installationId = positiveInteger(env.SKILLD_READ_APP_INSTALLATION_ID) + if (installationId === null) + return misconfigured('SKILLD_READ_APP_INSTALLATION_ID is not an installation ID') + const privateKeyPkcs8 = pkcs8PemBytes(env.SKILLD_READ_APP_PRIVATE_KEY_PKCS8 ?? '') + if (!privateKeyPkcs8) + return misconfigured('SKILLD_READ_APP_PRIVATE_KEY_PKCS8 is not a PKCS #8 PEM private key') + return { _tag: 'app', app: { appId, installationId, privateKeyPkcs8 }, fallback } +} + +/** + * The GitHub credential a caller reads with: the read App's installation + * token, or the fallback when the App gives none. + * + * When the App cannot give a token, the read still runs on the fallback and + * the reason is reported, so a broken key shows up as an event instead of as + * a spent quota. + */ +export function createGithubCredential( + config: GithubCredentialConfig, + runtime: GithubCredentialRuntime, +): GithubCredential { + if (config._tag === 'anonymous') + return { current: async () => ({ token: undefined, isApp: false }), fallback: null } + if (config._tag === 'token') + return { current: async () => ({ token: config.token.token, isApp: false }), fallback: null } + const fallbackName = config.fallback?.name ?? 'anonymous' + const fallback = { token: config.fallback?.token, isApp: false } + if (config._tag === 'app-misconfigured') { + return { + current: async () => { + runtime.report({ outcome: 'app-misconfigured', reason: config.reason, fallback: fallbackName }) + return fallback + }, + fallback: null, + } + } + const app = config.app + const key = `${app.appId}:${app.installationId}` + return { + current: async () => { + const cached = runtime.tokenCache.tokens.get(key) + if (cached && cached.expiresAt - INSTALLATION_TOKEN_RENEW_SECONDS > runtime.now()) + return { token: cached.token, isApp: true } + let pending = runtime.tokenCache.pending.get(key) + if (!pending) { + pending = mintInstallationToken(app, runtime).finally(() => runtime.tokenCache.pending.delete(key)) + runtime.tokenCache.pending.set(key, pending) + } + const minted = await pending + if ('token' in minted) { + runtime.tokenCache.tokens.set(key, minted) + return { token: minted.token, isApp: true } + } + runtime.report({ outcome: 'app-token-unavailable', reason: minted.reason, fallback: fallbackName }) + return fallback + }, + fallback: config.fallback, + } +} + +/** + * Whether GitHub refused a read because it denies the App, as opposed to a + * spent quota. Organizations that restrict Apps answer 403 with quota left, + * and no rate limit message. `neondatabase` did on 2026-10-06. + * + * It reads the body of a 403, so pass a clone or a response whose body is + * not needed. + */ +export async function githubDeniedApp(response: Response): Promise { + if (response.status !== 403) + return false + if (response.headers.get('x-ratelimit-remaining') === '0' || response.headers.get('retry-after')) + return false + // A 403 body that will not read names no rate limit either, so it counts + // as a denial. The status already says the read failed. + const text = await response.text().catch(() => '') + return !/rate limit/i.test(text) +} + +const installationTokenSchema = z.object({ + token: z.string().min(1).max(2048), + expires_at: z.string().datetime(), +}).passthrough() + +export type InstallationTokenResult + = | { _tag: 'created', token: string, expiresAt: string } + /** GitHub answered 401, 403 or 404: the App, its key or the installation is gone. */ + | { _tag: 'refused', status: number } + +/** + * Exchange an App JWT for an installation access token. + * + * A refusal is a value. Any other failure, such as a 5xx or an unreadable + * answer, throws: it says nothing about the App. + */ +export async function requestInstallationToken(input: { + fetch: typeof globalThis.fetch + jwt: string + installationId: number + body: { repository_ids?: number[], permissions: Record } +}): Promise { + const response = await input.fetch(`${GITHUB_API}/app/installations/${input.installationId}/access_tokens`, { + method: 'POST', + headers: { + 'Accept': 'application/vnd.github+json', + 'Authorization': `Bearer ${input.jwt}`, + 'Content-Type': 'application/json', + 'User-Agent': 'skilld.dev', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + body: JSON.stringify(input.body), + // workerd accepts only `follow` and `manual`. A redirect here is an error. + redirect: 'manual', + signal: AbortSignal.timeout(GITHUB_REQUEST_TIMEOUT_MS), + }) + if (response.status >= 300 && response.status < 400) { + await response.body?.cancel() + throw new Error(`GitHub App request redirected ${response.status}`) + } + if (response.status === 401 || response.status === 403 || response.status === 404) { + await response.body?.cancel() + return { _tag: 'refused', status: response.status } + } + if (!response.ok) + throw new Error(`GitHub App request returned ${response.status}`) + const value = installationTokenSchema.safeParse(await readBoundedJson(response, MAX_TOKEN_RESPONSE_BYTES)) + if (!value.success) + throw new Error('GitHub App returned an invalid response') + return { _tag: 'created', token: value.data.token, expiresAt: value.data.expires_at } +} + +/** + * An RS256 GitHub App JWT. `issuer` is the App's client ID or its App ID; + * GitHub accepts either. + * + * It is valid for ten minutes from a minute in the past, which absorbs clock + * drift between the Worker and GitHub. Ten minutes is GitHub's maximum. + */ +export async function signGithubAppJwt(input: { + issuer: string + privateKeyPkcs8: Uint8Array + /** Unix seconds. */ + now: number +}): Promise { + const issuedAt = input.now - 60 + const header = base64Url(new TextEncoder().encode(JSON.stringify({ alg: 'RS256', typ: 'JWT' }))) + const payload = base64Url(new TextEncoder().encode(JSON.stringify({ + iat: issuedAt, + exp: issuedAt + 600, + iss: input.issuer, + }))) + const unsigned = `${header}.${payload}` + const imported = await crypto.subtle.importKey( + 'pkcs8', + Uint8Array.from(input.privateKeyPkcs8).buffer, + { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' }, + false, + ['sign'], + ) + const signature = await crypto.subtle.sign( + 'RSASSA-PKCS1-v1_5', + imported, + new TextEncoder().encode(unsigned), + ) + return `${unsigned}.${base64Url(new Uint8Array(signature))}` +} + +async function mintInstallationToken( + app: ReadAppConfig, + runtime: Pick, +): Promise { + const minted = await signGithubAppJwt({ issuer: String(app.appId), privateKeyPkcs8: app.privateKeyPkcs8, now: runtime.now() }) + .then(jwt => requestInstallationToken({ + fetch: runtime.fetch, + jwt, + installationId: app.installationId, + body: { permissions: { metadata: 'read' } }, + })) + // A network fault, a 5xx or a key WebCrypto rejects: the read uses the + // fallback token, and the reason reaches the report. + .catch((error: unknown) => ({ _tag: 'failed' as const, reason: error instanceof Error ? error.message : String(error) })) + if (minted._tag === 'failed') + return { _tag: 'unavailable', reason: minted.reason.slice(0, 200) } + if (minted._tag === 'refused') + return { _tag: 'unavailable', reason: `GitHub refused the installation token (${minted.status})` } + const expiresAt = Math.floor(Date.parse(minted.expiresAt) / 1000) + return { token: minted.token, expiresAt } +} + +function firstToken(env: GithubCredentialEnv, names: ReadonlyArray): FallbackToken | null { + for (const name of names) { + const token = env[name]?.trim() + if (token) + return { name, token } + } + return null +} + +function positiveInteger(value: string | undefined): number | null { + const trimmed = value?.trim() ?? '' + if (!/^[1-9]\d*$/.test(trimmed)) + return null + const parsed = Number(trimmed) + return Number.isSafeInteger(parsed) ? parsed : null +} + +/** + * The DER bytes of a PKCS #8 PEM private key, or null. + * + * A secret pasted through a dashboard or a JSON file can hold its line breaks + * as the two characters `\n`, so both forms parse. A PKCS #1 key + * (`BEGIN RSA PRIVATE KEY`, as GitHub downloads it) does not: WebCrypto + * imports PKCS #8 only. Convert it with `openssl pkcs8 -topk8 -nocrypt`. + */ +function pkcs8PemBytes(value: string): Uint8Array | null { + const text = value.replaceAll('\\n', '\n').trim() + const match = /^-----BEGIN PRIVATE KEY-----([A-Z0-9+/=\s]+)-----END PRIVATE KEY-----$/i.exec(text) + if (!match) + return null + const base64 = match[1]!.replace(/\s+/g, '') + if (!/^[A-Z0-9+/]+={0,2}$/i.test(base64) || base64.length % 4 !== 0) + return null + const binary = atob(base64) + return Uint8Array.from(binary, character => character.charCodeAt(0)) +} + +function base64Url(bytes: Uint8Array): string { + let binary = '' + for (let offset = 0; offset < bytes.length; offset += 0x8000) + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)) + return btoa(binary).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '') +} + +async function readBoundedJson(response: Response, maximumBytes: number): Promise { + const declared = Number(response.headers.get('content-length')) + if (Number.isFinite(declared) && declared > maximumBytes) + throw new Error('GitHub App response exceeded the byte limit') + if (!response.body) + throw new Error('GitHub App returned an empty response') + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let size = 0 + while (true) { + const next = await reader.read() + if (next.done) + break + size += next.value.byteLength + if (size > maximumBytes) { + await reader.cancel('response too large') + throw new Error('GitHub App response exceeded the byte limit') + } + chunks.push(next.value) + } + const bytes = new Uint8Array(size) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as unknown +} diff --git a/test/unit/artifact-github-read-app.test.ts b/test/unit/artifact-github-read-app.test.ts index d563e577..a52d0de4 100644 --- a/test/unit/artifact-github-read-app.test.ts +++ b/test/unit/artifact-github-read-app.test.ts @@ -1,8 +1,8 @@ import type { SourceRequest } from '../../layers/artifact-delivery/server/schemas/contracts' -import type { ArtifactGithubCredentialReport } from '../../layers/artifact-delivery/server/utils/github-read-credential' +import type { GithubCredentialReport } from '../../shared/server/github-app-credential' import { describe, expect, it } from 'vitest' -import { createInstallationTokenCache } from '../../layers/artifact-delivery/server/utils/github-read-credential' import { createArtifactGithubSource } from '../../layers/artifact-delivery/server/utils/queue' +import { createInstallationTokenCache } from '../../shared/server/github-app-credential' // Builds shared the site token with the hourly registry sync and every `/gh` // page view, so a run failed RATE_LIMITED whenever the sync spent it. The read @@ -90,7 +90,7 @@ describe('the GitHub credential public Artifact builds read with', () => { it('reads with the fallback token, and reports why, when GitHub refuses the installation token', async () => { const keys = await appKeys() const github = fakeGithub({ expiresAt: NOW + 3600, mintStatus: 401 }) - const reports: ArtifactGithubCredentialReport[] = [] + const reports: GithubCredentialReport[] = [] const source = createArtifactGithubSource(appEnv(keys.pem), runtime(github, { report: event => reports.push(event) })) await source.resolve(request) @@ -101,7 +101,7 @@ describe('the GitHub credential public Artifact builds read with', () => { it('reports an unreadable App key and reads with the fallback token', async () => { const github = fakeGithub({ expiresAt: NOW + 3600 }) - const reports: ArtifactGithubCredentialReport[] = [] + const reports: GithubCredentialReport[] = [] const env = { ...appEnv('-----BEGIN RSA PRIVATE KEY-----\nAAAA\n-----END RSA PRIVATE KEY-----'), ARTIFACT_GITHUB_TOKEN: '' } const source = createArtifactGithubSource(env, runtime(github, { report: event => reports.push(event) })) @@ -115,7 +115,7 @@ describe('the GitHub credential public Artifact builds read with', () => { it('never puts a token or the key in a report', async () => { const keys = await appKeys() const github = fakeGithub({ expiresAt: NOW + 3600, mintStatus: 500 }) - const reports: ArtifactGithubCredentialReport[] = [] + const reports: GithubCredentialReport[] = [] await createArtifactGithubSource(appEnv(keys.pem), runtime(github, { report: event => reports.push(event) })).resolve(request) const text = JSON.stringify(reports) @@ -128,7 +128,7 @@ describe('the GitHub credential public Artifact builds read with', () => { it('repeats a read the Repository denied the read App with the fallback token, and reports it', async () => { const keys = await appKeys() const github = fakeGithub({ expiresAt: NOW + 3600, answersApp: appDenied }) - const reports: ArtifactGithubCredentialReport[] = [] + const reports: GithubCredentialReport[] = [] const source = createArtifactGithubSource(appEnv(keys.pem), runtime(github, { report: event => reports.push(event) })) const result = await source.resolve(request) @@ -157,7 +157,7 @@ describe('the GitHub credential public Artifact builds read with', () => { ])('never reads with the fallback token after %s on the read App', async (_, answer) => { const keys = await appKeys() const github = fakeGithub({ expiresAt: NOW + 3600, answersApp: answer }) - const reports: ArtifactGithubCredentialReport[] = [] + const reports: GithubCredentialReport[] = [] const source = createArtifactGithubSource(appEnv(keys.pem), runtime(github, { report: event => reports.push(event) })) const result = await source.resolve(request) @@ -175,7 +175,11 @@ describe('the GitHub credential public Artifact builds read with', () => { const result = await createArtifactGithubSource(env, runtime(github)).resolve(request) expect(github.readAuthorizations).toEqual(['Bearer ghs_installation_1']) - expect(result).toMatchObject({ _tag: 'rejected', code: 'SOURCE_ACCESS_DENIED' }) + expect(result).toMatchObject({ + _tag: 'rejected', + code: 'SOURCE_ACCESS_DENIED', + summary: 'GitHub denies the skilld.dev read App access to this Repository, and no fallback token is set.', + }) }) }) @@ -215,7 +219,7 @@ function fakeGithub(options: { expiresAt: number, mintStatus?: number, answersAp function runtime( github: FakeGithub, - overrides: Partial<{ cache: ReturnType, now: () => number, report: (event: ArtifactGithubCredentialReport) => void }> = {}, + overrides: Partial<{ cache: ReturnType, now: () => number, report: (event: GithubCredentialReport) => void }> = {}, ) { return { fetch: github.fetch, diff --git a/test/unit/artifact-requester-github.test.ts b/test/unit/artifact-requester-github.test.ts index 3ef2c0a6..b4fa30af 100644 --- a/test/unit/artifact-requester-github.test.ts +++ b/test/unit/artifact-requester-github.test.ts @@ -89,6 +89,17 @@ describe('a signed-in run when the shared GitHub credential is rate limited', () expect(shared.resolve).toHaveBeenCalledOnce() }) + it('reads with the requester\'s token when GitHub denies the read App and no fallback token is set', async () => { + const github = userGithub() + const shared = sharedGithub({ code: 'SOURCE_ACCESS_DENIED' }) + const id = await requested('alice-run-0000008', ALICE) + + await processArtifactBuild(dependencies(github, shared), id) + + expect(github.authorizations).toContain('Bearer gho_alice') + expect(await getResolution(fixture.db, id)).toMatchObject({ state: 'failed', error_code: 'SOURCE_NOT_FOUND' }) + }) + it('reports RATE_LIMITED when the requester has no stored GitHub token', async () => { fixture.raw.prepare(`UPDATE users SET github_token_encrypted = NULL WHERE id = ?`).run(ALICE) const github = userGithub() diff --git a/test/unit/registry-github-app-credential.test.ts b/test/unit/registry-github-app-credential.test.ts new file mode 100644 index 00000000..2f50c7b4 --- /dev/null +++ b/test/unit/registry-github-app-credential.test.ts @@ -0,0 +1,196 @@ +import type { GithubCredentialReport } from '../../shared/server/github-app-credential' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + getRepo, + getRepoSummariesBatch, + getRepoSummary, + resolveGithubBindings, +} from '../../layers/registry/server/utils/github-client' +import { githubRateLimited } from '../../layers/registry/server/utils/github-rate-limited' +import { createInstallationTokenCache } from '../../shared/server/github-app-credential' + +// Every Worker GitHub read used `GITHUB_TOKEN`, a personal token whose quota +// the owner's own tools and CI runners also spend. The read App's +// installation has a bucket of its own. `GITHUB_TOKEN` is now read only where +// GitHub denies the App, such as an organization that restricts Apps. +const APP_ID = '5212127' +const INSTALLATION_ID = '168540695' +const NOW = 1_791_300_000 + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('registry GitHub reads with the read App', () => { + it('reads with the installation token, not GITHUB_TOKEN', async () => { + const github = await stubGithub({}) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const repo = await getRepo('nuxt', 'ui', bindings) + + expect(repo.status).toBe(200) + expect(github.mints).toBe(1) + expect(github.reads).toEqual(['Bearer ghs_app']) + }) + + it('repeats a read GitHub denied the App with GITHUB_TOKEN, and reports it', async () => { + const github = await stubGithub({ deniesApp: true }) + const reports: GithubCredentialReport[] = [] + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime(reports)) + + const repo = await getRepo('neondatabase', 'agent-skills', bindings) + + expect(repo.status).toBe(200) + expect(github.reads).toEqual(['Bearer ghs_app', 'Bearer site']) + expect(reports).toEqual([{ + outcome: 'app-denied', + reason: 'GitHub denied the read App for /repos/neondatabase/agent-skills', + fallback: 'GITHUB_TOKEN', + }]) + }) + + it('answers a denied read as denied, not rate limited, with no GITHUB_TOKEN', async () => { + const github = await stubGithub({ deniesApp: true }) + const reports: GithubCredentialReport[] = [] + const bindings = resolveGithubBindings(await appEnv({}), runtime(reports)) + + const repo = await getRepo('neondatabase', 'agent-skills', bindings) + + expect(github.reads).toEqual(['Bearer ghs_app']) + expect(repo).toMatchObject({ status: 403, denied: true, data: null }) + // The sync pauses every repository for a spent quota. A denial names one. + expect(githubRateLimited(repo)).toBe(false) + expect(reports).toEqual([{ + outcome: 'app-denied', + reason: 'GitHub denied the read App for /repos/neondatabase/agent-skills', + fallback: 'none', + }]) + }) + + it('keeps a secondary rate limit a rate limit, with no second read', async () => { + const github = await stubGithub({ secondaryLimit: true }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const repo = await getRepo('nuxt', 'ui', bindings) + + expect(github.reads).toEqual(['Bearer ghs_app']) + expect(githubRateLimited(repo)).toBe(true) + }) + + it('repeats a GraphQL summary GitHub answered FORBIDDEN with GITHUB_TOKEN', async () => { + const github = await stubGithub({ deniesApp: true }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const summary = await getRepoSummary('neondatabase', 'agent-skills', bindings) + + expect(summary.status).toBe(200) + expect(github.reads).toEqual(['Bearer ghs_app', 'Bearer site']) + }) + + it('leaves a FORBIDDEN alias to the per-repository sync and keeps the rest of the batch', async () => { + const github = await stubGithub({ deniesApp: true }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const batch = await getRepoSummariesBatch([ + { owner: 'nuxt', repo: 'ui' }, + { owner: 'neondatabase', repo: 'agent-skills' }, + ], bindings) + + expect(batch).toMatchObject({ _tag: 'read', summaries: [{ meta: { full_name: 'nuxt/ui' } }, null] }) + expect(github.reads).toEqual(['Bearer ghs_app']) + }) + + it('reads with GITHUB_TOKEN when no App secret is set, as in local development', async () => { + const github = await stubGithub({}) + const bindings = resolveGithubBindings({ GITHUB_TOKEN: 'local' }, runtime()) + + await getRepo('nuxt', 'ui', bindings) + + expect(github.mints).toBe(0) + expect(github.reads).toEqual(['Bearer local']) + }) +}) + +function runtime(reports: GithubCredentialReport[] = []) { + return { tokenCache: createInstallationTokenCache(), now: () => NOW, report: (event: GithubCredentialReport) => reports.push(event) } +} + +async function appEnv(tokens: { GITHUB_TOKEN?: string }) { + const pair = await crypto.subtle.generateKey( + { name: 'RSASSA-PKCS1-v1_5', modulusLength: 2048, publicExponent: new Uint8Array([1, 0, 1]), hash: 'SHA-256' }, + true, + ['sign', 'verify'], + ) as CryptoKeyPair + const der = new Uint8Array(await crypto.subtle.exportKey('pkcs8', pair.privateKey) as ArrayBuffer) + const pem = `-----BEGIN PRIVATE KEY-----\n${btoa(String.fromCharCode(...der))}\n-----END PRIVATE KEY-----\n` + return { + SKILLD_READ_APP_ID: APP_ID, + SKILLD_READ_APP_INSTALLATION_ID: INSTALLATION_ID, + SKILLD_READ_APP_PRIVATE_KEY_PKCS8: pem, + ...tokens, + } +} + +/** + * GitHub as the App and a personal token see it. `neondatabase` denies the + * App on every read, as it did on 2026-10-06, and answers a personal token. + */ +async function stubGithub(options: { deniesApp?: boolean, secondaryLimit?: boolean }) { + const state = { mints: 0, reads: [] as Array } + vi.stubGlobal('fetch', vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + const headers = new Headers(init?.headers) + if (url.endsWith(`/app/installations/${INSTALLATION_ID}/access_tokens`)) { + state.mints++ + return Response.json({ token: 'ghs_app', expires_at: new Date((NOW + 3600) * 1000).toISOString() }, { status: 201 }) + } + const authorization = headers.get('authorization') + state.reads.push(authorization) + const asApp = authorization === 'Bearer ghs_app' + if (options.secondaryLimit) + return Response.json({ message: 'You have exceeded a secondary rate limit. Please wait a few minutes before you try again.' }, { status: 403 }) + if (url.endsWith('/graphql')) { + const body = JSON.parse(String(init?.body)) as { variables: Record } + const owners = Object.entries(body.variables).filter(([key]) => key === 'owner' || /^o\d+$/.test(key)) + const denied = options.deniesApp && asApp && owners.some(([, owner]) => owner === 'neondatabase') + const repository = (owner: string, name: string) => ({ + databaseId: owner.length * 1000 + name.length, + name, + nameWithOwner: `${owner}/${name}`, + url: `https://github.com/${owner}/${name}`, + owner: { login: owner }, + description: null, + stargazerCount: 1, + forkCount: 0, + pushedAt: '2026-10-01T00:00:00Z', + createdAt: '2026-01-01T00:00:00Z', + isArchived: false, + isFork: false, + defaultBranchRef: { name: 'main', target: { oid: 'a'.repeat(40), tree: { oid: 'b'.repeat(40) } } }, + }) + if ('owner' in body.variables) { + return denied + ? Response.json({ data: { repository: null }, errors: [{ type: 'FORBIDDEN', message: 'Resource not accessible by integration' }] }) + : Response.json({ data: { repository: repository(body.variables.owner!, body.variables.repo!) } }) + } + const data: Record = {} + const errors: Array<{ type: string, path: string[] }> = [] + for (const [key, owner] of owners) { + const alias = `r${key.slice(1)}` + const name = body.variables[`n${key.slice(1)}`]! + if (owner === 'neondatabase' && denied) { + data[alias] = null + errors.push({ type: 'FORBIDDEN', path: [alias] }) + } + else { + data[alias] = repository(owner, name) + } + } + return Response.json(errors.length ? { data, errors } : { data }) + } + if (options.deniesApp && asApp && url.includes('/neondatabase/')) + return Response.json({ message: 'Resource not accessible by integration' }, { status: 403, headers: { 'x-ratelimit-remaining': '4990' } }) + return Response.json({ name: 'repo', full_name: 'owner/repo', owner: { login: 'owner' }, default_branch: 'main' }, { status: 200 }) + })) + return state +} From 6e7e3bf38b429b5993e086ea51defcffb2cfb278 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 13:18:40 +1100 Subject: [PATCH 2/3] fix(github): renew a rejected App token and contain fallback reads A 401 on the cached installation token sent every read to GITHUB_TOKEN until the token's stated expiry, and in the registry it paused the whole sync. The read now drops the token and mints a new one once; only a denial, or a new token GitHub rejects too, reaches the fallback. A read GITHUB_TOKEN repeats for a Repository that denies the App now answers as that Repository's denial when the token is spent or expired, and reports the App's quota. Its owner's bucket paused every Repository before. The REST, GraphQL and build paths share one rate limit message rule, so an abuse detection limit, or a FORBIDDEN error that names a limit, never reaches GITHUB_TOKEN. A summary batch never falls back as a whole. --- .../server/utils/github-read-credential.ts | 50 +++-- .../server/utils/github-source.ts | 5 +- .../artifact-delivery/server/utils/queue.ts | 3 +- layers/registry/server/utils/github-client.ts | 129 ++++++----- shared/server/github-app-credential.ts | 151 ++++++++++--- test/unit/artifact-github-read-app.test.ts | 40 +++- .../registry-github-app-credential.test.ts | 204 +++++++++++++++--- 7 files changed, 452 insertions(+), 130 deletions(-) diff --git a/layers/artifact-delivery/server/utils/github-read-credential.ts b/layers/artifact-delivery/server/utils/github-read-credential.ts index 151ae7c1..1d36ea3a 100644 --- a/layers/artifact-delivery/server/utils/github-read-credential.ts +++ b/layers/artifact-delivery/server/utils/github-read-credential.ts @@ -1,6 +1,6 @@ import type { GithubCredential, GithubCredentialConfig, GithubCredentialEnv, GithubCredentialReport } from '#shared/server/github-app-credential' import type { LoadSourceResult, PublicGithubSourceClient, ResolveSourceResult } from './github-source' -import { parseGithubCredentialConfig } from '#shared/server/github-app-credential' +import { parseGithubCredentialConfig, readWithGithubCredential } from '#shared/server/github-app-credential' /** * Where public Artifact builds get their GitHub credential: the read App's @@ -17,32 +17,50 @@ export function parseArtifactGithubCredentialConfig(env: GithubCredentialEnv): G * Some organizations deny the read App on a public Repository that a personal * token reads: `neondatabase/agent-skills` answered SOURCE_ACCESS_DENIED to * every run on 2026-10-06. A read the App was denied repeats once with the - * fallback token. Every denial is reported, with or without a fallback. + * fallback token. A 401 first repeats with a new installation token. Every + * use of the fallback is reported, and so is a refusal with no fallback. */ export function withGithubCredential( credential: GithubCredential, - create: (token: string | undefined) => PublicGithubSourceClient, + fetch: typeof globalThis.fetch, + create: (token: string | undefined, fetch: typeof globalThis.fetch) => PublicGithubSourceClient, report: (event: GithubCredentialReport) => void, ): PublicGithubSourceClient { const read = async ( repository: { owner: string, repository: string }, run: (client: PublicGithubSourceClient) => Promise, ): Promise => { - const current = await credential.current() - const first = await run(create(current.token)) - if (!current.isApp || first._tag !== 'rejected' || first.code !== 'SOURCE_ACCESS_DENIED') - return first - const fallback = credential.fallback - report({ - outcome: 'app-denied', - reason: `GitHub denied the read App for ${repository.owner}/${repository.repository}`, - fallback: fallback?.name ?? 'none', + const settled = await readWithGithubCredential(credential, { + label: `${repository.owner}/${repository.repository}`, + send: async (token) => { + // A rejection code cannot tell a 401, which names the token, from a 403. + const seen = { unauthorized: false } + const result = await run(create(token, async (input, init) => { + const response = await fetch(input, init) + if (response.status === 401) + seen.unauthorized = true + return response + })) + return { result, unauthorized: seen.unauthorized } + }, + refusal: async ({ result, unauthorized }) => { + if (result._tag !== 'rejected') + return null + if (unauthorized) + return 'unauthorized' + return result.code === 'SOURCE_ACCESS_DENIED' ? 'denied' : null + }, + report, + fallbackOnDenial: true, }) - if (fallback) - return await run(create(fallback.token)) + const result = settled.answer.result + if (settled._tag !== 'refused' || result._tag !== 'rejected') + return result return { - ...first, - summary: 'GitHub denies the skilld.dev read App access to this Repository, and no fallback token is set.', + ...result, + summary: settled.refusal === 'denied' + ? 'GitHub denies the skilld.dev read App access to this Repository, and no fallback token is set.' + : 'GitHub rejected the skilld.dev read App\'s token, and no fallback token is set.', } } return { diff --git a/layers/artifact-delivery/server/utils/github-source.ts b/layers/artifact-delivery/server/utils/github-source.ts index 06d5e309..5faf8ed8 100644 --- a/layers/artifact-delivery/server/utils/github-source.ts +++ b/layers/artifact-delivery/server/utils/github-source.ts @@ -4,6 +4,7 @@ import type { PackedFile, SkillFileReader } from './artifact-pack' import type { GithubReadTry } from './github-read' import type { GitTreeEntry, LinkSourceReader, LinkText, SkillEntry, SymbolicLinkNote } from './symbolic-links' import { z } from 'zod' +import { GITHUB_RATE_LIMIT_MESSAGE } from '#shared/server/github-app-credential' import { canonicalSkillFolder, isRegistrySkillPath, slugifySkillName } from '#shared/skill-path' import { createGithubArchiveReader } from './archive-reader' import { splitUstarPath } from './checks' @@ -1393,8 +1394,6 @@ interface GithubRateLimit { resetAt: number | null } -/** The words GitHub uses for a primary limit, a secondary limit, and the older abuse limit. */ -const RATE_LIMIT_MESSAGE = /\brate limit\b|\babuse detection\b/i const MAX_GITHUB_ERROR_BYTES = 64 * 1024 const githubErrorSchema = z.object({ message: z.string() }) @@ -1420,7 +1419,7 @@ export function githubRateLimit( return { _tag: 'rate-limited', resetAt: now + retryAfter } if (response.headers.get('x-ratelimit-remaining') === '0') return { _tag: 'rate-limited', resetAt: epochHeader(response.headers, 'x-ratelimit-reset') } - return response.status === 429 || (message !== null && RATE_LIMIT_MESSAGE.test(message)) + return response.status === 429 || (message !== null && GITHUB_RATE_LIMIT_MESSAGE.test(message)) ? { _tag: 'rate-limited', resetAt: null } : null } diff --git a/layers/artifact-delivery/server/utils/queue.ts b/layers/artifact-delivery/server/utils/queue.ts index 55fd516e..1ee18b08 100644 --- a/layers/artifact-delivery/server/utils/queue.ts +++ b/layers/artifact-delivery/server/utils/queue.ts @@ -174,7 +174,8 @@ export function createArtifactGithubSource( const credential = createGithubCredential(parseArtifactGithubCredentialConfig(env), runtime) return withGithubCredential( credential, - token => createPublicGithubSourceClient({ fetch: runtime.fetch, token, onReadFailure: reportGithubReadFailure, cache }), + runtime.fetch, + (token, fetch) => createPublicGithubSourceClient({ fetch, token, onReadFailure: reportGithubReadFailure, cache }), runtime.report, ) } diff --git a/layers/registry/server/utils/github-client.ts b/layers/registry/server/utils/github-client.ts index 64bdb98e..e801ef51 100644 --- a/layers/registry/server/utils/github-client.ts +++ b/layers/registry/server/utils/github-client.ts @@ -1,16 +1,19 @@ /// import type { + GithubAppRefusal, GithubCredential, GithubCredentialReport, - GithubReadCredential, InstallationTokenCache, } from '#shared/server/github-app-credential' import { createGithubCredential, - githubDeniedApp, + GITHUB_RATE_LIMIT_MESSAGE, + githubAppRefusal, isolateInstallationTokenCache, parseGithubCredentialConfig, + readWithGithubCredential, + staticGithubCredential, } from '#shared/server/github-app-credential' const API_BASE = 'https://api.github.com' @@ -111,22 +114,29 @@ function reportGithubCredential(event: GithubCredentialReport): void { })) } -async function readCredential(bindings: GithubBindings): Promise { - return bindings.credential ? await bindings.credential.current() : { token: bindings.GITHUB_TOKEN, isApp: false } +function credentialOf(bindings: GithubBindings): GithubCredential { + return bindings.credential ?? staticGithubCredential(bindings.GITHUB_TOKEN) +} + +function readAsApp( + bindings: GithubBindings, + read: { + label: string + send: (token: string | undefined) => Promise + refusal: (answer: A) => Promise + fallbackOnDenial: boolean + }, +) { + return readWithGithubCredential(credentialOf(bindings), { ...read, report: bindings.reportCredential ?? reportGithubCredential }) } /** - * Report a read GitHub denied the App, and answer the token to repeat it - * with, or null when there is none. Every use of the fallback is reported. + * Whether GITHUB_TOKEN failed a read it repeated for a Repository that + * denies the App. Its quota and expiry belong to its owner, so the failure + * names that Repository and never pauses the sync. */ -function fallbackAfterDenial(bindings: GithubBindings, label: string): string | null { - const fallback = bindings.credential?.fallback ?? null - ;(bindings.reportCredential ?? reportGithubCredential)({ - outcome: 'app-denied', - reason: `GitHub denied the read App for ${label}`, - fallback: fallback?.name ?? 'none', - }) - return fallback?.token ?? null +function fallbackRefusedStatus(status: number): boolean { + return status === 401 || status === 403 || status === 429 } function withAuthorization(headers: Headers, token: string | undefined): Headers { @@ -265,16 +275,24 @@ async function ghRequest( headers.set('If-None-Match', cached.etag) } - const send = (token: string | undefined) => fetch(url, { ...init, headers: withAuthorization(headers, token) }) - const credential = await readCredential(bindings) - let res = await send(credential.token) - if (credential.isApp && await githubDeniedApp(res)) { - const fallback = fallbackAfterDenial(bindings, new URL(url).pathname) - if (fallback === null) - return { status: 403, data: null, rateLimit: parseRateLimit(res.headers), notModified: false, denied: true } - res = await send(fallback) + const read = await readAsApp(bindings, { + label: new URL(url).pathname, + send: token => fetch(url, { ...init, headers: withAuthorization(headers, token) }), + refusal: githubAppRefusal, + fallbackOnDenial: true, + }) + if (read._tag === 'refused') { + const rateLimit = parseRateLimit(read.answer.headers) + return read.refusal === 'denied' + ? { status: 403, data: null, rateLimit, notModified: false, denied: true } + : { status: 401, data: null, rateLimit, notModified: false } } - const rateLimit = parseRateLimit(res.headers) + const res = read.answer + // The sync paces itself on the App's quota, not on GITHUB_TOKEN's. + const deniedRead = read._tag === 'fallback' && read.refusal === 'denied' + const rateLimit = parseRateLimit(deniedRead ? read.refused.headers : res.headers) + if (deniedRead && fallbackRefusedStatus(res.status)) + return { status: 403, data: null, rateLimit, notModified: false, denied: true } if (res.status === 304 && cached) { return { status: 304, data: cached.body, rateLimit, notModified: true } @@ -403,7 +421,7 @@ export async function getRepoSummary( ${REPO_SUMMARY_FIELDS} } }` - const out = await gqlRequest<{ repository: RepoSummaryGql | null }>(query, { owner, repo }, bindings, { label: `${owner}/${repo}`, retryForbidden: true }) + const out = await gqlRequest<{ repository: RepoSummaryGql | null }>(query, { owner, repo }, bindings, { label: `${owner}/${repo}`, oneRepository: true }) if (out._tag === 'failed') { return out.denied ? { status: out.status, data: null, rateLimit: out.rateLimit, notModified: false, denied: true } @@ -477,7 +495,7 @@ export async function getRepoSummariesBatch( variables[`n${i}`] = request.repo }) sent++ - const out = await gqlRequest>(`query(${varDecls.join(',')}){${aliases}}`, variables, bindings, { label: 'a summary batch', retryForbidden: false }) + const out = await gqlRequest>(`query(${varDecls.join(',')}){${aliases}}`, variables, bindings, { label: 'a summary batch', oneRepository: false }) rateLimit = out.rateLimit ?? rateLimit if (out._tag === 'failed') return { _tag: 'failed', status: out.status, rateLimit, requests: sent } @@ -547,34 +565,51 @@ type GqlRequestOutcome * * A query about one Repository that GitHub denied the read App, with a 403 * or a FORBIDDEN error, repeats once with the fallback token. A batch over - * many Repositories passes `retryForbidden: false`: one denied alias says + * many Repositories passes `oneRepository: false`: one denied alias says * nothing about the others, and the per-repository sync repeats it. */ async function gqlRequest( query: string, variables: Record, bindings: GithubBindings, - options: { label: string, retryForbidden: boolean }, + options: { label: string, oneRepository: boolean }, ): Promise> { - const credential = await readCredential(bindings) - const first = await gqlSend(query, variables, credential.token, credential.isApp) - const deniedApp = first._tag === 'failed' - ? first.denied === true - : options.retryForbidden && first.errors.some(error => error.type === 'FORBIDDEN') - if (!credential.isApp || !deniedApp) - return first - const fallback = fallbackAfterDenial(bindings, options.label) - if (fallback === null) - return { _tag: 'failed', status: 403, rateLimit: first.rateLimit, denied: true } - return await gqlSend(query, variables, fallback, false) + const read = await readAsApp(bindings, { + label: options.label, + send: token => gqlSend(query, variables, token), + refusal: async sent => sent.refusal + ?? (options.oneRepository && sent.outcome._tag === 'answered' && gqlDeniedApp(sent.outcome.errors) ? 'denied' : null), + fallbackOnDenial: options.oneRepository, + }) + const out = read.answer.outcome + if (read._tag === 'refused') { + return read.refusal === 'denied' + ? { _tag: 'failed', status: 403, rateLimit: out.rateLimit, denied: true } + : { _tag: 'failed', status: 401, rateLimit: out.rateLimit } + } + if (read._tag === 'read' || read.refusal === 'unauthorized') + return out + // The sync paces itself on the App's quota, not on GITHUB_TOKEN's. + const rateLimit = read.refused.outcome.rateLimit + const fallbackFailed = out._tag === 'failed' + ? fallbackRefusedStatus(out.status) + : out.errors.some(error => error.type === 'RATE_LIMITED' || error.type === 'FORBIDDEN') + return fallbackFailed ? { _tag: 'failed', status: 403, rateLimit, denied: true } : { ...out, rateLimit } +} + +/** + * Whether GraphQL errors say GitHub denies the App the Repository. The type + * FORBIDDEN alone does not: a message that names a rate limit is a limit. + */ +function gqlDeniedApp(errors: GqlError[]): boolean { + return errors.some(error => error.type === 'FORBIDDEN' && !GITHUB_RATE_LIMIT_MESSAGE.test(error.message ?? '')) } async function gqlSend( query: string, variables: Record, token: string | undefined, - asApp: boolean, -): Promise> { +): Promise<{ outcome: GqlRequestOutcome, refusal: GithubAppRefusal | null }> { const headers = new Headers() headers.set('Accept', 'application/vnd.github+json') headers.set('Content-Type', 'application/json') @@ -587,10 +622,8 @@ async function gqlSend( body: JSON.stringify({ query, variables }), }) const rateLimit = parseRateLimit(res.headers) - if (!res.ok) { - const denied = asApp && await githubDeniedApp(res) - return denied ? { _tag: 'failed', status: res.status, rateLimit, denied: true } : { _tag: 'failed', status: res.status, rateLimit } - } + if (!res.ok) + return { outcome: { _tag: 'failed', status: res.status, rateLimit }, refusal: await githubAppRefusal(res) } // A gateway can answer 200 with a truncated or non-JSON body. Parsing that // eagerly threw `Unexpected end of JSON input` out of the client and reached // the sync summary as an opaque reason with no status attached. @@ -599,8 +632,8 @@ async function gqlSend( return null }) as { data?: T, errors?: GqlError[] } | null if (!body) - return { _tag: 'failed', status: 502, rateLimit } - return { _tag: 'answered', data: body.data ?? null, errors: body.errors ?? [], rateLimit } + return { outcome: { _tag: 'failed', status: 502, rateLimit }, refusal: null } + return { outcome: { _tag: 'answered', data: body.data ?? null, errors: body.errors ?? [], rateLimit }, refusal: null } } async function gqlPost( @@ -609,7 +642,7 @@ async function gqlPost( bindings: GithubBindings, label: string, ): Promise<{ status: number, data: T | null, rateLimit: RateLimitInfo | null, denied?: true }> { - const out = await gqlRequest(query, variables, bindings, { label, retryForbidden: true }) + const out = await gqlRequest(query, variables, bindings, { label, oneRepository: true }) if (out._tag === 'failed') return out.denied ? { status: out.status, data: null, rateLimit: out.rateLimit, denied: true } : { status: out.status, data: null, rateLimit: out.rateLimit } if (out.errors.length) { @@ -750,7 +783,7 @@ export async function getRawFile( ): Promise { const headers = new Headers() headers.set('User-Agent', 'skilld.dev') - const credential = await readCredential(bindings) + const credential = await credentialOf(bindings).current() if (credential.token) headers.set('Authorization', `Bearer ${credential.token}`) const res = await fetch(`${RAW_BASE}/${owner}/${repo}/${ref}/${path}`, { headers }) diff --git a/shared/server/github-app-credential.ts b/shared/server/github-app-credential.ts index 03dd0973..30be7363 100644 --- a/shared/server/github-app-credential.ts +++ b/shared/server/github-app-credential.ts @@ -56,7 +56,7 @@ export interface GithubCredentialEnv { * secrets and paths, never a token value. */ export interface GithubCredentialReport { - outcome: 'app-misconfigured' | 'app-token-unavailable' | 'app-denied' + outcome: 'app-misconfigured' | 'app-token-unavailable' | 'app-denied' | 'app-token-rejected' reason: string fallback: FallbackToken['name'] | 'anonymous' | 'none' } @@ -108,6 +108,12 @@ export interface GithubReadCredential { export interface GithubCredential { current: () => Promise + /** + * The credential after GitHub answered 401 to `rejected`. The cached + * installation token is dropped and a new one minted, once for every read + * that held it. + */ + renew: (rejected: GithubReadCredential) => Promise /** The token for a read GitHub denied the App, or null. */ fallback: FallbackToken | null } @@ -152,61 +158,138 @@ export function createGithubCredential( runtime: GithubCredentialRuntime, ): GithubCredential { if (config._tag === 'anonymous') - return { current: async () => ({ token: undefined, isApp: false }), fallback: null } + return staticGithubCredential(undefined) if (config._tag === 'token') - return { current: async () => ({ token: config.token.token, isApp: false }), fallback: null } + return staticGithubCredential(config.token.token) const fallbackName = config.fallback?.name ?? 'anonymous' const fallback = { token: config.fallback?.token, isApp: false } if (config._tag === 'app-misconfigured') { - return { - current: async () => { - runtime.report({ outcome: 'app-misconfigured', reason: config.reason, fallback: fallbackName }) - return fallback - }, - fallback: null, + const current = async (): Promise => { + runtime.report({ outcome: 'app-misconfigured', reason: config.reason, fallback: fallbackName }) + return fallback } + return { current, renew: current, fallback: null } } const app = config.app const key = `${app.appId}:${app.installationId}` + const current = async (): Promise => { + const cached = runtime.tokenCache.tokens.get(key) + if (cached && cached.expiresAt - INSTALLATION_TOKEN_RENEW_SECONDS > runtime.now()) + return { token: cached.token, isApp: true } + let pending = runtime.tokenCache.pending.get(key) + if (!pending) { + pending = mintInstallationToken(app, runtime).finally(() => runtime.tokenCache.pending.delete(key)) + runtime.tokenCache.pending.set(key, pending) + } + const minted = await pending + if ('token' in minted) { + runtime.tokenCache.tokens.set(key, minted) + return { token: minted.token, isApp: true } + } + runtime.report({ outcome: 'app-token-unavailable', reason: minted.reason, fallback: fallbackName }) + return fallback + } return { - current: async () => { - const cached = runtime.tokenCache.tokens.get(key) - if (cached && cached.expiresAt - INSTALLATION_TOKEN_RENEW_SECONDS > runtime.now()) - return { token: cached.token, isApp: true } - let pending = runtime.tokenCache.pending.get(key) - if (!pending) { - pending = mintInstallationToken(app, runtime).finally(() => runtime.tokenCache.pending.delete(key)) - runtime.tokenCache.pending.set(key, pending) - } - const minted = await pending - if ('token' in minted) { - runtime.tokenCache.tokens.set(key, minted) - return { token: minted.token, isApp: true } - } - runtime.report({ outcome: 'app-token-unavailable', reason: minted.reason, fallback: fallbackName }) - return fallback + current, + renew: async (rejected) => { + // Another read that held the same token may have renewed it already. + if (runtime.tokenCache.tokens.get(key)?.token === rejected.token) + runtime.tokenCache.tokens.delete(key) + return await current() }, fallback: config.fallback, } } +/** A credential of one token, or anonymous reads, with no App and no fallback. */ +export function staticGithubCredential(token: string | undefined): GithubCredential { + const current = async (): Promise => ({ token, isApp: false }) + return { current, renew: current, fallback: null } +} + +/** The words GitHub uses for a primary limit, a secondary limit, and the older abuse limit. */ +export const GITHUB_RATE_LIMIT_MESSAGE = /\brate limit\b|\babuse detection\b/i + +/** + * Why GitHub refused a read made as the App: `unauthorized` names the token, + * and `denied` names the Repository. + */ +export type GithubAppRefusal = 'unauthorized' | 'denied' + /** - * Whether GitHub refused a read because it denies the App, as opposed to a - * spent quota. Organizations that restrict Apps answer 403 with quota left, - * and no rate limit message. `neondatabase` did on 2026-10-06. + * How GitHub refused an HTTP read made as the App, or null when it did not. * - * It reads the body of a 403, so pass a clone or a response whose body is - * not needed. + * A 403 with quota left and no rate limit message is an organization that + * denies the App, as `neondatabase` did on 2026-10-06. A spent quota is no + * refusal: the read stands as rate limited. + * + * It reads the body of a 403, so pass a response whose body is not needed. */ -export async function githubDeniedApp(response: Response): Promise { +export async function githubAppRefusal(response: Response): Promise { + if (response.status === 401) + return 'unauthorized' if (response.status !== 403) - return false + return null if (response.headers.get('x-ratelimit-remaining') === '0' || response.headers.get('retry-after')) - return false + return null // A 403 body that will not read names no rate limit either, so it counts // as a denial. The status already says the read failed. const text = await response.text().catch(() => '') - return !/rate limit/i.test(text) + return GITHUB_RATE_LIMIT_MESSAGE.test(text) ? null : 'denied' +} + +/** One read through a {@link GithubCredential}, settled. */ +export type GithubCredentialRead + = | { _tag: 'read', answer: A } + /** GitHub refused the App, and the fallback token made the read again. */ + | { _tag: 'fallback', answer: A, refused: A, refusal: GithubAppRefusal } + /** GitHub refused the App, and no fallback token may make the read. */ + | { _tag: 'refused', answer: A, refusal: GithubAppRefusal } + +/** + * Make one read as the read App, and settle a refusal. + * + * A 401 drops the installation token and repeats the read once with a new + * one, so a revoked token does not move every read to the fallback. Only a + * denial, or a new token GitHub rejects too, reaches the fallback, and each + * of those is reported. + */ +export async function readWithGithubCredential( + credential: GithubCredential, + read: { + /** The Repository or API path, for the report. Never a token or a query string. */ + label: string + send: (token: string | undefined) => Promise + /** How GitHub refused a read made as the App, or null when it did not. */ + refusal: (answer: A) => Promise + report: (event: GithubCredentialReport) => void + /** False for a read of many Repositories: one denial there names none of them. */ + fallbackOnDenial: boolean + }, +): Promise> { + let current = await credential.current() + let answer = await read.send(current.token) + let refusal = current.isApp ? await read.refusal(answer) : null + if (refusal === 'unauthorized') { + current = await credential.renew(current) + answer = await read.send(current.token) + refusal = current.isApp ? await read.refusal(answer) : null + } + if (refusal === null) + return { _tag: 'read', answer } + if (refusal === 'denied' && !read.fallbackOnDenial) + return { _tag: 'refused', answer, refusal } + const fallback = credential.fallback + read.report({ + outcome: refusal === 'denied' ? 'app-denied' : 'app-token-rejected', + reason: refusal === 'denied' + ? `GitHub denied the read App for ${read.label}` + : `GitHub rejected a new read App installation token for ${read.label}`, + fallback: fallback?.name ?? 'none', + }) + if (!fallback) + return { _tag: 'refused', answer, refusal } + return { _tag: 'fallback', answer: await read.send(fallback.token), refused: answer, refusal } } const installationTokenSchema = z.object({ diff --git a/test/unit/artifact-github-read-app.test.ts b/test/unit/artifact-github-read-app.test.ts index a52d0de4..7f65b4a9 100644 --- a/test/unit/artifact-github-read-app.test.ts +++ b/test/unit/artifact-github-read-app.test.ts @@ -183,6 +183,40 @@ describe('the GitHub credential public Artifact builds read with', () => { }) }) +// An installation token can stop working before the expiry GitHub gave it. +// A 401 names the token, not the Repository, so one new token fixes it. +describe('a build read GitHub answered 401 for the cached installation token', () => { + it('mints one new token and repeats the read with it, not with a fallback token', async () => { + const keys = await appKeys() + const github = fakeGithub({ expiresAt: NOW + 3600, rejects: ['ghs_installation_1'] }) + const reports: GithubCredentialReport[] = [] + const source = createArtifactGithubSource(appEnv(keys.pem), runtime(github, { report: event => reports.push(event) })) + + const result = await source.resolve(request) + + expect(github.mints).toHaveLength(2) + expect(github.readAuthorizations).toEqual(['Bearer ghs_installation_1', 'Bearer ghs_installation_2']) + expect(result).toMatchObject({ _tag: 'rejected', code: 'SOURCE_NOT_FOUND' }) + expect(reports).toEqual([]) + }) + + it('reads with the fallback token, and reports it, when GitHub rejects the new token too', async () => { + const keys = await appKeys() + const github = fakeGithub({ expiresAt: NOW + 3600, rejects: ['ghs_installation_1', 'ghs_installation_2'] }) + const reports: GithubCredentialReport[] = [] + const source = createArtifactGithubSource(appEnv(keys.pem), runtime(github, { report: event => reports.push(event) })) + + await source.resolve(request) + + expect(github.readAuthorizations).toEqual(['Bearer ghs_installation_1', 'Bearer ghs_installation_2', 'Bearer runs']) + expect(reports).toEqual([{ + outcome: 'app-token-rejected', + reason: 'GitHub rejected a new read App installation token for skilld-dev/skills', + fallback: 'ARTIFACT_GITHUB_TOKEN', + }]) + }) +}) + interface FakeGithub { fetch: typeof fetch mints: Array<{ url: string, authorization: string, body: unknown }> @@ -195,8 +229,8 @@ function appDenied(): Response { return Response.json({ message: 'Resource not accessible by integration' }, { status: 403, headers: { 'x-ratelimit-remaining': '4321' } }) } -/** Answers every read 404, and every read with an installation token `answersApp` when set. */ -function fakeGithub(options: { expiresAt: number, mintStatus?: number, answersApp?: () => Response }): FakeGithub { +/** Answers every read 404, every read with an installation token `answersApp` when set, and 401 to a token in `rejects`. */ +function fakeGithub(options: { expiresAt: number, mintStatus?: number, answersApp?: () => Response, rejects?: string[] }): FakeGithub { const mints: FakeGithub['mints'] = [] const readAuthorizations: FakeGithub['readAuthorizations'] = [] const fetcher = async (input: RequestInfo | URL, init?: RequestInit): Promise => { @@ -210,6 +244,8 @@ function fakeGithub(options: { expiresAt: number, mintStatus?: number, answersAp } const authorization = headers.get('authorization') readAuthorizations.push(authorization) + if (authorization && options.rejects?.includes(authorization.slice('Bearer '.length))) + return Response.json({ message: 'Bad credentials' }, { status: 401 }) if (options.answersApp && authorization?.startsWith('Bearer ghs_')) return options.answersApp() return Response.json({ message: 'Not Found' }, { status: 404 }) diff --git a/test/unit/registry-github-app-credential.test.ts b/test/unit/registry-github-app-credential.test.ts index 2f50c7b4..3afdf017 100644 --- a/test/unit/registry-github-app-credential.test.ts +++ b/test/unit/registry-github-app-credential.test.ts @@ -21,6 +21,9 @@ afterEach(() => { vi.unstubAllGlobals() }) +const readRepo = (bindings: ReturnType) => getRepo('neondatabase', 'agent-skills', bindings) +const readSummary = (bindings: ReturnType) => getRepoSummary('neondatabase', 'agent-skills', bindings) + describe('registry GitHub reads with the read App', () => { it('reads with the installation token, not GITHUB_TOKEN', async () => { const github = await stubGithub({}) @@ -30,7 +33,7 @@ describe('registry GitHub reads with the read App', () => { expect(repo.status).toBe(200) expect(github.mints).toBe(1) - expect(github.reads).toEqual(['Bearer ghs_app']) + expect(github.reads).toEqual(['Bearer ghs_app_1']) }) it('repeats a read GitHub denied the App with GITHUB_TOKEN, and reports it', async () => { @@ -38,10 +41,10 @@ describe('registry GitHub reads with the read App', () => { const reports: GithubCredentialReport[] = [] const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime(reports)) - const repo = await getRepo('neondatabase', 'agent-skills', bindings) + const repo = await readRepo(bindings) expect(repo.status).toBe(200) - expect(github.reads).toEqual(['Bearer ghs_app', 'Bearer site']) + expect(github.reads).toEqual(['Bearer ghs_app_1', 'Bearer site']) expect(reports).toEqual([{ outcome: 'app-denied', reason: 'GitHub denied the read App for /repos/neondatabase/agent-skills', @@ -54,9 +57,9 @@ describe('registry GitHub reads with the read App', () => { const reports: GithubCredentialReport[] = [] const bindings = resolveGithubBindings(await appEnv({}), runtime(reports)) - const repo = await getRepo('neondatabase', 'agent-skills', bindings) + const repo = await readRepo(bindings) - expect(github.reads).toEqual(['Bearer ghs_app']) + expect(github.reads).toEqual(['Bearer ghs_app_1']) expect(repo).toMatchObject({ status: 403, denied: true, data: null }) // The sync pauses every repository for a spent quota. A denial names one. expect(githubRateLimited(repo)).toBe(false) @@ -67,24 +70,41 @@ describe('registry GitHub reads with the read App', () => { }]) }) - it('keeps a secondary rate limit a rate limit, with no second read', async () => { - const github = await stubGithub({ secondaryLimit: true }) + // GITHUB_TOKEN is a personal token. A limit on the App is no denial, so it + // never reaches that token, on REST or on GraphQL. + it.each([ + ['a secondary limit', 'REST', readRepo, 'You have exceeded a secondary rate limit. Please wait a few minutes before you try again.'], + ['a secondary limit', 'GraphQL', readSummary, 'You have exceeded a secondary rate limit. Please wait a few minutes before you try again.'], + ['the abuse detection limit', 'REST', readRepo, 'You have triggered an abuse detection mechanism. Please wait a few minutes before you try again.'], + ['the abuse detection limit', 'GraphQL', readSummary, 'You have triggered an abuse detection mechanism. Please wait a few minutes before you try again.'], + ] as const)('keeps %s on %s a rate limit, with no second read', async (_, __, read, message) => { + const github = await stubGithub({ limitMessage: message }) const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) - const repo = await getRepo('nuxt', 'ui', bindings) + const outcome = await read(bindings) + + expect(github.reads).toEqual(['Bearer ghs_app_1']) + expect(githubRateLimited(outcome)).toBe(true) + }) + + it('never repeats a GraphQL RATE_LIMITED answer with GITHUB_TOKEN', async () => { + const github = await stubGithub({ graphqlRateLimited: true }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const summary = await readSummary(bindings) - expect(github.reads).toEqual(['Bearer ghs_app']) - expect(githubRateLimited(repo)).toBe(true) + expect(github.reads).toEqual(['Bearer ghs_app_1']) + expect(summary.data).toBeNull() }) it('repeats a GraphQL summary GitHub answered FORBIDDEN with GITHUB_TOKEN', async () => { const github = await stubGithub({ deniesApp: true }) const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) - const summary = await getRepoSummary('neondatabase', 'agent-skills', bindings) + const summary = await readSummary(bindings) expect(summary.status).toBe(200) - expect(github.reads).toEqual(['Bearer ghs_app', 'Bearer site']) + expect(github.reads).toEqual(['Bearer ghs_app_1', 'Bearer site']) }) it('leaves a FORBIDDEN alias to the per-repository sync and keeps the rest of the batch', async () => { @@ -97,7 +117,17 @@ describe('registry GitHub reads with the read App', () => { ], bindings) expect(batch).toMatchObject({ _tag: 'read', summaries: [{ meta: { full_name: 'nuxt/ui' } }, null] }) - expect(github.reads).toEqual(['Bearer ghs_app']) + expect(github.reads).toEqual(['Bearer ghs_app_1']) + }) + + it('never repeats a whole summary batch GitHub answered 403 with GITHUB_TOKEN', async () => { + const github = await stubGithub({ graphqlDeniesApp: true }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const batch = await getRepoSummariesBatch([{ owner: 'nuxt', repo: 'ui' }], bindings) + + expect(batch).toMatchObject({ _tag: 'failed', status: 403 }) + expect(github.reads).toEqual(['Bearer ghs_app_1']) }) it('reads with GITHUB_TOKEN when no App secret is set, as in local development', async () => { @@ -111,6 +141,97 @@ describe('registry GitHub reads with the read App', () => { }) }) +// GITHUB_TOKEN's quota belongs to its owner. A denied Repository that it +// cannot read either is a fact about that Repository, so the sync records +// one failure and keeps going on the App's quota. +describe('a read GITHUB_TOKEN repeats for a Repository that denies the App', () => { + it.each([ + ['a spent quota', 'REST', readRepo], + ['a spent quota', 'GraphQL', readSummary], + ['a rejected token', 'REST', readRepo], + ['a rejected token', 'GraphQL', readSummary], + ] as const)('answers %s on %s as denied, not rate limited or unauthorized', async (answer, _, read) => { + await stubGithub({ deniesApp: true, fallbackAnswers: answer === 'a spent quota' ? 'rate-limited' : 'unauthorized' }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const outcome = await read(bindings) + + expect(outcome).toMatchObject({ status: 403, denied: true, data: null }) + expect(githubRateLimited(outcome)).toBe(false) + }) + + it.each([ + ['REST', readRepo], + ['GraphQL', readSummary], + ] as const)('reports the App quota, not GITHUB_TOKEN\'s, on %s', async (_, read) => { + await stubGithub({ deniesApp: true, fallbackAnswers: 'low-quota' }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + + const outcome = await read(bindings) + + expect(outcome.status).toBe(200) + expect(outcome.rateLimit?.remaining).toBe(4990) + }) +}) + +// An installation token can stop working before the expiry GitHub gave it: +// the App key rotated, or the token was revoked. One new token fixes every +// later read, so GITHUB_TOKEN is not spent until that token fails too. +describe('a read GitHub answered 401 for the cached installation token', () => { + it.each([ + ['REST', readRepo], + ['GraphQL', readSummary], + ] as const)('mints one new token and repeats the %s read with it', async (_, read) => { + const github = await stubGithub({ rejects: ['ghs_app_1'] }) + const reports: GithubCredentialReport[] = [] + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime(reports)) + + const outcome = await read(bindings) + + expect(outcome.status).toBe(200) + expect(github.mints).toBe(2) + expect(github.reads).toEqual(['Bearer ghs_app_1', 'Bearer ghs_app_2']) + expect(reports).toEqual([]) + }) + + it('mints once for reads GitHub rejected together', async () => { + const github = await stubGithub({ rejects: ['ghs_app_1'] }) + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime()) + await getRepo('nuxt', 'ui', bindings) + github.reads.length = 0 + + await Promise.all([getRepo('nuxt', 'ui', bindings), getRepo('nuxt', 'nuxt', bindings), getRepo('nuxt', 'image', bindings)]) + + expect(github.mints).toBe(2) + expect(github.reads.filter(read => read === 'Bearer site')).toEqual([]) + }) + + it('reads with GITHUB_TOKEN, and reports it, when GitHub rejects the new token too', async () => { + const github = await stubGithub({ rejects: ['ghs_app_1', 'ghs_app_2'] }) + const reports: GithubCredentialReport[] = [] + const bindings = resolveGithubBindings(await appEnv({ GITHUB_TOKEN: 'site' }), runtime(reports)) + + const repo = await getRepo('nuxt', 'ui', bindings) + + expect(repo.status).toBe(200) + expect(github.reads).toEqual(['Bearer ghs_app_1', 'Bearer ghs_app_2', 'Bearer site']) + expect(reports).toEqual([{ + outcome: 'app-token-rejected', + reason: 'GitHub rejected a new read App installation token for /repos/nuxt/ui', + fallback: 'GITHUB_TOKEN', + }]) + }) + + it('answers 401 when GitHub rejects the new token and no GITHUB_TOKEN is set', async () => { + await stubGithub({ rejects: ['ghs_app_1', 'ghs_app_2'] }) + const bindings = resolveGithubBindings(await appEnv({}), runtime()) + + const repo = await getRepo('nuxt', 'ui', bindings) + + expect(repo).toMatchObject({ status: 401, data: null }) + }) +}) + function runtime(reports: GithubCredentialReport[] = []) { return { tokenCache: createInstallationTokenCache(), now: () => NOW, report: (event: GithubCredentialReport) => reports.push(event) } } @@ -131,25 +252,56 @@ async function appEnv(tokens: { GITHUB_TOKEN?: string }) { } } +interface StubOptions { + /** `neondatabase` denies the App on every read, as it did on 2026-10-06. */ + deniesApp?: boolean + /** Every read as the App answers 403 with this rate limit message. */ + limitMessage?: string + /** Every GraphQL read as the App answers 200 with a RATE_LIMITED error. */ + graphqlRateLimited?: boolean + /** Every GraphQL read as the App answers 403 with quota left. */ + graphqlDeniesApp?: boolean + /** How GitHub answers the personal token. It reads everything by default. */ + fallbackAnswers?: 'rate-limited' | 'unauthorized' | 'low-quota' + /** Installation tokens GitHub answers 401, as for a revoked token. */ + rejects?: string[] +} + /** - * GitHub as the App and a personal token see it. `neondatabase` denies the - * App on every read, as it did on 2026-10-06, and answers a personal token. + * GitHub as the App and a personal token see it. Each mint answers a new + * installation token: `ghs_app_1`, then `ghs_app_2`. */ -async function stubGithub(options: { deniesApp?: boolean, secondaryLimit?: boolean }) { +async function stubGithub(options: StubOptions) { const state = { mints: 0, reads: [] as Array } + const appQuota = { 'x-ratelimit-remaining': '4990', 'x-ratelimit-limit': '5000' } vi.stubGlobal('fetch', vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { const url = String(input) const headers = new Headers(init?.headers) if (url.endsWith(`/app/installations/${INSTALLATION_ID}/access_tokens`)) { state.mints++ - return Response.json({ token: 'ghs_app', expires_at: new Date((NOW + 3600) * 1000).toISOString() }, { status: 201 }) + return Response.json({ token: `ghs_app_${state.mints}`, expires_at: new Date((NOW + 3600) * 1000).toISOString() }, { status: 201 }) } const authorization = headers.get('authorization') state.reads.push(authorization) - const asApp = authorization === 'Bearer ghs_app' - if (options.secondaryLimit) - return Response.json({ message: 'You have exceeded a secondary rate limit. Please wait a few minutes before you try again.' }, { status: 403 }) - if (url.endsWith('/graphql')) { + const asApp = authorization?.startsWith('Bearer ghs_app_') === true + const graphql = url.endsWith('/graphql') + if (asApp && options.rejects?.includes(authorization!.slice('Bearer '.length))) + return Response.json({ message: 'Bad credentials' }, { status: 401 }) + if (asApp && options.limitMessage) + return Response.json({ message: options.limitMessage }, { status: 403, headers: appQuota }) + if (asApp && graphql && options.graphqlDeniesApp) + return Response.json({ message: 'Resource not accessible by integration' }, { status: 403, headers: appQuota }) + if (asApp && graphql && options.graphqlRateLimited) + return Response.json({ data: null, errors: [{ type: 'RATE_LIMITED', message: 'API rate limit exceeded for installation.' }] }, { headers: appQuota }) + if (!asApp && options.fallbackAnswers === 'unauthorized') + return Response.json({ message: 'Bad credentials' }, { status: 401 }) + if (!asApp && options.fallbackAnswers === 'rate-limited') { + return graphql + ? Response.json({ data: null, errors: [{ type: 'RATE_LIMITED', message: 'API rate limit exceeded for user ID 5326365.' }] }, { headers: { 'x-ratelimit-remaining': '0' } }) + : Response.json({ message: 'API rate limit exceeded for user ID 5326365.' }, { status: 403, headers: { 'x-ratelimit-remaining': '0' } }) + } + const quota = asApp ? appQuota : { 'x-ratelimit-remaining': options.fallbackAnswers === 'low-quota' ? '12' : '4000' } + if (graphql) { const body = JSON.parse(String(init?.body)) as { variables: Record } const owners = Object.entries(body.variables).filter(([key]) => key === 'owner' || /^o\d+$/.test(key)) const denied = options.deniesApp && asApp && owners.some(([, owner]) => owner === 'neondatabase') @@ -170,8 +322,8 @@ async function stubGithub(options: { deniesApp?: boolean, secondaryLimit?: boole }) if ('owner' in body.variables) { return denied - ? Response.json({ data: { repository: null }, errors: [{ type: 'FORBIDDEN', message: 'Resource not accessible by integration' }] }) - : Response.json({ data: { repository: repository(body.variables.owner!, body.variables.repo!) } }) + ? Response.json({ data: { repository: null }, errors: [{ type: 'FORBIDDEN', message: 'Resource not accessible by integration' }] }, { headers: quota }) + : Response.json({ data: { repository: repository(body.variables.owner!, body.variables.repo!) } }, { headers: quota }) } const data: Record = {} const errors: Array<{ type: string, path: string[] }> = [] @@ -186,11 +338,11 @@ async function stubGithub(options: { deniesApp?: boolean, secondaryLimit?: boole data[alias] = repository(owner, name) } } - return Response.json(errors.length ? { data, errors } : { data }) + return Response.json(errors.length ? { data, errors } : { data }, { headers: quota }) } if (options.deniesApp && asApp && url.includes('/neondatabase/')) - return Response.json({ message: 'Resource not accessible by integration' }, { status: 403, headers: { 'x-ratelimit-remaining': '4990' } }) - return Response.json({ name: 'repo', full_name: 'owner/repo', owner: { login: 'owner' }, default_branch: 'main' }, { status: 200 }) + return Response.json({ message: 'Resource not accessible by integration' }, { status: 403, headers: quota }) + return Response.json({ name: 'repo', full_name: 'owner/repo', owner: { login: 'owner' }, default_branch: 'main' }, { status: 200, headers: quota }) })) return state } From b0b5497f31c48c6cd24237d05e3468af5980561e Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 13:20:16 +1100 Subject: [PATCH 3/3] docs(github): describe token renewal and fallback failure in the runbook The runbook and two comments still named GITHUB_TOKEN as the sync's bucket. The App's installation bucket paces the sync now, and a rejected token is renewed before the fallback. --- docs/ops/artifact-delivery-rollout.md | 6 ++++-- layers/registry/server/utils/github-sync-control.ts | 3 +-- layers/registry/server/utils/ungh-repo.ts | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/ops/artifact-delivery-rollout.md b/docs/ops/artifact-delivery-rollout.md index eddf17dc..7b1c20c3 100644 --- a/docs/ops/artifact-delivery-rollout.md +++ b/docs/ops/artifact-delivery-rollout.md @@ -112,14 +112,16 @@ Preserve these existing values: - `NUXT_OAUTH_GITHUB_CLIENT_SECRET` - `NUXT_TOKEN_KEY` -Every Worker GitHub read uses the read App `skilld-dev-registry-reads`: the registry sync, repository maintenance, mention checks, `/gh` pages and public Artifact builds. Set `SKILLD_READ_APP_ID`, `SKILLD_READ_APP_INSTALLATION_ID` and `SKILLD_READ_APP_PRIVATE_KEY_PKCS8` (a PKCS #8 PEM). The App has metadata read permission only, and its installation has its own quota. +Every Worker [GitHub](https://github.com) read uses the read App `skilld-dev-registry-reads`: the registry sync, repository maintenance, mention checks, `/gh` pages and public Artifact builds. Set `SKILLD_READ_APP_ID`, `SKILLD_READ_APP_INSTALLATION_ID` and `SKILLD_READ_APP_PRIVATE_KEY_PKCS8` (a PKCS #8 PEM). The App has metadata read permission only, and its installation has its own quota. Personal tokens are fallbacks only. They repeat a read GitHub denied the App, such as an organization that restricts Apps: 1. Builds try `ARTIFACT_GITHUB_TOKEN`, then `GITHUB_TOKEN`, then the token of the signed-in account that asked for the run. 2. Every other caller tries `GITHUB_TOKEN`. -Each denial emits an `app-denied` event: `github-credential` for registry reads and `artifact-github-credential` for builds. The `github.credential` field names the fallback, or `none`. Without a fallback, the read fails for that Repository only: a build answers `SOURCE_ACCESS_DENIED`, and the sync records a failure without pausing. +Each denial emits an `app-denied` event: `github-credential` for registry reads and `artifact-github-credential` for builds. The `github.credential` field names the fallback, or `none`. Without a fallback, the read fails for that Repository only: a build answers `SOURCE_ACCESS_DENIED`, and the sync records a failure without pausing. If the fallback is spent or expired, the sync also records a failure for that Repository without pausing. The sync paces itself on the App's quota, never on the fallback's. + +If GitHub answers 401 to the cached installation token, the read mints one new token and repeats. If GitHub rejects the new token too, the read uses the fallback and emits an `app-token-rejected` event. If the App secrets are set but unusable, reads use the fallback and emit an event with the reason. Keep the read App apart from the `GITHUB_APP_*` secrets below. Those belong to private delivery. diff --git a/layers/registry/server/utils/github-sync-control.ts b/layers/registry/server/utils/github-sync-control.ts index 3ee37942..9e51fb13 100644 --- a/layers/registry/server/utils/github-sync-control.ts +++ b/layers/registry/server/utils/github-sync-control.ts @@ -60,8 +60,7 @@ export function githubRatePauseUntil( /** * Requests the sync leaves unspent in a GitHub bucket before it pauses. * - * `GITHUB_TOKEN` is a personal token, and GitHub counts its quota per - * account, so every tool on that account draws on the same bucket. The sync + * Sync and page views share the read App installation's bucket. The sync * spends about 10 REST requests and 130 GraphQL points an hour, and page * views about 50 REST requests at most. 200 covers page views for the rest * of any hour once the sync stops. diff --git a/layers/registry/server/utils/ungh-repo.ts b/layers/registry/server/utils/ungh-repo.ts index 2673261f..3774ef4f 100644 --- a/layers/registry/server/utils/ungh-repo.ts +++ b/layers/registry/server/utils/ungh-repo.ts @@ -23,7 +23,7 @@ export type UnghRepoResult * Public repository metadata from ungh.cc, the unjs GitHub proxy that caches * anonymous reads. * - * A page-view fallback for when `GITHUB_TOKEN` is spent or GitHub is down. It + * A page-view fallback for when the GitHub quota is spent or GitHub is down. It * answers metadata only. It never serves a tree or file content, and nothing * built or signed reads it. ungh.cc knows no archived or fork flag, so both * read false. Its 404 is no verdict about GitHub, so it reads as unavailable.