feat(artifact-delivery): read GitHub as the read App for builds

.dev.vars.example

@@ -14,6 +14,12 @@ GITHUB_TOKEN=
 # Public Artifact builds read GitHub with this token when it is set. Use a token
 # from another GitHub account, so runs keep a quota the registry sync cannot spend.
 ARTIFACT_GITHUB_TOKEN=
+# Public Artifact builds read GitHub as the read App: metadata read only,
+# installed on skilld-dev. Its installation has a quota nothing else spends.
+# The key is a PKCS #8 PEM. Without these, builds use the tokens above.
+SKILLD_READ_APP_ID=
+SKILLD_READ_APP_INSTALLATION_ID=
+SKILLD_READ_APP_PRIVATE_KEY_PKCS8=
 ARTIFACT_PRIVATE_ACCESS_ENABLED=false
 GITHUB_APP_ID=
 GITHUB_APP_CLIENT_ID=

docs/ops/artifact-delivery-rollout.md

@@ -113,7 +113,16 @@ Preserve these existing values:
 - `NUXT_TOKEN_KEY`
 - `GITHUB_TOKEN`
 
-The existing `GITHUB_TOKEN` reads public [GitHub](https://github.com) sources.
+The existing `GITHUB_TOKEN` reads public [GitHub](https://github.com) sources for the registry sync and `/gh` pages.
+
+Public Artifact builds read GitHub with a credential of their own, in this order:
+
+1. The read App `skilld-dev-registry-reads`: `SKILLD_READ_APP_ID`, `SKILLD_READ_APP_INSTALLATION_ID` and `SKILLD_READ_APP_PRIVATE_KEY_PKCS8` (a PKCS #8 PEM). It has metadata read permission only, and its installation has its own quota.
+2. `ARTIFACT_GITHUB_TOKEN`.
+3. `GITHUB_TOKEN`.
+
+If the App secrets are set but unusable, builds read with the next credential and emit an `artifact-github-credential` event with the reason.
+Keep the read App apart from the `GITHUB_APP_*` secrets below. Those belong to private delivery.
 
 Changing `NUXT_TOKEN_KEY` breaks stored GitHub credentials.
 

layers/artifact-delivery/server/utils/github-app.ts

@@ -491,22 +491,16 @@ export function createGithubAppClient(config: GithubAppConfig): GithubAppClient
     },
 
     async createRepositoryToken(installationId, repositoryId) {
-      const jwt = await createGithubAppJwt(config)
-      const result = await request(
-        `/app/installations/${installationId}/access_tokens`,
-        jwt,
-        installationTokenSchema,
-        {
-          method: 'POST',
-          body: {
-            repository_ids: [repositoryId],
-            permissions: { contents: 'read', metadata: 'read' },
-          },
+      const result = await requestInstallationToken({
+        fetch: config.fetch,
+        jwt: await createGithubAppJwt(config),
+        installationId,
+        body: {
+          repository_ids: [repositoryId],
+          permissions: { contents: 'read', metadata: 'read' },
         },
-      )
-      return result._tag === 'ok'
-        ? { _tag: 'created', token: result.value.token, expiresAt: result.value.expires_at }
-        : result
+      })
+      return result._tag === 'created' ? result : { _tag: 'not-found' }
     },
 
     async userCanAccessRepository(userToken, installationId, repositoryId) {
@@ -547,21 +541,81 @@ export async function verifyGithubWebhookSignature(
   return different === 0
 }
 
+export type InstallationTokenResult
+  = | { _tag: 'created', token: string, expiresAt: string }
+    /** GitHub answered 401, 403 or 404: the App, its key or the installation is gone. */
+    | { _tag: 'refused', status: number }
+
+/**
+ * Exchange an App JWT for an installation access token.
+ *
+ * A refusal is a value. Any other failure, such as a 5xx or an unreadable
+ * answer, throws: it says nothing about the App.
+ */
+export async function requestInstallationToken(input: {
+  fetch: typeof globalThis.fetch
+  jwt: string
+  installationId: number
+  body: { repository_ids?: number[], permissions: Record<string, 'read'> }
+}): Promise<InstallationTokenResult> {
+  const fetched = await fetchNoRedirect(input.fetch, `${GITHUB_API}/app/installations/${input.installationId}/access_tokens`, {
+    method: 'POST',
+    headers: {
+      'Accept': 'application/vnd.github+json',
+      'Authorization': `Bearer ${input.jwt}`,
+      'Content-Type': 'application/json',
+      'User-Agent': 'skilld.dev',
+      'X-GitHub-Api-Version': GITHUB_API_VERSION,
+    },
+    body: JSON.stringify(input.body),
+    signal: AbortSignal.timeout(GITHUB_REQUEST_TIMEOUT_MS),
+  })
+  if (fetched._tag === 'unexpected-redirect')
+    throw new Error(`GitHub App request redirected ${fetched.status}`)
+  const response = fetched.response
+  if (response.status === 401 || response.status === 403 || response.status === 404) {
+    await response.body?.cancel()
+    return { _tag: 'refused', status: response.status }
+  }
+  if (!response.ok)
+    throw new Error(`GitHub App request returned ${response.status}`)
+  const value = installationTokenSchema.safeParse(await readBoundedJson(response, MAX_GITHUB_TOKEN_RESPONSE_BYTES))
+  if (!value.success)
+    throw new Error('GitHub App returned an invalid response')
+  return { _tag: 'created', token: value.data.token, expiresAt: value.data.expires_at }
+}
+
 async function createGithubAppJwt(config: GithubAppConfig): Promise<string> {
   const privateKey = decodeCanonicalBase64Url(config.privateKeyPkcs8)
   if (!privateKey)
     throw new Error('GitHub App private key is malformed')
-  const issuedAt = config.now() - 60
+  return await signGithubAppJwt({ issuer: config.clientId, privateKeyPkcs8: privateKey, now: config.now() })
+}
+
+/**
+ * An RS256 GitHub App JWT. `issuer` is the App's client ID or its App ID;
+ * GitHub accepts either.
+ *
+ * It is valid for ten minutes from a minute in the past, which absorbs clock
+ * drift between the Worker and GitHub. Ten minutes is GitHub's maximum.
+ */
+export async function signGithubAppJwt(input: {
+  issuer: string
+  privateKeyPkcs8: Uint8Array
+  /** Unix seconds. */
+  now: number
+}): Promise<string> {
+  const issuedAt = input.now - 60
   const header = bytesToBase64Url(new TextEncoder().encode(JSON.stringify({ alg: 'RS256', typ: 'JWT' })))
   const payload = bytesToBase64Url(new TextEncoder().encode(JSON.stringify({
     iat: issuedAt,
     exp: issuedAt + 600,
-    iss: config.clientId,
+    iss: input.issuer,
   })))
   const unsigned = `${header}.${payload}`
   const imported = await crypto.subtle.importKey(
     'pkcs8',
-    toArrayBuffer(privateKey),
+    toArrayBuffer(input.privateKeyPkcs8),
     { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' },
     false,
     ['sign'],

layers/artifact-delivery/server/utils/github-read-credential.ts

@@ -0,0 +1,214 @@
+import type { PublicGithubSourceClient } from './github-source'
+import { requestInstallationToken, signGithubAppJwt } from './github-app'
+
+/**
+ * Renew an installation token this long before GitHub says it expires, so a
+ * build that starts with it never sees it lapse part way through.
+ */
+export const INSTALLATION_TOKEN_RENEW_SECONDS = 5 * 60
+
+/** The read App: metadata read only, installed on skilld-dev. */
+export interface ReadAppConfig {
+  appId: number
+  installationId: number
+  privateKeyPkcs8: Uint8Array
+}
+
+/** The token a build reads with when the App gives none, by its secret name. */
+export interface FallbackToken {
+  name: 'ARTIFACT_GITHUB_TOKEN' | 'GITHUB_TOKEN'
+  token: string
+}
+
+/**
+ * Where public Artifact builds get their GitHub credential, parsed once from
+ * the Worker env.
+ *
+ * GitHub counts a personal token's quota per account, so every token of one
+ * account shares one bucket. An App installation has a bucket of its own.
+ */
+export type ArtifactGithubCredentialConfig
+  = | { _tag: 'app', app: ReadAppConfig, fallback: FallbackToken | null }
+    /** App secrets are set but unusable. Builds read with the fallback and report why. */
+    | { _tag: 'app-misconfigured', reason: string, fallback: FallbackToken | null }
+    | { _tag: 'token', token: FallbackToken }
+    | { _tag: 'anonymous' }
+
+export interface ArtifactGithubCredentialEnv {
+  SKILLD_READ_APP_ID?: string
+  SKILLD_READ_APP_INSTALLATION_ID?: string
+  SKILLD_READ_APP_PRIVATE_KEY_PKCS8?: string
+  ARTIFACT_GITHUB_TOKEN?: string
+  GITHUB_TOKEN?: string
+}
+
+/** Why a build read with a fallback token. It names secrets, never a value. */
+export interface ArtifactGithubCredentialReport {
+  outcome: 'app-misconfigured' | 'app-token-unavailable'
+  reason: string
+  fallback: FallbackToken['name'] | 'anonymous'
+}
+
+interface CachedInstallationToken {
+  token: string
+  /** Unix seconds. */
+  expiresAt: number
+}
+
+/**
+ * Installation tokens kept between builds in one isolate. A token lives an
+ * hour; minting one per build would add a request and a signature to each.
+ */
+export interface InstallationTokenCache {
+  tokens: Map<string, CachedInstallationToken>
+  pending: Map<string, Promise<CachedInstallationToken | { _tag: 'unavailable', reason: string }>>
+}
+
+export function createInstallationTokenCache(): InstallationTokenCache {
+  return { tokens: new Map(), pending: new Map() }
+}
+
+export interface ArtifactGithubCredentialRuntime {
+  fetch: typeof globalThis.fetch
+  /** Unix seconds. */
+  now: () => number
+  tokenCache: InstallationTokenCache
+  report: (event: ArtifactGithubCredentialReport) => void
+}
+
+/** The credential for one GitHub read. Undefined reads anonymously. */
+export type ArtifactGithubCredential = () => Promise<string | undefined>
+
+export function parseArtifactGithubCredentialConfig(env: ArtifactGithubCredentialEnv): ArtifactGithubCredentialConfig {
+  const fallback = fallbackToken(env)
+  const appSecrets = [env.SKILLD_READ_APP_ID, env.SKILLD_READ_APP_INSTALLATION_ID, env.SKILLD_READ_APP_PRIVATE_KEY_PKCS8]
+  if (appSecrets.every(value => !value?.trim()))
+    return fallback ? { _tag: 'token', token: fallback } : { _tag: 'anonymous' }
+  const misconfigured = (reason: string): ArtifactGithubCredentialConfig => ({ _tag: 'app-misconfigured', reason, fallback })
+  const appId = positiveInteger(env.SKILLD_READ_APP_ID)
+  if (appId === null)
+    return misconfigured('SKILLD_READ_APP_ID is not a GitHub App ID')
+  const installationId = positiveInteger(env.SKILLD_READ_APP_INSTALLATION_ID)
+  if (installationId === null)
+    return misconfigured('SKILLD_READ_APP_INSTALLATION_ID is not an installation ID')
+  const privateKeyPkcs8 = pkcs8PemBytes(env.SKILLD_READ_APP_PRIVATE_KEY_PKCS8 ?? '')
+  if (!privateKeyPkcs8)
+    return misconfigured('SKILLD_READ_APP_PRIVATE_KEY_PKCS8 is not a PKCS #8 PEM private key')
+  return { _tag: 'app', app: { appId, installationId, privateKeyPkcs8 }, fallback }
+}
+
+/**
+ * The GitHub credential public Artifact builds read with: the read App's
+ * installation token, then `ARTIFACT_GITHUB_TOKEN`, then `GITHUB_TOKEN`.
+ *
+ * The registry sync and `/gh` page views keep `GITHUB_TOKEN`. When the App
+ * cannot give a token, the build still runs on the fallback and the reason is
+ * reported, so a broken key shows up as an event instead of as a spent quota.
+ */
+export function createArtifactGithubCredential(
+  config: ArtifactGithubCredentialConfig,
+  runtime: ArtifactGithubCredentialRuntime,
+): ArtifactGithubCredential {
+  if (config._tag === 'anonymous')
+    return async () => undefined
+  if (config._tag === 'token')
+    return async () => config.token.token
+  const fallbackName = config.fallback?.name ?? 'anonymous'
+  if (config._tag === 'app-misconfigured') {
+    return async () => {
+      runtime.report({ outcome: 'app-misconfigured', reason: config.reason, fallback: fallbackName })
+      return config.fallback?.token
+    }
+  }
+  const app = config.app
+  const key = `${app.appId}:${app.installationId}`
+  return async () => {
+    const cached = runtime.tokenCache.tokens.get(key)
+    if (cached && cached.expiresAt - INSTALLATION_TOKEN_RENEW_SECONDS > runtime.now())
+      return cached.token
+    let pending = runtime.tokenCache.pending.get(key)
+    if (!pending) {
+      pending = mintInstallationToken(app, runtime).finally(() => runtime.tokenCache.pending.delete(key))
+      runtime.tokenCache.pending.set(key, pending)
+    }
+    const minted = await pending
+    if ('token' in minted) {
+      runtime.tokenCache.tokens.set(key, minted)
+      return minted.token
+    }
+    runtime.report({ outcome: 'app-token-unavailable', reason: minted.reason, fallback: fallbackName })
+    return config.fallback?.token
+  }
+}
+
+/**
+ * A GitHub source client whose every read asks for the current credential, so
+ * a long-lived client never reads with an expired installation token.
+ */
+export function withGithubCredential(
+  credential: ArtifactGithubCredential,
+  create: (token: string | undefined) => PublicGithubSourceClient,
+): PublicGithubSourceClient {
+  return {
+    resolve: async request => await create(await credential()).resolve(request),
+    load: async source => await create(await credential()).load(source),
+  }
+}
+
+async function mintInstallationToken(
+  app: ReadAppConfig,
+  runtime: Pick<ArtifactGithubCredentialRuntime, 'fetch' | 'now'>,
+): Promise<CachedInstallationToken | { _tag: 'unavailable', reason: string }> {
+  const minted = await signGithubAppJwt({ issuer: String(app.appId), privateKeyPkcs8: app.privateKeyPkcs8, now: runtime.now() })
+    .then(jwt => requestInstallationToken({
+      fetch: runtime.fetch,
+      jwt,
+      installationId: app.installationId,
+      body: { permissions: { metadata: 'read' } },
+    }))
+    // A network fault, a 5xx or a key WebCrypto rejects: the build reads with
+    // the fallback token, and the reason reaches the report.
+    .catch((error: unknown) => ({ _tag: 'failed' as const, reason: error instanceof Error ? error.message : String(error) }))
+  if (minted._tag === 'failed')
+    return { _tag: 'unavailable', reason: minted.reason.slice(0, 200) }
+  if (minted._tag === 'refused')
+    return { _tag: 'unavailable', reason: `GitHub refused the installation token (${minted.status})` }
+  const expiresAt = Math.floor(Date.parse(minted.expiresAt) / 1000)
+  return { token: minted.token, expiresAt }
+}
+
+function fallbackToken(env: ArtifactGithubCredentialEnv): FallbackToken | null {
+  if (env.ARTIFACT_GITHUB_TOKEN?.trim())
+    return { name: 'ARTIFACT_GITHUB_TOKEN', token: env.ARTIFACT_GITHUB_TOKEN.trim() }
+  if (env.GITHUB_TOKEN?.trim())
+    return { name: 'GITHUB_TOKEN', token: env.GITHUB_TOKEN.trim() }
+  return null
+}
+
+function positiveInteger(value: string | undefined): number | null {
+  const trimmed = value?.trim() ?? ''
+  if (!/^[1-9]\d*$/.test(trimmed))
+    return null
+  const parsed = Number(trimmed)
+  return Number.isSafeInteger(parsed) ? parsed : null
+}
+
+/**
+ * The DER bytes of a PKCS #8 PEM private key, or null.
+ *
+ * A secret pasted through a dashboard or a JSON file can hold its line breaks
+ * as the two characters `\n`, so both forms parse. A PKCS #1 key
+ * (`BEGIN RSA PRIVATE KEY`, as GitHub downloads it) does not: WebCrypto
+ * imports PKCS #8 only. Convert it with `openssl pkcs8 -topk8 -nocrypt`.
+ */
+function pkcs8PemBytes(value: string): Uint8Array | null {
+  const text = value.replaceAll('\\n', '\n').trim()
+  const match = /^-----BEGIN PRIVATE KEY-----([A-Z0-9+/=\s]+)-----END PRIVATE KEY-----$/i.exec(text)
+  if (!match)
+    return null
+  const base64 = match[1]!.replace(/\s+/g, '')
+  if (!/^[A-Z0-9+/]+={0,2}$/i.test(base64) || base64.length % 4 !== 0)
+    return null
+  const binary = atob(base64)
+  return Uint8Array.from(binary, character => character.charCodeAt(0))
+}