feat(artifact-delivery): fall back to the requester's GitHub token

layers/artifact-delivery/server/api/v1/resolutions/index.post.ts

@@ -49,6 +49,7 @@ export default withArtifactProblems(defineApiHandler({
             repositoryId: privateAccess.repositoryId,
           }
         : { visibility: 'public' },
+      requesterAccountId: user?.id ?? null,
     })
     if (result._tag === 'idempotency-conflict') {
       throw createError({

layers/artifact-delivery/server/utils/build.ts

@@ -29,6 +29,7 @@ import {
 import { checkArtifactSource, checksBlockArtifact, checksPermitSigning } from './checks'
 import { canonicalJson, digestHex } from './encoding'
 import { isRetryableProblem, storedFilesPassLoadRules } from './github-source'
+import { withRequesterFallback } from './requester-github'
 import {
   ARTIFACT_POLICY_VERSION,
   BYTE_COMPATIBLE_POLICY_VERSIONS,
@@ -46,6 +47,12 @@ export interface ArtifactBuildDependencies {
   db: D1Database
   github: PublicGithubSourceClient
   privateGithub?: (row: ResolutionRow) => Promise<PublicGithubSourceClient | SourceRejection>
+  /**
+   * The GitHub client of the signed-in account that asked for a public
+   * Resolution, or null. A build uses it only for a read the shared
+   * credential's quota refused, and only for that account's own Resolution.
+   */
+  requesterGithub?: (row: ResolutionRow) => Promise<PublicGithubSourceClient | null>
   bucket: R2Bucket
   privateArtifacts?: {
     put: (input: {
@@ -824,8 +831,12 @@ async function githubForResolution(
   dependencies: ArtifactBuildDependencies,
   row: ResolutionRow,
 ): Promise<PublicGithubSourceClient | SourceRejection> {
-  if (row.visibility === 'public')
-    return dependencies.github
+  if (row.visibility === 'public') {
+    const requesterGithub = dependencies.requesterGithub
+    return requesterGithub
+      ? withRequesterFallback(dependencies.github, () => requesterGithub(row))
+      : dependencies.github
+  }
   if (!dependencies.privateGithub) {
     return {
       _tag: 'rejected',

layers/artifact-delivery/server/utils/queue.ts

@@ -26,6 +26,7 @@ import { createGithubSourceClient, createPublicGithubSourceClient } from './gith
 import { createD1PrivateArtifactKeyProvider, privateArtifactWrappingKeysFromEnv } from './private-crypto'
 import { privateArtifactAccessEnabled } from './private-feature'
 import { putPrivateArtifact } from './private-storage'
+import { createRequesterGithubSource, forgetResolutionRequester } from './requester-github'
 import { getResolution } from './state'
 import { parseTrustedRoot } from './trusted-root'
 
@@ -93,6 +94,10 @@ export async function consumeArtifactBuildBatch(
       continue
     }
     if (outcome._tag === 'ok') {
+      // A settled build never reads GitHub again, so it no longer needs to
+      // know who asked. A superseded one leaves that to the build that won.
+      if (outcome.value._tag !== 'superseded')
+        await forgetResolutionRequester(dependencies.db, parsed.data.resolutionId)
       message.ack()
       continue
     }
@@ -112,6 +117,7 @@ export async function consumeArtifactBuildBatch(
     const row = await getResolution(dependencies.db, parsed.data.resolutionId)
     if (row)
       await failResolution(dependencies, row, 'SERVICE_UNAVAILABLE', true)
+    await forgetResolutionRequester(dependencies.db, parsed.data.resolutionId)
     message.ack()
   }
 }
@@ -209,9 +215,23 @@ export function createArtifactBuildDependencies(env: Cloudflare.Env): ArtifactBu
   const privateDependencies = privateArtifactAccessEnabled(env)
     ? createPrivateBuildDependencies(env, runtimeFetch)
     : {}
+  const githubObjects = createKvGithubObjectCache(env.KV_CACHE)
   return {
     db: env.DB,
-    github: createArtifactGithubSource(env, defaultGithubSourceRuntime(), createKvGithubObjectCache(env.KV_CACHE)),
+    github: createArtifactGithubSource(env, defaultGithubSourceRuntime(), githubObjects),
+    requesterGithub: createRequesterGithubSource({
+      db: env.DB,
+      tokenKey: () => env.NUXT_TOKEN_KEY,
+      fetch: runtimeFetch,
+      now: () => Math.floor(Date.now() / 1000),
+      cache: githubObjects,
+      onReadFailure: reportGithubReadFailure,
+      onUnusableToken: reason => emitOperationalEvent(createWideEvent({
+        operation: 'artifact-github-credential',
+        outcome: 'requester-token-unusable',
+        reason,
+      })),
+    }),
     bucket: env.PUBLIC_ARTIFACTS,
     signer: createArtifactSigner(env.ARTIFACT_SIGNER),
     trustedRoot: parseTrustedRoot(env.ARTIFACT_TRUSTED_ROOT_JSON, Math.floor(Date.now() / 1000)),

layers/artifact-delivery/server/utils/request-resolution.ts

@@ -3,6 +3,7 @@ import type { SourceRequest } from '../schemas/contracts'
 import type { FetchAdmittedSkillIdentity } from './admitted-identity'
 import type { CreateResolutionResult } from './state'
 import { admittedSourceRequest } from './admitted-identity'
+import { recordResolutionRequester } from './requester-github'
 import { createResolution, resolutionRequestIdentity } from './state'
 
 export interface ResolutionRequestDependencies {
@@ -48,14 +49,30 @@ export function fetchAdmittedSkillIdentity(context?: H3EventContext): FetchAdmit
  */
 export async function requestResolution(
   dependencies: ResolutionRequestDependencies,
-  input: { source: SourceRequest, idempotencyKey: string, access: ResolutionAccess },
+  input: {
+    source: SourceRequest
+    idempotencyKey: string
+    access: ResolutionAccess
+    /**
+     * The signed-in account that sent a public request, or null. Its build may
+     * read GitHub with that account's own token once the shared quota is spent.
+     */
+    requesterAccountId?: number | null
+  },
 ): Promise<CreateResolutionResult> {
   const accountId = input.access.visibility === 'private' ? input.access.accountId : undefined
   const identity = await resolutionRequestIdentity(input.source, input.idempotencyKey, accountId)
   const source = input.access.visibility === 'private'
     ? input.source
     : await admittedSourceRequest(input.source, dependencies.lookupAdmitted)
   const result = await createResolution(dependencies.db, source, identity, dependencies.now(), input.access)
+  if (result._tag === 'created' && input.access.visibility === 'public' && input.requesterAccountId) {
+    await recordResolutionRequester(dependencies.db, {
+      resolutionId: result.row.id,
+      accountId: input.requesterAccountId,
+      now: dependencies.now(),
+    })
+  }
   if (result._tag !== 'idempotency-conflict' && result.row.state === 'requested')
     await dependencies.enqueue(result.row.id)
   return result

layers/artifact-delivery/server/utils/requester-github.ts

@@ -0,0 +1,120 @@
+import type { GithubObjectCache, GithubReadFailure, PublicGithubSourceClient } from './github-source'
+import type { ResolutionRow } from './state'
+import { decryptToken } from '#layers/identity/server/utils/crypto'
+import { createPublicGithubSourceClient } from './github-source'
+
+/**
+ * Record the signed-in account that asked for a public Resolution.
+ *
+ * Only that Resolution's build reads it, and only after the shared GitHub
+ * credentials are rate limited. The row goes when the build settles.
+ */
+export async function recordResolutionRequester(
+  db: D1Database,
+  input: { resolutionId: string, accountId: number, now: number },
+): Promise<void> {
+  await db.prepare(
+    `INSERT OR IGNORE INTO artifact_resolution_requesters (resolution_id, account_id, created_at)
+     VALUES (?1, ?2, ?3)`,
+  ).bind(input.resolutionId, input.accountId, input.now).run()
+}
+
+/** Forget who asked for a Resolution. Its build has settled. */
+export async function forgetResolutionRequester(db: D1Database, resolutionId: string): Promise<void> {
+  await db.prepare(`DELETE FROM artifact_resolution_requesters WHERE resolution_id = ?1`).bind(resolutionId).run()
+}
+
+export interface RequesterGithubDependencies {
+  db: D1Database
+  /**
+   * `NUXT_TOKEN_KEY`, which encrypts the stored GitHub OAuth tokens. It is read
+   * only when a requester's token is needed.
+   */
+  tokenKey: () => string
+  fetch: typeof globalThis.fetch
+  /** Unix seconds. */
+  now: () => number
+  /** Commits and trees by SHA, shared with the build's own client. */
+  cache?: GithubObjectCache
+  onReadFailure?: (failure: GithubReadFailure) => void
+  /** Called when a stored token cannot be used. It never receives the token. */
+  onUnusableToken?: (reason: string) => void
+}
+
+/**
+ * The GitHub client of the account that asked for one public Resolution, or
+ * null when nobody signed in asked for it, or the account holds no usable
+ * GitHub token.
+ *
+ * The token is the account's GitHub sign-in token. Its scopes, `read:user` and
+ * `user:email`, read public Repositories at that account's own quota, and no
+ * private one. The public client also refuses any Repository GitHub reports
+ * as private, so a build read with it can only produce public content.
+ */
+export function createRequesterGithubSource(
+  dependencies: RequesterGithubDependencies,
+): (row: ResolutionRow) => Promise<PublicGithubSourceClient | null> {
+  return async (row) => {
+    if (row.visibility !== 'public')
+      return null
+    const stored = await dependencies.db.prepare(
+      `SELECT u.github_token_encrypted, u.github_token_expires_at
+       FROM artifact_resolution_requesters r
+       JOIN users u ON u.id = r.account_id
+       WHERE r.resolution_id = ?1
+       LIMIT 1`,
+    ).bind(row.id).first<{ github_token_encrypted: string | null, github_token_expires_at: number | null }>()
+    if (!stored?.github_token_encrypted)
+      return null
+    if (stored.github_token_expires_at !== null && stored.github_token_expires_at <= dependencies.now() + 60)
+      return null
+    const token = await decryptToken(stored.github_token_encrypted, dependencies.tokenKey()).then(
+      value => ({ _tag: 'token' as const, value }),
+      (error: unknown) => ({ _tag: 'unreadable' as const, reason: error instanceof Error ? error.message : String(error) }),
+    )
+    if (token._tag === 'unreadable') {
+      dependencies.onUnusableToken?.(`The stored GitHub token did not decrypt: ${token.reason}`.slice(0, 200))
+      return null
+    }
+    return createPublicGithubSourceClient({
+      fetch: dependencies.fetch,
+      token: token.value,
+      cache: dependencies.cache,
+      onReadFailure: dependencies.onReadFailure,
+    })
+  }
+}
+
+/**
+ * A client that reads with the shared credential, and repeats a read the
+ * shared quota refused with the requester's own client.
+ *
+ * Every other answer, success or rejection, stays the shared one, so a
+ * requester's token is spent only when nothing else could read.
+ */
+export function withRequesterFallback(
+  shared: PublicGithubSourceClient,
+  requester: () => Promise<PublicGithubSourceClient | null>,
+): PublicGithubSourceClient {
+  let own: Promise<PublicGithubSourceClient | null> | null = null
+  const ownClient = () => {
+    own ??= requester()
+    return own
+  }
+  return {
+    async resolve(request) {
+      const first = await shared.resolve(request)
+      if (first._tag !== 'rejected' || first.code !== 'RATE_LIMITED')
+        return first
+      const client = await ownClient()
+      return client ? await client.resolve(request) : first
+    },
+    async load(source) {
+      const first = await shared.load(source)
+      if (first._tag !== 'rejected' || first.code !== 'RATE_LIMITED')
+        return first
+      const client = await ownClient()
+      return client ? await client.load(source) : first
+    },
+  }
+}

layers/identity/server/utils/account-deletion.ts

@@ -27,6 +27,7 @@ const ACCOUNT_ROW_DELETES = [
   `DELETE FROM digest_runs WHERE user_id = ?1`,
   `DELETE FROM weekly_runs WHERE user_id = ?1`,
   `DELETE FROM email_preference_events WHERE user_id = ?1`,
+  `DELETE FROM artifact_resolution_requesters WHERE account_id = ?1`,
   `DELETE FROM artifact_download_grants WHERE account_id = ?1`,
   `DELETE FROM private_artifact_attestations WHERE account_id = ?1`,
   `DELETE FROM private_artifacts WHERE account_id = ?1`,

layers/identity/server/utils/data-retention.ts

@@ -14,6 +14,11 @@ export const RETENTION_SECONDS = {
   cliSignIn: DAY_SECONDS,
   /** Delivery history and preference changes support three months of reporting. */
   emailHistory: 90 * DAY_SECONDS,
+  /**
+   * The account that asked for a public Artifact build. The build deletes it
+   * when it settles; this removes one a dead build left behind.
+   */
+  runRequester: DAY_SECONDS,
 } as const
 
 export interface RetentionPurgeResult {
@@ -23,6 +28,7 @@ export interface RetentionPurgeResult {
   digestRuns: number
   weeklyRuns: number
   emailPreferenceEvents: number
+  runRequesters: number
 }
 
 /**
@@ -40,6 +46,7 @@ export async function purgeRetainedPersonalData(
   const endedTokenCutoff = now - RETENTION_SECONDS.endedCliToken
   const signInCutoff = now - RETENTION_SECONDS.cliSignIn
   const historyCutoff = now - RETENTION_SECONDS.emailHistory
+  const requesterCutoff = now - RETENTION_SECONDS.runRequester
 
   const results = await db.batch([
     db.prepare(
@@ -62,6 +69,7 @@ export async function purgeRetainedPersonalData(
     ).bind(historyCutoff),
     db.prepare(`DELETE FROM weekly_runs WHERE window_end <= ?1`).bind(historyCutoff),
     db.prepare(`DELETE FROM email_preference_events WHERE occurred_at <= ?1`).bind(historyCutoff),
+    db.prepare(`DELETE FROM artifact_resolution_requesters WHERE created_at <= ?1`).bind(requesterCutoff),
   ])
 
   const changes = results.map(result => result.meta.changes)
@@ -72,5 +80,6 @@ export async function purgeRetainedPersonalData(
     digestRuns: changes[3] ?? 0,
     weeklyRuns: changes[4] ?? 0,
     emailPreferenceEvents: changes[5] ?? 0,
+    runRequesters: changes[6] ?? 0,
   }
 }

migrations/0144_artifact_resolution_requesters.sql

@@ -0,0 +1,13 @@
+-- The signed-in account that asked for a public Resolution, kept only while
+-- its build runs. When the shared GitHub credentials are rate limited, the
+-- build reads GitHub with this account's own token, for this build only.
+-- The build deletes the row when it settles; the daily personal data purge
+-- removes any row a dead build left behind.
+CREATE TABLE artifact_resolution_requesters (
+  resolution_id TEXT PRIMARY KEY REFERENCES artifact_resolutions(id) ON DELETE CASCADE,
+  account_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  created_at INTEGER NOT NULL
+);
+
+CREATE INDEX idx_artifact_resolution_requesters_created_at
+  ON artifact_resolution_requesters(created_at);

test/unit/account-deletion.test.ts

@@ -190,6 +190,14 @@ function seedAccount(raw: DatabaseSync, account: Account, githubToken: string |
     id,
     installationId,
   )
+  run(
+    `INSERT INTO artifact_resolutions (
+       id, request_fingerprint, state, requested_owner, requested_repository, selector_type, selector_value, visibility
+     ) VALUES (?, ?, 'resolving', 'acme', 'skills', 'named-skill', 'tdd', 'public')`,
+    `public-run-${login}`,
+    `public-fingerprint-${login}`,
+  )
+  run(`INSERT INTO artifact_resolution_requesters (resolution_id, account_id, created_at) VALUES (?, ?, ?)`, `public-run-${login}`, id, NOW)
   run(`INSERT INTO artifact_check_results (resolution_id, name, version, outcome, required) VALUES (?, 'skill-structure', '1', 'pass', 1)`, resolutionId)
   run(
     `INSERT INTO artifacts (id, content_sha256, content_bytes, format, r2_key) VALUES (?, ?, 128, 'skilld-tar-v1', ?)`,

test/unit/artifact-build-reuse.test.ts

@@ -43,6 +43,7 @@ const MIGRATIONS = [
   'migrations/0112_private_artifact_keys.sql',
   'migrations/0122_artifact_resolution_retry_after.sql',
   'migrations/0131_artifact_build_reuse.sql',
+  'migrations/0144_artifact_resolution_requesters.sql',
 ]
 const COMMIT = '0123456789abcdef0123456789abcdef01234567'
 const NEXT_COMMIT = 'fedcba9876543210fedcba9876543210fedcba98'

test/unit/artifact-delivery.test.ts

@@ -41,6 +41,7 @@ const ARTIFACT_MIGRATIONS = [
   'migrations/0111_github_app_delivery.sql',
   'migrations/0112_private_artifact_keys.sql',
   'migrations/0122_artifact_resolution_retry_after.sql',
+  'migrations/0144_artifact_resolution_requesters.sql',
 ]
 const sourceRequest: SourceRequest = {
   provider: 'github',

test/unit/artifact-rate-limit.test.ts

@@ -22,6 +22,7 @@ const ARTIFACT_MIGRATIONS = [
   'migrations/0111_github_app_delivery.sql',
   'migrations/0112_private_artifact_keys.sql',
   'migrations/0122_artifact_resolution_retry_after.sql',
+  'migrations/0144_artifact_resolution_requesters.sql',
 ]
 const resolutionRequest: SourceRequest = {
   provider: 'github',