From 388710ee7f622c534773701f7958f999470e5c18 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 16:29:13 +1100 Subject: [PATCH 1/6] fix(artifacts): verify the key window at the statement creation time The skilld CLI refuses a signature over a statement created outside the key window. The site checked only the current time, so it could store, reuse and serve an attestation every CLI refuses until the key expired. A signing key cutover made that case reachable. --- .../server/utils/attestation.ts | 19 ++++- test/unit/artifact-attestation-window.test.ts | 85 +++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 test/unit/artifact-attestation-window.test.ts diff --git a/layers/artifact-delivery/server/utils/attestation.ts b/layers/artifact-delivery/server/utils/attestation.ts index 06eb7a2b..855ea4dc 100644 --- a/layers/artifact-delivery/server/utils/attestation.ts +++ b/layers/artifact-delivery/server/utils/attestation.ts @@ -77,8 +77,18 @@ export async function createAttestationSignaturePayload(statementBytes: Uint8Arr return payload } -export async function verifyAttestationSignature( +/** + * A trusted key verifies a statement only while its window is open now, and + * only for a statement created inside that window. + * + * The skilld CLI applies the second rule and never the first. Without the + * second, the site stored and served a statement every CLI refuses. The first + * is the site's own: once a key's window closes, a reuse misses and the next + * request builds and signs again under the current key. + */ +async function verifyAttestationSignature( statementBytes: Uint8Array, + createdAt: string, signature: z.infer, trustedRoot: TrustedRoot, now: number, @@ -86,7 +96,10 @@ export async function verifyAttestationSignature( const trustedKey = trustedRoot.keys.find(key => key.keyId === signature.keyId) if (!trustedKey || trustedKey.status === 'retired' || trustedKey.status === 'revoked') return false - if (now < Date.parse(trustedKey.notBefore) / 1000 || now >= Date.parse(trustedKey.notAfter) / 1000) + const notBefore = Date.parse(trustedKey.notBefore) / 1000 + const notAfter = Date.parse(trustedKey.notAfter) / 1000 + const created = Date.parse(createdAt) / 1000 + if (now < notBefore || now >= notAfter || !(created >= notBefore && created <= notAfter)) return false const publicKeyBytes = decodeCanonicalBase64Url(trustedKey.publicKey) const signatureBytes = decodeCanonicalBase64Url(signature.value) @@ -130,7 +143,7 @@ export async function verifyArtifactAttestation( const { statement: _statement, signature, ...outerStatement } = attestation if (canonicalJson(parsed.data) !== canonicalJson(outerStatement)) return false - return await verifyAttestationSignature(statementBytes, signature, trustedRoot, now) + return await verifyAttestationSignature(statementBytes, parsed.data.createdAt, signature, trustedRoot, now) } /** diff --git a/test/unit/artifact-attestation-window.test.ts b/test/unit/artifact-attestation-window.test.ts new file mode 100644 index 00000000..bd5ea462 --- /dev/null +++ b/test/unit/artifact-attestation-window.test.ts @@ -0,0 +1,85 @@ +import type { ArtifactAttestation } from '../../layers/artifact-delivery/server/schemas/contracts' +import type { TrustedRoot } from '../../layers/artifact-delivery/server/utils/trusted-root' +import { describe, expect, it } from 'vitest' +import { + completeAttestation, + createAttestationSignaturePayload, + createAttestationStatement, + encodeAttestationStatement, + verifyArtifactAttestation, +} from '../../layers/artifact-delivery/server/utils/attestation' +import { bytesToBase64Url } from '../../layers/artifact-delivery/server/utils/encoding' + +const KEY_NOT_BEFORE = Date.parse('2026-11-01T00:00:00.000Z') / 1000 +const KEY_NOT_AFTER = Date.parse('2027-02-20T00:00:00.000Z') / 1000 + +describe('attestation signing key window', () => { + it('accepts a statement created inside the key window', async () => { + const signed = await signStatement(KEY_NOT_BEFORE + 60) + + expect(await verifyArtifactAttestation(signed.attestation, signed.trustedRoot, KEY_NOT_BEFORE + 120)).toBe(true) + }) + + // The skilld CLI accepts a signature only for a statement created inside + // the key window. A site that checked only the current time stored, reused + // and served a statement every CLI refuses. + it('rejects a statement created before the key window opened', async () => { + const signed = await signStatement(KEY_NOT_BEFORE - 60) + + expect(await verifyArtifactAttestation(signed.attestation, signed.trustedRoot, KEY_NOT_BEFORE + 120)).toBe(false) + }) + + it('stops accepting the key once its window closes, so the next request builds again', async () => { + const signed = await signStatement(KEY_NOT_BEFORE + 60) + + expect(await verifyArtifactAttestation(signed.attestation, signed.trustedRoot, KEY_NOT_AFTER - 1)).toBe(true) + expect(await verifyArtifactAttestation(signed.attestation, signed.trustedRoot, KEY_NOT_AFTER)).toBe(false) + }) +}) + +async function signStatement(createdAt: number): Promise<{ attestation: ArtifactAttestation, trustedRoot: TrustedRoot }> { + const keyPair = await crypto.subtle.generateKey('Ed25519', true, ['sign', 'verify']) + const publicKey = bytesToBase64Url(new Uint8Array(await crypto.subtle.exportKey('raw', keyPair.publicKey))) + const statement = encodeAttestationStatement(createAttestationStatement({ + artifactId: `sha256:${'a'.repeat(64)}`, + createdAt: new Date(createdAt * 1000).toISOString(), + source: { + provider: 'github', + repositoryId: 1, + owner: 'skilld-dev', + repository: 'skills', + visibility: 'public', + commitSha: '0123456789abcdef0123456789abcdef01234567', + treeSha: '89abcdef0123456789abcdef0123456789abcdef', + skillPath: 'skills/demo', + }, + contentSha256: 'a'.repeat(64), + contentBytes: 1024, + files: [{ path: 'SKILL.md', mode: 420, size: 1, sha256: 'b'.repeat(64) }], + checkResults: [{ name: 'path-policy', version: '1', outcome: 'pass', required: true }], + })) + const payload = await createAttestationSignaturePayload(new TextEncoder().encode(statement)) + const signature = new Uint8Array(await crypto.subtle.sign('Ed25519', keyPair.privateKey, Uint8Array.from(payload).buffer)) + const attestation = completeAttestation(statement, { + algorithm: 'Ed25519', + keyId: 'skilld-production-2026-11', + value: bytesToBase64Url(signature), + }) + const trustedRoot: TrustedRoot = { + version: 1, + rootKeyId: 'skilld-root-2026', + rootPublicKey: publicKey, + keys: [{ + keyId: 'skilld-production-2026-11', + algorithm: 'Ed25519', + publicKey, + notBefore: new Date(KEY_NOT_BEFORE * 1000).toISOString(), + notAfter: new Date(KEY_NOT_AFTER * 1000).toISOString(), + status: 'active', + statement: bytesToBase64Url(new TextEncoder().encode('{}')), + rootSignature: 'C'.repeat(86), + }], + fetchedAt: new Date(KEY_NOT_BEFORE * 1000).toISOString(), + } + return { attestation, trustedRoot } +} From 99d63ceeedbe56c3180e9c723ac074f8596ad850 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 16:29:14 +1100 Subject: [PATCH 2/6] feat(artifact-signer): sign from two key slots to overlap rotations One key ID var set and one secret could not change together: a secret put deployed at once with the old key ID, so every build failed until the vars deploy landed. A staged secret in the free slot now signs nothing until its vars deploy. The signer picks the newest key whose window holds both the statement creation time and now. --- test/unit/artifact-signer.test.ts | 135 +++++++++++++++++- workers/artifact-signer/.dev.vars.example | 4 +- workers/artifact-signer/src/handler.ts | 84 +++++++---- workers/artifact-signer/src/slots.ts | 65 +++++++++ .../artifact-signer/worker-configuration.d.ts | 5 +- 5 files changed, 259 insertions(+), 34 deletions(-) create mode 100644 workers/artifact-signer/src/slots.ts diff --git a/test/unit/artifact-signer.test.ts b/test/unit/artifact-signer.test.ts index e64a700c..201a1360 100644 --- a/test/unit/artifact-signer.test.ts +++ b/test/unit/artifact-signer.test.ts @@ -258,7 +258,111 @@ describe('artifact signing Worker', () => { }) }) +describe('artifact signing key slots', () => { + const DAY = 86_400 + + it('signs with the newest key whose window holds the statement and now', async () => { + const fixture = await createSignerFixture({ + notBefore: NOW - 30 * DAY, + secondary: { notBefore: NOW - 120, notAfter: NOW + 90 * DAY }, + }) + + const signature = await readSuccess(await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW)) + const payload = await createAttestationSignaturePayload(new TextEncoder().encode(fixture.statement)) + + expect(signature.keyId).toBe('skilld-production-2026-11') + expect(await crypto.subtle.verify('Ed25519', fixture.secondaryPublicKey, decodeBase64Url(signature.value), payload)).toBe(true) + fixture.close() + }) + + // The skilld CLI refuses a statement created before its key window opened. + it('keeps the old key for a statement created before the new key window opened', async () => { + const fixture = await createSignerFixture({ + notBefore: NOW - 30 * DAY, + createdAt: NOW - 60, + secondary: { notBefore: NOW - 30, notAfter: NOW + 90 * DAY }, + }) + + const signature = await readSuccess(await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW)) + + expect(signature.keyId).toBe('skilld-production-2026-08') + fixture.close() + }) + + it('signs with the new key once the old key window closes', async () => { + const fixture = await createSignerFixture({ + notBefore: NOW - 30 * DAY, + notAfter: NOW - 1, + secondary: { notBefore: NOW - DAY, notAfter: NOW + 90 * DAY }, + }) + + const signature = await readSuccess(await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW)) + + expect(signature.keyId).toBe('skilld-production-2026-11') + fixture.close() + }) + + it('signs with the secondary slot alone once the primary slot is emptied', async () => { + const fixture = await createSignerFixture({ + primary: false, + secondary: { notBefore: NOW - DAY, notAfter: NOW + 90 * DAY }, + }) + + const signature = await readSuccess(await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW)) + + expect(signature.keyId).toBe('skilld-production-2026-11') + fixture.close() + }) + + it('signs nothing with a staged secret that no vars name', async () => { + const fixture = await createSignerFixture({ + secondary: { notBefore: NOW - DAY, notAfter: NOW + 90 * DAY, vars: false }, + }) + + const signature = await readSuccess(await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW)) + + expect(signature.keyId).toBe('skilld-production-2026-08') + fixture.close() + }) + + it.each([ + ['a named slot has no secret', { notBefore: NOW - DAY, notAfter: NOW + 90 * DAY, secret: false }], + ['both slots name one key ID', { keyId: 'skilld-production-2026-08', notBefore: NOW - DAY, notAfter: NOW + 90 * DAY }], + ['a slot window ends before it starts', { notBefore: NOW + DAY, notAfter: NOW - DAY }], + ] as const)('refuses to sign when %s', async (_name, secondary) => { + const fixture = await createSignerFixture({ secondary }) + + const response = await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW) + + expect(await readCode(response)).toBe('SIGNING_KEY_INVALID') + fixture.close() + }) + + it('refuses to sign when no slot names a key', async () => { + const fixture = await createSignerFixture({ primary: false }) + + const response = await handleArtifactSignerRequest(attestRequest(fixture.artifactId), fixture.bindings, () => NOW) + + expect(await readCode(response)).toBe('SIGNING_KEY_INVALID') + fixture.close() + }) +}) + +interface SecondarySlotOptions { + keyId?: string + notBefore: number + notAfter: number + /** False leaves the slot's secret unset. */ + secret?: boolean + /** False leaves the slot's vars unset, so only its secret is staged. */ + vars?: boolean +} + interface SignerFixtureOptions { + createdAt?: number + /** False leaves the primary slot empty. */ + primary?: boolean + secondary?: SecondarySlotOptions changeStateAfterObjectRead?: boolean checkOutcome?: 'pass' | 'fail' | 'error' privateKey?: string @@ -306,7 +410,7 @@ async function createSignerFixture(options: SignerFixtureOptions = {}) { const statement = encodeAttestationStatement({ ...createAttestationStatement({ artifactId, - createdAt: new Date((NOW - 60) * 1000).toISOString(), + createdAt: new Date((options.createdAt ?? NOW - 60) * 1000).toISOString(), source: options.privateArtifact ? { ...source, visibility: 'private' } : source, contentSha256, contentBytes: artifactBytes.byteLength, @@ -345,7 +449,7 @@ async function createSignerFixture(options: SignerFixtureOptions = {}) { r2Key, JSON.stringify(checks), statement, - NOW - 60, + options.createdAt ?? NOW - 60, options.updatedAt ?? NOW, options.privateArtifact ? 'private' : 'public', options.privateArtifact ? 1 : null, @@ -389,22 +493,39 @@ async function createSignerFixture(options: SignerFixtureOptions = {}) { body: new Blob([Uint8Array.from(body)]).stream(), } as R2ObjectBody }) - const bindings = { + const secondaryKeyPair = await crypto.subtle.generateKey('Ed25519', true, ['sign', 'verify']) + const secondary = options.secondary + const bindings: ArtifactSignerBindings = { DB: sqlite.db, PUBLIC_ARTIFACTS: { get: getObject } as R2Bucket, PRIVATE_ARTIFACTS: { get: getObject } as R2Bucket, - ARTIFACT_SIGNING_KEY_ID: 'skilld-production-2026-08', - ARTIFACT_SIGNING_KEY_NOT_BEFORE: new Date((options.notBefore ?? NOW - 60) * 1000).toISOString(), - ARTIFACT_SIGNING_KEY_NOT_AFTER: new Date((options.notAfter ?? NOW + 3600) * 1000).toISOString(), ARTIFACT_SIGNING_MAX_AGE_SECONDS: '300', ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8: privateKey, - } satisfies ArtifactSignerBindings + ...(options.primary === false + ? {} + : { + ARTIFACT_SIGNING_KEY_ID: 'skilld-production-2026-08', + ARTIFACT_SIGNING_KEY_NOT_BEFORE: new Date((options.notBefore ?? NOW - 60) * 1000).toISOString(), + ARTIFACT_SIGNING_KEY_NOT_AFTER: new Date((options.notAfter ?? NOW + 3600) * 1000).toISOString(), + }), + ...(secondary && secondary.secret !== false + ? { ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8: bytesToBase64Url(new Uint8Array(await crypto.subtle.exportKey('pkcs8', secondaryKeyPair.privateKey))) } + : {}), + ...(secondary && secondary.vars !== false + ? { + ARTIFACT_SIGNING_SECONDARY_KEY_ID: secondary.keyId ?? 'skilld-production-2026-11', + ARTIFACT_SIGNING_SECONDARY_KEY_NOT_BEFORE: new Date(secondary.notBefore * 1000).toISOString(), + ARTIFACT_SIGNING_SECONDARY_KEY_NOT_AFTER: new Date(secondary.notAfter * 1000).toISOString(), + } + : {}), + } return { artifactId, bindings, close: sqlite.close, getObject, publicKey: keyPair.publicKey, + secondaryPublicKey: secondaryKeyPair.publicKey, raw: sqlite.raw, statement, } diff --git a/workers/artifact-signer/.dev.vars.example b/workers/artifact-signer/.dev.vars.example index e2a79735..228238db 100644 --- a/workers/artifact-signer/.dev.vars.example +++ b/workers/artifact-signer/.dev.vars.example @@ -1,2 +1,4 @@ -# Binding name only. Use a base64url PKCS8 Ed25519 private key in .dev.vars. +# Binding names only. Each value is a base64url PKCS8 Ed25519 private key in .dev.vars. ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8= +# The secondary slot. It signs nothing until wrangler.jsonc names its key ID. +ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8= diff --git a/workers/artifact-signer/src/handler.ts b/workers/artifact-signer/src/handler.ts index 589f4368..2a3c9bf2 100644 --- a/workers/artifact-signer/src/handler.ts +++ b/workers/artifact-signer/src/handler.ts @@ -1,4 +1,5 @@ import type { ArtifactAttestationStatement, CheckResult } from '../../../layers/artifact-delivery/server/schemas/contracts' +import type { ArtifactSigningKeyBindingName, ArtifactSigningKeySlot, ArtifactSigningKeyWindow } from './slots' import { createHash } from 'node:crypto' import { z } from 'zod' import { @@ -13,18 +14,17 @@ import { createAttestationSignaturePayload, encodeAttestationStatement } from '. import { checksPermitSigning, SIGNABLE_ARTIFACT_POLICIES } from '../../../layers/artifact-delivery/server/utils/checks' import { base64ToBytes, bytesToBase64Url, canonicalJson, digestHex } from '../../../layers/artifact-delivery/server/utils/encoding' import { ARTIFACT_POLICY_VERSION } from '../../../layers/artifact-delivery/server/utils/state' +import { ARTIFACT_SIGNING_KEY_SLOTS, selectSigningKey } from './slots' type ArtifactSignerBindingName - = 'ARTIFACT_SIGNING_KEY_ID' - | 'ARTIFACT_SIGNING_KEY_NOT_AFTER' - | 'ARTIFACT_SIGNING_KEY_NOT_BEFORE' - | 'ARTIFACT_SIGNING_MAX_AGE_SECONDS' - | 'ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8' + = 'ARTIFACT_SIGNING_MAX_AGE_SECONDS' | 'DB' | 'PRIVATE_ARTIFACTS' | 'PUBLIC_ARTIFACTS' +/** Either slot may be empty, so its vars and secret are optional here. */ export type ArtifactSignerBindings = Pick + & Partial> export const ARTIFACT_SIGNER_MAX_REQUEST_BYTES = 512 const MAX_ATTESTATION_STATEMENT_BYTES = 6_291_456 @@ -40,14 +40,23 @@ const signerRequestSchema = z.object({ artifactId: artifactIdSchema, }).strict() -const signerConfigSchema = z.object({ +const maximumAgeSecondsSchema = z.string().regex(/^[1-9]\d{0,3}$/).transform(Number).pipe(z.number().int().min(1).max(3600)) + +const signingKeySchema = z.object({ keyId: z.string().min(1).max(100), notBefore: z.string().datetime(), notAfter: z.string().datetime(), - maximumAgeSeconds: z.string().regex(/^[1-9]\d{0,3}$/).transform(Number).pipe(z.number().int().min(1).max(3600)), privateKey: z.string().min(2).max(512).regex(/^[\w-]+$/), }).strict() +interface SigningKey extends ArtifactSigningKeyWindow { + privateKey: string +} + +type SignerConfig + = { _tag: 'valid', maximumAgeSeconds: number, keys: SigningKey[] } + | SignerFailure + const signingRowSchema = z.object({ id: resolutionIdSchema, state: z.string(), @@ -148,22 +157,12 @@ async function routeArtifactSignerRequest( if (!parsedRequest.success) return failureResponse(failures.invalidRequest()) - const parsedConfig = signerConfigSchema.safeParse({ - keyId: env.ARTIFACT_SIGNING_KEY_ID, - notBefore: env.ARTIFACT_SIGNING_KEY_NOT_BEFORE, - notAfter: env.ARTIFACT_SIGNING_KEY_NOT_AFTER, - maximumAgeSeconds: env.ARTIFACT_SIGNING_MAX_AGE_SECONDS, - privateKey: env.ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8, - }) - if (!parsedConfig.success) - return failureResponse(failures.signingKeyInvalid()) + const config = parseSignerConfig(env) + if (config._tag === 'failure') + return failureResponse(config) const timestamp = now() - const keyNotBefore = Date.parse(parsedConfig.data.notBefore) / 1000 - const keyNotAfter = Date.parse(parsedConfig.data.notAfter) / 1000 - if (keyNotBefore >= keyNotAfter) - return failureResponse(failures.signingKeyInvalid()) - if (timestamp < keyNotBefore || timestamp >= keyNotAfter) + if (!config.keys.some(key => key.notBefore <= timestamp && timestamp < key.notAfter)) return failureResponse(failures.signingKeyNotActive()) const initialRow = await loadSigningRow(env.DB, parsedRequest.data.resolutionId) @@ -174,7 +173,7 @@ async function routeArtifactSignerRequest( if (initialRow.artifact_id !== parsedRequest.data.artifactId) return failureResponse(failures.artifactIdMismatch()) - const validated = validateSigningRow(initialRow, parsedConfig.data.maximumAgeSeconds, timestamp) + const validated = validateSigningRow(initialRow, config.maximumAgeSeconds, timestamp) if (validated._tag === 'failure') return failureResponse(validated) const objectMatches = initialRow.visibility === 'private' @@ -187,7 +186,10 @@ async function routeArtifactSignerRequest( if (!currentRow || canonicalJson(currentRow) !== canonicalJson(initialRow)) return failureResponse(failures.stateChanged()) - const privateKeyBytes = decodeCanonicalBase64Url(parsedConfig.data.privateKey) + const signingKey = selectSigningKey(config.keys, initialRow.created_at, timestamp) + if (!signingKey) + return failureResponse(failures.signingKeyNotActive()) + const privateKeyBytes = decodeCanonicalBase64Url(signingKey.privateKey) if (!privateKeyBytes || privateKeyBytes.byteLength > 256) return failureResponse(failures.signingKeyInvalid()) const imported = await crypto.subtle.importKey( @@ -204,11 +206,45 @@ async function routeArtifactSignerRequest( const signature = await crypto.subtle.sign('Ed25519', imported.key, Uint8Array.from(payload).buffer) return jsonResponse({ algorithm: 'Ed25519', - keyId: parsedConfig.data.keyId, + keyId: signingKey.keyId, value: bytesToBase64Url(new Uint8Array(signature)), }, 200) } +function parseSignerConfig(env: ArtifactSignerBindings): SignerConfig { + const maximumAgeSeconds = maximumAgeSecondsSchema.safeParse(env.ARTIFACT_SIGNING_MAX_AGE_SECONDS) + if (!maximumAgeSeconds.success) + return failures.signingKeyInvalid() + const keys: SigningKey[] = [] + for (const slot of Object.keys(ARTIFACT_SIGNING_KEY_SLOTS) as ArtifactSigningKeySlot[]) { + const names = ARTIFACT_SIGNING_KEY_SLOTS[slot] + // A slot with no vars is empty. Its secret alone is a staged key. + if (env[names.keyId] === undefined && env[names.notBefore] === undefined && env[names.notAfter] === undefined) + continue + const parsed = signingKeySchema.safeParse({ + keyId: env[names.keyId], + notBefore: env[names.notBefore], + notAfter: env[names.notAfter], + privateKey: env[names.privateKey], + }) + if (!parsed.success) + return failures.signingKeyInvalid() + const key: SigningKey = { + slot, + keyId: parsed.data.keyId, + notBefore: Date.parse(parsed.data.notBefore) / 1000, + notAfter: Date.parse(parsed.data.notAfter) / 1000, + privateKey: parsed.data.privateKey, + } + if (key.notBefore >= key.notAfter || keys.some(other => other.keyId === key.keyId)) + return failures.signingKeyInvalid() + keys.push(key) + } + if (keys.length === 0) + return failures.signingKeyInvalid() + return { _tag: 'valid', maximumAgeSeconds: maximumAgeSeconds.data, keys } +} + function validateSigningRow(row: SigningRow, maximumAgeSeconds: number, now: number): ValidationResult { if ( row.artifact_id !== `sha256:${row.content_sha256}` diff --git a/workers/artifact-signer/src/slots.ts b/workers/artifact-signer/src/slots.ts new file mode 100644 index 00000000..51a25d3f --- /dev/null +++ b/workers/artifact-signer/src/slots.ts @@ -0,0 +1,65 @@ +/** + * The signer holds up to two signing keys, one per slot, so a rotation can + * overlap the old key with the new one. Each slot is three vars in + * `wrangler.jsonc` and one secret. + * + * A slot counts only once its vars name a key ID. A secret with no vars is a + * staged key and signs nothing, so the secret can be put before the deploy + * that names it. Rotations alternate slots: the next key goes in the slot the + * last rotation emptied. `docs/runbooks/signing-key-rotation.md` has the order. + */ +export const ARTIFACT_SIGNING_KEY_SLOTS = { + primary: { + keyId: 'ARTIFACT_SIGNING_KEY_ID', + notBefore: 'ARTIFACT_SIGNING_KEY_NOT_BEFORE', + notAfter: 'ARTIFACT_SIGNING_KEY_NOT_AFTER', + privateKey: 'ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8', + }, + secondary: { + keyId: 'ARTIFACT_SIGNING_SECONDARY_KEY_ID', + notBefore: 'ARTIFACT_SIGNING_SECONDARY_KEY_NOT_BEFORE', + notAfter: 'ARTIFACT_SIGNING_SECONDARY_KEY_NOT_AFTER', + privateKey: 'ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8', + }, +} as const + +export type ArtifactSigningKeySlot = keyof typeof ARTIFACT_SIGNING_KEY_SLOTS + +type SlotNames = typeof ARTIFACT_SIGNING_KEY_SLOTS[ArtifactSigningKeySlot] + +export type ArtifactSigningKeyBindingName = SlotNames[keyof SlotNames] + +export interface ArtifactSigningKeyWindow { + slot: ArtifactSigningKeySlot + keyId: string + /** Unix seconds. */ + notBefore: number + /** Unix seconds. */ + notAfter: number +} + +/** + * The key the signer uses for a statement created at `createdAt` and signed + * at `now`: the newest key whose window holds both times. + * + * Both times count because each verifier checks one of them. The site + * refuses a key whose window is closed now. The skilld CLI refuses a + * statement created outside the key window. A Resolution created just before + * the new key's window opens still gets the old key. + */ +export function selectSigningKey( + keys: readonly Key[], + createdAt: number, + now: number, +): Key | null { + const earliest = Math.min(createdAt, now) + const latest = Math.max(createdAt, now) + let selected: Key | null = null + for (const key of keys) { + if (key.notBefore > earliest || latest >= key.notAfter) + continue + if (!selected || key.notBefore > selected.notBefore || (key.notBefore === selected.notBefore && key.notAfter > selected.notAfter)) + selected = key + } + return selected +} diff --git a/workers/artifact-signer/worker-configuration.d.ts b/workers/artifact-signer/worker-configuration.d.ts index bb17c76c..b44b686b 100644 --- a/workers/artifact-signer/worker-configuration.d.ts +++ b/workers/artifact-signer/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --config=wrangler.jsonc --env-file=.dev.vars.example --env-interface=ArtifactSignerEnv` (hash: b435ee032fde03ae9ccda451d68c341c) +// Generated by Wrangler by running `wrangler types --config=wrangler.jsonc --env-file=.dev.vars.example --env-interface=ArtifactSignerEnv` (hash: afcefeb6b8f445e9d8051085c3e6c6fe) // Runtime types generated with workerd@1.20260828.1 2026-08-20 nodejs_compat interface __BaseEnv_ArtifactSignerEnv { PUBLIC_ARTIFACTS: R2Bucket; @@ -10,6 +10,7 @@ interface __BaseEnv_ArtifactSignerEnv { ARTIFACT_SIGNING_KEY_NOT_AFTER: "2026-11-20T00:00:00.000Z"; ARTIFACT_SIGNING_MAX_AGE_SECONDS: "300"; ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8: string; + ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8: string; } declare namespace Cloudflare { interface GlobalProps { @@ -22,7 +23,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } // Begin runtime types From f28adedd6f3e5f254a4e323196f0f3d2df9589a7 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 16:29:15 +1100 Subject: [PATCH 3/6] feat(scripts): add the trusted root signing ceremony tool The site never checks a root signature, and every released CLI does. A hand-built key entry with a bad signature stops every skilld run with an upgrade message no upgrade fixes. The tool signs entries with the offline root key and checks a root by the CLI rules before it reaches the secret. --- scripts/lib/trusted-root-ceremony.ts | 216 ++++++++++++++++++++++++ scripts/trusted-root.ts | 146 ++++++++++++++++ test/unit/trusted-root-ceremony.test.ts | 133 +++++++++++++++ 3 files changed, 495 insertions(+) create mode 100644 scripts/lib/trusted-root-ceremony.ts create mode 100644 scripts/trusted-root.ts create mode 100644 test/unit/trusted-root-ceremony.test.ts diff --git a/scripts/lib/trusted-root-ceremony.ts b/scripts/lib/trusted-root-ceremony.ts new file mode 100644 index 00000000..4d5c0f37 --- /dev/null +++ b/scripts/lib/trusted-root-ceremony.ts @@ -0,0 +1,216 @@ +import type { KeyObject } from 'node:crypto' +import type { z } from 'zod' +import { createHash, createPrivateKey, createPublicKey, generateKeyPairSync, sign, verify } from 'node:crypto' +import { trustedKeyStatementSchema, trustedRootConfigSchema, trustedRootSchema } from '../../layers/artifact-delivery/server/schemas/contracts' +import { canonicalJson } from '../../layers/artifact-delivery/server/utils/encoding' +import { parseTrustedRoot } from '../../layers/artifact-delivery/server/utils/trusted-root' + +/** + * The signing ceremony for `ARTIFACT_TRUSTED_ROOT_JSON`: generate a signing + * key, sign its statement with the offline root key, and check a root the way + * the skilld CLI does. + * + * The site never checks a root signature. Every released CLI does, and a bad + * one stops every `skilld run` with an upgrade message that no upgrade fixes. + * `verifyTrustedRoot` is the gate before a root reaches production. + */ + +export type TrustedRootConfig = z.infer +export type TrustedKeyStatus = z.infer['status'] + +export interface RootPin { + rootKeyId: string + rootPublicKey: string +} + +export interface NewTrustedKey { + keyId: string + publicKey: string + notBefore: string + notAfter: string + status: Extract +} + +export type VerifyResult + = { _tag: 'valid' } + | { _tag: 'invalid', problems: string[] } + +const TRUSTED_KEY_DOMAIN = Buffer.from('skilld-trusted-key-v1\0') +const STATEMENT_FIELDS = ['algorithm', 'keyId', 'notAfter', 'notBefore', 'publicKey', 'rootKeyId', 'status', 'version'] + +/** Reads a stored root, or a live `/api/v1/trusted-root` answer, as the secret value. */ +export function parseRootConfig(value: unknown): TrustedRootConfig { + const live = trustedRootSchema.safeParse(value) + if (live.success) { + const { fetchedAt: _fetchedAt, ...config } = live.data + return config + } + return trustedRootConfigSchema.parse(value) +} + +/** A new Ed25519 signing key: the signer secret value and its public key. */ +export function generateSigningKey(): { privateKeyPkcs8: string, publicKey: string } { + const { privateKey } = generateKeyPairSync('ed25519') + return { + privateKeyPkcs8: privateKey.export({ format: 'der', type: 'pkcs8' }).toString('base64url'), + publicKey: publicKeyOf(privateKey), + } +} + +/** Reads a PKCS8 Ed25519 private key from PEM text or base64url DER, as the signer secret holds it. */ +export function readPrivateKey(text: string): KeyObject { + const value = text.trim() + const key = value.startsWith('-----BEGIN') + ? createPrivateKey(value) + : createPrivateKey({ key: Buffer.from(value, 'base64url'), format: 'der', type: 'pkcs8' }) + if (key.asymmetricKeyType !== 'ed25519') + throw new Error('The private key is not an Ed25519 key.') + return key +} + +/** The raw public key as canonical base64url, the form the trusted root holds. */ +export function publicKeyOf(key: KeyObject): string { + const jwk = createPublicKey(key).export({ format: 'jwk' }) + if (typeof jwk.x !== 'string') + throw new Error('The key has no Ed25519 public key.') + return jwk.x +} + +/** Adds one root-signed signing key. The root private key never leaves the caller. */ +export function addTrustedKey(root: TrustedRootConfig, rootPrivateKey: KeyObject, key: NewTrustedKey): TrustedRootConfig { + if (publicKeyOf(rootPrivateKey) !== root.rootPublicKey) + throw new Error('The root private key does not match rootPublicKey.') + if (root.keys.some(existing => existing.keyId === key.keyId)) + throw new Error(`The trusted root already holds ${key.keyId}.`) + const notBefore = normalizeTimestamp(key.notBefore) + const notAfter = normalizeTimestamp(key.notAfter) + if (Date.parse(notBefore) >= Date.parse(notAfter)) + throw new Error('notBefore must be earlier than notAfter.') + decodePublicKey(key.publicKey) + const statement = trustedKeyStatementSchema.parse({ + version: 1, + rootKeyId: root.rootKeyId, + keyId: key.keyId, + algorithm: 'Ed25519', + publicKey: key.publicKey, + notBefore, + notAfter, + status: key.status, + }) + const statementBytes = Buffer.from(canonicalJson(statement)) + const rootSignature = sign(null, signedMessage(statementBytes), rootPrivateKey) + return trustedRootConfigSchema.parse({ + ...root, + keys: [...root.keys, { + keyId: statement.keyId, + algorithm: statement.algorithm, + publicKey: statement.publicKey, + notBefore: statement.notBefore, + notAfter: statement.notAfter, + status: statement.status, + statement: statementBytes.toString('base64url'), + rootSignature: rootSignature.toString('base64url'), + }], + }) +} + +/** Removes one signing key. Removal needs no root signature. */ +export function removeTrustedKey(root: TrustedRootConfig, keyId: string): TrustedRootConfig { + if (!root.keys.some(key => key.keyId === keyId)) + throw new Error(`The trusted root does not hold ${keyId}.`) + const keys = root.keys.filter(key => key.keyId !== keyId) + if (keys.length === 0) + throw new Error('A trusted root must keep at least one signing key.') + return { ...root, keys } +} + +/** + * Checks a root by the rules every released skilld CLI applies, then by the + * site's own parser. Each failed rule is one problem. + */ +export function verifyTrustedRoot(root: TrustedRootConfig, pin: RootPin, now: number): VerifyResult { + const problems: string[] = [] + if (root.rootKeyId !== pin.rootKeyId || root.rootPublicKey !== pin.rootPublicKey) + problems.push('The root key differs from the pin the released CLIs carry.') + const rootKey = attempt(() => decodePublicKey(root.rootPublicKey)) + if (!rootKey) + problems.push('The root public key is not an Ed25519 key.') + const seen = new Set() + for (const key of root.keys) { + if (seen.has(key.keyId)) + problems.push(`${key.keyId}: the key ID appears twice.`) + seen.add(key.keyId) + const statementBytes = Buffer.from(key.statement, 'base64url') + if (statementBytes.toString('base64url') !== key.statement) { + problems.push(`${key.keyId}: the statement is not canonical base64url.`) + continue + } + const statement = parseJson(statementBytes.toString('utf8')) + const expected = { + version: 1, + rootKeyId: root.rootKeyId, + keyId: key.keyId, + algorithm: key.algorithm, + publicKey: key.publicKey, + notBefore: key.notBefore, + notAfter: key.notAfter, + status: key.status, + } + if ( + typeof statement !== 'object' || statement === null + || canonicalJson(Object.keys(statement).sort()) !== canonicalJson(STATEMENT_FIELDS) + || canonicalJson(statement) !== canonicalJson(expected) + ) { + problems.push(`${key.keyId}: the signed statement does not match the key fields.`) + } + if (!(Date.parse(key.notBefore) < Date.parse(key.notAfter))) + problems.push(`${key.keyId}: the validity window is invalid.`) + const publicKey = attempt(() => decodePublicKey(key.publicKey)) + if (!publicKey) + problems.push(`${key.keyId}: the public key is not an Ed25519 key.`) + const signature = Buffer.from(key.rootSignature, 'base64url') + if (rootKey && (signature.toString('base64url') !== key.rootSignature || !verify(null, signedMessage(statementBytes), rootKey, signature))) + problems.push(`${key.keyId}: the root signature does not verify.`) + } + const siteParse = attempt(() => parseTrustedRoot(JSON.stringify(root), now)) + if (!siteParse) + problems.push('The site parser rejects this root.') + return problems.length === 0 ? { _tag: 'valid' } : { _tag: 'invalid', problems } +} + +/** True when the private key is the one the trusted root lists for `keyId`. */ +export function signingKeyMatches(root: TrustedRootConfig, keyId: string, privateKey: KeyObject): boolean { + return root.keys.some(key => key.keyId === keyId && key.publicKey === publicKeyOf(privateKey)) +} + +function signedMessage(statementBytes: Uint8Array): Buffer { + return Buffer.concat([TRUSTED_KEY_DOMAIN, createHash('sha256').update(statementBytes).digest()]) +} + +function decodePublicKey(value: string): KeyObject { + const bytes = Buffer.from(value, 'base64url') + if (bytes.byteLength !== 32 || bytes.toString('base64url') !== value) + throw new Error('A public key must be 32 bytes of canonical base64url.') + return createPublicKey({ key: { kty: 'OKP', crv: 'Ed25519', x: value }, format: 'jwk' }) +} + +function normalizeTimestamp(value: string): string { + const parsed = Date.parse(value) + if (Number.isNaN(parsed) || !/(?:Z|[+-]\d{2}:\d{2})$/.test(value)) + throw new Error(`${value} is not a UTC timestamp.`) + return new Date(parsed).toISOString() +} + +function parseJson(value: string): unknown { + return attempt(() => JSON.parse(value) as unknown) +} + +/** The value, or null when `run` throws. Each caller turns null into a named problem. */ +function attempt(run: () => T): T | null { + try { + return run() + } + catch { + return null + } +} diff --git a/scripts/trusted-root.ts b/scripts/trusted-root.ts new file mode 100644 index 00000000..ae3e54a7 --- /dev/null +++ b/scripts/trusted-root.ts @@ -0,0 +1,146 @@ +// The Artifact signing key ceremony. docs/runbooks/signing-key-rotation.md +// gives the order. No command prints a private key: each one reads or writes +// key files with mode 600. +import type { RootPin, TrustedRootConfig } from './lib/trusted-root-ceremony' +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { parseArgs } from 'node:util' +import { + addTrustedKey, + generateSigningKey, + parseRootConfig, + readPrivateKey, + removeTrustedKey, + signingKeyMatches, + verifyTrustedRoot, +} from './lib/trusted-root-ceremony' + +const LIVE_ROOT = 'https://skilld.dev/api/v1/trusted-root' + +const USAGE = `Usage: pnpm exec tsx scripts/trusted-root.ts [options] + + signing-key --key-id --out + Write /.pkcs8 (the signer secret) and /.pub. + add-key --root --root-key --key-id --public-key-file + --not-before --not-after [--status active|overlapping] --out + Sign the key statement with the offline root key and write the new root. + remove-key --root --key-id --out + Write the root without that key. Removal needs no root key. + verify --root [--pin ] [--key-id --signing-key ] + Check the root as every released skilld CLI does. --pin defaults to ${LIVE_ROOT}. + With --signing-key, also check the private key matches that key ID.` + +const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + 'key-id': { type: 'string' }, + 'not-after': { type: 'string' }, + 'not-before': { type: 'string' }, + 'out': { type: 'string' }, + 'pin': { type: 'string' }, + 'public-key-file': { type: 'string' }, + 'root': { type: 'string' }, + 'root-key': { type: 'string' }, + 'signing-key': { type: 'string' }, + 'status': { type: 'string' }, + }, +}) + +function required(name: keyof typeof values): string { + const value = values[name] + if (typeof value !== 'string' || value.length === 0) + throw new Error(`Missing --${name}.\n\n${USAGE}`) + return value +} + +async function readRoot(source: string): Promise { + if (/^https?:\/\//.test(source)) { + const response = await fetch(source, { headers: { accept: 'application/json' } }) + if (!response.ok) + throw new Error(`${source} answered HTTP ${response.status}.`) + return parseRootConfig(await response.json()) + } + return parseRootConfig(JSON.parse(readFileSync(source, 'utf8'))) +} + +/** Writes a new file only. A key or root file is never overwritten. */ +function writeNew(path: string, value: string, mode: number): void { + if (existsSync(path)) + throw new Error(`${path} exists. Choose another path.`) + writeFileSync(path, value, { mode, flag: 'wx' }) +} + +function printKeys(root: TrustedRootConfig): void { + const now = Date.now() + for (const key of root.keys) { + const open = Date.parse(key.notBefore) <= now && now < Date.parse(key.notAfter) + console.log(` ${key.keyId} ${key.status} ${key.notBefore} to ${key.notAfter} ${open ? 'open now' : 'closed now'}`) + } +} + +async function main(command: string | undefined): Promise { + switch (command) { + case 'signing-key': { + const keyId = required('key-id') + const out = required('out') + const key = generateSigningKey() + writeNew(join(out, `${keyId}.pkcs8`), key.privateKeyPkcs8, 0o600) + writeNew(join(out, `${keyId}.pub`), key.publicKey, 0o644) + console.log(`Wrote ${join(out, `${keyId}.pkcs8`)} (private, mode 600) and ${join(out, `${keyId}.pub`)}.`) + return + } + case 'add-key': { + const root = await readRoot(required('root')) + const status = values.status ?? 'active' + if (status !== 'active' && status !== 'overlapping') + throw new Error('--status must be active or overlapping.') + const next = addTrustedKey(root, readPrivateKey(readFileSync(required('root-key'), 'utf8')), { + keyId: required('key-id'), + publicKey: readFileSync(required('public-key-file'), 'utf8').trim(), + notBefore: required('not-before'), + notAfter: required('not-after'), + status, + }) + writeNew(required('out'), JSON.stringify(next), 0o600) + console.log(`Wrote ${required('out')}:`) + printKeys(next) + return + } + case 'remove-key': { + const next = removeTrustedKey(await readRoot(required('root')), required('key-id')) + writeNew(required('out'), JSON.stringify(next), 0o600) + console.log(`Wrote ${required('out')}:`) + printKeys(next) + return + } + case 'verify': { + const root = await readRoot(required('root')) + const pinRoot = await readRoot(values.pin ?? LIVE_ROOT) + const pin: RootPin = { rootKeyId: pinRoot.rootKeyId, rootPublicKey: pinRoot.rootPublicKey } + const result = verifyTrustedRoot(root, pin, Math.floor(Date.now() / 1000)) + const problems = result._tag === 'invalid' ? [...result.problems] : [] + if (values['signing-key']) { + const keyId = required('key-id') + const privateKey = readPrivateKey(readFileSync(values['signing-key'], 'utf8')) + if (!signingKeyMatches(root, keyId, privateKey)) + problems.push(`${keyId}: the signing key file does not match the public key the root lists for that key ID.`) + } + console.log(`Root ${root.rootKeyId}, ${root.keys.length} signing keys:`) + printKeys(root) + if (problems.length > 0) { + console.error(`FAILED:\n${problems.map(problem => ` - ${problem}`).join('\n')}`) + process.exitCode = 1 + return + } + console.log('OK: the CLI pin, every statement, and every root signature verify.') + return + } + default: + throw new Error(USAGE) + } +} + +main(positionals[0]).catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 +}) diff --git a/test/unit/trusted-root-ceremony.test.ts b/test/unit/trusted-root-ceremony.test.ts new file mode 100644 index 00000000..1739f914 --- /dev/null +++ b/test/unit/trusted-root-ceremony.test.ts @@ -0,0 +1,133 @@ +import type { TrustedRootConfig } from '../../scripts/lib/trusted-root-ceremony' +import { generateKeyPairSync } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { parseTrustedRoot } from '../../layers/artifact-delivery/server/utils/trusted-root' +import { + addTrustedKey, + generateSigningKey, + parseRootConfig, + publicKeyOf, + readPrivateKey, + removeTrustedKey, + signingKeyMatches, + verifyTrustedRoot, +} from '../../scripts/lib/trusted-root-ceremony' + +const NOW = Date.parse('2026-10-07T00:00:00.000Z') / 1000 + +function ceremony() { + const { privateKey: rootPrivateKey } = generateKeyPairSync('ed25519') + const empty: TrustedRootConfig = { version: 1, rootKeyId: 'skilld-root-2026', rootPublicKey: publicKeyOf(rootPrivateKey), keys: [] } + const pin = { rootKeyId: empty.rootKeyId, rootPublicKey: empty.rootPublicKey } + const oldKey = generateSigningKey() + const newKey = generateSigningKey() + const current = addTrustedKey(empty, rootPrivateKey, { + keyId: 'skilld-production-2026-08', + publicKey: oldKey.publicKey, + notBefore: '2026-08-20T00:00:00.000Z', + notAfter: '2026-11-20T00:00:00.000Z', + status: 'active', + }) + return { rootPrivateKey, pin, oldKey, newKey, current } +} + +describe('trusted root ceremony', () => { + it('adds a root-signed key that the CLI rules and the site parser both accept', () => { + const { rootPrivateKey, pin, newKey, current } = ceremony() + + const next = addTrustedKey(current, rootPrivateKey, { + keyId: 'skilld-production-2026-11', + publicKey: newKey.publicKey, + notBefore: '2026-10-14T00:00:00Z', + notAfter: '2027-05-20T00:00:00Z', + status: 'active', + }) + + expect(verifyTrustedRoot(next, pin, NOW)).toEqual({ _tag: 'valid' }) + expect(parseTrustedRoot(JSON.stringify(next), NOW).keys.map(key => [key.keyId, key.notBefore])).toEqual([ + ['skilld-production-2026-08', '2026-08-20T00:00:00.000Z'], + ['skilld-production-2026-11', '2026-10-14T00:00:00.000Z'], + ]) + }) + + it('refuses a root private key that does not match the root public key', () => { + const { pin: _pin, newKey, current } = ceremony() + const { privateKey: otherRoot } = generateKeyPairSync('ed25519') + + expect(() => addTrustedKey(current, otherRoot, { + keyId: 'skilld-production-2026-11', + publicKey: newKey.publicKey, + notBefore: '2026-10-14T00:00:00Z', + notAfter: '2027-05-20T00:00:00Z', + status: 'active', + })).toThrow('does not match rootPublicKey') + }) + + it('reports a status changed without a new root signature', () => { + const { pin, current } = ceremony() + const changed = { ...current, keys: current.keys.map(key => ({ ...key, status: 'retired' as const })) } + + expect(verifyTrustedRoot(changed, pin, NOW)).toEqual({ + _tag: 'invalid', + problems: [ + 'skilld-production-2026-08: the signed statement does not match the key fields.', + 'The site parser rejects this root.', + ], + }) + }) + + it('reports a key signed by another root key', () => { + const { pin, newKey, current } = ceremony() + const { privateKey: otherRoot } = generateKeyPairSync('ed25519') + const forged = addTrustedKey({ ...current, rootPublicKey: publicKeyOf(otherRoot) }, otherRoot, { + keyId: 'skilld-production-2026-11', + publicKey: newKey.publicKey, + notBefore: '2026-10-14T00:00:00Z', + notAfter: '2027-05-20T00:00:00Z', + status: 'active', + }) + + const result = verifyTrustedRoot({ ...forged, rootPublicKey: pin.rootPublicKey }, pin, NOW) + + expect(result).toEqual({ _tag: 'invalid', problems: ['skilld-production-2026-11: the root signature does not verify.'] }) + }) + + it('reports a root key that differs from the CLI pin', () => { + const { current } = ceremony() + const { privateKey: otherRoot } = generateKeyPairSync('ed25519') + + const result = verifyTrustedRoot(current, { rootKeyId: current.rootKeyId, rootPublicKey: publicKeyOf(otherRoot) }, NOW) + + expect(result).toMatchObject({ _tag: 'invalid', problems: expect.arrayContaining(['The root key differs from the pin the released CLIs carry.']) }) + }) + + it('removes a key without the root key, and keeps at least one', () => { + const { rootPrivateKey, pin, newKey, current } = ceremony() + const both = addTrustedKey(current, rootPrivateKey, { + keyId: 'skilld-production-2026-11', + publicKey: newKey.publicKey, + notBefore: '2026-10-14T00:00:00Z', + notAfter: '2027-05-20T00:00:00Z', + status: 'active', + }) + + const next = removeTrustedKey(both, 'skilld-production-2026-08') + + expect(next.keys.map(key => key.keyId)).toEqual(['skilld-production-2026-11']) + expect(verifyTrustedRoot(next, pin, NOW)).toEqual({ _tag: 'valid' }) + expect(() => removeTrustedKey(next, 'skilld-production-2026-11')).toThrow('at least one') + }) + + it('reads the live answer as the secret value, without fetchedAt', () => { + const { current } = ceremony() + + expect(parseRootConfig({ ...current, fetchedAt: '2026-10-07T00:00:00.000Z' })).toEqual(current) + }) + + it('matches a signer secret to its key ID', () => { + const { oldKey, newKey, current } = ceremony() + + expect(signingKeyMatches(current, 'skilld-production-2026-08', readPrivateKey(oldKey.privateKeyPkcs8))).toBe(true) + expect(signingKeyMatches(current, 'skilld-production-2026-08', readPrivateKey(newKey.privateKeyPkcs8))).toBe(false) + }) +}) From 6843cd23fdc411192880c1ae7d48dead9a928910 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 16:29:16 +1100 Subject: [PATCH 4/6] feat(checkin): warn 30 days before Artifact signing stops skilld-production-2026-08 closes on 2026-11-20 and nothing watched it. The check reads the signer windows and the live trusted root, and finds when no key is both open and trusted. --- checks/_helpers/signing-key.ts | 128 ++++++++++++++++++ checks/external/signing-key.ts | 17 +++ shared/checkin-external.ts | 3 +- .../unit/external-checkin-signing-key.test.ts | 105 ++++++++++++++ 4 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 checks/_helpers/signing-key.ts create mode 100644 checks/external/signing-key.ts create mode 100644 test/unit/external-checkin-signing-key.test.ts diff --git a/checks/_helpers/signing-key.ts b/checks/_helpers/signing-key.ts new file mode 100644 index 00000000..f170186f --- /dev/null +++ b/checks/_helpers/signing-key.ts @@ -0,0 +1,128 @@ +import type { CheckResult } from '@harlan-zw/nuxt-checkin/external' +import type { ArtifactSigningKeyWindow } from '../../workers/artifact-signer/src/slots' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { fail, pass, readBoundedResponseText, warn } from '@harlan-zw/nuxt-checkin/external' +import { z } from 'zod' +import { ARTIFACT_SIGNING_KEY_SLOTS, selectSigningKey } from '../../workers/artifact-signer/src/slots' + +export const TRUSTED_ROOT_URL = 'https://skilld.dev/api/v1/trusted-root' +export const SIGNING_KEY_WARN_DAYS = 30 +export const SIGNING_KEY_FAIL_DAYS = 7 +const DAY_SECONDS = 86_400 +const RUNBOOK = 'docs/runbooks/signing-key-rotation.md' +const SIGNER_CONFIG = 'workers/artifact-signer/wrangler.jsonc' + +const timestamp = z.string().datetime().transform(value => Date.parse(value) / 1000) + +const rootSchema = z.object({ + keys: z.array(z.object({ + keyId: z.string().min(1), + status: z.enum(['active', 'overlapping', 'retired', 'revoked']), + notBefore: timestamp, + notAfter: timestamp, + })).min(1), +}) + +const signerSlotSchema = z.object({ + keyId: z.string().min(1), + notBefore: timestamp, + notAfter: timestamp, +}) + +export type RootKeyWindow = z.infer['keys'][number] + +export interface SigningKeyEvidence { + signer: ArtifactSigningKeyWindow[] + root: RootKeyWindow[] + /** Unix seconds. */ + now: number +} + +/** The signer key windows that `wrangler.jsonc` names, one per filled slot. */ +export function parseSignerKeys(vars: Readonly>): ArtifactSigningKeyWindow[] { + return Object.entries(ARTIFACT_SIGNING_KEY_SLOTS).flatMap(([slot, names]) => { + if (vars[names.keyId] === undefined && vars[names.notBefore] === undefined && vars[names.notAfter] === undefined) + return [] + const parsed = signerSlotSchema.parse({ + keyId: vars[names.keyId], + notBefore: vars[names.notBefore], + notAfter: vars[names.notAfter], + }) + return [{ slot: slot as ArtifactSigningKeyWindow['slot'], ...parsed }] + }) +} + +export function parseRootKeys(value: unknown): RootKeyWindow[] { + return rootSchema.parse(value).keys +} + +/** + * When Artifact signing stops, from the signer config and the live trusted root. + * + * A signer key signs usefully only inside both its signer window and its + * trusted root window, with the root status `active` or `overlapping`. Past + * the end of that coverage every build fails at verification, and every + * stored Artifact fails once its own key window closes. + */ +export function evaluateSigningKeys({ signer, root, now }: SigningKeyEvidence): CheckResult { + const trusted = new Map(root + .filter(key => key.status === 'active' || key.status === 'overlapping') + .map(key => [key.keyId, key])) + const usable = signer.flatMap((key) => { + const rootKey = trusted.get(key.keyId) + if (!rootKey) + return [] + const from = Math.max(key.notBefore, rootKey.notBefore) + const until = Math.min(key.notAfter, rootKey.notAfter) + return from < until ? [{ keyId: key.keyId, from, until }] : [] + }) + const signing = selectSigningKey(signer, now, now) + const evidence = { + signer: signer.map(key => ({ ...key, notBefore: iso(key.notBefore), notAfter: iso(key.notAfter) })), + root: root.map(key => ({ ...key, notBefore: iso(key.notBefore), notAfter: iso(key.notAfter) })), + signingKeyId: signing?.keyId ?? null, + warnDays: SIGNING_KEY_WARN_DAYS, + } + if (!signing) + return fail(`The Artifact signer has no key whose window holds now. Every Artifact build fails. Follow ${RUNBOOK}.`, evidence) + if (!usable.some(window => window.keyId === signing.keyId && window.from <= now && now < window.until)) + return fail(`The Artifact signer signs with ${signing.keyId}, and the live trusted root does not trust that key now. Every Artifact build fails. Follow ${RUNBOOK}.`, evidence) + + let endsAt = now + for (let extended = true; extended;) { + const next = Math.max(endsAt, ...usable.filter(window => window.from <= endsAt && endsAt < window.until).map(window => window.until)) + extended = next > endsAt + endsAt = next + } + const daysLeft = Math.floor((endsAt - now) / DAY_SECONDS) + const covered = { ...evidence, signingEndsAt: iso(endsAt), daysLeft } + const ending = `Artifact signing stops at ${iso(endsAt)}, in ${daysLeft} days. After that every skilld run fails. Follow ${RUNBOOK}.` + if (endsAt - now <= SIGNING_KEY_FAIL_DAYS * DAY_SECONDS) + return fail(ending, covered) + if (endsAt - now <= SIGNING_KEY_WARN_DAYS * DAY_SECONDS) + return warn(ending, covered) + + const staged = signer.filter(key => key.notBefore > now && !usable.some(window => window.keyId === key.keyId && window.from <= key.notBefore)) + if (staged.length > 0) { + const names = staged.map(key => `${key.keyId} (from ${iso(key.notBefore)})`).join(', ') + return warn(`The Artifact signer starts to sign with ${names}, and the live trusted root does not trust it from that date. Add it to the trusted root first. Follow ${RUNBOOK}.`, covered) + } + return pass(covered) +} + +export async function readSignerKeys(rootDir: string): Promise { + const config = JSON.parse(await readFile(join(rootDir, SIGNER_CONFIG), 'utf8')) as { vars?: Record } + return parseSignerKeys(config.vars ?? {}) +} + +export async function fetchRootKeys(signal: AbortSignal, request: typeof fetch = fetch): Promise { + const response = await request(TRUSTED_ROOT_URL, { signal, headers: { accept: 'application/json' } }) + if (!response.ok) + throw new Error(`The trusted root answered HTTP ${response.status}.`) + return parseRootKeys(JSON.parse(await readBoundedResponseText(response, 65_536))) +} + +function iso(seconds: number): string { + return new Date(seconds * 1000).toISOString() +} diff --git a/checks/external/signing-key.ts b/checks/external/signing-key.ts new file mode 100644 index 00000000..e989e6d3 --- /dev/null +++ b/checks/external/signing-key.ts @@ -0,0 +1,17 @@ +import { defineExternalCheck } from '@harlan-zw/nuxt-checkin/external' +import { withCause } from '../_helpers/cause.mjs' +import { evaluateSigningKeys, fetchRootKeys, readSignerKeys } from '../_helpers/signing-key' + +/** + * Warns 30 days before Artifact signing stops. The signer windows come from + * `workers/artifact-signer/wrangler.jsonc` on this checkout, which the deploy + * workflow ships. The trusted root comes from production. + */ +export default defineExternalCheck({ + id: 'skilld.signing-key', + run: withCause(async context => evaluateSigningKeys({ + signer: await readSignerKeys(context.rootDir), + root: await fetchRootKeys(context.signal), + now: Math.floor(context.now.getTime() / 1000), + })), +}) diff --git a/shared/checkin-external.ts b/shared/checkin-external.ts index 5e9a3f3f..520b033a 100644 --- a/shared/checkin-external.ts +++ b/shared/checkin-external.ts @@ -1,7 +1,7 @@ import type { ExternalOptions } from '@harlan-zw/nuxt-checkin/external' export const externalCheckin = { - required: ['skilld.baseline', 'skilld.git', 'skilld.deploy', 'skilld.ci', 'skilld.home', 'skilld.skills', 'skilld.database', 'skilld.workers', 'skilld.analytics', 'skilld.report', 'sentry.skilld', 'skilld.sentry-details'], + required: ['skilld.baseline', 'skilld.git', 'skilld.deploy', 'skilld.ci', 'skilld.home', 'skilld.skills', 'skilld.database', 'skilld.workers', 'skilld.analytics', 'skilld.report', 'sentry.skilld', 'skilld.sentry-details', 'skilld.signing-key'], prompts: [ { id: 'skilld.operations', @@ -18,6 +18,7 @@ Interpret this site evidence: - Use failed-job \`first_failed_at\` and \`last_failed_at\`, plus \`d1.recentJobBatches\` and \`d1.registryMaintenance\`, to distinguish an active incident from a recovered burst. Do not call a window clean because the latest batch passed, or active because an older batch failed. - \`d1.inventory.broken_repos\` is known cumulative inventory. Review \`d1.pipeline.newly_broken_repos_total\`, but gate health on \`d1.pipeline.newly_broken_repos_impacted\`. A source removal is impacting when it still backs a skill or appears in a star, subscription, or collection. - \`skilld.analytics\` holds command copies from Analytics Engine, which is the activation signal. Report \`commandCopies.run\` and \`commandCopies.install\` apart: a run copy reads a Skill once, an install copy keeps it. Analytics Engine samples, so treat a small day as a range, not an exact count. + - \`skilld.signing-key\` Warn or Fail means Artifact signing ends soon. Quote \`signingEndsAt\` and \`daysLeft\`, and point to \`docs/runbooks/signing-key-rotation.md\`. After that date every \`skilld run\` fails. - AI cost is only the recorded batch estimate. Unmeasured services are unknown, not $0.`, }, { diff --git a/test/unit/external-checkin-signing-key.test.ts b/test/unit/external-checkin-signing-key.test.ts new file mode 100644 index 00000000..75468ab1 --- /dev/null +++ b/test/unit/external-checkin-signing-key.test.ts @@ -0,0 +1,105 @@ +import type { RootKeyWindow } from '../../checks/_helpers/signing-key' +import type { ArtifactSigningKeyWindow } from '../../workers/artifact-signer/src/slots' +import { describe, expect, it } from 'vitest' +import { evaluateSigningKeys, fetchRootKeys, parseSignerKeys } from '../../checks/_helpers/signing-key' + +const seconds = (value: string) => Date.parse(value) / 1000 + +const OLD_SIGNER: ArtifactSigningKeyWindow = { + slot: 'primary', + keyId: 'skilld-production-2026-08', + notBefore: seconds('2026-08-20T00:00:00.000Z'), + notAfter: seconds('2026-11-20T00:00:00.000Z'), +} +const NEW_SIGNER: ArtifactSigningKeyWindow = { + slot: 'secondary', + keyId: 'skilld-production-2026-11', + notBefore: seconds('2026-10-20T00:00:00.000Z'), + notAfter: seconds('2027-05-20T00:00:00.000Z'), +} +const OLD_ROOT: RootKeyWindow = { keyId: OLD_SIGNER.keyId, status: 'active', notBefore: OLD_SIGNER.notBefore, notAfter: OLD_SIGNER.notAfter } +const NEW_ROOT: RootKeyWindow = { keyId: NEW_SIGNER.keyId, status: 'active', notBefore: NEW_SIGNER.notBefore, notAfter: NEW_SIGNER.notAfter } + +describe('signing key check', () => { + it('passes with more than 30 days of signing left', () => { + const result = evaluateSigningKeys({ signer: [OLD_SIGNER], root: [OLD_ROOT], now: seconds('2026-10-07T00:00:00.000Z') }) + + expect(result).toMatchObject({ _tag: 'Pass', evidence: { signingKeyId: OLD_SIGNER.keyId, signingEndsAt: '2026-11-20T00:00:00.000Z', daysLeft: 44 } }) + }) + + it.each([ + ['2026-10-20T23:59:59.000Z', 'Pass'], + ['2026-10-21T00:00:00.000Z', 'Warn'], + ['2026-11-13T00:00:00.000Z', 'Fail'], + ] as const)('reads %s as %s', (now, tag) => { + expect(evaluateSigningKeys({ signer: [OLD_SIGNER], root: [OLD_ROOT], now: seconds(now) })._tag).toBe(tag) + }) + + it('fails once no signer key window holds now', () => { + const result = evaluateSigningKeys({ signer: [OLD_SIGNER], root: [OLD_ROOT], now: seconds('2026-11-20T00:00:00.000Z') }) + + expect(result).toMatchObject({ _tag: 'Fail', reason: expect.stringContaining('no key whose window holds now') }) + }) + + it('counts the overlapping next key as more signing time', () => { + const result = evaluateSigningKeys({ signer: [OLD_SIGNER, NEW_SIGNER], root: [OLD_ROOT, NEW_ROOT], now: seconds('2026-11-15T00:00:00.000Z') }) + + expect(result).toMatchObject({ _tag: 'Pass', evidence: { signingKeyId: NEW_SIGNER.keyId, signingEndsAt: '2027-05-20T00:00:00.000Z' } }) + }) + + it('ends signing where the trusted root window ends, when it ends before the signer window', () => { + const result = evaluateSigningKeys({ + signer: [NEW_SIGNER], + root: [{ ...NEW_ROOT, notAfter: seconds('2027-02-20T00:00:00.000Z') }], + now: seconds('2027-02-01T00:00:00.000Z'), + }) + + expect(result).toMatchObject({ _tag: 'Warn', evidence: { signingEndsAt: '2027-02-20T00:00:00.000Z', daysLeft: 19 } }) + }) + + it('warns when the signer will start a key the trusted root does not trust', () => { + const result = evaluateSigningKeys({ signer: [OLD_SIGNER, NEW_SIGNER], root: [OLD_ROOT], now: seconds('2026-10-10T00:00:00.000Z') }) + + expect(result).toMatchObject({ _tag: 'Warn', reason: expect.stringContaining('skilld-production-2026-11') }) + }) + + it.each([ + ['missing', [OLD_ROOT]], + ['retired', [OLD_ROOT, { ...NEW_ROOT, status: 'retired' as const }]], + ])('fails when the signer signs with a key the trusted root lists as %s', (_name, root) => { + const result = evaluateSigningKeys({ signer: [OLD_SIGNER, NEW_SIGNER], root, now: seconds('2026-10-25T00:00:00.000Z') }) + + expect(result).toMatchObject({ _tag: 'Fail', reason: expect.stringContaining('does not trust that key now') }) + }) +}) + +describe('signing key evidence', () => { + it('reads each filled signer slot and skips an empty one', () => { + expect(parseSignerKeys({ + ARTIFACT_SIGNING_KEY_ID: OLD_SIGNER.keyId, + ARTIFACT_SIGNING_KEY_NOT_BEFORE: '2026-08-20T00:00:00.000Z', + ARTIFACT_SIGNING_KEY_NOT_AFTER: '2026-11-20T00:00:00.000Z', + ARTIFACT_SIGNING_MAX_AGE_SECONDS: '300', + })).toEqual([OLD_SIGNER]) + }) + + it('throws on a slot with a key ID and no window', () => { + expect(() => parseSignerKeys({ ARTIFACT_SIGNING_SECONDARY_KEY_ID: NEW_SIGNER.keyId })).toThrow() + }) + + it('reads key windows from the trusted root answer', async () => { + const request = async () => Response.json({ + version: 1, + rootKeyId: 'skilld-root-2026', + keys: [{ keyId: OLD_SIGNER.keyId, status: 'active', notBefore: '2026-08-20T00:00:00.000Z', notAfter: '2026-11-20T00:00:00.000Z', publicKey: 'B'.repeat(43) }], + }) + + expect(await fetchRootKeys(new AbortController().signal, request)).toEqual([OLD_ROOT]) + }) + + it('throws when the trusted root answers an error', async () => { + const request = async () => new Response('down', { status: 503 }) + + await expect(fetchRootKeys(new AbortController().signal, request)).rejects.toThrow('HTTP 503') + }) +}) From d9df472b72f1f413be337f141a0e5245da54f560 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 16:29:16 +1100 Subject: [PATCH 5/6] docs(runbooks): add the signing key rotation runbook The rollout guide's rotation steps swapped the secret and key ID non-atomically and retired the old key with an unsigned status change. --- docs/ops/artifact-delivery-rollout.md | 9 +- docs/runbooks/checkin.md | 9 + docs/runbooks/signing-key-rotation.md | 277 ++++++++++++++++++++++++++ 3 files changed, 288 insertions(+), 7 deletions(-) create mode 100644 docs/runbooks/signing-key-rotation.md diff --git a/docs/ops/artifact-delivery-rollout.md b/docs/ops/artifact-delivery-rollout.md index a562b2d3..a7ead141 100644 --- a/docs/ops/artifact-delivery-rollout.md +++ b/docs/ops/artifact-delivery-rollout.md @@ -292,13 +292,8 @@ If any private check fails, set the flag to `false` and redeploy. ## 11. Rotate keys -Rotate an Ed25519 signing key with an overlap window: - -1. Add the new root-signed public key with `overlapping` status. -2. Deploy the updated trusted root to the public Worker. -3. Deploy the signer with the new private key and key ID. -4. Confirm new Artifacts use the new key. -5. Mark the old key `retired` after its overlap window. +Rotate the Ed25519 Artifact signing key with [the signing key rotation runbook](../runbooks/signing-key-rotation.md). +It overlaps the old and new keys in two signer slots, so no build fails during the change. Start private key rotation only in v3.1. diff --git a/docs/runbooks/checkin.md b/docs/runbooks/checkin.md index 4e1e0751..48c7549c 100644 --- a/docs/runbooks/checkin.md +++ b/docs/runbooks/checkin.md @@ -60,6 +60,15 @@ The report must match the deployed Worker and be less than five minutes old. The daily health email cron and X API failure emails are removed. Customer digests remain unchanged. Historical email records stay in D1, but checks no longer depend on them. +## Artifact signing key + +The `skilld.signing-key` check reads the signer key windows from `workers/artifact-signer/wrangler.jsonc` and the live trusted root. +It finds the time when no signer key is both open and trusted. +It warns 30 days before that time and fails 7 days before it. +It also fails when the signer signs with a key the trusted root does not trust now. +It warns when a staged signer key starts before the trusted root trusts it. +Follow `docs/runbooks/signing-key-rotation.md` when it warns. + ## SEO recovery measurement The `skilld.seo-recovery` prompt item adds a weekly SEO recovery section. diff --git a/docs/runbooks/signing-key-rotation.md b/docs/runbooks/signing-key-rotation.md new file mode 100644 index 00000000..20dd871c --- /dev/null +++ b/docs/runbooks/signing-key-rotation.md @@ -0,0 +1,277 @@ +# Signing key rotation + +Every Artifact attestation carries an Ed25519 signature from the Artifact signer. +Each signing key has a validity window. +When the last usable window closes, the signer refuses to sign and the site stops trusting stored attestations. +After that, every `skilld run` fails. + +The key `skilld-production-2026-08` closes at `2026-11-20T00:00:00.000Z`. +This runbook replaces it with `skilld-production-2026-11` and keeps both keys trusted during an overlap. +The daily check-in result `skilld.signing-key` warns 30 days before the last window closes. + +## How signing works + +The site stages an attestation statement for a Resolution. +The signer Worker `workers/artifact-signer` loads that statement from D1, checks the R2 bytes, and signs a digest of the statement. + +The signer has two key slots. Each slot is three vars in `workers/artifact-signer/wrangler.jsonc` and one secret. + +| Slot | Vars | Secret | +| --- | --- | --- | +| primary | `ARTIFACT_SIGNING_KEY_ID`, `ARTIFACT_SIGNING_KEY_NOT_BEFORE`, `ARTIFACT_SIGNING_KEY_NOT_AFTER` | `ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8` | +| secondary | `ARTIFACT_SIGNING_SECONDARY_KEY_ID`, `ARTIFACT_SIGNING_SECONDARY_KEY_NOT_BEFORE`, `ARTIFACT_SIGNING_SECONDARY_KEY_NOT_AFTER` | `ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8` | + +A slot counts only when its vars name a key ID. +A secret with no vars signs nothing, so you can put a secret before the deploy that names it. +If a named slot has no valid secret, the signer refuses every request with `SIGNING_KEY_INVALID`. + +For each statement, the signer uses the newest key whose window holds both the statement's `createdAt` and the current time. +A Resolution created before a new window opens still gets the old key. + +The trusted root is the `ARTIFACT_TRUSTED_ROOT_JSON` secret on the public Worker. +It holds the root key ID, the root public key, and one entry per signing key. +Each entry carries a statement that the offline root key signed. +The site serves it at `/api/v1/trusted-root` with a five-minute cache. + +Each verifier applies these rules to a signing key: + +| Rule | Signer | Site | skilld CLI | +| --- | --- | --- | --- | +| The trusted root names the key with status `active` or `overlapping` | no | yes | yes | +| The root key signed the key entry | no | no | yes | +| The window holds the statement's `createdAt` | yes | yes | yes | +| The window holds the current time | yes | yes | no | + +The CLI fetches the trusted root on every run and keeps no copy. +It checks the root against a root key compiled into each release. +Every release from 3.0.0 through 3.6.4 carries the root key `skilld-root-2026` and no signing key. +A signing key rotation needs no CLI release. Only a root key change needs one. + +## What happens to stored Artifacts + +Once a key window closes, the site stops trusting every attestation that key signed. +The next request for that Skill commit misses the reuse check with `attestation-untrusted` and builds again. +The new build signs under the current key. +The bytes match, so the Artifact ID and the R2 object stay the same. Only a new attestation row is added. + +The signer cannot sign a stored statement again. It signs only a Resolution in `signing` state with fresh check results. +A rebuild is the only way to sign an Artifact again. + +The CLI keeps no attestation on disk. A lockfile names a commit, so `skilld install` resolves again and gets the new attestation. + +Private Artifacts follow the same path through `verifyArtifactAttestation`. + +## Before you start + +You need the offline root private key for `skilld-root-2026`, in PKCS8 PEM or base64url DER form. +Never copy it into the repository, [Cloudflare](https://cloudflare.com), or GitHub Actions. + +Work in a private directory outside the repository, for example on an encrypted volume. +Set `KEYS` to that directory. Run every command from the repository root. + +```sh +export KEYS=/path/to/private/dir +``` + +`scripts/trusted-root.ts` runs the ceremony. It never prints a private key. +It writes new files with mode 600 and refuses to overwrite a file. + +## Timeline + +All times are UTC. Each date is the latest safe date. Earlier is better. + +| Step | Action | Latest safe date | +| --- | --- | --- | +| 1 | Deploy slot support | 2026-11-02 | +| 2 | Run the ceremony | 2026-11-06 | +| 3 | Put the new trusted root on the site | 2026-11-09 | +| 4 | Put the new private key in the free slot | 2026-11-09 | +| 5 | Name the new key in the signer vars | 2026-11-13 | +| 6 | Remove the old key from the trusted root | 2026-11-19 | +| 7 | Empty the old slot | after 2026-11-20 | + +Step 3 must reach production at least 10 minutes before step 5, so that cached roots expire. +If step 5 slips past 2026-11-13, a failed deploy has less than a week to recover. +If step 5 does not deploy before `2026-11-20T00:00:00.000Z`, every `skilld run` fails until it does. + +## 1. Deploy slot support + +Confirm that `workers/artifact-signer/src/slots.ts` exists on `main`. +Confirm that the last `Deploy to Cloudflare` run on `main` passed after that file arrived. + +```sh +git fetch origin main +git log -1 --format='%H %cs' origin/main -- workers/artifact-signer/src/slots.ts +gh run list --repo skilld-dev/skilld.dev --workflow deploy-cloudflare.yml --branch main --limit 3 +``` + +## 2. Run the ceremony + +Save the live root. It is the rollback value and the pin for the checks below. + +```sh +curl -fsS https://skilld.dev/api/v1/trusted-root | jq -c 'del(.fetchedAt)' > "$KEYS/root-before.json" +``` + +Generate the new signing key. + +```sh +pnpm exec tsx scripts/trusted-root.ts signing-key --key-id skilld-production-2026-11 --out "$KEYS" +``` + +Sign its statement with the root key. +Set `--not-before` to the ceremony day at 00:00. Set `--not-after` six months later. +A longer window means fewer ceremonies. A shorter one limits the damage of a leaked key. + +```sh +pnpm exec tsx scripts/trusted-root.ts add-key \ + --root "$KEYS/root-before.json" \ + --root-key /path/to/root-private-key \ + --key-id skilld-production-2026-11 \ + --public-key-file "$KEYS/skilld-production-2026-11.pub" \ + --not-before 2026-10-14T00:00:00Z \ + --not-after 2027-05-20T00:00:00Z \ + --status active \ + --out "$KEYS/root-next.json" +``` + +Use `active`. The verifiers treat `overlapping` the same way, and the status is part of the signed statement. + +Check the new root by the rules every released CLI applies, and check that the private key matches the entry: + +```sh +pnpm exec tsx scripts/trusted-root.ts verify \ + --root "$KEYS/root-next.json" \ + --pin "$KEYS/root-before.json" \ + --key-id skilld-production-2026-11 \ + --signing-key "$KEYS/skilld-production-2026-11.pkcs8" +``` + +Stop unless it prints `OK`. + +## 3. Put the new trusted root on the site + +```sh +pnpm exec wrangler secret put ARTIFACT_TRUSTED_ROOT_JSON --config wrangler.jsonc < "$KEYS/root-next.json" +``` + +The put deploys a new public Worker version at once. + +Verify the live root: + +```sh +pnpm exec tsx scripts/trusted-root.ts verify --root https://skilld.dev/api/v1/trusted-root --pin "$KEYS/root-before.json" +``` + +It must list both keys and print `OK`. + +Run the oldest and newest released CLIs. Each run checks every root signature as shipped: + +```sh +npx -y skilld@3.0.0 run skilld-dev/skills/find-skill > /dev/null && echo ok +npx -y skilld@latest run skilld-dev/skills/find-skill > /dev/null && echo ok +``` + +If either run fails with a `TRUSTED_ROOT_` code, roll back this step at once. + +## 4. Put the new private key in the free slot + +The free slot is the one whose vars name no key. For this rotation it is the secondary slot. + +```sh +pnpm --filter @skilld/artifact-signer-worker exec wrangler secret put ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8 --config wrangler.jsonc < "$KEYS/skilld-production-2026-11.pkcs8" +pnpm --filter @skilld/artifact-signer-worker exec wrangler secret list --config wrangler.jsonc +``` + +The list must name `ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8`. +The signer still signs with the old key, because no vars name the secondary slot. + +## 5. Name the new key in the signer vars + +Open a pull request that adds these vars to `workers/artifact-signer/wrangler.jsonc`. +Use the same window as the trusted root entry, or a window inside it. + +```jsonc +"ARTIFACT_SIGNING_SECONDARY_KEY_ID": "skilld-production-2026-11", +"ARTIFACT_SIGNING_SECONDARY_KEY_NOT_BEFORE": "2026-10-14T00:00:00.000Z", +"ARTIFACT_SIGNING_SECONDARY_KEY_NOT_AFTER": "2027-05-20T00:00:00.000Z" +``` + +Regenerate the signer types in the same pull request: + +```sh +pnpm --filter @skilld/artifact-signer-worker types +``` + +Before you merge, confirm step 3 is live and step 4 lists the secret. +The merge runs `Deploy to Cloudflare`. It deploys the signer first. +From that deploy, each new build signs with `skilld-production-2026-11`. + +Count attestations per key. New builds must show the new key ID: + +```sh +pnpm exec wrangler d1 execute DB --remote --config wrangler.jsonc --command \ + "SELECT json_extract(attestation_json, '$.signature.keyId') AS key_id, COUNT(*) AS attestations, datetime(MAX(created_at), 'unixepoch') AS newest FROM artifact_attestations GROUP BY key_id" +``` + +Check [Sentry](https://sentry.io) for `Artifact signer returned an invalid signature` and the signer logs for `SIGNING_KEY_INVALID`. +Either one means the signer and the trusted root disagree. Roll back this step. + +## 6. Remove the old key from the trusted root + +Do this at a time you can watch, at least one day before the old window closes. +Every stored Artifact that the old key signed then builds again on its next request. +The same rebuilds happen at `2026-11-20T00:00:00.000Z` anyway. Before that time, you can still roll back. + +Removal needs no root key. + +```sh +pnpm exec tsx scripts/trusted-root.ts remove-key --root "$KEYS/root-next.json" --key-id skilld-production-2026-08 --out "$KEYS/root-after.json" +pnpm exec tsx scripts/trusted-root.ts verify --root "$KEYS/root-after.json" --pin "$KEYS/root-before.json" +pnpm exec wrangler secret put ARTIFACT_TRUSTED_ROOT_JSON --config wrangler.jsonc < "$KEYS/root-after.json" +pnpm exec tsx scripts/trusted-root.ts verify --root https://skilld.dev/api/v1/trusted-root --pin "$KEYS/root-before.json" +``` + +Watch the build queue, failed jobs, and [GitHub](https://github.com) read quota in the daily check-in for one hour. +A CLI that held a Resolution across the change fails once with `SIGNING_KEY_UNKNOWN`. Its next run passes. + +## 7. Empty the old slot + +After `2026-11-20T00:00:00.000Z`, open a pull request that removes the three primary slot vars from `workers/artifact-signer/wrangler.jsonc`. +Regenerate the signer types in it. +After its deploy, delete the old secret: + +```sh +pnpm --filter @skilld/artifact-signer-worker exec wrangler secret delete ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8 --config wrangler.jsonc +``` + +Never delete a secret while vars still name its slot. The signer then refuses every request. + +Store the new private key file offline, or delete it. Cloudflare holds the only copy the signer needs. + +## Next rotation + +The next rotation uses the slot this one emptied, so the slots alternate. +Repeat steps 2 to 7 with the slot names swapped. +`skilld.signing-key` warns 30 days before `skilld-production-2026-11` closes. + +## Roll back + +| Step | Roll back | Valid until | +| --- | --- | --- | +| 3 | Put `$KEYS/root-before.json` back as `ARTIFACT_TRUSTED_ROOT_JSON`. | step 5 deploys | +| 4 | Delete `ARTIFACT_SIGNING_SECONDARY_PRIVATE_KEY_PKCS8`. | step 5 deploys | +| 5 | Revert the vars pull request. The signer signs with the old key again. Keep the new key in the trusted root, because it signed live attestations. | `2026-11-20T00:00:00.000Z` | +| 6 | Put `$KEYS/root-next.json` back as `ARTIFACT_TRUSTED_ROOT_JSON`. | `2026-11-20T00:00:00.000Z` | + +Never remove a key from the trusted root while the signer still names it. +The site then rejects every new signature, and every build fails. + +## Rules for every released CLI + +- Never change the root key without a CLI release. Each release compiles in the root key and refuses any other root. +- Never add a field to a trusted root key entry or statement. The CLI rejects unknown fields there. +- Use only the statuses `active`, `overlapping`, `retired`, and `revoked`. +- A status change needs a new root signature. Remove a key instead of retiring it. +- Run `scripts/trusted-root.ts verify` before every `wrangler secret put ARTIFACT_TRUSTED_ROOT_JSON`. + The site never checks a root signature. A bad one stops every CLI with an upgrade message that no upgrade fixes. From bc9cc654cb5b4d2c979ac7ee099b751da00baa19 Mon Sep 17 00:00:00 2001 From: Harlan GitHub Agent Date: Thu, 8 Oct 2026 13:49:12 +1100 Subject: [PATCH 6/6] docs(runbooks): make the signing key vars snippet valid JSONC --- docs/runbooks/signing-key-rotation.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/runbooks/signing-key-rotation.md b/docs/runbooks/signing-key-rotation.md index 20dd871c..9c2fa054 100644 --- a/docs/runbooks/signing-key-rotation.md +++ b/docs/runbooks/signing-key-rotation.md @@ -192,9 +192,13 @@ Open a pull request that adds these vars to `workers/artifact-signer/wrangler.js Use the same window as the trusted root entry, or a window inside it. ```jsonc -"ARTIFACT_SIGNING_SECONDARY_KEY_ID": "skilld-production-2026-11", -"ARTIFACT_SIGNING_SECONDARY_KEY_NOT_BEFORE": "2026-10-14T00:00:00.000Z", -"ARTIFACT_SIGNING_SECONDARY_KEY_NOT_AFTER": "2027-05-20T00:00:00.000Z" +{ + "vars": { + "ARTIFACT_SIGNING_SECONDARY_KEY_ID": "skilld-production-2026-11", + "ARTIFACT_SIGNING_SECONDARY_KEY_NOT_BEFORE": "2026-10-14T00:00:00.000Z", + "ARTIFACT_SIGNING_SECONDARY_KEY_NOT_AFTER": "2027-05-20T00:00:00.000Z" + } +} ``` Regenerate the signer types in the same pull request: