From 9c8ca11fa58aecf312c25a397c27e67a11a0eac2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:32:07 +0000 Subject: [PATCH 01/23] Lock the local API to the app Any web page could send no-preflight POSTs to 127.0.0.1:4317 (the server parsed text/plain bodies as JSON), and DNS rebinding gave a hostile page full read and write access, including pointing a model tier at its own host with any environment variable as the bearer key. - Require a per-launch token on every API request. The server injects it into the index.html it serves; `pnpm dev` shares one token between the API and Vite through a small launcher that replaces concurrently. - Refuse non-loopback Host headers and foreign Origins; send CORS headers only for explicitly allowed origins. - Accept only application/json request bodies, capped at 20 MB. - Validate git refs and branch names before they reach git, so a value like `--output=...` cannot become an option. - Restrict api_key_env to *_API_KEY names and base_url to http(s) URLs. - Open a new book before closing the current one, so a failed open leaves the app usable. - Validate the export format, return 500 for programming errors, use unique job ids, and resolve the sample fixture with fileURLToPath (Windows, spaces). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- README.md | 1 + apps/desktop/package.json | 3 +- apps/desktop/scripts/dev.mjs | 29 +++++ apps/desktop/src/lib/api.ts | 6 +- apps/desktop/vite.config.ts | 20 ++- packages/core/src/git.ts | 37 ++++++ packages/core/src/schemas/index.ts | 11 +- packages/server/src/index.ts | 141 ++++++++++++++++++--- packages/server/test/server.test.ts | 95 +++++++++++++- pnpm-lock.yaml | 188 +++------------------------- scripts/e2e-ui.ts | 4 +- scripts/screenshots.ts | 2 +- scripts/ui-refresh.ts | 2 +- 13 files changed, 334 insertions(+), 205 deletions(-) create mode 100644 apps/desktop/scripts/dev.mjs diff --git a/README.md b/README.md index 15b3e4b..d308542 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ packages/cli/ the `quillgon` command fixtures/novella/ "The Tidekeeper's Ledger": 12 scenes, 4 characters, 3 threads, 2 plot points ``` +- **The local API is locked to the app.** The server binds to 127.0.0.1 and requires a per-launch token on every API request; the UI reads it from the page the server serves. Requests with a non-loopback `Host` (DNS rebinding) or a foreign `Origin` are refused, request bodies must be JSON, and git refs from the UI are validated before they reach git. Set `QUILLGON_TOKEN` to choose the token yourself, for example to script against `pnpm serve`. A book's `api_key_env` must name a variable ending in `_API_KEY`, so a shared book file cannot send another secret to a model endpoint. - **Only the graph store writes to the manuscript.** Agents return structured output, validated with Zod, and the store writes files, updates the index and commits. - **A book is a git repo of Markdown and YAML** (`book.yaml`, `intent.md`, `plot-points/`, `threads/`, `characters//{bio.md,voice.md,state.yaml}`, `scenes//{card.yaml,prose.md,facts.yaml}`, `obligations/`, `idea.yaml`, `structure.yaml`, `timeline.yaml`, `style/` (samples, banned patterns, `style.yaml`), `reports/`). You can edit any of it in another editor; the app reloads. - **The index is disposable.** `.engine/index.sqlite` holds entities, references, facts and scene embeddings. Delete it and it rebuilds. Embeddings are local and deterministic (hashed term vectors), so impact analysis needs no network. diff --git a/apps/desktop/package.json b/apps/desktop/package.json index bc253c0..bd56fd6 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -5,7 +5,7 @@ "type": "module", "main": "electron/main.cjs", "scripts": { - "dev": "concurrently -k -n api,ui -c gray,magenta \"node --import tsx ../../packages/cli/src/main.ts serve --port 4317\" \"vite\"", + "dev": "node scripts/dev.mjs", "ui": "vite", "build": "vite build", "preview": "vite preview", @@ -35,7 +35,6 @@ "@types/react": "^19.2.0", "@types/react-dom": "^19.2.0", "@vitejs/plugin-react": "^6.1.2", - "concurrently": "^10.0.6", "electron": "^44.7.0", "typescript": "^5.9.0", "vite": "^8.3.4" diff --git a/apps/desktop/scripts/dev.mjs b/apps/desktop/scripts/dev.mjs new file mode 100644 index 0000000..16fbae7 --- /dev/null +++ b/apps/desktop/scripts/dev.mjs @@ -0,0 +1,29 @@ +// `pnpm dev`: the API server and the Vite dev server, sharing one per-launch API token. +import { spawn } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const app = path.resolve(here, '..'); +const root = path.resolve(app, '..', '..'); +const require = createRequire(path.join(app, 'package.json')); +// Resolve everything before starting anything, so a failure cannot orphan a child. +const viteBin = path.join(path.dirname(require.resolve('vite/package.json')), 'bin', 'vite.js'); +const env = { ...process.env, QUILLGON_TOKEN: process.env.QUILLGON_TOKEN || randomBytes(24).toString('base64url') }; + +const children = [ + spawn(process.execPath, ['--import', 'tsx', path.join(root, 'packages', 'cli', 'src', 'main.ts'), 'serve', '--port', '4317'], { cwd: root, env, stdio: 'inherit' }), + spawn(process.execPath, [viteBin], { cwd: app, env, stdio: 'inherit' }), +]; + +let stopping = false; +const stop = (code) => { + if (stopping) return; + stopping = true; + for (const c of children) if (c.exitCode === null) c.kill(); + process.exitCode = code ?? 0; +}; +for (const c of children) c.on('exit', (code) => stop(code)); +for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => stop(0)); diff --git a/apps/desktop/src/lib/api.ts b/apps/desktop/src/lib/api.ts index 93ea8e1..4459572 100644 --- a/apps/desktop/src/lib/api.ts +++ b/apps/desktop/src/lib/api.ts @@ -7,6 +7,8 @@ declare global { } const BASE = window.quillgon?.apiBase ?? ''; +/** The server injects a per-launch API token into index.html (the dev server does the same). */ +const TOKEN = document.querySelector('meta[name="quillgon-token"]')?.content ?? ''; export class ApiError extends Error { constructor(public status: number, message: string, public data?: any) { @@ -17,7 +19,7 @@ export class ApiError extends Error { export async function api(method: string, path: string, body?: unknown): Promise { const res = await fetch(`${BASE}/api${path}`, { method, - headers: body !== undefined ? { 'content-type': 'application/json' } : undefined, + headers: { 'x-quillgon-token': TOKEN, ...(body !== undefined ? { 'content-type': 'application/json' } : {}) }, body: body !== undefined ? JSON.stringify(body) : undefined, }); const json = await res.json().catch(() => ({})); @@ -37,7 +39,7 @@ let source: EventSource | null = null; export function subscribe(fn: Listener): () => void { listeners.add(fn); if (!source) { - source = new EventSource(`${BASE}/api/events`); + source = new EventSource(`${BASE}/api/events?token=${encodeURIComponent(TOKEN)}`); for (const t of ['store', 'reloaded', 'job', 'hello']) { source.addEventListener(t, (e) => { const data = JSON.parse((e as MessageEvent).data || '{}'); diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 7be343d..5c9c91d 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -1,8 +1,24 @@ -import { defineConfig } from 'vite'; +import { defineConfig, type Plugin } from 'vite'; import react from '@vitejs/plugin-react'; +/** + * In development the UI is served by Vite, not the Quillgon server, so Vite injects the + * API token the server expects. `pnpm dev` (scripts/dev.mjs) gives both the same token. + */ +function apiToken(): Plugin { + return { + name: 'quillgon-api-token', + apply: 'serve', + transformIndexHtml(html) { + const token = process.env.QUILLGON_TOKEN; + if (!token) return html; + return html.replace('', ` \n `); + }, + }; +} + export default defineConfig({ - plugins: [react()], + plugins: [react(), apiToken()], base: './', server: { port: 5173, diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index 5a19480..fc4c441 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -2,6 +2,34 @@ import fs from 'node:fs'; import path from 'node:path'; import { simpleGit, type SimpleGit } from 'simple-git'; +/** + * Refs arrive from the UI (branch names, hashes from history, compare targets). Anything + * git could read as an option, or that is not a plausible ref, is refused before it reaches + * the command line. + */ +export function assertRef(ref: unknown, what = 'ref'): string { + const ok = + typeof ref === 'string' && + ref.length <= 200 && + /^[A-Za-z0-9][A-Za-z0-9._/~^-]*$/.test(ref) && + !ref.includes('..') && + !ref.includes('//') && + !ref.endsWith('/') && + !ref.endsWith('.') && + !ref.endsWith('.lock'); + if (!ok) throw new InvalidRefError(`invalid ${what}: ${JSON.stringify(ref)}`); + return ref; +} + +/** A branch name: a ref without revision suffixes. */ +export function assertBranch(name: unknown): string { + const ref = assertRef(name, 'branch name'); + if (/[~^]/.test(ref)) throw new InvalidRefError(`invalid branch name: ${JSON.stringify(name)}`); + return ref; +} + +export class InvalidRefError extends Error {} + export interface Commit { hash: string; date: string; @@ -64,6 +92,7 @@ export class Repo { } async show(hash: string): Promise<{ files: string[]; patch: string }> { + assertRef(hash, 'commit'); const files = (await this.git.raw(['show', '--name-only', '--format=', hash])).split('\n').filter(Boolean); const patch = await this.git.raw(['show', '--format=', hash]); return { files, patch }; @@ -86,20 +115,24 @@ export class Repo { /** Start a "what if": commit pending work, then branch from here. */ async createBranch(name: string) { + assertBranch(name); await this.commit(`Checkpoint before what-if "${name}"`); await this.git.checkoutLocalBranch(name); } async switchBranch(name: string) { + assertBranch(name); await this.commit(`Checkpoint before switching to "${name}"`); await this.git.checkout(name); } async deleteBranch(name: string) { + assertBranch(name); await this.git.deleteLocalBranch(name, true); } async merge(from: string): Promise<{ ok: boolean; conflicts: string[] }> { + assertBranch(from); try { await this.git.merge([from, '--no-ff', '-m', `Merge what-if "${from}"`]); return { ok: true, conflicts: [] }; @@ -111,11 +144,13 @@ export class Repo { } async revert(hash: string) { + assertRef(hash, 'commit'); await this.git.revert(hash, ['--no-edit']); } /** Read a file as it is on another branch (for scene-by-scene comparison). */ async readAt(ref: string, rel: string): Promise { + assertRef(ref); try { return await this.git.show([`${ref}:${rel}`]); } catch { @@ -124,6 +159,8 @@ export class Repo { } async changedBetween(a: string, b: string): Promise { + assertRef(a); + assertRef(b); const out = await this.git.raw(['diff', '--name-only', `${a}...${b}`]); return out.split('\n').filter(Boolean); } diff --git a/packages/core/src/schemas/index.ts b/packages/core/src/schemas/index.ts index 271587b..64324bf 100644 --- a/packages/core/src/schemas/index.ts +++ b/packages/core/src/schemas/index.ts @@ -22,11 +22,18 @@ export type Lock = z.infer; export const ModelTier = z.enum(['frontier', 'mid', 'small', 'reader']); export type ModelTier = z.infer; +const blankToUndefined = (v: unknown) => (typeof v === 'string' && v.trim() === '' ? undefined : v); + export const ModelConfig = z.object({ provider: z.enum(['openai', 'codex', 'anthropic', 'openai-compatible', 'offline']).default('openai'), model: z.string(), - base_url: z.string().optional(), - api_key_env: z.string().optional(), + base_url: z.preprocess(blankToUndefined, z.string().regex(/^https?:\/\/\S+$/, 'must be an http(s) URL').optional()), + /** + * The environment variable holding the key. Restricted to *_API_KEY names so a book file + * (which may come from someone else) cannot send an arbitrary secret, like a cloud or + * GitHub credential, to a model endpoint. + */ + api_key_env: z.preprocess(blankToUndefined, z.string().regex(/^[A-Z][A-Z0-9_]*_API_KEY$/, 'must name an environment variable ending in _API_KEY').optional()), effort: z.enum(['low', 'medium', 'high', 'xhigh', 'max']).optional(), /** USD per million tokens, used for cost logging and `novel estimate`. */ price_in: z.number().optional(), diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 9dfbbc0..f7132db 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -3,6 +3,8 @@ import http from 'node:http'; import os from 'node:os'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; +import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; import { AgentName, analyseImpact, @@ -21,6 +23,7 @@ import { suggestRules, suggestSheet, SchemaError, + InvalidRefError, sceneOrder, type Change, type ReaderReport, @@ -57,10 +60,28 @@ export class HttpError extends Error { } const RECENT = path.join(os.homedir(), '.quillgon', 'recent.json'); +const MAX_BODY = 20 * 1024 * 1024; +const TOKEN_HEADER = 'x-quillgon-token'; +const LOOPBACK_HOST = /^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/i; +const EXPORT_FORMATS = new Set(['md', 'docx', 'epub']); + +export interface ServerOptions { + staticDir?: string; + /** Required on every API request. Defaults to QUILLGON_TOKEN, else a random per-launch value. */ + token?: string; + /** Extra browser origins allowed to call the API (same-origin requests are always allowed). */ + allowedOrigins?: string[]; +} /** * The local boundary between the UI and the core. JSON over HTTP, server-sent events for * job progress and file changes. Single user, binds to localhost only. + * + * Binding to localhost is not enough on its own: any web page the author visits can send + * requests to 127.0.0.1, and DNS rebinding can make those requests same-origin. So every + * API request must carry the per-launch token (injected into the UI's index.html), the + * Host header must be a loopback name, browser requests must come from an allowed origin, + * and request bodies must be JSON. */ export class NovelServer { engine: Engine | null = null; @@ -71,8 +92,12 @@ export class NovelServer { private quietUntil = 0; private reloadTimer: NodeJS.Timeout | null = null; readonly http: http.Server; + readonly token: string; + private allowedOrigins: Set; - constructor(private opts: { staticDir?: string } = {}) { + constructor(private opts: ServerOptions = {}) { + this.token = opts.token || process.env.QUILLGON_TOKEN || randomBytes(24).toString('base64url'); + this.allowedOrigins = new Set(opts.allowedOrigins ?? []); this.http = http.createServer((req, res) => this.handle(req, res)); this.register(); } @@ -93,12 +118,16 @@ export class NovelServer { // ------------------------------------------------------------------ project lifecycle open(dir: string) { - const abs = path.resolve(dir); + if (typeof dir !== 'string' || !dir.trim()) throw new HttpError(400, 'dir is required'); + const abs = path.resolve(expandHome(dir)); if (!fs.existsSync(path.join(abs, paths.book))) throw new HttpError(404, `no book.yaml in ${abs}`); - this.watcher?.close(); - this.engine?.store.close(); + // Open the new book before closing the current one, so a book that fails to load + // leaves the current one usable. const store = GraphStore.open(abs); const engine = new Engine(store, createSuite(store)); + this.watcher?.close(); + this.watcher = null; + this.engine?.store.close(); this.engine = engine; store.on((e) => { this.quietUntil = Date.now() + 800; @@ -135,7 +164,7 @@ export class NovelServer { /** Run a long operation in the background; progress and completion go out over SSE. */ job(name: string, fn: (progress: (stage: string, detail?: string) => void) => Promise): { job: string } { - const id = `${name}-${Date.now().toString(36)}`; + const id = `${name}-${randomUUID().slice(0, 8)}`; const job: Job = { id, name, state: 'running', stage: 'starting', detail: '', started: Date.now() }; this.jobs.set(id, job); this.broadcast('job', job); @@ -151,7 +180,7 @@ export class NovelServer { }) .catch((e: Error) => { job.state = 'error'; - job.error = e instanceof SchemaError ? e.message : e.message; + job.error = e.message; }) .finally(() => { this.broadcast('job', job); @@ -169,12 +198,41 @@ export class NovelServer { this.routes.push({ method, pattern, keys, handler, open }); } + /** A browser origin is allowed if it is this server itself or explicitly configured. */ + private originAllowed(origin: string | undefined, host: string): boolean { + if (!origin) return true; // not a browser, or a same-origin GET + return origin === `http://${host}` || this.allowedOrigins.has(origin); + } + + private tokenMatches(given: string | null | undefined): boolean { + if (!given) return false; + const a = Buffer.from(given); + const b = Buffer.from(this.token); + return a.length === b.length && timingSafeEqual(a, b); + } + private async handle(req: http.IncomingMessage, res: http.ServerResponse) { const url = new URL(req.url ?? '/', 'http://localhost'); - res.setHeader('access-control-allow-origin', req.headers.origin && /^(http:\/\/(localhost|127\.0\.0\.1)(:\d+)?|file:\/\/|app:\/\/)/.test(req.headers.origin) ? req.headers.origin : 'http://localhost'); - res.setHeader('access-control-allow-headers', 'content-type'); - res.setHeader('access-control-allow-methods', 'GET,POST,PUT,DELETE,OPTIONS'); - if (req.method === 'OPTIONS') return void res.end(); + const host = req.headers.host ?? ''; + // DNS rebinding: a page on attacker.example resolved to 127.0.0.1 still sends its own Host. + if (!LOOPBACK_HOST.test(host)) return send(res, 421, { error: 'unexpected Host header' }); + const origin = req.headers.origin; + const allowed = this.originAllowed(origin, host); + if (origin && allowed && origin !== `http://${host}`) { + res.setHeader('access-control-allow-origin', origin); + res.setHeader('vary', 'origin'); + res.setHeader('access-control-allow-headers', `content-type, ${TOKEN_HEADER}`); + res.setHeader('access-control-allow-methods', 'GET,POST,PUT,DELETE,OPTIONS'); + } + if (req.method === 'OPTIONS') return void res.writeHead(allowed ? 204 : 403).end(); + + if (!url.pathname.startsWith('/api/')) return this.serveStatic(url.pathname, res); + + if (!allowed) return send(res, 403, { error: 'origin not allowed' }); + const given = req.headers[TOKEN_HEADER]; + // EventSource cannot set headers, so the event stream also accepts ?token=. + const token = typeof given === 'string' ? given : url.pathname === '/api/events' ? url.searchParams.get('token') : null; + if (!this.tokenMatches(token)) return send(res, 401, { error: 'missing or invalid API token' }); if (url.pathname === '/api/events') { res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache', connection: 'keep-alive' }); @@ -196,6 +254,7 @@ export class NovelServer { if (!m) continue; try { const body = req.method === 'GET' ? {} : await readBody(req); + if (body === null || typeof body !== 'object' || Array.isArray(body)) throw new HttpError(400, 'request body must be a JSON object'); if (!r.open && !this.engine) throw new HttpError(409, 'no project is open'); const params = Object.fromEntries(r.keys.map((k, i) => [k, decodeURIComponent(m[i + 1])])); const out = await r.handler({ params, query: url.searchParams, body, engine: this.engine!, server: this }); @@ -203,6 +262,12 @@ export class NovelServer { } catch (e) { if (e instanceof SchemaError) return send(res, 422, { error: e.message, issues: e.issues }); if (e instanceof HttpError) return send(res, e.status, { error: e.message, data: e.data }); + if (e instanceof InvalidRefError) return send(res, 400, { error: e.message }); + // Programming errors are the server's fault; everything else is a refused request. + if (e instanceof TypeError || e instanceof ReferenceError || e instanceof RangeError) { + console.error(e); + return send(res, 500, { error: (e as Error).message }); + } return send(res, 400, { error: (e as Error).message }); } } @@ -215,9 +280,17 @@ export class NovelServer { res.writeHead(200, { 'content-type': 'text/plain' }); return res.end('Quillgon API. Build the UI with `pnpm build` or run `pnpm dev`.'); } - let file = path.join(dir, path.normalize(p).replace(/^(\.\.[/\\])+/, '')); - if (!file.startsWith(dir) || !fs.existsSync(file) || fs.statSync(file).isDirectory()) file = path.join(dir, 'index.html'); - const types: Record = { '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css', '.svg': 'image/svg+xml', '.woff2': 'font/woff2', '.png': 'image/png', '.json': 'application/json' }; + const root = path.resolve(dir); + let file = path.join(root, path.normalize(decodeURIComponentSafe(p)).replace(/^(\.\.[/\\])+/, '')); + if (!file.startsWith(root + path.sep) || !fs.existsSync(file) || fs.statSync(file).isDirectory()) file = path.join(root, 'index.html'); + const types: Record = { '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css', '.svg': 'image/svg+xml', '.woff2': 'font/woff2', '.woff': 'font/woff', '.png': 'image/png', '.json': 'application/json' }; + if (path.extname(file) === '.html') { + // The UI reads its API token from here. Only same-machine pages can read it: the Host + // check refuses rebound DNS names, and there are no CORS headers for other origins. + const html = fs.readFileSync(file, 'utf8').replace('', ` \n `); + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' }); + return void res.end(html); + } res.writeHead(200, { 'content-type': types[path.extname(file)] ?? 'application/octet-stream' }); fs.createReadStream(file).pipe(res); } @@ -248,7 +321,7 @@ export class NovelServer { async ({ server }) => { const dir = path.join(os.homedir(), 'Novels', 'The-Tidekeepers-Ledger'); if (!fs.existsSync(path.join(dir, paths.book))) { - const here = path.dirname(new URL(import.meta.url).pathname); + const here = path.dirname(fileURLToPath(import.meta.url)); const fixture = path.resolve(here, '../../../fixtures/novella'); fs.mkdirSync(path.dirname(dir), { recursive: true }); fs.cpSync(fixture, dir, { recursive: true, filter: (src) => !src.includes(`${path.sep}.engine`) }); @@ -629,7 +702,11 @@ export class NovelServer { }); // --- export - r('GET', '/api/export', ({ engine, query }) => exportManuscript(engine.store, (query.get('format') as 'md' | 'docx' | 'epub') ?? 'md')); + r('GET', '/api/export', ({ engine, query }) => { + const format = query.get('format') ?? 'md'; + if (!EXPORT_FORMATS.has(format)) throw new HttpError(400, `unknown export format "${format}"`); + return exportManuscript(engine.store, format as 'md' | 'docx' | 'epub'); + }); } } @@ -642,19 +719,45 @@ function send(res: http.ServerResponse, status: number, body: unknown) { function readBody(req: http.IncomingMessage): Promise { return new Promise((resolve, reject) => { - let data = ''; - req.on('data', (c) => (data += c)); + const chunks: Buffer[] = []; + let size = 0; + let failed = false; + const fail = (e: HttpError) => { + if (failed) return; + failed = true; + req.resume(); // drain, so the error response can be sent + reject(e); + }; + req.on('data', (c: Buffer) => { + size += c.length; + if (size > MAX_BODY) fail(new HttpError(413, 'request body too large')); + else if (!failed) chunks.push(c); + }); req.on('end', () => { + if (failed) return; + const data = Buffer.concat(chunks).toString('utf8'); + if (!data) return resolve({}); + // JSON only: a cross-site form or text/plain "simple request" never reaches a handler. + const type = String(req.headers['content-type'] ?? '').split(';')[0].trim().toLowerCase(); + if (type !== 'application/json') return fail(new HttpError(415, 'request body must be application/json')); try { - resolve(data ? JSON.parse(data) : {}); + resolve(JSON.parse(data)); } catch { - reject(new HttpError(400, 'invalid JSON body')); + fail(new HttpError(400, 'invalid JSON body')); } }); req.on('error', reject); }); } +function decodeURIComponentSafe(p: string) { + try { + return decodeURIComponent(p); + } catch { + return p; + } +} + function expandHome(p: string) { return p.startsWith('~') ? path.join(os.homedir(), p.slice(1)) : p; } diff --git a/packages/server/test/server.test.ts b/packages/server/test/server.test.ts index fada616..4375f0f 100644 --- a/packages/server/test/server.test.ts +++ b/packages/server/test/server.test.ts @@ -1,3 +1,7 @@ +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { GraphStore } from '@quillgon/core'; import { NovelServer } from '../src/index.js'; @@ -8,7 +12,7 @@ let server: NovelServer; let base = ''; async function api(method: string, path: string, body?: unknown) { - const res = await fetch(base + path, { method, headers: { 'content-type': 'application/json' }, body: body ? JSON.stringify(body) : undefined }); + const res = await fetch(base + path, { method, headers: { 'content-type': 'application/json', 'x-quillgon-token': server.token }, body: body ? JSON.stringify(body) : undefined }); const json: any = await res.json(); if (!res.ok) throw Object.assign(new Error(json.error), { status: res.status, json }); return json; @@ -149,3 +153,92 @@ describe('server API', () => { expect((await api('GET', '/api/prompts')).find((p: any) => p.name === 'reader')).toMatchObject({ overridden: true, text: '# Reader\nBe brief.' }); }); }); + +/** Raw HTTP, so tests can forge the Host and Origin headers a browser would send. */ +function raw(opts: { method?: string; path: string; headers?: Record; body?: string; port?: number }): Promise<{ status: number; body: string; headers: http.IncomingHttpHeaders }> { + return new Promise((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port: opts.port ?? Number(new URL(base).port), method: opts.method ?? 'GET', path: opts.path, headers: opts.headers }, (res) => { + let body = ''; + res.on('data', (c) => { + body += c; + // The event stream never ends; the first event is enough. + if (String(res.headers['content-type']).startsWith('text/event-stream')) req.destroy(); + }); + res.on('end', () => resolve({ status: res.statusCode!, body, headers: res.headers })); + res.on('close', () => resolve({ status: res.statusCode!, body, headers: res.headers })); + }); + req.on('error', reject); + if (opts.body) req.write(opts.body); + req.end(); + }); +} + +describe('server API boundary', () => { + const auth = () => ({ 'x-quillgon-token': server.token }); + + it('refuses API requests without the per-launch token', async () => { + expect((await raw({ path: '/api/project' })).status).toBe(401); + expect((await raw({ path: '/api/project', headers: { 'x-quillgon-token': 'nope' } })).status).toBe(401); + expect((await raw({ path: '/api/project', headers: auth() })).status).toBe(200); + }); + + it('refuses a cross-site text/plain POST, the request a hostile page can send without a preflight', async () => { + const r = await raw({ method: 'POST', path: '/api/commit', headers: { 'content-type': 'text/plain', origin: 'https://evil.example' }, body: '{"message":"pwned"}' }); + expect(r.status).toBe(403); + expect(r.headers['access-control-allow-origin']).toBeUndefined(); + // Even with the token, a non-JSON body never reaches a handler. + expect((await raw({ method: 'POST', path: '/api/commit', headers: { ...auth(), 'content-type': 'text/plain' }, body: '{"message":"x"}' })).status).toBe(415); + }); + + it('refuses DNS-rebound requests by Host, and look-alike origins', async () => { + expect((await raw({ path: '/api/project', headers: { ...auth(), host: 'attacker.example:4317' } })).status).toBe(421); + expect((await raw({ path: '/', headers: { host: 'attacker.example' } })).status).toBe(421); + expect((await raw({ path: '/api/project', headers: { ...auth(), origin: 'http://localhost.evil.com' } })).status).toBe(403); + expect((await raw({ method: 'OPTIONS', path: '/api/commit', headers: { origin: 'http://localhost:9999' } })).status).toBe(403); + // Same origin is fine. + expect((await raw({ path: '/api/project', headers: { ...auth(), origin: base } })).status).toBe(200); + }); + + it('streams events only with the token', async () => { + expect((await raw({ path: '/api/events' })).status).toBe(401); + const ok = await raw({ path: `/api/events?token=${encodeURIComponent(server.token)}` }); + expect(ok.status).toBe(200); + expect(ok.body).toContain('event: hello'); + }); + + it('refuses git refs that could be read as options', async () => { + await expect(api('POST', '/api/branches', { name: '--output=/tmp/x' })).rejects.toMatchObject({ status: 400 }); + await expect(api('GET', `/api/history/${encodeURIComponent('--output=/tmp/x')}`)).rejects.toMatchObject({ status: 400 }); + await expect(api('GET', '/api/branches/compare?a=-p&b=main')).rejects.toMatchObject({ status: 400 }); + await expect(api('GET', '/api/export?format=../../x')).rejects.toMatchObject({ status: 400 }); + }); + + it('refuses a model config that would send an arbitrary secret to an endpoint', async () => { + const p = await api('GET', '/api/project'); + const models = { ...p.book.models, frontier: { provider: 'openai-compatible', model: 'x', base_url: 'https://evil.example/v1', api_key_env: 'AWS_SECRET_ACCESS_KEY' } }; + await expect(api('PUT', '/api/book', { book: { models } })).rejects.toMatchObject({ status: 422 }); + expect((await api('GET', '/api/project')).book.models.frontier?.api_key_env).toBeUndefined(); + }); + + it('keeps the current book open when another fails to open', async () => { + const bad = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-bad-')); + fs.mkdirSync(path.join(bad, 'book.yaml')); // exists, but cannot be read + await expect(api('POST', '/api/open', { dir: bad })).rejects.toBeTruthy(); + expect((await api('GET', '/api/project')).book.title).toBe("The Tidekeeper's Ledger"); + }); + + it('injects the token into the UI it serves', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-ui-')); + fs.writeFileSync(path.join(dir, 'index.html'), 'x'); + const ui = new NovelServer({ staticDir: dir, token: 'fixed-token' }); + const port = await ui.listen(0); + try { + const r = await raw({ path: '/', port }); + expect(r.body).toContain(''); + expect(r.headers['cache-control']).toBe('no-store'); + expect((await raw({ path: '/../../etc/passwd', port })).body).toContain('quillgon-token'); + } finally { + ui.close(); + } + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e48ffbf..05a6314 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,12 +99,9 @@ importers: '@vitejs/plugin-react': specifier: ^6.1.2 version: 6.1.2(vite@8.3.4(@types/node@24.19.1)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) - concurrently: - specifier: ^10.0.6 - version: 10.0.6 electron: specifier: ^44.7.0 - version: 44.7.0(supports-color@10.2.2) + version: 44.7.0 typescript: specifier: ^5.9.0 version: 5.9.3 @@ -166,7 +163,7 @@ importers: version: 8.0.4 simple-git: specifier: ^3.28.0 - version: 3.36.0(supports-color@10.2.2) + version: 3.36.0 yaml: specifier: ^2.8.1 version: 2.9.1 @@ -717,42 +714,36 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.2.13': resolution: {integrity: sha512-xU+Kvv9N2A3SY35zFh1FWWNjxbadE/BKeqnkXe7vScU4GZsw757BlVmDLITZEq8DPpk6m7uoDYkK7P+x/uU5Ow==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.2.13': resolution: {integrity: sha512-XMk/hnhh25oQ756Lclb1Q11h4ll0xC0Esa5w4Vzp+mAE2S3AGJgeM0SonrIgY8fHhjarb6hcoTuwoFG0VVP2ig==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.2.13': resolution: {integrity: sha512-U1BoTHYmlywE2n3HUWjzZ+8EQsKr+OwpxYuDWaubx42tUL8QSL7nOqRz/NVHanibbFP3M1Rxuprz/wNcbMKjqw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.2.13': resolution: {integrity: sha512-J4cm80Pc6wEwV5AZ42o62sIQgDk+Y7R2s646jIwuvO+4V46QB58N4vVuMPWCBaNgursXLpmUWJV//WQAkPvHBQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-musl@1.2.13': resolution: {integrity: sha512-W0euZS40U4V6JkmEnVCIX9mcsL6cbXQ4X8c2w+MMOIjFg68TXUW+u3RT+2e1t1pv9mg/EP3pDtK3POXdQ7Rxyw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@rolldown/binding-openharmony-arm64@1.2.13': resolution: {integrity: sha512-do/AF/4n9MRshia3a5aXoy8xM2c6mgfT6D7H/dY6my5z+hfCIR4ab7mk4IRj++nsEQpKy39kd0DQnDq+r692tw==} @@ -858,14 +849,6 @@ packages: '@vitest/utils@4.1.11': resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} - ansi-regex@6.4.0: - resolution: {integrity: sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==} - engines: {node: '>=12'} - - ansi-styles@6.2.3: - resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} - engines: {node: '>=12'} - aria-hidden@1.2.6: resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} engines: {node: '>=10'} @@ -894,28 +877,15 @@ packages: resolution: {integrity: sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==} engines: {node: '>=18'} - chalk@5.6.2: - resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} - engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} - chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - cliui@9.0.1: - resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} - engines: {node: '>=20'} - cmdk@1.1.1: resolution: {integrity: sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==} peerDependencies: react: ^18 || ^19 || ^19.0.0-rc react-dom: ^18 || ^19 || ^19.0.0-rc - concurrently@10.0.6: - resolution: {integrity: sha512-qgICXXFp2/3nAebRLmxcINKEXM1mLK3ij/y31P02VTs/KIkddIeC05n73DxOXsXMB9LFSqNU1/na3lDlOhcbIw==} - engines: {node: '>=22'} - hasBin: true - convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -958,9 +928,6 @@ packages: engines: {node: '>= 22.12.0'} hasBin: true - emoji-regex@10.6.0: - resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} - end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} @@ -976,10 +943,6 @@ packages: engines: {node: '>=18'} hasBin: true - escalade@3.2.0: - resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} - engines: {node: '>=6'} - estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} @@ -1014,14 +977,6 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - get-caller-file@2.0.5: - resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} - engines: {node: 6.* || 8.* || >= 10.*} - - get-east-asian-width@1.7.0: - resolution: {integrity: sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==} - engines: {node: '>=18'} - get-nonce@1.0.1: resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} engines: {node: '>=6'} @@ -1080,28 +1035,24 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.33.0: resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.33.0: resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.33.0: resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.33.0: resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} @@ -1271,9 +1222,6 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true - rxjs@7.8.2: - resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} - safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -1285,10 +1233,6 @@ packages: engines: {node: '>=10'} hasBin: true - shell-quote@1.12.0: - resolution: {integrity: sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==} - engines: {node: '>= 0.4'} - siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -1314,17 +1258,9 @@ packages: std-env@4.3.0: resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} - string-width@7.2.0: - resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} - engines: {node: '>=18'} - string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - strip-ansi@7.2.0: - resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} - engines: {node: '>=12'} - strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} @@ -1336,10 +1272,6 @@ packages: resolution: {integrity: sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==} engines: {node: '>= 8.0'} - supports-color@10.2.2: - resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} - engines: {node: '>=18'} - tar-fs@2.1.5: resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==} @@ -1362,10 +1294,6 @@ packages: resolution: {integrity: sha512-LgO3D9yZJjApUiuUfl9iFAwrtaX4+lok3wJIqttGoKCHlWUqHqbQpnxCf82L8FjgKsh4iGo78hqJwgL8F6To2A==} engines: {node: '>=14.0.0'} - tree-kill@1.2.2: - resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} - hasBin: true - ts-algebra@2.0.0: resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} @@ -1510,30 +1438,14 @@ packages: engines: {node: '>=8'} hasBin: true - wrap-ansi@9.0.2: - resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} - engines: {node: '>=18'} - wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - y18n@5.0.8: - resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} - engines: {node: '>=10'} - yaml@2.9.1: resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} engines: {node: '>= 14.6'} hasBin: true - yargs-parser@22.0.0: - resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} - engines: {node: ^20.19.0 || ^22.12.0 || >=23} - - yargs@18.0.0: - resolution: {integrity: sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==} - engines: {node: ^20.19.0 || ^22.12.0 || >=23} - zod@4.6.5: resolution: {integrity: sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==} @@ -1658,14 +1570,14 @@ snapshots: '@electron-internal/extract-zip@1.0.5': {} - '@electron/get@5.1.0(supports-color@10.2.2)': + '@electron/get@5.1.0': dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3 env-paths: 3.0.0 graceful-fs: 4.2.11 progress: 2.0.3 semver: 7.8.5 - sumchecker: 3.0.1(supports-color@10.2.2) + sumchecker: 3.0.1 optionalDependencies: undici: 7.30.0 transitivePeerDependencies: @@ -1757,9 +1669,9 @@ snapshots: '@jridgewell/sourcemap-codec@1.6.0': {} - '@kwsites/file-exists@1.1.1(supports-color@10.2.2)': + '@kwsites/file-exists@1.1.1': dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -2102,10 +2014,6 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.2.0 - ansi-regex@6.4.0: {} - - ansi-styles@6.2.3: {} - aria-hidden@1.2.6: dependencies: tslib: 2.8.1 @@ -2136,16 +2044,8 @@ snapshots: chai@6.3.0: {} - chalk@5.6.2: {} - chownr@1.1.4: {} - cliui@9.0.1: - dependencies: - string-width: 7.2.0 - strip-ansi: 7.2.0 - wrap-ansi: 9.0.2 - cmdk@1.1.1(@types/react-dom@19.3.0(@types/react@19.3.0))(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: '@radix-ui/react-compose-refs': 1.1.5(@types/react@19.3.0)(react@19.3.0) @@ -2158,26 +2058,15 @@ snapshots: - '@types/react' - '@types/react-dom' - concurrently@10.0.6: - dependencies: - chalk: 5.6.2 - rxjs: 7.8.2 - shell-quote: 1.12.0 - supports-color: 10.2.2 - tree-kill: 1.2.2 - yargs: 18.0.0 - convert-source-map@2.0.0: {} crelt@1.0.7: {} csstype@3.2.3: {} - debug@4.4.3(supports-color@10.2.2): + debug@4.4.3: dependencies: ms: 2.1.3 - optionalDependencies: - supports-color: 10.2.2 decompress-response@6.0.0: dependencies: @@ -2191,16 +2080,14 @@ snapshots: diff@8.0.4: {} - electron@44.7.0(supports-color@10.2.2): + electron@44.7.0: dependencies: '@electron-internal/extract-zip': 1.0.5 - '@electron/get': 5.1.0(supports-color@10.2.2) + '@electron/get': 5.1.0 '@types/node': 24.19.1 transitivePeerDependencies: - supports-color - emoji-regex@10.6.0: {} - end-of-stream@1.4.5: dependencies: once: 1.4.0 @@ -2238,8 +2125,6 @@ snapshots: '@esbuild/win32-ia32': 0.28.2 '@esbuild/win32-x64': 0.28.2 - escalade@3.2.0: {} - estree-walker@3.0.3: dependencies: '@types/estree': 1.0.9 @@ -2261,10 +2146,6 @@ snapshots: fsevents@2.3.3: optional: true - get-caller-file@2.0.5: {} - - get-east-asian-width@1.7.0: {} - get-nonce@1.0.1: {} github-from-package@0.0.0: {} @@ -2472,18 +2353,12 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.2.13 '@rolldown/binding-win32-x64-msvc': 1.2.13 - rxjs@7.8.2: - dependencies: - tslib: 2.8.1 - safe-buffer@5.2.1: {} scheduler@0.28.0: {} semver@7.8.5: {} - shell-quote@1.12.0: {} - siginfo@2.0.0: {} simple-concat@1.0.1: {} @@ -2494,13 +2369,13 @@ snapshots: once: 1.4.0 simple-concat: 1.0.1 - simple-git@3.36.0(supports-color@10.2.2): + simple-git@3.36.0: dependencies: - '@kwsites/file-exists': 1.1.1(supports-color@10.2.2) + '@kwsites/file-exists': 1.1.1 '@kwsites/promise-deferred': 1.1.1 '@simple-git/args-pathspec': 1.0.4 '@simple-git/argv-parser': 1.1.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -2515,32 +2390,20 @@ snapshots: std-env@4.3.0: {} - string-width@7.2.0: - dependencies: - emoji-regex: 10.6.0 - get-east-asian-width: 1.7.0 - strip-ansi: 7.2.0 - string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 - strip-ansi@7.2.0: - dependencies: - ansi-regex: 6.4.0 - strip-json-comments@2.0.1: {} style-mod@4.1.4: {} - sumchecker@3.0.1(supports-color@10.2.2): + sumchecker@3.0.1: dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3 transitivePeerDependencies: - supports-color - supports-color@10.2.2: {} - tar-fs@2.1.5: dependencies: chownr: 1.1.4 @@ -2567,8 +2430,6 @@ snapshots: tinyrainbow@3.2.0: {} - tree-kill@1.2.2: {} - ts-algebra@2.0.0: {} tslib@2.8.1: {} @@ -2671,27 +2532,8 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 - wrap-ansi@9.0.2: - dependencies: - ansi-styles: 6.2.3 - string-width: 7.2.0 - strip-ansi: 7.2.0 - wrappy@1.0.2: {} - y18n@5.0.8: {} - yaml@2.9.1: {} - yargs-parser@22.0.0: {} - - yargs@18.0.0: - dependencies: - cliui: 9.0.1 - escalade: 3.2.0 - get-caller-file: 2.0.5 - string-width: 7.2.0 - y18n: 5.0.8 - yargs-parser: 22.0.0 - zod@4.6.5: {} diff --git a/scripts/e2e-ui.ts b/scripts/e2e-ui.ts index 6e929ed..9544dbe 100644 --- a/scripts/e2e-ui.ts +++ b/scripts/e2e-ui.ts @@ -19,7 +19,7 @@ process.env.QUILLGON_OFFLINE = '1'; const { NovelServer } = await import('@quillgon/server'); const server = new NovelServer({ staticDir: path.resolve(here, '..', 'apps', 'desktop', 'dist') }); const base = `http://127.0.0.1:${await server.listen(0)}`; -const api = async (p: string) => (await fetch(`${base}/api${p}`)).json() as Promise; +const api = async (p: string) => (await fetch(`${base}/api${p}`, { headers: { 'x-quillgon-token': server.token } })).json() as Promise; const browser = await chromium.launch({ executablePath: process.env.CHROME ?? '/opt/pw-browsers/chromium-1194/chrome-linux/chrome' }); const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); const errors: string[] = []; @@ -88,7 +88,7 @@ try { // ---------------------------------------------------------------- B console.log('B. A new book through every gate in the UI'); - await page.evaluate(() => fetch('/api/init', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ dir: '~/Novels/Glasshouse', title: 'The Glasshouse', target_words: 12000, offline: true }) })); + await page.evaluate(() => fetch('/api/init', { method: 'POST', headers: { 'content-type': 'application/json', 'x-quillgon-token': document.querySelector('meta[name="quillgon-token"]')!.content }, body: JSON.stringify({ dir: '~/Novels/Glasshouse', title: 'The Glasshouse', target_words: 12000, offline: true }) })); await page.goto(`${base}/#/build`); await page.getByPlaceholder(/harbourmaster drowns/).fill('A glasshouse whose orchids are watered every night, though the house has been empty for a year.'); await page.getByRole('button', { name: 'Show me ideas' }).click(); diff --git a/scripts/screenshots.ts b/scripts/screenshots.ts index a2fd5f9..4ebafea 100644 --- a/scripts/screenshots.ts +++ b/scripts/screenshots.ts @@ -19,7 +19,7 @@ const server = new NovelServer({ staticDir: path.resolve(here, '..', 'apps', 'de const port = await server.listen(0); const base = `http://127.0.0.1:${port}`; const api = async (method: string, p: string, body?: unknown) => { - const r = await fetch(`${base}/api${p}`, { method, headers: { 'content-type': 'application/json' }, body: body ? JSON.stringify(body) : undefined }); + const r = await fetch(`${base}/api${p}`, { method, headers: { 'content-type': 'application/json', 'x-quillgon-token': server.token }, body: body ? JSON.stringify(body) : undefined }); const j: any = await r.json(); if (!r.ok) throw new Error(`${p}: ${j.error}`); return j; diff --git a/scripts/ui-refresh.ts b/scripts/ui-refresh.ts index 3b5aa1c..415eca3 100644 --- a/scripts/ui-refresh.ts +++ b/scripts/ui-refresh.ts @@ -28,7 +28,7 @@ const page = await browser.newPage({ viewport: { width: 1440, height: 1000 }, co const errors: string[] = []; page.on('pageerror', e => errors.push(e.message)); const api = async (method: string, route: string, body?: unknown) => { - const r = await fetch(`${base}/api${route}`, { method, headers: { 'content-type': 'application/json' }, body: body ? JSON.stringify(body) : undefined }); + const r = await fetch(`${base}/api${route}`, { method, headers: { 'content-type': 'application/json', 'x-quillgon-token': server.token }, body: body ? JSON.stringify(body) : undefined }); assert.ok(r.ok, `${method} ${route}: ${r.status}`); return r.json(); }; From 8493a06480ebee4381ba3d087880e42acf056256 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:38:16 +0000 Subject: [PATCH 02/23] Commit only what each change wrote Every commit ran `git add -A`, so it swept in whatever else was dirty: edits made in another editor, and prose the author had autosaved but not committed. A ripple's origin commit could therefore contain an hour of unrelated typing, and reverting the ripple threw it away. The same sweep committed an edited card before minimal regeneration read "the committed card", so the reviser was given identical old and new cards. - GraphStore.commit stages only the files the store wrote (deletions included); GraphStore.checkpoint commits the whole tree for explicit checkpoints such as a new book or the step before a revert. - A ripple first checkpoints other work, then commits the originating edit alone (changeFiles maps each change kind to its files). Each round commit holds only what the round wrote, tracked by write sequence. - Minimal regeneration finds the card version whose hash matches the prose's generated_from in history, instead of trusting HEAD. - The repo staging area is reset before each commit; a file written and removed before any commit no longer errors; dirty state survives a failed or non-repo commit. - Prompt overrides are written through the store, so they are committed. - .engine/ and export/ are added to .gitignore of pre-existing repos. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- packages/agents/test/helpers.ts | 2 +- packages/agents/test/history.test.ts | 99 ++++++++++++++++++++++++++++ packages/core/src/engine/pipeline.ts | 24 +++++-- packages/core/src/engine/ripple.ts | 16 +++-- packages/core/src/git.ts | 56 +++++++++++++--- packages/core/src/impact.ts | 34 +++++++++- packages/core/src/init.ts | 2 +- packages/core/src/store.ts | 75 ++++++++++++++++++--- packages/core/test/store.test.ts | 2 +- packages/server/src/index.ts | 11 +--- packages/server/test/server.test.ts | 2 +- scripts/ui-refresh.ts | 2 +- 12 files changed, 281 insertions(+), 44 deletions(-) create mode 100644 packages/agents/test/history.test.ts diff --git a/packages/agents/test/helpers.ts b/packages/agents/test/helpers.ts index 46cda02..a423fa3 100644 --- a/packages/agents/test/helpers.ts +++ b/packages/agents/test/helpers.ts @@ -9,7 +9,7 @@ export async function fixtureEngine(overrides: Partial = {}) { const dir = copyFixture(); const store = GraphStore.open(dir, { memoryIndex: true }); await store.repo.init(); - await store.commit('Fixture'); + await store.checkpoint('Fixture'); const agents = new ScriptedSuite(overrides); return { dir, store, agents, engine: new Engine(store, agents) }; } diff --git a/packages/agents/test/history.test.ts b/packages/agents/test/history.test.ts new file mode 100644 index 0000000..4ffc852 --- /dev/null +++ b/packages/agents/test/history.test.ts @@ -0,0 +1,99 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { GraphStore, paths } from '@quillgon/core'; +import { copyFixture, fixtureEngine } from './helpers.js'; + +const read = (dir: string, rel: string) => fs.readFileSync(path.join(dir, rel), 'utf8'); + +/** Commits only ever contain what they are about, so reverting one cannot undo other work. */ +describe('scoped commits', () => { + it('a store commit leaves edits made in another editor uncommitted', async () => { + const { dir, store } = await fixtureEngine(); + fs.appendFileSync(path.join(dir, paths.prose('s02')), '\nWritten in another editor.\n'); + store.saveCard({ ...store.scene('s03').card, purpose: 'A new purpose.' }); + const hash = await store.commit('Edit s03'); + expect((await store.repo.show(hash!)).files).toEqual([paths.card('s03')]); + expect((await store.repo.git.status()).modified).toEqual([paths.prose('s02')]); + // A checkpoint takes everything. + await store.checkpoint('Checkpoint'); + expect((await store.repo.git.status()).isClean()).toBe(true); + }); + + it('reverting a ripple keeps prose the author wrote before it started', async () => { + const { dir, store, engine } = await fixtureEngine(); + // An hour of typing in s02: autosaved through the store, never committed. + const typed = store.scene('s02').prose!.body + '\n\nA paragraph the author wrote by hand.\n'; + store.saveProse('s02', typed, 'author'); + // Then a card edit elsewhere starts a ripple. + const before = structuredClone(store.project.plotPoints.get('pp-drowning')!); + const after = store.savePlotPoint({ ...before, description: 'Walter Penrose drowns on a flat-calm night.' }); + const ripple = await engine.startRipple({ kind: 'plot', before, after }); + expect((await store.repo.show(ripple.data.originCommit!)).files).toEqual([paths.plotPoint('pp-drowning')]); + + await ripple.revert(); + expect(store.project.plotPoints.get('pp-drowning')!.description).toBe(before.description); + expect(read(dir, paths.prose('s02'))).toContain('A paragraph the author wrote by hand.'); + }); + + it('a ripple round commit holds only what the round wrote', async () => { + const { store, engine } = await fixtureEngine(); + const before = structuredClone(store.project.plotPoints.get('pp-drowning')!); + const after = store.savePlotPoint({ ...before, description: 'Walter Penrose drowns on a flat-calm night.' }); + const ripple = await engine.startRipple({ kind: 'plot', before, after }); + await ripple.propose(); + await ripple.resolveConflict('s10', 'keep'); + // The author keeps writing elsewhere while reviewing. + store.saveProse('s05', store.scene('s05').prose!.body + '\n\nTyped during review.\n', 'author'); + const round = await ripple.apply(); + expect((await store.repo.show(round.commit!)).files).toEqual(['reports/known-inconsistencies.md']); + }); + + it('a deleted scene is committed as a deletion', async () => { + const { store } = await fixtureEngine(); + store.deleteScene('s12'); + const hash = await store.commit('Delete s12'); + const files = (await store.repo.show(hash!)).files; + expect(files).toContain(paths.card('s12')); + expect(files).toContain(paths.structure); + expect((await store.repo.git.status()).isClean()).toBe(true); + }); + + it('a file written and removed before any commit is not an error', async () => { + const { store } = await fixtureEngine(); + store.savePrompt('reader', 'temporary'); + store.savePrompt('reader', null); + expect(await store.commit('Nothing')).toBeNull(); + expect(store.hasUncommitted()).toBe(false); + }); +}); + +describe('minimal regeneration', () => { + it('revises from the card the prose was written against, even after the new card is committed', async () => { + let seen: { old_card: { purpose: string }; new_card: { purpose: string } } | null = null; + const { store, engine } = await fixtureEngine({ + reviseProse: async (input) => { + seen = input as any; + return { prose: input.old_prose, summary: '', needs_decision: [] }; + }, + }); + const old = store.scene('s03').card.purpose; + store.saveCard({ ...store.scene('s03').card, purpose: 'Mara decides to trust Tom after all.' }); + // Something else commits first (an approval, a checkpoint): the new card is now at HEAD. + await store.checkpoint('Unrelated checkpoint'); + await engine.pipeline.regenerate('s03', 'minimal'); + expect(seen!.old_card.purpose).toBe(old); + expect(seen!.new_card.purpose).toBe('Mara decides to trust Tom after all.'); + }); +}); + +describe('opening an existing repository', () => { + it('adds the index and exports to .gitignore', async () => { + const dir = copyFixture(); + fs.writeFileSync(path.join(dir, '.gitignore'), 'notes/\n'); + execFileSync('git', ['init', '-q'], { cwd: dir }); + GraphStore.open(dir, { memoryIndex: true }).close(); + expect(read(dir, '.gitignore')).toBe('notes/\n.engine/\nexport/\n'); + }); +}); diff --git a/packages/core/src/engine/pipeline.ts b/packages/core/src/engine/pipeline.ts index fdc9b5c..67c765f 100644 --- a/packages/core/src/engine/pipeline.ts +++ b/packages/core/src/engine/pipeline.ts @@ -11,7 +11,7 @@ import type { import { Intent, SceneCard, type PipelineStage, type StateSnapshot } from '../schemas/index.js'; import type { GraphStore } from '../store.js'; import { paths, sceneOrder } from '../project.js'; -import { canGenerate, sceneSync } from '../sync.js'; +import { canGenerate, cardHash, sceneSync } from '../sync.js'; import { writerContext } from '../context.js'; import { fromYaml, slugify, uniq } from '../util.js'; import { runStylePasses, type Progress, type StylePassReport } from './style-passes.js'; @@ -405,12 +405,24 @@ export class Pipeline { return { before, after: out.prose }; } - /** The card as last committed: what the current prose was written against, if the card has since moved. */ + /** + * The card the current prose was written against: the version in history whose hash is + * the prose's `generated_from`. HEAD is not reliable, because the edited card may already + * have been committed (by a checkpoint, or any later commit) before regenerating. + */ private async committedCard(sceneId: string): Promise { - const text = await this.store.repo.readAt('HEAD', paths.card(sceneId)).catch(() => null); - if (!text) return null; - const r = SceneCard.safeParse(fromYaml(text)); - return r.success ? r.data : null; + const rel = paths.card(sceneId); + const want = this.store.scene(sceneId).prose?.meta.generated_from ?? null; + let head: SceneCard | null = null; + for (const hash of await this.store.repo.history(rel, 50)) { + const text = await this.store.repo.readAt(hash, rel).catch(() => null); + if (!text) continue; + const r = SceneCard.safeParse(fromYaml(text)); + if (!r.success) continue; + head ??= r.data; + if (!want || cardHash(r.data) === want) return r.data; + } + return head; } async acceptProse(sceneId: string, prose: string) { diff --git a/packages/core/src/engine/ripple.ts b/packages/core/src/engine/ripple.ts index 3b63959..f4dbe1e 100644 --- a/packages/core/src/engine/ripple.ts +++ b/packages/core/src/engine/ripple.ts @@ -4,7 +4,7 @@ import type { AgentSuite } from '../agents/contracts.js'; import { SceneCard } from '../schemas/index.js'; import type { GraphStore } from '../store.js'; import { paths, sceneOrder, stateBefore } from '../project.js'; -import { analyseImpact, cardDiffSummary, partitionImpacts, type Change, type Impact } from '../impact.js'; +import { analyseImpact, cardDiffSummary, changeFiles, partitionImpacts, type Change, type Impact } from '../impact.js'; import { applyExtraction, runExtraction } from './extraction.js'; import type { Progress } from './style-passes.js'; import { cardHash } from '../sync.js'; @@ -102,13 +102,16 @@ export class Ripple { /** * Start a ripple for a change the author has already made through the store. - * The originating edit is committed first, so the whole ripple can be reverted. + * The originating edit is committed first, so the whole ripple can be reverted. Any other + * uncommitted work is checkpointed separately beforehand, so a revert cannot undo it. */ static async start(store: GraphStore, agents: AgentSuite, change: Change, opts: { hopLimit?: number; roundCap?: number } = {}): Promise { const existing = Ripple.current(store, agents); if (existing) existing.abandon(); const impact = analyseImpact(store.project, change, store.index); - const originCommit = await store.commit(`Edit: ${impact.description}`); + const files = changeFiles(change); + await store.checkpoint('Work in progress before a ripple', { exclude: files }); + const originCommit = await store.commit(`Edit: ${impact.description}`, { files }); const r = new Ripple(store, agents, { id: timestamp(), created: new Date().toISOString(), @@ -366,6 +369,9 @@ export class Ripple { const blockers = this.blockers(); if (blockers.length) throw new Error(`round ${this.round.n} still needs decisions:\n${blockers.join('\n')}`); const round = this.round; + // Commit the round alone: earlier work (an autosave elsewhere during review) goes first. + await this.store.commit('Work in progress before a ripple round'); + const mark = this.store.writeMark(); const changed: { scene: string; before: SceneCard; after: SceneCard }[] = []; for (const it of round.items) { if ((it.status === 'accepted') && it.card?.after && it.kind === 'card') { @@ -396,7 +402,7 @@ export class Ripple { ...(kept.length ? [`Kept locked: ${kept.join(', ')}`] : []), ...round.items.filter((i) => i.status === 'no_change').map((i) => `No change needed in ${i.scene}: ${i.card?.justification ?? ''}`), ].join('\n'); - round.commit = await this.store.commit(msg); + round.commit = await this.store.commit(msg, { files: this.store.writtenSince(mark) }); round.applied = true; // Cascade: impact of the accepted card changes. @@ -424,7 +430,7 @@ export class Ripple { /** Revert every commit this ripple made, including the originating edit. */ async revert() { const hashes = [...this.data.rounds.map((r) => r.commit).filter((h): h is string => !!h).reverse(), ...(this.data.originCommit ? [this.data.originCommit] : [])]; - await this.store.commit('Checkpoint before reverting ripple'); + await this.store.checkpoint('Checkpoint before reverting ripple'); for (const h of hashes) await this.store.repo.revert(h); this.store.reload(); if (!this.data.originCommit) { diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index fc4c441..856eeda 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -53,13 +53,21 @@ export class Repo { } async init() { - if (this.isRepo()) return; + if (this.isRepo()) return this.ensureIgnored(); await this.git.init(['-b', 'main']); - const ignore = path.join(this.dir, '.gitignore'); - if (!fs.existsSync(ignore)) fs.writeFileSync(ignore, '.engine/\nexport/\n'); + this.ensureIgnored(); await this.ensureIdentity(); } + /** The index, proposals and exports never belong in history, even in a pre-existing repo. */ + ensureIgnored() { + const ignore = path.join(this.dir, '.gitignore'); + const text = fs.existsSync(ignore) ? fs.readFileSync(ignore, 'utf8') : ''; + const lines = new Set(text.split(/\r?\n/).map((l) => l.trim())); + const missing = ['.engine/', 'export/'].filter((e) => !lines.has(e) && !lines.has(e.slice(0, -1)) && !lines.has('/' + e)); + if (missing.length) fs.writeFileSync(ignore, text + (text && !text.endsWith('\n') ? '\n' : '') + missing.join('\n') + '\n'); + } + private async ensureIdentity() { const name = await this.git.getConfig('user.name'); if (!name.value) await this.git.addConfig('user.name', 'Quillgon'); @@ -67,20 +75,48 @@ export class Repo { if (!email.value) await this.git.addConfig('user.email', 'engine@localhost'); } - /** Stage everything (or just `files`) and commit. Returns the hash, or null if nothing changed. */ - async commit(message: string, files?: string[]): Promise { + /** + * Commit. With no options, a checkpoint: everything in the working tree. With `files`, + * only those paths (deletions included), whatever else is modified. With `exclude`, + * everything except those paths. Returns the hash, or null if there was nothing to commit. + */ + async commit(message: string, opts: string[] | { files?: string[]; exclude?: string[] } = {}): Promise { if (!this.isRepo()) return null; + const { files, exclude } = Array.isArray(opts) ? { files: opts, exclude: undefined } : opts; await this.ensureIdentity(); - if (files?.length) await this.git.add(files); - else await this.git.add(['-A']); - const status = await this.git.status(); - if (status.staged.length === 0 && status.created.length === 0 && status.deleted.length === 0 && status.renamed.length === 0) { - if (status.files.length === 0) return null; + const hasHead = await this.git.raw(['rev-parse', '--verify', '-q', 'HEAD']).then(() => true, () => false); + // Start from a clean index, so nothing staged earlier rides along. + if (hasHead) await this.git.raw(['reset', '-q']); + if (files) { + if (!files.length) return null; + const literal = files.map((f) => `:(literal)${f}`); + // A path that was written and removed again before any commit is unknown to git. + const tracked = new Set((await this.git.raw(['ls-files', '-z', '--', ...literal])).split('\0').filter(Boolean)); + const known = files.filter((f) => fs.existsSync(path.join(this.dir, f)) || [...tracked].some((t) => t === f || t.startsWith(f + '/'))); + if (!known.length) return null; + await this.git.raw(['add', '-A', '--', ...known.map((f) => `:(literal)${f}`)]); + } else if (exclude?.length) { + await this.git.raw(['add', '-A', '--', '.', ...exclude.map((f) => `:(exclude,literal)${f}`)]); + } else { + await this.git.add(['-A']); } + const staged = (await this.git.raw(['diff', '--cached', '--name-only'])).trim(); + if (!staged) return null; const r = await this.git.commit(message); return r.commit || null; } + /** Commits that touched a path, newest first. */ + async history(rel: string, limit = 50): Promise { + if (!this.isRepo()) return []; + try { + const out = await this.git.raw(['log', `--max-count=${limit}`, '--format=%H', '--', `:(literal)${rel}`]); + return out.split('\n').filter(Boolean); + } catch { + return []; + } + } + async log(limit = 50): Promise { if (!this.isRepo()) return []; try { diff --git a/packages/core/src/impact.ts b/packages/core/src/impact.ts index 827d842..81aee18 100644 --- a/packages/core/src/impact.ts +++ b/packages/core/src/impact.ts @@ -1,5 +1,5 @@ import type { Fact, Obligation, PlotPoint, SceneCard, StateSnapshot, Thread } from './schemas/index.js'; -import { sceneOrder, type Project } from './project.js'; +import { paths, sceneOrder, type Project } from './project.js'; import type { ProjectIndex } from './index-db.js'; import { uniq } from './util.js'; @@ -23,6 +23,38 @@ export type Change = | { kind: 'reorder'; before: string[]; after: string[]; description?: string } | { kind: 'delete-scene'; scene: string; card: SceneCard; orderBefore: string[]; description?: string }; +/** + * The files an edit of this kind writes. A ripple commits these as its origin commit, + * apart from any other uncommitted work, so reverting the ripple reverts exactly the edit. + */ +export function changeFiles(change: Change): string[] { + switch (change.kind) { + case 'card': + return [paths.card(change.scene)]; + case 'state': + return [paths.state(change.character)]; + case 'character': + return [paths.bio(change.character), paths.voice(change.character)]; + case 'thread': + return [...new Set([change.before?.id, change.after?.id].filter((x): x is string => !!x))].map(paths.thread); + case 'plot': + return [...new Set([change.before?.id, change.after?.id].filter((x): x is string => !!x))].map(paths.plotPoint); + case 'obligation': { + const ids = [change.before?.id, change.after?.id].filter((x): x is string => !!x); + // Moving where an obligation is planted or paid also rewrites those scenes' cards. + const scenes = [change.before, change.after].flatMap((o) => (o ? [o.setup_scene, o.payoff_scene] : [])).filter((x): x is string => !!x); + return [...new Set([...ids.map(paths.obligation), ...scenes.map(paths.card)])]; + } + case 'reorder': + return [paths.structure]; + case 'delete-scene': { + // Deleting a scene also drops it from the structure and from its cast's state files. + const cast = [...new Set([change.card.pov, ...change.card.characters])]; + return [`scenes/${change.scene}`, paths.structure, ...cast.map(paths.state)]; + } + } +} + export interface Impact { scene: string; hop: 0 | 1 | 2 | 3; diff --git a/packages/core/src/init.ts b/packages/core/src/init.ts index f7db3bc..c4d3956 100644 --- a/packages/core/src/init.ts +++ b/packages/core/src/init.ts @@ -54,6 +54,6 @@ export async function initProject(dir: string, opts: { title: string; genre?: st if (!fs.readdirSync(path.join(root, d)).length) fs.writeFileSync(keep, ''); } await store.repo.init(); - await store.commit(`New book: ${opts.title}`); + await store.checkpoint(`New book: ${opts.title}`); return store; } diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 3091bc0..934ab8c 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -57,8 +57,9 @@ export class GraphStore { private fileIssues: Issue[] = []; index: ProjectIndex | null = null; readonly repo: Repo; - /** Files written since the last commit. */ - private dirty = new Set(); + /** Files written since they were last committed, with the write sequence of the latest write. */ + private dirty = new Map(); + private writeSeq = 0; private listeners = new Set<(e: StoreEvent) => void>(); private constructor(readonly dir: string, private opts: OpenOptions) { @@ -67,6 +68,8 @@ export class GraphStore { static open(dir: string, opts: OpenOptions = {}): GraphStore { const s = new GraphStore(path.resolve(dir), opts); + // Checkpoints stage the whole tree, so the index must be ignored even in an older repo. + if (s.repo.isRepo()) s.repo.ensureIgnored(); s.reload(); return s; } @@ -137,18 +140,18 @@ export class GraphStore { const abs = path.join(this.dir, rel); fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.writeFileSync(abs, text); - this.dirty.add(rel); + this.dirty.set(rel, ++this.writeSeq); } private remove(rel: string) { const abs = path.join(this.dir, rel); if (fs.existsSync(abs)) fs.rmSync(abs, { recursive: true, force: true }); - this.dirty.add(rel); + this.dirty.set(rel, ++this.writeSeq); } private after(entity: string) { // File-level problems persist until that file is rewritten; reference problems are recomputed. - for (const rel of this.dirty) this.fileIssues = this.fileIssues.filter((i) => i.file !== rel); + for (const rel of this.dirty.keys()) this.fileIssues = this.fileIssues.filter((i) => i.file !== rel); this.issues = [...this.fileIssues, ...validateReferences(this.project)]; this.index?.update(this.project); this.emit({ type: 'changed', entity }); @@ -438,22 +441,76 @@ export class GraphStore { const abs = path.join(this.dir, paths.reports, name); fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.appendFileSync(abs, line.endsWith('\n') ? line : line + '\n'); - this.dirty.add(`${paths.reports}/${name}`); + this.dirty.set(`${paths.reports}/${name}`, ++this.writeSeq); } // --------------------------------------------------------------- git - async commit(message: string): Promise { - const hash = await this.repo.commit(message); - this.dirty.clear(); + /** + * Commit what the store has written (or just `files`), and nothing else. Edits made in + * another editor, or files the author is still working on elsewhere, never ride along + * in an unrelated commit, so reverting that commit cannot undo them. + */ + async commit(message: string, opts: { files?: string[] } = {}): Promise { + if (!this.repo.isRepo()) return null; + const files = opts.files ?? [...this.dirty.keys()]; + const seq = this.writeSeq; + const hash = await this.repo.commit(message, { files }); + this.settle(files, seq); + if (hash) this.emit({ type: 'committed', hash, message }); + return hash; + } + + /** + * Commit everything in the working tree, including edits made outside the app (or + * everything except `exclude`). For explicit checkpoints: a new book, before a revert. + */ + async checkpoint(message: string, opts: { exclude?: string[] } = {}): Promise { + if (!this.repo.isRepo()) return null; + const seq = this.writeSeq; + const exclude = opts.exclude ?? []; + const hash = await this.repo.commit(message, exclude.length ? { exclude } : {}); + const ex = new Set(exclude); + this.settle([...this.dirty.keys()].filter((f) => !ex.has(f)), seq); if (hash) this.emit({ type: 'committed', hash, message }); return hash; } + /** Forget files that are now committed, unless they were written again while committing. */ + private settle(files: string[], seq: number) { + for (const f of files) if ((this.dirty.get(f) ?? Infinity) <= seq) this.dirty.delete(f); + } + hasUncommitted() { return this.dirty.size > 0; } + /** A point in the write history; see writtenSince. */ + writeMark(): number { + return this.writeSeq; + } + + /** Uncommitted files written after a writeMark. */ + writtenSince(mark: number): string[] { + return [...this.dirty].filter(([, seq]) => seq > mark).map(([f]) => f); + } + + /** Files written and not yet committed. */ + uncommitted(): string[] { + return [...this.dirty.keys()]; + } + + // --------------------------------------------------------------- prompts + + /** Save (or with null, remove) the book's override of an agent's prompt. */ + savePrompt(agent: string, text: string | null) { + if (!/^[a-z][a-z-]*$/.test(agent)) throw new Error(`invalid agent name "${agent}"`); + const rel = `prompts/${agent}.md`; + if (text === null) this.remove(rel); + else this.write(rel, text); + this.emit({ type: 'changed', entity: `prompt:${agent}` }); + } + stamp() { return timestamp(); } diff --git a/packages/core/test/store.test.ts b/packages/core/test/store.test.ts index bf9ae00..3c79978 100644 --- a/packages/core/test/store.test.ts +++ b/packages/core/test/store.test.ts @@ -73,7 +73,7 @@ describe('phase 1: schemas and store', () => { const dir = copyFixture(); const store = GraphStore.open(dir, { memoryIndex: true }); await store.repo.init(); - await store.commit('Initial'); + await store.checkpoint('Initial'); const card = { ...store.scene('s03').card, location: 'harbour' }; store.saveCard(card); expect(store.sceneSync('s03').state).toBe('card_ahead'); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index f7132db..5a04543 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -327,7 +327,7 @@ export class NovelServer { fs.cpSync(fixture, dir, { recursive: true, filter: (src) => !src.includes(`${path.sep}.engine`) }); const store = GraphStore.open(dir, { noIndex: true }); await store.repo.init(); - await store.commit('The Tidekeeper’s Ledger (sample)'); + await store.checkpoint('The Tidekeeper’s Ledger (sample)'); store.close(); } server.open(dir); @@ -649,7 +649,7 @@ export class NovelServer { r('GET', '/api/history', async ({ engine, query }) => engine.store.repo.log(Number(query.get('limit')) || 100)); r('GET', '/api/history/:hash', async ({ engine, params }) => engine.store.repo.show(params.hash)); r('POST', '/api/history/revert', ({ engine, body, server }) => server.job('revert', async () => { - await engine.store.commit('Checkpoint before revert'); + await engine.store.checkpoint('Checkpoint before revert'); await engine.store.repo.revert(body.hash); engine.store.reload(); return { ok: true }; @@ -691,12 +691,7 @@ export class NovelServer { ); r('PUT', '/api/prompts/:name', async ({ engine, params, body }) => { if (!AgentName.safeParse(params.name).success) throw new HttpError(404, `no agent "${params.name}"`); - const file = path.join(engine.store.dir, 'prompts', `${params.name}.md`); - if (body.reset) fs.rmSync(file, { force: true }); - else { - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, String(body.text)); - } + engine.store.savePrompt(params.name, body.reset ? null : String(body.text)); await engine.store.commit(`${body.reset ? 'Reset' : 'Edit'} ${params.name} prompt`); return { ok: true }; }); diff --git a/packages/server/test/server.test.ts b/packages/server/test/server.test.ts index 4375f0f..35c6615 100644 --- a/packages/server/test/server.test.ts +++ b/packages/server/test/server.test.ts @@ -32,7 +32,7 @@ beforeAll(async () => { const dir = copyFixture(); const s = GraphStore.open(dir, { memoryIndex: true }); await s.repo.init(); - await s.commit('Fixture'); + await s.checkpoint('Fixture'); s.close(); server = new NovelServer(); server.open(dir); diff --git a/scripts/ui-refresh.ts b/scripts/ui-refresh.ts index 415eca3..d96388e 100644 --- a/scripts/ui-refresh.ts +++ b/scripts/ui-refresh.ts @@ -19,7 +19,7 @@ fs.mkdirSync(out, { recursive: true }); fs.cpSync(path.join(root, 'fixtures/novella'), dir, { recursive: true }); const seed = GraphStore.open(dir, { noIndex: true }); await seed.repo.init(); -await seed.commit('Sample manuscript'); +await seed.checkpoint('Sample manuscript'); seed.close(); const server = new NovelServer({ staticDir: path.join(root, 'apps/desktop/dist') }); const base = `http://127.0.0.1:${await server.listen(0)}`; From 0c95869cd1c912992d94cdabdf0b243e48010cf5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:42:10 +0000 Subject: [PATCH 03/23] Make ripple reverts all-or-nothing and retry failed proposals - Revert a ripple's commits as one commit with `git revert --no-commit`. If any of them conflicts with later work, abort, put HEAD and the tree back, and raise RevertConflictError naming the files (HTTP 409). Previously a conflict left the book mid-revert with the ripple still open. - The history revert route uses the same all-or-nothing path. - A proposal that failed (network, rate limit) is retried by proposing again; before, an errored item blocked the round and only rejecting it unblocked it. - Accepting an edited card proposal keeps the scene's status and lock instead of resetting an approved or locked scene to Zod defaults. - Ripple ids get a random suffix; second-granular ids collided. - The revert fallback for edits committed before the ripple started now covers scene deletions and character edits, restoring their files from history. - A wording-only prose edit on a scene whose card had also moved no longer marks it in sync, which hid the card change from the prose. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- packages/agents/test/ripple-state.test.ts | 121 ++++++++++++++++++++++ packages/core/src/engine/pipeline.ts | 4 +- packages/core/src/engine/ripple.ts | 55 ++++++++-- packages/core/src/git.ts | 54 +++++++++- packages/server/src/index.ts | 2 + 5 files changed, 227 insertions(+), 9 deletions(-) create mode 100644 packages/agents/test/ripple-state.test.ts diff --git a/packages/agents/test/ripple-state.test.ts b/packages/agents/test/ripple-state.test.ts new file mode 100644 index 0000000..49acb33 --- /dev/null +++ b/packages/agents/test/ripple-state.test.ts @@ -0,0 +1,121 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { paths, RevertConflictError, sceneOrder } from '@quillgon/core'; +import { fixtureEngine, fixtureExtractor } from './helpers.js'; + +const read = (dir: string, rel: string) => fs.readFileSync(path.join(dir, rel), 'utf8'); + +/** The ripple state machine's failure paths. */ +describe('ripple state machine', () => { + async function appliedRipple() { + const env = await fixtureEngine(); + (env.engine.agents as any).extract = fixtureExtractor(env.store); + const snap = env.store.project.characters.get('mara')!.state['s04']; + const ripple = await env.engine.startRipple(env.engine.editState('mara', 's04', { ...snap, knows: snap.knows.filter((f) => f !== 'eliza-holds-debts') })); + await ripple.propose(); + ripple.accept('s06'); + ripple.reject('s11'); + ripple.rejectProse('s06'); + await ripple.apply(); + return { ...env, ripple }; + } + + it('a revert that conflicts with later work changes nothing and names the files', async () => { + const { dir, store, ripple } = await appliedRipple(); + // The author edits the same lines of the card after the ripple, and commits. + // (Edits elsewhere in the file merge cleanly and the revert goes ahead.) + store.saveCard({ ...store.scene('s06').card, mentions: [] }); + await store.commit('Hand edit'); + const head = await store.repo.head(); + const card = read(dir, paths.card('s06')); + + const err = await ripple.revert().catch((e) => e); + expect(err).toBeInstanceOf(RevertConflictError); + expect(err.files).toContain(paths.card('s06')); + // Nothing moved: same HEAD, same files, clean tree, no revert in progress. + expect(await store.repo.head()).toBe(head); + expect(read(dir, paths.card('s06'))).toBe(card); + expect((await store.repo.git.status()).isClean()).toBe(true); + expect(fs.existsSync(path.join(dir, '.git', 'REVERT_HEAD'))).toBe(false); + expect(ripple.data.state).not.toBe('reverted'); + }); + + it('a clean revert is one commit', async () => { + const { store, ripple } = await appliedRipple(); + const before = (await store.repo.log(50)).length; + await ripple.revert(); + const log = await store.repo.log(50); + // A checkpoint (only if there was pending work) and one revert commit. + expect(log[0].message).toMatch(/^Revert ripple: /); + expect(log.length - before).toBeLessThanOrEqual(2); + }); + + it('a proposal that failed is retried by proposing again', async () => { + let fail = true; + const { store, engine } = await fixtureEngine({ + reviseCard: async (i) => { + if (fail) throw new Error('rate limited'); + return { action: 'no_change', justification: `fine: ${i.card.id}`, needs_decision: [] }; + }, + }); + const snap = store.project.characters.get('mara')!.state['s04']; + const ripple = await engine.startRipple(engine.editState('mara', 's04', { ...snap, knows: snap.knows.filter((f) => f !== 'eliza-holds-debts') })); + await ripple.propose(); + expect(ripple.item('s06')).toMatchObject({ status: 'error', error: 'rate limited' }); + expect(ripple.blockers()).toContain('s06: card error'); + fail = false; + await ripple.propose(); + expect(ripple.item('s06')).toMatchObject({ status: 'no_change', error: null }); + }); + + it("accepting an edited proposal keeps the scene's status and lock", async () => { + const { store, engine } = await fixtureEngine(); + store.setSceneStatus('s06', { status: 'approved', lock: 'card' }); + store.setSceneStatus('s06', { lock: 'none' }); + expect(store.scene('s06').card.status).toBe('approved'); + const snap = store.project.characters.get('mara')!.state['s04']; + const ripple = await engine.startRipple(engine.editState('mara', 's04', { ...snap, knows: snap.knows.filter((f) => f !== 'eliza-holds-debts') })); + await ripple.propose(); + const { status: _s, lock: _l, ...edited } = ripple.item('s06').card!.after!; + ripple.accept('s06', { ...edited, purpose: 'Edited by the author in review.' }); + expect(ripple.item('s06').card!.after).toMatchObject({ status: 'approved', purpose: 'Edited by the author in review.' }); + }); + + it('two ripples started in the same second get different ids', async () => { + const { store, engine } = await fixtureEngine(); + const before = structuredClone(store.project.plotPoints.get('pp-drowning')!); + const a = await engine.startRipple({ kind: 'plot', before, after: store.savePlotPoint({ ...before, description: 'One.' }) }); + const b = await engine.startRipple({ kind: 'plot', before, after: store.savePlotPoint({ ...before, description: 'Two.' }) }); + expect(a.data.id).not.toBe(b.data.id); + }); + + it('reverting a scene deletion committed before the ripple restores the scene in place', async () => { + const { dir, store, engine } = await fixtureEngine(); + const card = structuredClone(store.scene('s07').card); + const prose = read(dir, paths.prose('s07')); + const orderBefore = sceneOrder(store.project); + store.deleteScene('s07'); + await store.commit('Delete s07'); + await new Promise((r) => setTimeout(r, 1100)); // the ripple starts after the deletion commit + const ripple = await engine.startRipple({ kind: 'delete-scene', scene: 's07', card, orderBefore }); + expect(ripple.data.originCommit).toBeNull(); + await ripple.revert(); + expect(sceneOrder(store.project)).toEqual(orderBefore); + expect(read(dir, paths.prose('s07'))).toBe(prose); + expect((await store.repo.git.status()).isClean()).toBe(true); + }); +}); + +describe('prose edits', () => { + it('a wording-only edit to a scene whose card moved leaves it card_ahead', async () => { + const { store, engine } = await fixtureEngine(); + (engine.agents as any).extract = fixtureExtractor(store); + store.saveCard({ ...store.scene('s03').card, purpose: 'A changed purpose the prose does not reflect yet.' }); + store.saveProse('s03', store.scene('s03').prose!.body.replace('A gull landed on the bollard', 'A gull dropped onto the bollard'), 'author'); + expect(store.sceneSync('s03').state).toBe('conflict'); + const r = await engine.pipeline.reconcileProseEdit('s03'); + expect(r.presentationOnly).toBe(true); + expect(store.sceneSync('s03').state).toBe('card_ahead'); + }); +}); diff --git a/packages/core/src/engine/pipeline.ts b/packages/core/src/engine/pipeline.ts index 67c765f..f92400e 100644 --- a/packages/core/src/engine/pipeline.ts +++ b/packages/core/src/engine/pipeline.ts @@ -448,8 +448,10 @@ export class Pipeline { const ex = await runExtraction(this.store, this.agents, sceneId); if (!ex.diff.graphLevel) { applyExtraction(this.store, ex); + // Only the prose side is reconciled. If the card had moved too, the scene stays + // card_ahead: the prose still needs to follow the card. const s = this.store.scene(sceneId); - if (sceneSync(s).state !== 'in_sync') this.store.markInSync(sceneId); + if (sceneSync(s).state === 'prose_ahead') this.store.markInSync(sceneId); await this.store.commit(`Edit prose of ${sceneId} (wording only)`); return { presentationOnly: true, extraction: ex }; } diff --git a/packages/core/src/engine/ripple.ts b/packages/core/src/engine/ripple.ts index f4dbe1e..18de149 100644 --- a/packages/core/src/engine/ripple.ts +++ b/packages/core/src/engine/ripple.ts @@ -1,3 +1,4 @@ +import { randomBytes } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import type { AgentSuite } from '../agents/contracts.js'; @@ -113,7 +114,8 @@ export class Ripple { await store.checkpoint('Work in progress before a ripple', { exclude: files }); const originCommit = await store.commit(`Edit: ${impact.description}`, { files }); const r = new Ripple(store, agents, { - id: timestamp(), + // Second-granular timestamps collide when two ripples start in the same second. + id: `${timestamp()}-${randomBytes(2).toString('hex')}`, created: new Date().toISOString(), description: impact.description, change, @@ -199,9 +201,11 @@ export class Ripple { /** Ask the reviser for each pending card, in story order (or just one scene). */ async propose(scene?: string, progress: Progress = () => undefined) { const p = this.store.project; - const targets = this.round.items.filter((i) => i.kind === 'card' && i.status === 'pending' && (!scene || i.scene === scene)); + // A failed proposal (network, model refusal) is retried by proposing again. + const targets = this.round.items.filter((i) => i.kind === 'card' && (i.status === 'pending' || i.status === 'error') && (!scene || i.scene === scene)); for (const it of targets) { progress('propose', `Reviewing ${it.scene} "${it.title}"`); + it.error = null; const card = this.store.scene(it.scene).card; try { const out = await this.agents.reviseCard({ @@ -238,7 +242,8 @@ export class Ripple { const it = this.item(scene); if (it.kind === 'conflict') throw new Error(`${scene} is a conflict; resolve it first`); if (!it.card) throw new Error(`${scene} has no proposal yet`); - if (edited) it.card.after = SceneCard.parse({ ...(edited as object), id: scene }); + // An edited proposal keeps the scene's own status and lock: the reviser never changes those. + if (edited) it.card.after = SceneCard.parse({ ...(edited as object), id: scene, status: it.card.before.status, lock: it.card.before.lock }); if (!it.card.after) throw new Error(`${scene}: nothing to accept (no_change)`); it.status = 'accepted'; const s = this.store.scene(scene); @@ -427,12 +432,19 @@ export class Ripple { return round; } - /** Revert every commit this ripple made, including the originating edit. */ + /** + * Revert every commit this ripple made, including the originating edit, as one commit. + * All or nothing: if the author has since changed the same files, nothing is reverted + * and the error names them. + */ async revert() { const hashes = [...this.data.rounds.map((r) => r.commit).filter((h): h is string => !!h).reverse(), ...(this.data.originCommit ? [this.data.originCommit] : [])]; await this.store.checkpoint('Checkpoint before reverting ripple'); - for (const h of hashes) await this.store.repo.revert(h); - this.store.reload(); + try { + await this.store.repo.revertAll(hashes, `Revert ripple: ${this.data.description}`); + } finally { + this.store.reload(); + } if (!this.data.originCommit) { // The originating edit was committed before the ripple started: restore it from the change. const c = this.data.change; @@ -451,6 +463,25 @@ export class Ripple { let i = 0; for (const a of st.acts) for (const ch of a.chapters) ch.scenes = ch.scenes.map(() => c.before[i++]); this.store.saveStructure(st); + } else if (c.kind === 'delete-scene' || c.kind === 'character') { + // No "before" is kept in the change for these: take the files from just before the + // commit that last touched them before this ripple started. + const files = changeFiles(c).filter((f) => f !== paths.structure); + const before = await this.commitBefore(files[0]); + if (before) await this.store.repo.restore(before, files); + this.store.reload(); + if (c.kind === 'delete-scene' && before) { + const st = structuredClone(this.store.project.structure); + const chapters = st.acts.flatMap((a) => a.chapters); + const prevIdx = c.orderBefore.indexOf(c.scene) - 1; + const prev = prevIdx >= 0 ? c.orderBefore[prevIdx] : null; + const ch = (prev && chapters.find((x) => x.scenes.includes(prev))) || chapters[0]; + if (ch && !chapters.some((x) => x.scenes.includes(c.scene))) { + ch.scenes.splice(prev ? ch.scenes.indexOf(prev) + 1 : 0, 0, c.scene); + this.store.saveStructure(st); + } + } + await this.store.checkpoint(`Revert: ${this.data.description}`); } await this.store.commit(`Revert: ${this.data.description}`); } @@ -458,6 +489,18 @@ export class Ripple { this.save(); } + /** The parent of the newest commit touching `rel` made before this ripple started. */ + private async commitBefore(rel: string | undefined): Promise { + if (!rel) return null; + const started = Date.parse(this.data.created); + const log = await this.store.repo.git.raw(['log', '--format=%H %cI', '--', `:(literal)${rel}`]).catch(() => ''); + for (const line of log.split('\n').filter(Boolean)) { + const [hash, date] = line.split(' '); + if (Date.parse(date) <= started + 1000) return `${hash}~1`; + } + return null; + } + abandon() { this.data.state = 'abandoned'; this.save(); diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index 856eeda..641d96a 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -30,6 +30,13 @@ export function assertBranch(name: unknown): string { export class InvalidRefError extends Error {} +/** A revert that would overwrite later work. Nothing was changed. */ +export class RevertConflictError extends Error { + constructor(readonly files: string[]) { + super(`these files changed again since, so the revert was not applied (nothing was changed): ${files.join(', ')}`); + } +} + export interface Commit { hash: string; date: string; @@ -179,9 +186,52 @@ export class Repo { } } - async revert(hash: string) { + async head(): Promise { + return this.git.raw(['rev-parse', '--verify', '-q', 'HEAD']).then((h) => h.trim() || null, () => null); + } + + /** Revert one commit. All or nothing: on a conflict the tree is put back and this throws. */ + async revert(hash: string): Promise { assertRef(hash, 'commit'); - await this.git.revert(hash, ['--no-edit']); + const subject = (await this.git.raw(['log', '-1', '--format=%s', hash])).trim(); + return this.revertAll([hash], `Revert "${subject}"`); + } + + /** + * Revert several commits (newest first) as one new commit. All or nothing: if any of + * them conflicts with later work, the tree is restored and RevertConflictError names + * the files. The working tree must be clean (checkpoint first). + */ + async revertAll(hashes: string[], message: string): Promise { + for (const h of hashes) assertRef(h, 'commit'); + if (!hashes.length) return null; + const head = await this.head(); + if (!head) throw new Error('nothing to revert: the book has no commits'); + if ((await this.git.raw(['status', '--porcelain', '--untracked-files=no'])).trim()) { + throw new Error('commit or checkpoint pending changes before reverting'); + } + try { + await this.git.raw(['revert', '--no-commit', ...hashes]); + } catch (e) { + const conflicted = (await this.git.status()).conflicted; + await this.git.raw(['revert', '--abort']).catch(() => this.git.raw(['reset', '-q', '--hard', head])); + // Belt and braces: whatever abort did, HEAD and the tree are as they were. + if ((await this.head()) !== head) await this.git.raw(['reset', '-q', '--hard', head]); + throw new RevertConflictError(conflicted.length ? conflicted : [(e as Error).message]); + } + if (!(await this.git.raw(['diff', '--cached', '--name-only'])).trim()) { + await this.git.raw(['revert', '--quit']).catch(() => undefined); + return null; + } + const r = await this.git.commit(message); + await this.git.raw(['revert', '--quit']).catch(() => undefined); + return r.commit || null; + } + + /** Put files back as they were at a commit (e.g. a deleted scene, from before its deletion). */ + async restore(ref: string, rels: string[]) { + assertRef(ref); + if (rels.length) await this.git.raw(['checkout', ref, '--', ...rels.map((r) => `:(literal)${r}`)]); } /** Read a file as it is on another branch (for scene-by-scene comparison). */ diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 5a04543..2914034 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -24,6 +24,7 @@ import { suggestSheet, SchemaError, InvalidRefError, + RevertConflictError, sceneOrder, type Change, type ReaderReport, @@ -263,6 +264,7 @@ export class NovelServer { if (e instanceof SchemaError) return send(res, 422, { error: e.message, issues: e.issues }); if (e instanceof HttpError) return send(res, e.status, { error: e.message, data: e.data }); if (e instanceof InvalidRefError) return send(res, 400, { error: e.message }); + if (e instanceof RevertConflictError) return send(res, 409, { error: e.message, data: { files: e.files } }); // Programming errors are the server's fault; everything else is a refused request. if (e instanceof TypeError || e instanceof ReferenceError || e instanceof RangeError) { console.error(e); From 987f6c480fa69091bf696179585e7488af2c5958 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:46:50 +0000 Subject: [PATCH 04/23] Retry transient model failures and log every call's cost - Retry rate limits, overload, 5xx and dropped connections with exponential backoff (honouring retry-after), on top of the SDKs' own retries. Mid-stream failures, which the SDKs do not retry, are now retried. Refusals, bad requests, missing keys and exhausted quotas are not. - Log usage when a call fails, including tokens already billed by earlier attempts; costs.jsonl under-counted exactly the expensive failure cases. - Cache read and write prices are configurable per tier (price_cache_read, price_cache_write). Read prices range from 0.025x to 0.1x of input across current models, so a fixed 0.1x was wrong for some tiers. - OpenAI strict mode now gets optional fields as required-but-nullable, and those nulls are stripped before Zod validation so defaults apply; strict mode used to force the model to invent a value for every optional field. Anthropic structured outputs keep optional fields optional. - Only a rejection of the output schema downgrades an Anthropic schema to prompt-only JSON, not any 400 whose message mentions "format". - The Codex child process gets an allowlisted environment (home, locale, temp, proxies, CA certificates) instead of everything except two OpenAI variables. Login advice is shown only for authentication errors, and a timeout says so. - The OpenAI-compatible backend has a timeout (QUILLGON_COMPAT_TIMEOUT_MS) and typed, retryable errors; the Responses backend reuses its client and explains an incomplete response caused by the output limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- packages/agents/src/adapter/anthropic.ts | 28 +--- packages/agents/src/adapter/codex.ts | 24 ++- packages/agents/src/adapter/openai.ts | 41 +++-- packages/agents/src/adapter/responses.ts | 21 ++- packages/agents/src/adapter/schema.ts | 75 ++++++++- packages/agents/src/adapter/structured.ts | 132 +++++++++++---- packages/agents/src/adapter/types.ts | 13 ++ packages/agents/src/index.ts | 5 +- .../agents/test/adapter-resilience.test.ts | 153 ++++++++++++++++++ packages/core/src/schemas/index.ts | 3 + 10 files changed, 417 insertions(+), 78 deletions(-) create mode 100644 packages/agents/test/adapter-resilience.test.ts diff --git a/packages/agents/src/adapter/anthropic.ts b/packages/agents/src/adapter/anthropic.ts index 606c2f8..de0abb4 100644 --- a/packages/agents/src/adapter/anthropic.ts +++ b/packages/agents/src/adapter/anthropic.ts @@ -1,6 +1,9 @@ import Anthropic from '@anthropic-ai/sdk'; import type { Backend, RawRequest, RawResponse } from './types.js'; import { RefusalError } from './types.js'; +import { strictify } from './schema.js'; + +export { strictify }; /** * Claude via the official SDK. Streams (long prose outputs), uses adaptive thinking with @@ -29,7 +32,9 @@ export class AnthropicBackend implements Backend { try { return await this.send(req, native); } catch (e) { - if (native && e instanceof Anthropic.BadRequestError && /schema|output_config|format/i.test(e.message)) { + // Only a rejection of the output schema itself downgrades this schema to prompt-only + // JSON; any other 400 (a too-long prompt, say) is a real error. + if (native && e instanceof Anthropic.BadRequestError && /output_config|json_schema|\bschema\b/i.test(e.message)) { this.noNative.add(req.schemaName); return this.send(req, false); } @@ -41,7 +46,7 @@ export class AnthropicBackend implements Backend { const client = this.client(req); const outputConfig: Record = {}; if (req.config.effort) outputConfig.effort = req.config.effort; - if (native) outputConfig.format = { type: 'json_schema', schema: strictify(req.jsonSchema) }; + if (native) outputConfig.format = { type: 'json_schema', schema: strictify(req.jsonSchema, 'optional') }; const fallbackCapable = /^claude-(opus-5|fable-5|sonnet-5-5)/.test(req.config.model); const params = { model: req.config.model, @@ -75,22 +80,3 @@ export class AnthropicBackend implements Backend { }; } } - -const DROP = new Set(['minLength', 'maxLength', 'minimum', 'maximum', 'exclusiveMinimum', 'exclusiveMaximum', 'pattern', 'format', 'maxItems', 'default', '$schema', 'propertyNames']); - -/** Native structured outputs accept a JSON Schema subset; drop constraints Zod still enforces locally. */ -export function strictify(schema: unknown): unknown { - if (Array.isArray(schema)) return schema.map(strictify); - if (!schema || typeof schema !== 'object') return schema; - const out: Record = {}; - for (const [k, v] of Object.entries(schema as Record)) { - if (DROP.has(k)) continue; - if (k === 'minItems' && typeof v === 'number' && v > 1) continue; - out[k] = strictify(v); - } - if (out.type === 'object' && out.properties) { - out.additionalProperties = false; - out.required = Object.keys(out.properties as object); - } - return out; -} diff --git a/packages/agents/src/adapter/codex.ts b/packages/agents/src/adapter/codex.ts index d329a86..2e6ae67 100644 --- a/packages/agents/src/adapter/codex.ts +++ b/packages/agents/src/adapter/codex.ts @@ -4,6 +4,18 @@ import path from 'node:path'; import { Codex } from '@openai/codex-sdk'; import type { Backend, RawRequest, RawResponse } from './types.js'; import { nativeSchema } from './schema.js'; +import { BackendError } from './types.js'; + +/** + * What the Codex child process may see: enough to find its home, its saved ChatGPT login, + * temp space, locale and the network (proxies, CA certificates). Nothing else: no API keys, + * no cloud or GitHub credentials, whatever else is in the environment that launched Quillgon. + */ +const CODEX_ENV = /^(PATH|PATHEXT|HOME|USERPROFILE|HOMEDRIVE|HOMEPATH|APPDATA|LOCALAPPDATA|PROGRAMDATA|SYSTEMROOT|SYSTEMDRIVE|WINDIR|COMSPEC|TEMP|TMP|TMPDIR|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|TERM|TZ|LC_[A-Z]+|XDG_[A-Z_]+|CODEX_HOME|(HTTPS?|ALL|NO)_PROXY|SSL_CERT_(FILE|DIR)|NODE_EXTRA_CA_CERTS)$/i; + +export function codexEnv(env: NodeJS.ProcessEnv = process.env): Record { + return Object.fromEntries(Object.entries(env).filter((entry): entry is [string, string] => entry[1] !== undefined && CODEX_ENV.test(entry[0]))); +} /** Runs locally with the user's saved ChatGPT login, never with Quillgon's API key. */ export class CodexBackend implements Backend { @@ -12,7 +24,7 @@ export class CodexBackend implements Backend { async call(req: RawRequest): Promise { const workingDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'quillgon-codex-')); try { - const env = Object.fromEntries(Object.entries(process.env).filter((entry): entry is [string, string] => entry[1] !== undefined && !/^(OPENAI_API_KEY|OPENAI_BASE_URL|CODEX_API_KEY)$/i.test(entry[0]))); + const env = codexEnv(); const codex = new Codex({ env, config: { forced_login_method: 'chatgpt', model_provider: 'openai', features: { shell_tool: false, apps: false } }, @@ -35,7 +47,15 @@ export class CodexBackend implements Backend { const cached = result.usage?.cached_input_tokens ?? 0; return { text: result.finalResponse, inputTokens: (result.usage?.input_tokens ?? 0) - cached, cacheReadTokens: cached, outputTokens: result.usage?.output_tokens ?? 0, model: req.config.model || 'codex-account-default' }; } catch (error) { - throw new Error(`Codex subscription: ${(error as Error).message}. Run codex login and sign in with ChatGPT on this computer. Your plan's model access and usage limits apply; Quillgon will not fall back to API billing.`); + const message = (error as Error).message; + if ((error as Error).name === 'TimeoutError' || /aborted|timed? ?out/i.test(message)) { + throw new BackendError(`Codex subscription: no answer after 15 minutes (${message}). Quillgon will not fall back to API billing.`, false); + } + const auth = /log ?in|auth|unauthori[sz]ed|401|403|token|credential/i.test(message); + throw new BackendError( + `Codex subscription: ${message}.${auth ? ' Run codex login and sign in with ChatGPT on this computer.' : ''} Your plan's model access and usage limits apply; Quillgon will not fall back to API billing.`, + false, + ); } finally { await fs.rm(workingDirectory, { recursive: true, force: true }); } diff --git a/packages/agents/src/adapter/openai.ts b/packages/agents/src/adapter/openai.ts index a01e244..d1b8f68 100644 --- a/packages/agents/src/adapter/openai.ts +++ b/packages/agents/src/adapter/openai.ts @@ -1,4 +1,8 @@ import type { Backend, RawRequest, RawResponse } from './types.js'; +import { BackendError } from './types.js'; + +/** Local models can take minutes for a long scene; this bounds a hung request. */ +const TIMEOUT_MS = Number(process.env.QUILLGON_COMPAT_TIMEOUT_MS) || 15 * 60 * 1000; /** * Any OpenAI-compatible chat completions endpoint (another vendor for the simulated reader, @@ -8,19 +12,30 @@ export class OpenAICompatibleBackend implements Backend { readonly provider = 'openai-compatible'; async call(req: RawRequest): Promise { - const base = (req.config.base_url ?? process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1').replace(/\/$/, ''); - const key = process.env[req.config.api_key_env ?? 'OPENAI_API_KEY']; - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', ...(key ? { authorization: `Bearer ${key}` } : {}) }, - body: JSON.stringify({ - model: req.config.model, - max_tokens: req.maxTokens, - messages: [{ role: 'system', content: req.system }, ...req.messages], - response_format: { type: 'json_object' }, - }), - }); - if (!res.ok) throw new Error(`${base}: HTTP ${res.status} ${(await res.text()).slice(0, 300)}`); + const base = (req.config.base_url || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1').replace(/\/$/, ''); + const key = process.env[req.config.api_key_env || 'OPENAI_API_KEY']; + let res: Response; + try { + res = await fetch(`${base}/chat/completions`, { + method: 'POST', + signal: AbortSignal.timeout(TIMEOUT_MS), + headers: { 'content-type': 'application/json', ...(key ? { authorization: `Bearer ${key}` } : {}) }, + body: JSON.stringify({ + model: req.config.model, + max_tokens: req.maxTokens, + messages: [{ role: 'system', content: req.system }, ...req.messages], + response_format: { type: 'json_object' }, + }), + }); + } catch (e) { + // Connection refused or reset (a local server restarting) is worth a retry; a timeout is not. + const timedOut = (e as Error).name === 'TimeoutError'; + throw new BackendError(`${base}: ${timedOut ? `no response after ${Math.round(TIMEOUT_MS / 60000)} minutes` : (e as Error).message}`, !timedOut); + } + if (!res.ok) { + const retryAfter = Number(res.headers.get('retry-after')); + throw new BackendError(`${base}: HTTP ${res.status} ${(await res.text()).slice(0, 300)}`, res.status === 408 || res.status === 429 || res.status >= 500, Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : undefined, res.status); + } const body = (await res.json()) as { model?: string; choices: { message: { content: string } }[]; diff --git a/packages/agents/src/adapter/responses.ts b/packages/agents/src/adapter/responses.ts index a1462ce..3af626b 100644 --- a/packages/agents/src/adapter/responses.ts +++ b/packages/agents/src/adapter/responses.ts @@ -1,17 +1,22 @@ import OpenAI from 'openai'; import type { Backend, RawRequest, RawResponse } from './types.js'; -import { RefusalError } from './types.js'; +import { BackendError, RefusalError } from './types.js'; import { nativeSchema } from './schema.js'; /** OpenAI's native Responses API; the compatible Chat Completions adapter is separate. */ export class OpenAIBackend implements Backend { readonly provider = 'openai'; + /** One client per key and endpoint, so connections are reused across calls. */ + private clients = new Map(); async call(req: RawRequest): Promise { const variable = req.config.api_key_env || 'OPENAI_API_KEY'; const apiKey = process.env[variable]; if (!apiKey) throw new Error(`Set ${variable} in the environment that launches Quillgon, then restart. Or select Codex subscription in Settings → Models.`); - const client = new OpenAI({ apiKey, baseURL: req.config.base_url || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1' }); + const baseURL = req.config.base_url || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'; + const key = `${baseURL}|${variable}|${apiKey.length}:${apiKey.slice(-6)}`; + let client = this.clients.get(key); + if (!client) this.clients.set(key, (client = new OpenAI({ apiKey, baseURL }))); const schema = nativeSchema(req.jsonSchema); const response = await client.responses.create({ model: req.config.model, @@ -26,14 +31,20 @@ export class OpenAIBackend implements Backend { let final: OpenAI.Responses.Response | undefined; for await (const event of response) { if (event.type === 'response.completed' || event.type === 'response.incomplete' || event.type === 'response.failed') final = event.response; - if (event.type === 'error') throw new Error(`OpenAI: ${event.message}`); + // A failure mid-stream is not retried by the SDK; the adapter retries it. + if (event.type === 'error') throw new BackendError(`OpenAI: ${event.message}`, true); } - if (!final) throw new Error('OpenAI response ended before completion. Try again.'); + if (!final) throw new BackendError('OpenAI response ended before completion.', true); for (const item of final.output) { if (item.type !== 'message') continue; for (const content of item.content) if (content.type === 'refusal') throw new RefusalError(null, content.refusal); } - if (final.status !== 'completed') throw new Error(`OpenAI response ${final.status}: ${final.error?.message ?? final.incomplete_details?.reason ?? 'try again'}`); + if (final.status === 'incomplete') { + // Usually the output limit: retrying the same request would fail the same way. + const reason = final.incomplete_details?.reason ?? 'unknown reason'; + throw new BackendError(`OpenAI response incomplete (${reason})${reason === 'max_output_tokens' ? ': the scene needs more output tokens than allowed; lower the effort for this tier or shorten the scene' : ''}`, false); + } + if (final.status !== 'completed') throw new BackendError(`OpenAI response ${final.status}: ${final.error?.message ?? 'try again'}`, final.status === 'failed'); const cached = final.usage?.input_tokens_details.cached_tokens ?? 0; return { text: final.output.filter((item) => item.type === 'message').flatMap((item) => item.content).filter((content) => content.type === 'output_text').map((content) => content.text).join(''), diff --git a/packages/agents/src/adapter/schema.ts b/packages/agents/src/adapter/schema.ts index 8f6b4df..1ad96c2 100644 --- a/packages/agents/src/adapter/schema.ts +++ b/packages/agents/src/adapter/schema.ts @@ -1,7 +1,44 @@ -import { strictify } from './anthropic.js'; +/** + * Native structured output takes a JSON Schema subset. Two dialects: + * - 'optional' (Anthropic): every object closed with additionalProperties: false; + * optional properties stay optional. + * - 'nullable' (OpenAI strict mode, Codex): every property must be listed as required, + * so an optional property becomes required-but-nullable. Without this, strict mode makes + * the model invent a value for every optional field. stripInventedNulls removes the + * nulls again before Zod validation, so Zod defaults apply. + * Constraints Zod still enforces locally (lengths, ranges, patterns) are dropped. + */ +export type SchemaDialect = 'optional' | 'nullable'; + +const DROP = new Set(['minLength', 'maxLength', 'minimum', 'maximum', 'exclusiveMinimum', 'exclusiveMaximum', 'multipleOf', 'pattern', 'format', 'maxItems', 'default', '$schema', 'propertyNames']); + +export function strictify(schema: unknown, dialect: SchemaDialect = 'optional'): unknown { + if (Array.isArray(schema)) return schema.map((s) => strictify(s, dialect)); + if (!schema || typeof schema !== 'object') return schema; + const out: Record = {}; + for (const [k, v] of Object.entries(schema as Record)) { + if (DROP.has(k)) continue; + if (k === 'minItems' && typeof v === 'number' && v > 1) continue; + out[k] = strictify(v, dialect); + } + if (out.type === 'object' && out.properties) { + out.additionalProperties = false; + const props = out.properties as Record; + const required = new Set(Array.isArray(out.required) ? (out.required as string[]) : []); + if (dialect === 'nullable') { + for (const key of Object.keys(props)) if (!required.has(key) && !acceptsNull(props[key])) props[key] = { anyOf: [props[key], { type: 'null' }] }; + out.required = Object.keys(props); + } else if (required.size) { + out.required = [...required]; + } else { + delete out.required; + } + } + return out; +} /** Arbitrary-key records cannot be closed without losing their keys. Keep Zod validation. */ -export function nativeSchema(schema: Record): Record | undefined { +export function nativeSchema(schema: Record, dialect: SchemaDialect = 'nullable'): Record | undefined { function hasRecord(value: unknown): boolean { if (!value || typeof value !== 'object') return false; if (Array.isArray(value)) return value.some(hasRecord); @@ -9,5 +46,37 @@ export function nativeSchema(schema: Record): Record; + return hasRecord(schema) ? undefined : (strictify(schema, dialect) as Record); +} + +function acceptsNull(s: unknown): boolean { + if (!s || typeof s !== 'object') return false; + const n = s as Record; + if (n.type === 'null' || (Array.isArray(n.type) && n.type.includes('null')) || n.const === null) return true; + if (Array.isArray(n.enum) && n.enum.includes(null)) return true; + return [n.anyOf, n.oneOf].some((xs) => Array.isArray(xs) && xs.some(acceptsNull)); +} + +/** + * Drop `null` from properties the original schema marks optional and not nullable: those + * nulls only mean "absent" (see the 'nullable' dialect). Everything else is left alone. + */ +export function stripInventedNulls(value: unknown, schema: unknown): unknown { + if (!schema || typeof schema !== 'object' || value === null || typeof value !== 'object') return value; + const s = schema as Record; + if (Array.isArray(value)) return s.items ? value.map((v) => stripInventedNulls(v, s.items)) : value; + const branches = [s.anyOf, s.oneOf].find(Array.isArray) as unknown[] | undefined; + if (!s.properties && branches) { + const obj = branches.find((b) => !!b && typeof b === 'object' && (b as Record).properties); + return obj ? stripInventedNulls(value, obj) : value; + } + const props = s.properties as Record | undefined; + if (!props) return value; + const required = new Set(Array.isArray(s.required) ? (s.required as string[]) : []); + const out: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + if (v === null && !required.has(k) && props[k] && !acceptsNull(props[k])) continue; + out[k] = stripInventedNulls(v, props[k]); + } + return out; } diff --git a/packages/agents/src/adapter/structured.ts b/packages/agents/src/adapter/structured.ts index d39a0dc..ace535b 100644 --- a/packages/agents/src/adapter/structured.ts +++ b/packages/agents/src/adapter/structured.ts @@ -1,7 +1,8 @@ import { z, type ZodType } from 'zod'; import { DEFAULT_MODELS, type Book, type ModelConfig, type ModelTier } from '@quillgon/core'; -import type { Backend, Message, UsageRecord } from './types.js'; -import { SchemaFailure } from './types.js'; +import type { Backend, Message, RawRequest, RawResponse, UsageRecord } from './types.js'; +import { BackendError, RefusalError, SchemaFailure } from './types.js'; +import { stripInventedNulls } from './schema.js'; export interface CompleteOptions { agent: string; @@ -49,17 +50,72 @@ export function formatZodError(e: z.ZodError): string { .join('\n'); } +const TRANSIENT_STATUS = new Set([408, 409, 425, 429, 500, 502, 503, 504, 529]); +const TRANSIENT_CODES = new Set(['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE', 'EAI_AGAIN', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']); + +/** + * Worth retrying? Rate limits, overload, server errors and dropped connections are; a + * refusal, a bad request, a missing key or an exhausted quota are not. + */ +export function isTransient(e: unknown): boolean { + if (e instanceof RefusalError || e instanceof SchemaFailure) return false; + if (e instanceof BackendError) return e.transient; + const err = e as { status?: number; code?: string; name?: string; cause?: { code?: string } }; + if (typeof err?.status === 'number') return TRANSIENT_STATUS.has(err.status) || err.status >= 500; + if (err?.name === 'APIConnectionError' || err?.name === 'APIConnectionTimeoutError') return true; + const code = err?.code ?? err?.cause?.code; + return !!code && TRANSIENT_CODES.has(code); +} + +function retryAfterMs(e: unknown): number | undefined { + if (e instanceof BackendError) return e.retryAfterMs; + const h = (e as { headers?: unknown })?.headers; + const raw = h && typeof (h as Headers).get === 'function' ? (h as Headers).get('retry-after') : (h as Record | undefined)?.['retry-after']; + const secs = raw ? Number(raw) : NaN; + return Number.isFinite(secs) ? secs * 1000 : undefined; +} + +export interface AdapterOptions { + /** Extra attempts after a transient failure (the provider SDKs already retry twice). */ + networkRetries?: number; + /** First backoff delay; doubles each time, with jitter, capped at a minute. */ + backoffMs?: number; + sleep?: (ms: number) => Promise; +} + /** * The one adapter the agents use: complete(messages, schema, options) -> parsed object. * Validates with Zod; on failure re-prompts with the validation error (max 2 retries). - * Logs model, tokens and cost for every call. + * Retries transient backend failures with backoff. Logs model, tokens and cost for every + * call, failed ones included, so the cost report counts what was actually spent. */ export class ModelAdapter { + private retries: number; + private backoff: number; + private sleep: (ms: number) => Promise; + constructor( private backends: Record, private book: () => Book, private onUsage: (u: UsageRecord) => void = () => undefined, - ) {} + opts: AdapterOptions = {}, + ) { + this.retries = opts.networkRetries ?? 2; + this.backoff = opts.backoffMs ?? 2000; + this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))); + } + + private async call(backend: Backend, req: RawRequest): Promise { + for (let n = 0; ; n++) { + try { + return await backend.call(req); + } catch (e) { + if (n >= this.retries || !isTransient(e)) throw e; + const base = Math.min(60_000, this.backoff * 4 ** n); + await this.sleep(Math.min(120_000, retryAfterMs(e) ?? base * (0.75 + Math.random() * 0.5))); + } + } + } configFor(tier: ModelTier): ModelConfig { return { ...DEFAULT_MODELS[tier], ...(this.book().models[tier] ?? {}) } as ModelConfig; @@ -79,41 +135,52 @@ export class ModelAdapter { let cacheW = 0; let cacheR = 0; let model = config.model; - for (let attempt = 1; attempt <= 3; attempt++) { - const res = await backend.call({ config, system, messages, jsonSchema, schemaName: opts.schemaName, maxTokens: opts.maxTokens ?? 32000 }); - inTok += res.inputTokens; - outTok += res.outputTokens; - cacheW += res.cacheWriteTokens ?? 0; - cacheR += res.cacheReadTokens ?? 0; - model = res.model; - lastText = res.text; - let parsed: unknown; - try { - parsed = extractJson(res.text); - } catch (e) { - lastError = (e as Error).message; - messages.push({ role: 'assistant', content: res.text }, { role: 'user', content: `That was not valid JSON (${lastError}). Reply again with only the JSON object.` }); - continue; - } - const r = opts.schema.safeParse(parsed); - if (r.success) { - this.log(opts, config, model, inTok, outTok, attempt, true, cacheW, cacheR); - return r.data; + let attempt = 0; + try { + for (attempt = 1; attempt <= 3; attempt++) { + const res = await this.call(backend, { config, system, messages, jsonSchema, schemaName: opts.schemaName, maxTokens: opts.maxTokens ?? 32000 }); + inTok += res.inputTokens; + outTok += res.outputTokens; + cacheW += res.cacheWriteTokens ?? 0; + cacheR += res.cacheReadTokens ?? 0; + model = res.model; + lastText = res.text; + let parsed: unknown; + try { + parsed = stripInventedNulls(extractJson(res.text), jsonSchema); + } catch (e) { + lastError = (e as Error).message; + messages.push({ role: 'assistant', content: res.text }, { role: 'user', content: `That was not valid JSON (${lastError}). Reply again with only the JSON object.` }); + continue; + } + const r = opts.schema.safeParse(parsed); + if (r.success) { + this.log(opts, config, model, inTok, outTok, attempt, true, cacheW, cacheR); + return r.data; + } + lastError = formatZodError(r.error); + messages.push( + { role: 'assistant', content: res.text }, + { role: 'user', content: `Your JSON did not validate:\n${lastError}\n\nReply again with the corrected JSON object only. Change nothing else.` }, + ); } - lastError = formatZodError(r.error); - messages.push( - { role: 'assistant', content: res.text }, - { role: 'user', content: `Your JSON did not validate:\n${lastError}\n\nReply again with the corrected JSON object only. Change nothing else.` }, - ); + } catch (e) { + // A backend failure part-way (after earlier attempts were billed) is still logged. + this.log(opts, config, model, inTok, outTok, Math.max(attempt, 1), false, cacheW, cacheR, (e as Error).message); + throw e; } - this.log(opts, config, model, inTok, outTok, 3, false, cacheW, cacheR); + this.log(opts, config, model, inTok, outTok, 3, false, cacheW, cacheR, lastError); throw new SchemaFailure(opts.agent, 3, lastError, lastText); } - private log(opts: CompleteOptions, config: ModelConfig, model: string, inTok: number, outTok: number, attempts: number, ok: boolean, cacheW = 0, cacheR = 0) { + private log(opts: CompleteOptions, config: ModelConfig, model: string, inTok: number, outTok: number, attempts: number, ok: boolean, cacheW = 0, cacheR = 0, error?: string) { const pin = config.price_in ?? 0; + // Cache prices differ by model (reads are 0.025x to 0.1x input): configurable per tier, + // with the common 1.25x write / 0.1x read as the fallback. + const pcw = config.price_cache_write ?? pin * 1.25; + const pcr = config.price_cache_read ?? pin * 0.1; // Subscription use has no per-token API charge; quota use is still logged. - const cost = config.provider === 'codex' ? 0 : (inTok * pin + cacheW * pin * 1.25 + cacheR * pin * 0.1 + outTok * (config.price_out ?? 0)) / 1e6; + const cost = config.provider === 'codex' ? 0 : (inTok * pin + cacheW * pcw + cacheR * pcr + outTok * (config.price_out ?? 0)) / 1e6; this.onUsage({ ts: new Date().toISOString(), agent: opts.agent, @@ -127,6 +194,7 @@ export class ModelAdapter { cost_usd: Math.round(cost * 1e6) / 1e6, attempts, ok, + ...(error ? { error: error.slice(0, 500) } : {}), }); } } diff --git a/packages/agents/src/adapter/types.ts b/packages/agents/src/adapter/types.ts index c08c82d..f62e451 100644 --- a/packages/agents/src/adapter/types.ts +++ b/packages/agents/src/adapter/types.ts @@ -45,6 +45,8 @@ export interface UsageRecord { cost_usd: number; attempts: number; ok: boolean; + /** Why the call failed, when ok is false. */ + error?: string; } export class RefusalError extends Error { @@ -60,3 +62,14 @@ export class SchemaFailure extends Error { this.name = 'SchemaFailure'; } } + +/** + * A backend failure. `transient` failures (rate limits, overload, a dropped stream) are + * worth retrying; the adapter does so with backoff. `retryAfterMs` honours the provider. + */ +export class BackendError extends Error { + constructor(message: string, public transient: boolean, public retryAfterMs?: number, public status?: number) { + super(message); + this.name = 'BackendError'; + } +} diff --git a/packages/agents/src/index.ts b/packages/agents/src/index.ts index 37136c0..69221cd 100644 --- a/packages/agents/src/index.ts +++ b/packages/agents/src/index.ts @@ -1,9 +1,10 @@ export * from './adapter/types.js'; export * from './adapter/structured.js'; -export { AnthropicBackend, strictify } from './adapter/anthropic.js'; +export { AnthropicBackend } from './adapter/anthropic.js'; +export { nativeSchema, strictify, stripInventedNulls, type SchemaDialect } from './adapter/schema.js'; export { OpenAICompatibleBackend } from './adapter/openai.js'; export { OpenAIBackend } from './adapter/responses.js'; -export { CodexBackend } from './adapter/codex.js'; +export { CodexBackend, codexEnv } from './adapter/codex.js'; export { RecordReplayBackend, requestKey } from './adapter/record.js'; export * from './agents/index.js'; export { ModelSuite, type SuiteContext } from './suite.js'; diff --git a/packages/agents/test/adapter-resilience.test.ts b/packages/agents/test/adapter-resilience.test.ts new file mode 100644 index 0000000..4958969 --- /dev/null +++ b/packages/agents/test/adapter-resilience.test.ts @@ -0,0 +1,153 @@ +import http from 'node:http'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { Book } from '@quillgon/core'; +import { BackendError, codexEnv, isTransient, ModelAdapter, OpenAICompatibleBackend, RefusalError, strictify, stripInventedNulls, type Backend, type RawResponse, type UsageRecord } from '../src/index.js'; + +const book = Book.parse({ title: 'T' }); +const Out = z.object({ answer: z.string() }); +const ok = (text = '{"answer":"yes"}'): RawResponse => ({ text, inputTokens: 1000, outputTokens: 200, model: 'm' }); + +/** A backend that plays a script: a response, or an error to throw. */ +function scripted(steps: (RawResponse | Error)[]): Backend & { calls: number } { + const b = { + provider: 'openai', + calls: 0, + async call() { + b.calls++; + const s = steps.shift(); + if (!s) throw new Error('script exhausted'); + if (s instanceof Error) throw s; + return s; + }, + }; + return b; +} + +function adapter(b: Backend, usage: UsageRecord[] = [], sleeps: number[] = [], b2: Book = book) { + return new ModelAdapter({ openai: b, 'openai-compatible': b }, () => b2, (u) => usage.push(u), { sleep: async (ms) => void sleeps.push(ms), backoffMs: 1000 }); +} +const ask = { agent: 'writer', tier: 'frontier' as const, system: 'S', messages: [], schema: Out, schemaName: 'writer' }; + +describe('model adapter resilience', () => { + it('retries a transient failure with backoff, then succeeds', async () => { + const usage: UsageRecord[] = []; + const sleeps: number[] = []; + const b = scripted([new BackendError('overloaded', true), ok()]); + expect(await adapter(b, usage, sleeps).complete(ask)).toEqual({ answer: 'yes' }); + expect(b.calls).toBe(2); + expect(sleeps).toHaveLength(1); + expect(sleeps[0]).toBeGreaterThanOrEqual(750); + expect(usage).toEqual([expect.objectContaining({ ok: true, attempts: 1 })]); + }); + + it("honours the provider's retry-after", async () => { + const sleeps: number[] = []; + const rateLimited = Object.assign(new Error('429'), { status: 429, headers: new Headers({ 'retry-after': '7' }) }); + await adapter(scripted([rateLimited, ok()]), [], sleeps).complete(ask); + expect(sleeps).toEqual([7000]); + }); + + it('gives up after the retry budget and logs the failure', async () => { + const usage: UsageRecord[] = []; + const b = scripted([new BackendError('503', true), new BackendError('503', true), new BackendError('503', true)]); + await expect(adapter(b, usage).complete(ask)).rejects.toThrow('503'); + expect(b.calls).toBe(3); + expect(usage).toEqual([expect.objectContaining({ ok: false, error: '503' })]); + }); + + it('logs the tokens already spent when a later attempt fails', async () => { + const usage: UsageRecord[] = []; + // Attempt 1 is billed but invalid; attempt 2 hits a hard error. + const b = scripted([ok('not json'), Object.assign(new Error('Bad request'), { status: 400 })]); + await expect(adapter(b, usage).complete(ask)).rejects.toThrow('Bad request'); + expect(b.calls).toBe(2); + expect(usage).toEqual([expect.objectContaining({ ok: false, attempts: 2, input_tokens: 1000, output_tokens: 200, error: 'Bad request' })]); + expect(usage[0].cost_usd).toBeGreaterThan(0); + }); + + it('never retries a refusal', async () => { + const sleeps: number[] = []; + const b = scripted([new RefusalError('cyber', 'declined'), ok()]); + await expect(adapter(b, [], sleeps).complete(ask)).rejects.toBeInstanceOf(RefusalError); + expect(b.calls).toBe(1); + expect(sleeps).toEqual([]); + }); + + it('classifies errors', () => { + expect(isTransient(Object.assign(new Error(), { status: 529 }))).toBe(true); + expect(isTransient(Object.assign(new Error(), { status: 401 }))).toBe(false); + expect(isTransient(Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } }))).toBe(true); + expect(isTransient(new Error('Set OPENAI_API_KEY'))).toBe(false); + }); + + it('prices cache tokens per tier when configured', async () => { + const usage: UsageRecord[] = []; + const priced = Book.parse({ title: 'T', models: { frontier: { provider: 'openai', model: 'm', price_in: 10, price_out: 50, price_cache_read: 0.25, price_cache_write: 12.5 } } }); + const b = scripted([{ ...ok(), inputTokens: 0, outputTokens: 0, cacheReadTokens: 1_000_000, cacheWriteTokens: 1_000_000 }]); + await adapter(b, usage, [], priced).complete(ask); + expect(usage[0].cost_usd).toBeCloseTo(12.75); + }); +}); + +describe('structured output schemas', () => { + const S = z.object({ a: z.string(), b: z.string().optional(), c: z.string().nullable(), d: z.number().default(3) }); + const json = z.toJSONSchema(S, { io: 'input' }) as Record; + + it('OpenAI strict mode: every field required, optional ones nullable', () => { + const s = strictify(json, 'nullable') as any; + expect(s.required).toEqual(['a', 'b', 'c', 'd']); + expect(s.properties.b).toEqual({ anyOf: [{ type: 'string' }, { type: 'null' }] }); + expect(s.properties.d.anyOf).toContainEqual({ type: 'null' }); + expect(s.properties.c).toEqual(json.properties.c); // already nullable + expect(s.additionalProperties).toBe(false); + }); + + it('Anthropic: optional fields stay optional', () => { + const s = strictify(json, 'optional') as any; + expect(s.required).toEqual(['a', 'c']); + expect(s.properties.b).toEqual({ type: 'string' }); + expect(s.additionalProperties).toBe(false); + }); + + it('nulls that only mean "absent" are dropped before validation, so defaults apply', () => { + const parsed = stripInventedNulls({ a: 'x', b: null, c: null, d: null }, json); + expect(parsed).toEqual({ a: 'x', c: null }); + expect(S.parse(parsed)).toEqual({ a: 'x', c: null, d: 3 }); + }); + + it('end to end through the adapter', async () => { + const b = scripted([ok('{"a":"x","b":null,"c":null,"d":null}')]); + const r = await adapter(b).complete({ ...ask, schema: S, schemaName: 's' }); + expect(r).toEqual({ a: 'x', c: null, d: 3 }); + }); +}); + +describe('backends', () => { + it('the Codex child sees only what it needs from the environment', () => { + const env = codexEnv({ PATH: '/bin', HOME: '/h', HTTPS_PROXY: 'http://p', LC_ALL: 'C', ANTHROPIC_API_KEY: 'x', OPENAI_API_KEY: 'x', CODEX_API_KEY: 'x', AWS_SECRET_ACCESS_KEY: 'x', GITHUB_TOKEN: 'x' }); + expect(Object.keys(env).sort()).toEqual(['HOME', 'HTTPS_PROXY', 'LC_ALL', 'PATH']); + }); + + it('the compatible backend marks overload as transient and passes retry-after on', async () => { + const srv = http.createServer((_req, res) => res.writeHead(503, { 'retry-after': '2' }).end('busy')); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + const port = (srv.address() as { port: number }).port; + try { + const err = await new OpenAICompatibleBackend() + .call({ config: { provider: 'openai-compatible', model: 'local', base_url: `http://127.0.0.1:${port}/v1` }, system: 'S', messages: [], jsonSchema: {}, schemaName: 's', maxTokens: 10 }) + .catch((e) => e); + expect(err).toBeInstanceOf(BackendError); + expect(err).toMatchObject({ transient: true, retryAfterMs: 2000, status: 503 }); + } finally { + srv.close(); + } + }); + + it('the compatible backend treats a refused connection as transient', async () => { + const err = await new OpenAICompatibleBackend() + .call({ config: { provider: 'openai-compatible', model: 'local', base_url: 'http://127.0.0.1:1/v1' }, system: 'S', messages: [], jsonSchema: {}, schemaName: 's', maxTokens: 10 }) + .catch((e) => e); + expect(err).toMatchObject({ transient: true }); + }); +}); diff --git a/packages/core/src/schemas/index.ts b/packages/core/src/schemas/index.ts index 64324bf..e8f2573 100644 --- a/packages/core/src/schemas/index.ts +++ b/packages/core/src/schemas/index.ts @@ -38,6 +38,9 @@ export const ModelConfig = z.object({ /** USD per million tokens, used for cost logging and `novel estimate`. */ price_in: z.number().optional(), price_out: z.number().optional(), + /** USD per million prompt-cache tokens. Default to 1.25x (write) and 0.1x (read) of price_in. */ + price_cache_write: z.number().optional(), + price_cache_read: z.number().optional(), }); export type ModelConfig = z.infer; From ccc7dc7318ed109239b9f17e4223461ca8954671 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:53:28 +0000 Subject: [PATCH 05/23] Keep prompts bounded as the book grows Measured with a new scale check (pnpm scale-check: the sample grown to 96 scenes and 409 facts), the extractor, card planner and card reviser each sent the whole fact glossary (about 12k tokens at that size, growing with every extraction), and the simulated reader sent its entire log to every scene, so a reader run cost 743k input tokens and grew quadratically. At the last scene, approx. tokens, before -> after: writer 8,310 -> 5,900 extractor 11,969 -> 3,018 card planner 12,744 -> 3,804 card reviser 11,368 -> 2,700 reader 14,996 -> 1,372 (whole run 743k -> 132k) - Facts: once the glossary passes 80, send every fact the task names plus the most similar ones (local embeddings); the planner prefers facts no scene has revealed yet. Small books still get the whole glossary. - Reader: the latest six log entries in full, plus the predictions and still open confusions from older scenes. - Writer: a deterministic "story so far" from chapter summaries and earlier outcomes (bounded, oldest chapters dropped first), and style samples clipped to 6,000 characters. - One words-per-scene rule for the planner and the writer; the writer's target ballooned when chapters were carded one at a time. - New scene numbers ignore id suffixes ("s04-2" was read as 42). - The cost estimate measures the writer's real context, prices the line editor as span edits rather than a whole-scene rewrite, and counts the reader window. - `pnpm scale-check --max-prompt N` fails when any prompt exceeds N tokens. Store writes stay at 9 ms (96 scenes) and 22 ms (300 scenes), so the index is left as a full refresh per write. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- package.json | 3 +- packages/agents/src/agents/reader/index.ts | 22 +++- packages/agents/src/agents/writer/index.ts | 1 + packages/agents/test/scale.test.ts | 85 ++++++++++++++ packages/core/src/agents/contracts.ts | 8 ++ packages/core/src/context.ts | 7 +- packages/core/src/engine/estimate.ts | 16 ++- packages/core/src/engine/extraction.ts | 13 ++- packages/core/src/engine/pipeline.ts | 17 ++- packages/core/src/engine/ripple.ts | 33 ++++-- packages/core/src/index.ts | 1 + packages/core/src/quality/reader.ts | 5 +- packages/core/src/scope.ts | 129 +++++++++++++++++++++ scripts/scale-check.ts | 113 ++++++++++++++++++ 14 files changed, 420 insertions(+), 33 deletions(-) create mode 100644 packages/agents/test/scale.test.ts create mode 100644 packages/core/src/scope.ts create mode 100644 scripts/scale-check.ts diff --git a/package.json b/package.json index 27ec5d0..10d0c39 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "test:ui-refresh": "pnpm build && tsx scripts/ui-refresh.ts", "site:build": "node apps/site/build.mjs", "site:check": "node apps/site/check.mjs", - "site:dev": "node apps/site/serve.mjs" + "site:dev": "node apps/site/serve.mjs", + "scale-check": "tsx scripts/scale-check.ts" }, "devDependencies": { "@quillgon/agents": "workspace:*", diff --git a/packages/agents/src/agents/reader/index.ts b/packages/agents/src/agents/reader/index.ts index a17336a..256f4ef 100644 --- a/packages/agents/src/agents/reader/index.ts +++ b/packages/agents/src/agents/reader/index.ts @@ -6,13 +6,24 @@ export const reader = defineAgent({ task: 'read', input: ReaderInput, output: ReaderOutput, - render: (i) => - [ + render: (i) => { + // With a long book only the latest entries are shown; number them by their real scene. + const first = i.scene_number - i.log.length; + const e = i.earlier; + return [ + e + ? `## What you remember from the first ${e.scenes_read} scenes\n${[ + e.predictions.length ? `Predictions you made: ${e.predictions.join(' | ')}` : 'No predictions yet.', + e.open_confusions.length ? `Still unclear: ${e.open_confusions.map((c) => `[${c.id}] ${c.text}`).join(' ')}` : '', + ] + .filter(Boolean) + .join('\n')}` + : '', i.log.length - ? `## Your reading log so far\n${i.log + ? `## Your reading log${e ? ', latest scenes' : ' so far'}\n${i.log .map( (e, n) => - `Scene ${n + 1}: believed: ${e.believes} Expected: ${e.expects} Engagement ${e.engagement}/5.${e.confusions.length ? ` Confused by: ${e.confusions.map((c) => `[${c.id}] ${c.text}`).join(' ')}` : ''}${e.predictions.length ? ` Predicted: ${e.predictions.join(' | ')}` : ''}`, + `Scene ${first + n}: believed: ${e.believes} Expected: ${e.expects} Engagement ${e.engagement}/5.${e.confusions.length ? ` Confused by: ${e.confusions.map((c) => `[${c.id}] ${c.text}`).join(' ')}` : ''}${e.predictions.length ? ` Predicted: ${e.predictions.join(' | ')}` : ''}`, ) .join('\n')}` : '## Your reading log so far\n(This is the first scene.)', @@ -20,5 +31,6 @@ export const reader = defineAgent({ i.fairness_question ? `## One question\n${i.fairness_question}` : '', ] .filter(Boolean) - .join('\n\n'), + .join('\n\n'); + }, }); diff --git a/packages/agents/src/agents/writer/index.ts b/packages/agents/src/agents/writer/index.ts index 4f78d50..2a6555b 100644 --- a/packages/agents/src/agents/writer/index.ts +++ b/packages/agents/src/agents/writer/index.ts @@ -44,6 +44,7 @@ export const writer = defineAgent({ .join('\n\n')}`, ); if (i.threads.length) parts.push(`## Threads this scene touches\n${i.threads.map((t) => `- ${t.title || t.id} (${t.kind}): ${t.description}`).join('\n')}`); + if (i.story_so_far) parts.push(`## The story so far\n${i.story_so_far}`); if (i.recent_cards.length) parts.push(`## The scenes just before (cards)\n${i.recent_cards.map((r) => `- ${r.id} ${r.title}: ${r.outcome}`).join('\n')}`); if (i.previous_prose) parts.push(`## The previous scene, in full\n${i.previous_prose}`); const ob = i.obligations; diff --git a/packages/agents/test/scale.test.ts b/packages/agents/test/scale.test.ts new file mode 100644 index 0000000..f9073b5 --- /dev/null +++ b/packages/agents/test/scale.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest'; +import { BUDGET, estimate, extractInput, readerWindow, relevantFacts, sceneOrder, storySoFar, wordsPerScene, writerContext, type ReaderLogEntry } from '@quillgon/core'; +import { fixtureEngine } from './helpers.js'; + +const entry = (scene: string, extra: Partial = {}): ReaderLogEntry => ({ scene, believes: 'b', expects: 'e', confusions: [], resolved_confusions: [], engagement: 3, predictions: [], fairness: null, ...extra }); + +/** Prompts grow with the scene, not with the book. */ +describe('context budgets', () => { + it('a small glossary is sent whole; a large one is cut to what the task names and what is relevant', async () => { + const { store } = await fixtureEngine(); + const p = store.project; + expect(relevantFacts(p, { query: 'anything' })).toEqual(p.factGlossary.facts); + store.addFacts(Array.from({ length: 300 }, (_, i) => ({ id: `noise-${i}`, statement: `Weather report ${i}: wind and rain over the moor.` }))); + const facts = relevantFacts(store.project, { must: ['walter-letter'], query: 'Eliza holds the debts of the boats' }); + expect(facts.length).toBe(BUDGET.facts); + expect(facts.map((f) => f.id)).toEqual(expect.arrayContaining(['walter-letter', 'eliza-holds-debts'])); + // The extractor always sees the facts the scene's card and cast already carry. + const ex = extractInput(store, 's06'); + expect(ex.facts.length).toBeLessThanOrEqual(BUDGET.facts); + const card = store.scene('s06').card; + for (const f of [...card.reveals, ...card.mentions.map((m) => m.fact)]) expect(ex.facts.map((x) => x.id)).toContain(f); + }); + + it("the writer gets the story so far, oldest chapters dropped first when it's long", async () => { + const { store } = await fixtureEngine(); + const p = store.project; + const last = sceneOrder(p).at(-1)!; + const sofar = storySoFar(p, last); + const chapters = p.structure.acts.flatMap((a) => a.chapters); + expect(sofar).toContain(chapters[0].title); + expect(writerContext(p, last, null).story_so_far).toBe(sofar); + const short = storySoFar(p, last, 200); + expect(short.length).toBeLessThanOrEqual(220); + expect(short).toContain('earlier chapters omitted'); + expect(short).toContain(chapters.at(-2)!.title); + expect(storySoFar(p, sceneOrder(p)[0])).toBe(''); + }); + + it('long style samples are clipped for the writer', async () => { + const { store } = await fixtureEngine(); + store.saveStyle({ samples: 'A paragraph of the voice.\n\n'.repeat(2000), banned: '' }); + expect(writerContext(store.project, 's03', null).style_samples.length).toBeLessThanOrEqual(BUDGET.styleSamples + 10); + }); + + it("the reader keeps the latest entries and only what still matters from older ones", () => { + const log = Array.from({ length: 20 }, (_, i) => entry(`s${i}`, { predictions: i === 2 ? ['Eliza forged it'] : [], confusions: i === 3 ? [{ id: 'c1', text: 'who is Jory?' }] : i === 4 ? [{ id: 'c2', text: 'why the storm?' }] : [], resolved_confusions: i === 15 ? ['c2'] : [] })); + const { recent, earlier } = readerWindow(log); + expect(recent.map((e) => e.scene)).toEqual(['s14', 's15', 's16', 's17', 's18', 's19']); + expect(earlier).toEqual({ scenes_read: 14, predictions: ['Eliza forged it'], open_confusions: [{ id: 'c1', text: 'who is Jory?' }] }); + expect(readerWindow(log.slice(0, 4)).earlier).toBeNull(); + }); + + it('runs the reader with a bounded log', async () => { + const seen: number[] = []; + const { engine } = await fixtureEngine({ + read: async (i) => { + seen.push(i.log.length); + return { believes: 'b', expects: 'e', confusions: [], resolved_confusions: [], engagement: 3, predictions: [], fairness: null }; + }, + }); + await engine.reader({}); + expect(Math.max(...seen)).toBe(BUDGET.readerWindow); + expect(seen).toHaveLength(12); + }); + + it('the planner and the writer agree on scene length', async () => { + const { store, engine } = await fixtureEngine(); + const ch = engine.pipeline.chapters()[1]; + expect(engine.pipeline.cardsInput(ch.id).words_per_scene).toBe(wordsPerScene(store.project)); + expect(writerContext(store.project, ch.scenes[0], null).target_words).toBe(wordsPerScene(store.project)); + }); + + it('numbers new scenes after the highest s, ignoring suffixes', async () => { + const { store, engine } = await fixtureEngine(); + store.saveCard({ ...store.scene('s04').card, id: 's04-2' }); + expect(engine.pipeline.cardsInput(engine.pipeline.chapters()[0].id).first_scene_number).toBe(13); + }); + + it('the estimate prices line edits, not whole-scene rewrites', async () => { + const { store } = await fixtureEngine(); + const e = estimate(store.project); + expect(e.lines.map((l) => l.stage)).not.toContain('humaniser'); + expect(e.lines.find((l) => l.stage === 'reader run')).toBeTruthy(); + }); +}); diff --git a/packages/core/src/agents/contracts.ts b/packages/core/src/agents/contracts.ts index a1406b2..82716e3 100644 --- a/packages/core/src/agents/contracts.ts +++ b/packages/core/src/agents/contracts.ts @@ -142,6 +142,8 @@ export const WriterInput = z.object({ book: z.object({ title: z.string(), pov_mode: z.string(), tense: z.string() }), card: SceneCard, previous_prose: z.string().nullable(), + /** The book before this scene, compressed (chapter summaries, earlier outcomes). */ + story_so_far: z.string().default(''), recent_cards: z.array(SceneCard), states: z.array(z.object({ character: Id, name: z.string(), snapshot: StateSnapshot.nullable() })), voices: z.array(z.object({ character: Id, name: z.string(), voice: CharacterVoice })), @@ -309,7 +311,13 @@ export const ReaderInput = z.object({ scene: Id, scene_number: z.number().int(), prose: z.string(), + /** The latest entries of the reader's log, in full. */ log: z.array(ReaderLogEntry), + /** What the reader still carries from older scenes, once the log is longer than the window. */ + earlier: z + .object({ scenes_read: z.number().int(), predictions: z.array(z.string()), open_confusions: z.array(z.object({ id: z.string(), text: z.string() })) }) + .nullable() + .default(null), /** Set when this scene reveals something: ask whether it was fair in hindsight. */ fairness_question: z.string().nullable().default(null), /** 'prose' for the manuscript; 'outline' when reading scene synopses before anything is written. */ diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts index f1a064b..73cb3dc 100644 --- a/packages/core/src/context.ts +++ b/packages/core/src/context.ts @@ -4,6 +4,7 @@ import type { ProjectIndex } from './index-db.js'; import type { WriterInput } from './agents/contracts.js'; import { obligationsForScene } from './obligations.js'; import { styleRules } from './style-sheet.js'; +import { BUDGET, clip, storySoFar, wordsPerScene } from './scope.js'; /** * Writer context per scene. Deliberately scoped: full-manuscript context makes the @@ -36,21 +37,21 @@ export function writerContext(p: Project, sceneId: string, index: ProjectIndex | .filter((s) => s?.prose) .map((s) => ({ id: s!.id, excerpt: excerpt(s!.prose!.body, 900) })) : []; - const scenesTotal = Math.max(order.length, 1); return { intent: p.intent, book: { title: p.book.title, pov_mode: p.book.pov_mode, tense: p.book.tense }, card, previous_prose: prev?.prose?.body ?? null, + story_so_far: storySoFar(p, sceneId), recent_cards: recent, states, voices, threads, facts, - style_samples: p.style.samples, + style_samples: clip(p.style.samples, BUDGET.styleSamples), banned_patterns: p.style.banned.map((b) => b.source), similar_scenes: similar, - target_words: Math.round(p.book.target_words / scenesTotal / 50) * 50 || 1200, + target_words: wordsPerScene(p), obligations: obligationsForScene(p, sceneId, order), style_rules: styleRules(p), }; diff --git a/packages/core/src/engine/estimate.ts b/packages/core/src/engine/estimate.ts index 5994f80..b9beea7 100644 --- a/packages/core/src/engine/estimate.ts +++ b/packages/core/src/engine/estimate.ts @@ -2,6 +2,8 @@ import { DEFAULT_AGENT_TIERS, DEFAULT_MODELS, type AgentName, type ModelTier } f import type { Project } from '../project.js'; import { sceneOrder } from '../project.js'; import type { GraphStore } from '../store.js'; +import { writerContext } from '../context.js'; +import { BUDGET } from '../scope.js'; const TOKENS_PER_WORD = 1.35; @@ -51,13 +53,19 @@ export function estimate(p: Project): Estimate { lines.push(line('scene cards (2 alternatives per chapter)', 'planner', n, 12000, 6000)); } if (unwritten) { - const ctx = 6000 + sceneOut * 2 + 3000; // card, states, voices + previous scene + similar excerpts + // Measure the writer's real context on scenes that have cards; fall back to a formula. + const sample = order.filter((s) => !p.scenes.get(s)!.prose).slice(0, 5); + const measured = sample.map((s) => Math.round(JSON.stringify(writerContext(p, s, null)).length / 4) + 1500); // + prompts + const ctx = measured.length ? Math.round(measured.reduce((a, b) => a + b, 0) / measured.length) + sceneOut : 6000 + sceneOut * 2 + 3000; + const factsCtx = Math.min(p.factGlossary.facts.length, BUDGET.facts) * 25; lines.push(line('prose', 'writer', unwritten, ctx, sceneOut)); lines.push(line('voice check', 'voice', unwritten, 2500, 600)); - lines.push(line('humaniser', 'humaniser', unwritten, sceneOut + 2000, sceneOut)); - lines.push(line('extraction', 'extractor', unwritten, sceneOut + 3500, 1500)); + // Span-level line edits: the scene in, a list of edits out (not the scene rewritten). + lines.push(line('line edit', 'humaniser', unwritten, sceneOut + 2000, 800)); + lines.push(line('extraction', 'extractor', unwritten, sceneOut + 3500 + factsCtx, 1500)); } - lines.push(line('reader run', 'reader', unwritten || order.length, sceneOut + 2500, 500)); + // The reader sees one scene plus a fixed window of its log, so this is linear in scenes. + lines.push(line('reader run', 'reader', unwritten || order.length, sceneOut + 400 * BUDGET.readerWindow + 1500, 500)); const total = lines.reduce((n, l) => n + l.cost_usd, 0); return { lines, total_usd: Math.round(total * 100) / 100, scenes_to_write: unwritten, words_to_write: Math.round(unwritten * wordsPerScene) }; } diff --git a/packages/core/src/engine/extraction.ts b/packages/core/src/engine/extraction.ts index 2c9a9b6..27d75ae 100644 --- a/packages/core/src/engine/extraction.ts +++ b/packages/core/src/engine/extraction.ts @@ -1,4 +1,5 @@ import type { AgentSuite, ExtractInput, ExtractOutput } from '../agents/contracts.js'; +import { relevantFacts } from '../scope.js'; import type { Contradiction, Fact, SceneCard, StateSnapshot } from '../schemas/index.js'; import type { GraphStore } from '../store.js'; import { sceneOrder, stateBefore } from '../project.js'; @@ -10,13 +11,19 @@ export function extractInput(store: GraphStore, sceneId: string, prose?: string) const p = store.project; const s = store.scene(sceneId); const order = sceneOrder(p); + const text = prose ?? s.prose?.body ?? ''; + const statesBefore = s.card.characters.map((c) => ({ character: c, snapshot: stateBefore(p, c, sceneId, order)?.snapshot ?? null })); return { card: s.card, - prose: prose ?? s.prose?.body ?? '', + prose: text, characters: [...p.characters.values()].map((c) => ({ id: c.id, name: c.bio.name, aliases: c.bio.aliases })), - facts: p.factGlossary.facts, + // The glossary the prose is most likely to touch, so known facts keep their ids. + facts: relevantFacts(p, { + must: [...s.card.reveals, ...s.card.mentions.map((m) => m.fact), ...statesBefore.flatMap((b) => b.snapshot?.knows ?? [])], + query: `${s.card.purpose} ${s.card.outcome} ${text.slice(0, 6000)}`, + }), locations: p.locations.locations.map((l) => ({ id: l.id, name: l.name })), - states_before: s.card.characters.map((c) => ({ character: c, snapshot: stateBefore(p, c, sceneId, order)?.snapshot ?? null })), + states_before: statesBefore, obligations: [...p.obligations.values()].filter( (o) => s.card.sets_up.includes(o.id) || s.card.pays_off.includes(o.id) || (o.status === 'open' && o.setup_scene && order.indexOf(o.setup_scene) < order.indexOf(sceneId)), ), diff --git a/packages/core/src/engine/pipeline.ts b/packages/core/src/engine/pipeline.ts index f92400e..d06536a 100644 --- a/packages/core/src/engine/pipeline.ts +++ b/packages/core/src/engine/pipeline.ts @@ -13,6 +13,7 @@ import type { GraphStore } from '../store.js'; import { paths, sceneOrder } from '../project.js'; import { canGenerate, cardHash, sceneSync } from '../sync.js'; import { writerContext } from '../context.js'; +import { relevantFacts, unrevealedFacts, wordsPerScene } from '../scope.js'; import { fromYaml, slugify, uniq } from '../util.js'; import { runStylePasses, type Progress, type StylePassReport } from './style-passes.js'; import { resolveObligationScenes } from '../obligations.js'; @@ -213,8 +214,10 @@ export class Pipeline { const ch = chapters[ci]; const before = chapters.slice(0, ci).flatMap((c) => c.scenes); const lastBefore = before[before.length - 1]; - const plannedTotal = chapters.reduce((n, c) => n + (c.scenes.length || c.planned_scenes || 1), 0); - const maxNum = Math.max(0, ...[...this.p.scenes.keys()].map((id) => Number(id.replace(/\D+/g, '')) || 0)); + // Scene ids are s, possibly with a suffix ("s04-2"): only the leading number counts. + const maxNum = Math.max(0, ...[...this.p.scenes.keys()].map((id) => Number(/^s(\d+)/.exec(id)?.[1]) || 0)); + const previous = before.slice(-3).map((id) => this.store.scene(id).card); + const plotPoints = [...this.p.plotPoints.values()]; return { intent: this.p.intent, chapter: { id: ch.id, title: ch.title, summary: ch.summary, scene_count: ch.planned_scenes || 2 }, @@ -230,10 +233,14 @@ export class Pipeline { knows: lastBefore ? (this.store.stateAt(c.id, lastBefore)?.snapshot.knows ?? []) : [], })), locations: this.p.locations.locations.map((l) => ({ id: l.id, name: l.name })), - facts: this.p.factGlossary.facts, + facts: relevantFacts(this.p, { + must: previous.flatMap((c) => [...c.reveals, ...c.mentions.map((m) => m.fact)]), + query: `${ch.title} ${ch.summary} ${plotPoints.map((pp) => pp.description).join(' ')} ${previous.map((c) => c.outcome).join(' ')}`, + prefer: unrevealedFacts(this.p), + }), obligations: [...this.p.obligations.values()].filter((o) => o.status !== 'paid' && o.status !== 'dropped'), - previous_cards: before.slice(-3).map((id) => this.store.scene(id).card), - words_per_scene: Math.round(this.p.book.target_words / plannedTotal / 50) * 50 || 1200, + previous_cards: previous, + words_per_scene: wordsPerScene(this.p), }; } diff --git a/packages/core/src/engine/ripple.ts b/packages/core/src/engine/ripple.ts index 18de149..0de179d 100644 --- a/packages/core/src/engine/ripple.ts +++ b/packages/core/src/engine/ripple.ts @@ -1,15 +1,16 @@ import { randomBytes } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import type { AgentSuite } from '../agents/contracts.js'; +import type { AgentSuite, ReviseCardInput } from '../agents/contracts.js'; import { SceneCard } from '../schemas/index.js'; import type { GraphStore } from '../store.js'; -import { paths, sceneOrder, stateBefore } from '../project.js'; +import { paths, sceneOrder, stateBefore, type Project } from '../project.js'; import { analyseImpact, cardDiffSummary, changeFiles, partitionImpacts, type Change, type Impact } from '../impact.js'; import { applyExtraction, runExtraction } from './extraction.js'; import type { Progress } from './style-passes.js'; import { cardHash } from '../sync.js'; import { timestamp } from '../util.js'; +import { relevantFacts } from '../scope.js'; export type ItemStatus = 'pending' | 'proposed' | 'accepted' | 'rejected' | 'no_change' | 'conflict' | 'resolved' | 'error'; export type ProseStatus = 'none' | 'pending' | 'proposed' | 'accepted' | 'rejected' | 'locked'; @@ -208,15 +209,7 @@ export class Ripple { it.error = null; const card = this.store.scene(it.scene).card; try { - const out = await this.agents.reviseCard({ - intent: p.intent, - change: this.data.description, - reasons: it.reasons, - card, - facts: p.factGlossary.facts, - characters: [...p.characters.values()].map((c) => ({ id: c.id, name: c.bio.name })), - states_before: card.characters.map((c) => ({ character: c, knows: stateBefore(p, c, it.scene)?.snapshot.knows ?? [] })), - }); + const out = await this.agents.reviseCard(reviseCardInput(p, it.scene, this.data.description, it.reasons)); if (out.action === 'no_change' || !out.card) { it.status = 'no_change'; it.card = { before: card, after: null, justification: out.justification, needs_decision: out.needs_decision }; @@ -524,3 +517,21 @@ export class Ripple { }; } } + +/** What the card reviser sees for one scene: the card, who knows what, and the facts in play. */ +export function reviseCardInput(p: Project, sceneId: string, change: string, reasons: string[]): ReviseCardInput { + const card = p.scenes.get(sceneId)!.card; + const statesBefore = card.characters.map((c) => ({ character: c, knows: stateBefore(p, c, sceneId)?.snapshot.knows ?? [] })); + return { + intent: p.intent, + change, + reasons, + card, + facts: relevantFacts(p, { + must: [...card.reveals, ...card.mentions.map((m) => m.fact), ...statesBefore.flatMap((s) => s.knows)], + query: `${change} ${reasons.join(' ')} ${card.purpose} ${card.outcome}`, + }), + characters: [...p.characters.values()].map((c) => ({ id: c.id, name: c.bio.name })), + states_before: statesBefore, + }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e8c6987..3c30aa7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -11,6 +11,7 @@ export * from './store.js'; export * from './impact.js'; export * from './agents/contracts.js'; export * from './context.js'; +export * from './scope.js'; export * from './engine/style-passes.js'; export * from './engine/extraction.js'; export * from './engine/pipeline.js'; diff --git a/packages/core/src/quality/reader.ts b/packages/core/src/quality/reader.ts index cb9a731..c8cbca2 100644 --- a/packages/core/src/quality/reader.ts +++ b/packages/core/src/quality/reader.ts @@ -4,6 +4,7 @@ import { sceneOrder, scenePlaces } from '../project.js'; import { cosine, LocalEmbedder } from '../embed.js'; import { timestamp } from '../util.js'; import { synopsis } from '../engine/critics.js'; +import { readerWindow } from '../scope.js'; export interface PredictionHit { prediction: string; @@ -97,11 +98,13 @@ export async function runReader( opts.progress?.('read', `Reading ${mode === 'outline' ? 'the outline of ' : ''}scene ${i + 1} of ${order.length}`); const text = mode === 'outline' ? synopsis(p, sid) : p.scenes.get(sid)!.prose!.body; const reveals = i > 0 ? revealsIn(sid) : []; + const { recent, earlier } = readerWindow(log); const out = await agents.read({ scene: sid, scene_number: i + 1, prose: text, - log, + log: recent, + earlier, mode, fairness_question: reveals.length ? `This scene revealed: ${reveals.join('; ')}. Looking back at everything you read before it, how earned did that feel? 1 = out of nowhere, 5 = surprising but inevitable.` : null, }); diff --git a/packages/core/src/scope.ts b/packages/core/src/scope.ts new file mode 100644 index 0000000..068255b --- /dev/null +++ b/packages/core/src/scope.ts @@ -0,0 +1,129 @@ +import type { ReaderLogEntry } from './agents/contracts.js'; +import { cosine, LocalEmbedder } from './embed.js'; +import type { Project } from './project.js'; +import { sceneOrder } from './project.js'; + +/** + * Context budgets. Prompts must grow with the scene, not with the book: at novel length + * the fact glossary, the reader's log and the style samples would otherwise dominate + * every call (see scripts/scale-check.ts). + */ +export const BUDGET = { + /** Facts sent to the extractor, planner and reviser once the glossary outgrows this. */ + facts: 80, + /** Characters of style samples sent to the writer. */ + styleSamples: 6000, + /** Characters of the writer's "story so far". */ + storySoFar: 5000, + /** Reader log entries shown in full; older ones are folded into a summary. */ + readerWindow: 6, +}; + +const embedder = new LocalEmbedder(256); +const vectors = new Map(); +const vec = (text: string) => { + let v = vectors.get(text); + if (!v) { + if (vectors.size > 20000) vectors.clear(); + vectors.set(text, (v = embedder.embedOne(text))); + } + return v; +}; + +/** + * The facts worth sending for a task: every fact the task names (`must`), then the + * glossary ranked by similarity to `query`, up to `limit`. Small glossaries go whole. + * `prefer` facts (e.g. ones no scene has revealed yet) get a ranking boost. + */ +export function relevantFacts(p: Project, opts: { must?: Iterable; query: string; limit?: number; prefer?: Set }) { + const all = p.factGlossary.facts; + const limit = opts.limit ?? BUDGET.facts; + if (all.length <= limit) return all; + const must = new Set(opts.must ?? []); + const picked = all.filter((f) => must.has(f.id)); + const room = limit - picked.length; + if (room <= 0) return picked; + const q = vec(opts.query); + const ranked = all + .filter((f) => !must.has(f.id)) + .map((f) => ({ f, score: cosine(q, vec(`${f.id.replace(/-/g, ' ')} ${f.statement}`)) + (opts.prefer?.has(f.id) ? 0.15 : 0) })) + .sort((a, b) => b.score - a.score) + .slice(0, room) + .map((x) => x.f); + const keep = new Set([...picked, ...ranked].map((f) => f.id)); + return all.filter((f) => keep.has(f.id)); // glossary order, for stable prompts +} + +/** Facts no scene reveals yet: the planner most needs these to place reveals. */ +export function unrevealedFacts(p: Project): Set { + const revealed = new Set([...p.scenes.values()].flatMap((s) => s.card.reveals)); + return new Set(p.factGlossary.facts.map((f) => f.id).filter((id) => !revealed.has(id))); +} + +/** Cut text to about n characters at a paragraph (or word) boundary. */ +export function clip(text: string, n: number): string { + if (text.length <= n) return text; + const cut = text.slice(0, n); + const para = cut.lastIndexOf('\n\n'); + return (para > n * 0.5 ? cut.slice(0, para) : cut.slice(0, cut.lastIndexOf(' '))).trimEnd() + '\n\n[…]'; +} + +/** + * The book before this scene, compressed: each earlier chapter as its title and summary, + * then this chapter's earlier scenes as their planned outcomes (the three scenes right + * before are sent as cards separately). Deterministic, from the graph: no model call. + * When it is over budget, the oldest chapters go first. + */ +export function storySoFar(p: Project, sceneId: string, maxChars = BUDGET.storySoFar): string { + const chapters = p.structure.acts.flatMap((a) => a.chapters.map((c) => ({ ...c, act: a.title || a.id }))); + const ci = chapters.findIndex((c) => c.scenes.includes(sceneId)); + if (ci < 0) return ''; + const order = sceneOrder(p); + const recent = new Set(order.slice(Math.max(0, order.indexOf(sceneId) - 3), order.indexOf(sceneId))); + const lines: string[] = []; + for (const c of chapters.slice(0, ci)) { + const outcomes = c.summary ? '' : c.scenes.map((s) => p.scenes.get(s)?.card.outcome).filter(Boolean).join(' '); + lines.push(`- ${c.title || c.id}: ${c.summary || outcomes || '(no summary)'}`); + } + const here = chapters[ci]; + const before = here.scenes.slice(0, here.scenes.indexOf(sceneId)).filter((s) => !recent.has(s)); + if (before.length) lines.push(`- ${here.title || here.id}, so far: ${before.map((s) => p.scenes.get(s)?.card.outcome || p.scenes.get(s)?.card.title).filter(Boolean).join(' ')}`); + let out = lines.join('\n'); + while (out.length > maxChars && lines.length > 1) { + lines.shift(); + out = `- (earlier chapters omitted)\n${lines.join('\n')}`; + } + return out.length > maxChars ? clip(out, maxChars) : out; +} + +/** Target length per scene, the same everywhere: the book's target over its planned scenes. */ +export function wordsPerScene(p: Project): number { + const chapters = p.structure.acts.flatMap((a) => a.chapters); + const planned = chapters.reduce((n, c) => n + (c.scenes.length || c.planned_scenes || 1), 0); + return Math.round(p.book.target_words / Math.max(planned, 1) / 50) * 50 || 1200; +} + +export interface ReaderMemory { + scenes_read: number; + predictions: string[]; + open_confusions: { id: string; text: string }[]; +} + +/** + * The reader's memory for the next scene: the latest entries in full, and from everything + * older only what still matters (predictions, confusions not yet resolved). Keeps the + * reader's cost linear in the book's length rather than quadratic. + */ +export function readerWindow(log: ReaderLogEntry[], window = BUDGET.readerWindow): { recent: ReaderLogEntry[]; earlier: ReaderMemory | null } { + if (log.length <= window) return { recent: log, earlier: null }; + const older = log.slice(0, log.length - window); + const resolved = new Set(log.flatMap((e) => e.resolved_confusions)); + return { + recent: log.slice(-window), + earlier: { + scenes_read: older.length, + predictions: [...new Set(older.flatMap((e) => e.predictions))].slice(-15), + open_confusions: older.flatMap((e) => e.confusions).filter((c) => !resolved.has(c.id)).slice(-10), + }, + }; +} diff --git a/scripts/scale-check.ts b/scripts/scale-check.ts new file mode 100644 index 0000000..3655b91 --- /dev/null +++ b/scripts/scale-check.ts @@ -0,0 +1,113 @@ +/** + * Scale check: grows the sample novella to novel length and reports how big each agent's + * prompt gets and how long store writes take. Context that grows with the book (rather + * than with the scene) is what makes an 80,000-word draft slow and expensive. + * + * pnpm scale-check # 96 scenes (8 copies of the sample), 400 extra facts + * pnpm scale-check --copies 4 # smaller + * pnpm scale-check --json # machine-readable + * pnpm scale-check --max-prompt 8000 # exit 1 if any prompt at the last scene is bigger (CI) + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Engine, extractInput, GraphStore, readerWindow, reviseCardInput, sceneOrder, writerContext } from '@quillgon/core'; +import { OfflineSuite } from '@quillgon/agents'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const arg = (name: string, def: number) => { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? Number(process.argv[i + 1]) : def; +}; +const copies = arg('copies', 8); +const extraFacts = arg('facts', 400); +const json = process.argv.includes('--json'); +const maxPrompt = arg('max-prompt', 0); +const tokens = (x: unknown) => Math.round((typeof x === 'string' ? x : JSON.stringify(x)).length / 4); + +// ---------------------------------------------------------------- build a long book +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-scale-')); +fs.cpSync(path.resolve(here, '..', 'fixtures', 'novella'), dir, { recursive: true, filter: (s) => !s.includes(`${path.sep}.engine`) }); +const store = GraphStore.open(dir, { memoryIndex: true }); +const base = sceneOrder(store.project); +const st = structuredClone(store.project.structure); +let n = 100; +for (let k = 1; k < copies; k++) { + const act = { id: `act-x${k}`, title: `Act ${k + 1}`, chapters: [] as typeof st.acts[number]['chapters'] }; + for (const ch of st.acts.flatMap((a) => a.chapters).filter((c) => !c.id.includes('-x'))) { + const scenes: string[] = []; + for (const sid of ch.scenes) { + const s = store.scene(sid); + const id = `s${n++}`; + store.saveCard({ ...s.card, id, title: `${s.card.title} (${k + 1})` }); + if (s.prose) store.saveProse(id, s.prose.body, 'generated'); + scenes.push(id); + } + act.chapters.push({ ...ch, id: `${ch.id}-x${k}`, scenes }); + } + st.acts.push(act); +} +store.saveStructure(st); +// A long book accumulates facts: every extraction can add to the glossary. +store.addFacts(Array.from({ length: extraFacts }, (_, i) => ({ id: `fact-${i}`, statement: `Synthetic fact number ${i} about the harbour, the ledger and the tide tables.` }))); +store.saveStyle({ samples: store.project.style.samples.repeat(Math.max(1, Math.ceil(20000 / Math.max(1, store.project.style.samples.length)))).slice(0, 20000), banned: store.project.style.banned.map((b) => b.source).join('\n') }); + +const p = store.project; +const order = sceneOrder(p); +const words = order.reduce((w, s) => w + (p.scenes.get(s)!.prose?.meta.word_count ?? 0), 0); +const last = order[order.length - 1]; +const first = order[1]; + +// ---------------------------------------------------------------- prompt sizes +const engine = new Engine(store, new OfflineSuite()); +const cardsInputFor = (ch: string) => engine.pipeline.cardsInput(ch); +const chapters = engine.pipeline.chapters(); +const reviseCard = (sid: string) => reviseCardInput(p, sid, 'Mara no longer knows who holds the debts.', ['she speaks of it here']); +// The reader's input at scene i, as runReader builds it: one log entry per scene read so far. +const logEntry = { scene: 's01', believes: 'x'.repeat(240), expects: 'x'.repeat(160), confusions: [], resolved_confusions: [], engagement: 3, predictions: ['x'.repeat(80)], fairness: null }; +const readerAt = (i: number) => { + const { recent, earlier } = readerWindow(Array.from({ length: i }, () => logEntry)); + return { prose: p.scenes.get(order[i])!.prose?.body ?? '', log: recent, earlier }; +}; + +const rows = [ + { prompt: 'writer', early: tokens(writerContext(p, first, store.index)), late: tokens(writerContext(p, last, store.index)) }, + { prompt: 'extractor', early: tokens(extractInput(store, first)), late: tokens(extractInput(store, last)) }, + { prompt: 'planner (cards)', early: tokens(cardsInputFor(chapters[1].id)), late: tokens(cardsInputFor(chapters[chapters.length - 1].id)) }, + { prompt: 'reviser (card)', early: tokens(reviseCard(first)), late: tokens(reviseCard(last)) }, + { prompt: 'reader', early: tokens(readerAt(1)), late: tokens(readerAt(order.length - 1)) }, +]; +const readerTotal = order.reduce((t, _s, i) => t + tokens(readerAt(i)), 0); + +// ---------------------------------------------------------------- write latency +const time = (fn: () => void, times = 10) => { + const t = performance.now(); + for (let i = 0; i < times; i++) fn(); + return Math.round(((performance.now() - t) / times) * 10) / 10; +}; +const card = store.scene(last).card; +const saveCardMs = time(() => store.saveCard({ ...card, purpose: card.purpose + ' ' })); +const t0 = performance.now(); +await engine.pipeline.writeScene(order[order.length - 2], { commit: false }); +const writeSceneMs = Math.round(performance.now() - t0); + +const result = { scenes: order.length, words, facts: p.factGlossary.facts.length, prompts: rows, reader_run_tokens: readerTotal, save_card_ms: saveCardMs, write_scene_offline_ms: writeSceneMs }; +store.close(); +fs.rmSync(dir, { recursive: true, force: true }); + +if (json) console.log(JSON.stringify(result, null, 2)); +else { + console.log(`Book: ${result.scenes} scenes, ${result.words.toLocaleString()} words, ${result.facts} facts\n`); + console.log('Prompt size (approx. tokens) 2nd scene last scene'); + for (const r of rows) console.log(` ${r.prompt.padEnd(28)} ${String(r.early).padStart(9)} ${String(r.late).padStart(13)}`); + console.log(`\nReader run, whole book: ${readerTotal.toLocaleString()} input tokens`); + console.log(`Store write (save a card): ${saveCardMs} ms`); + console.log(`Write one scene (offline): ${writeSceneMs} ms`); +} + +const over = maxPrompt ? rows.filter((r) => r.late > maxPrompt) : []; +if (over.length) { + console.error(`\nPrompt budget exceeded (${maxPrompt} tokens): ${over.map((r) => `${r.prompt} ${r.late}`).join(', ')}`); + process.exit(1); +} From 66a8bc77e3adb85fd54b24755d650733632ee883 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:54:35 +0000 Subject: [PATCH 06/23] Add a UI unit-test project and test the pure helpers `pnpm test` now also runs apps/desktop/src/**/*.test.{ts,tsx} in a happy-dom vitest project ("ui"), with @testing-library/react for hooks and components. Outline flatten/rebuild move to lib/outline.ts so they can be tested, and lib/util.ts gains deepEqual (used instead of JSON.stringify for dirty checks). Tests cover util and the outline row helpers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- apps/desktop/package.json | 2 + apps/desktop/src/lib/outline.test.ts | 39 ++++++ apps/desktop/src/lib/outline.ts | 34 +++++ apps/desktop/src/lib/util.test.ts | 72 ++++++++++ apps/desktop/src/lib/util.ts | 16 +++ apps/desktop/src/views/outline.tsx | 28 +--- package.json | 1 + pnpm-lock.yaml | 199 ++++++++++++++++++++++++++- vitest.config.ts | 8 ++ 9 files changed, 369 insertions(+), 30 deletions(-) create mode 100644 apps/desktop/src/lib/outline.test.ts create mode 100644 apps/desktop/src/lib/outline.ts create mode 100644 apps/desktop/src/lib/util.test.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index bd56fd6..c659fa1 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -32,6 +32,8 @@ }, "devDependencies": { "@quillgon/core": "workspace:*", + "@testing-library/dom": "^10.4.2", + "@testing-library/react": "^16.3.3", "@types/react": "^19.2.0", "@types/react-dom": "^19.2.0", "@vitejs/plugin-react": "^6.1.2", diff --git a/apps/desktop/src/lib/outline.test.ts b/apps/desktop/src/lib/outline.test.ts new file mode 100644 index 0000000..c14652e --- /dev/null +++ b/apps/desktop/src/lib/outline.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import type { Structure } from './types'; +import { flatten, rebuild } from './outline'; + +const structure = { + acts: [ + { id: 'a1', title: 'One', chapters: [{ id: 'c1', title: 'Arrival', scenes: ['s01', 's02'] }, { id: 'c2', title: 'Ledger', scenes: ['s03'] }] }, + { id: 'a2', title: 'Two', chapters: [{ id: 'c3', title: 'Storm', scenes: [] }] }, + ], +} as unknown as Structure; + +describe('outline rows', () => { + it('flattens acts, chapters and scenes in reading order', () => { + expect(flatten(structure).map((r) => r.key)).toEqual(['act:a1', 'ch:c1', 's01', 's02', 'ch:c2', 's03', 'act:a2', 'ch:c3']); + }); + + it('round-trips unchanged', () => { + expect(rebuild(structure, flatten(structure))).toEqual(structure); + }); + + it('moves a scene into the chapter above it, without touching the input', () => { + const rows = flatten(structure); + const s02 = rows.findIndex((r) => r.id === 's02'); + const [moved] = rows.splice(s02, 1); + rows.push(moved); // after ch:c3 + const next = rebuild(structure, rows); + expect(next.acts[0].chapters[0].scenes).toEqual(['s01']); + expect(next.acts[1].chapters[0].scenes).toEqual(['s02']); + expect(structure.acts[0].chapters[0].scenes).toEqual(['s01', 's02']); + }); + + it('puts scenes dragged above every chapter into the first chapter', () => { + const rows = flatten(structure); + const s03 = rows.findIndex((r) => r.id === 's03'); + const [moved] = rows.splice(s03, 1); + rows.unshift(moved); + expect(rebuild(structure, rows).acts[0].chapters[0].scenes).toEqual(['s03', 's01', 's02']); + }); +}); diff --git a/apps/desktop/src/lib/outline.ts b/apps/desktop/src/lib/outline.ts new file mode 100644 index 0000000..44e95f5 --- /dev/null +++ b/apps/desktop/src/lib/outline.ts @@ -0,0 +1,34 @@ +import type { Structure } from './types'; + +/** One row of the outline: an act heading, a chapter heading, or a scene. */ +export type Row = { key: string; kind: 'act' | 'chapter' | 'scene'; id: string }; + +/** The structure as the flat, draggable list the outline shows. */ +export function flatten(st: Structure): Row[] { + const rows: Row[] = []; + for (const a of st.acts) { + rows.push({ key: `act:${a.id}`, kind: 'act', id: a.id }); + for (const c of a.chapters) { + rows.push({ key: `ch:${c.id}`, kind: 'chapter', id: c.id }); + for (const s of c.scenes) rows.push({ key: s, kind: 'scene', id: s }); + } + } + return rows; +} + +/** + * Put a reordered flat list back into the structure: each scene belongs to the nearest chapter + * row above it (scenes above the first chapter go to the first chapter). Acts and chapters keep + * their order; only scene membership and order change. + */ +export function rebuild(st: Structure, rows: Row[]): Structure { + const next: Structure = structuredClone(st); + const chapters = new Map(next.acts.flatMap((a) => a.chapters).map((c) => [c.id, c])); + for (const c of chapters.values()) c.scenes = []; + let current: Structure['acts'][number]['chapters'][number] | undefined = next.acts[0]?.chapters[0]; + for (const r of rows) { + if (r.kind === 'chapter') current = chapters.get(r.id); + else if (r.kind === 'scene' && current) current.scenes.push(r.id); + } + return next; +} diff --git a/apps/desktop/src/lib/util.test.ts b/apps/desktop/src/lib/util.test.ts new file mode 100644 index 0000000..a865cca --- /dev/null +++ b/apps/desktop/src/lib/util.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ago, cx, deepEqual, factLabel, firstName, hue, loadPref, money, plural, savePref } from './util'; + +describe('util', () => { + afterEach(() => { + vi.useRealTimers(); + localStorage.clear(); + }); + + it('plural', () => { + expect(plural(1, 'scene')).toBe('1 scene'); + expect(plural(2, 'scene')).toBe('2 scenes'); + expect(plural(0, 'fix', 'fixes')).toBe('0 fixes'); + expect(plural(12000, 'word')).toBe('12,000 words'); + }); + + it('ago', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-03-12T12:00:00Z')); + expect(ago(null)).toBe('never'); + expect(ago(Date.now() - 5_000)).toBe('just now'); + expect(ago(Date.now() - 30_000)).toBe('30s ago'); + expect(ago(Date.now() - 5 * 60_000)).toBe('5m ago'); + expect(ago(new Date(Date.now() - 3 * 3600_000).toISOString())).toBe('3h ago'); + expect(ago('2026-01-02T12:00:00Z')).toMatch(/Jan|2/); + }); + + it('money', () => { + expect(money(0)).toBe('$0.00'); + expect(money(0.004)).toBe('<$0.01'); + expect(money(1.234)).toBe('$1.23'); + }); + + it('cx drops falsy parts', () => { + expect(cx('btn', false, null, undefined, 'primary')).toBe('btn primary'); + }); + + it('hue is stable and from the palette', () => { + expect(hue('mara')).toBe(hue('mara')); + expect([62, 150, 235, 300, 28, 190, 95, 330, 265, 120]).toContain(hue('anything')); + }); + + it('factLabel and firstName fall back to the id', () => { + const facts = [{ id: 'debt', statement: 'Eliza holds the debts' }]; + expect(factLabel(facts, 'debt')).toBe('Eliza holds the debts'); + expect(factLabel(facts, 'tide-ledger')).toBe('tide ledger'); + const chars = [{ id: 'mara', bio: { name: 'Mara Penrose' } }]; + expect(firstName(chars, 'mara')).toBe('Mara'); + expect(firstName(chars, 'jory')).toBe('jory'); + }); + + it('loadPref and savePref round-trip, with fallbacks', () => { + expect(loadPref('missing', 7)).toBe(7); + savePref('rail', true); + expect(loadPref('rail', false)).toBe(true); + localStorage.setItem('ne.theme', 'dark'); + expect(loadPref('theme', 'system')).toBe('dark'); + localStorage.setItem('ne.broken', '{not json'); + expect(loadPref('broken', 'fallback')).toBe('fallback'); + }); + + it('deepEqual compares structure, not identity or key order', () => { + expect(deepEqual({ a: 1, b: [1, { c: 'x' }] }, { b: [1, { c: 'x' }], a: 1 })).toBe(true); + expect(deepEqual({ a: 1 }, { a: 2 })).toBe(false); + expect(deepEqual([1, 2], [1, 2, 3])).toBe(false); + expect(deepEqual({ a: [] }, { a: {} })).toBe(false); + expect(deepEqual({ a: 1, b: undefined }, { a: 1 })).toBe(true); + expect(deepEqual({ a: 1 }, { b: 1 })).toBe(false); + expect(deepEqual(null, {})).toBe(false); + expect(deepEqual(NaN, NaN)).toBe(true); + }); +}); diff --git a/apps/desktop/src/lib/util.ts b/apps/desktop/src/lib/util.ts index bcb7138..055e6b7 100644 --- a/apps/desktop/src/lib/util.ts +++ b/apps/desktop/src/lib/util.ts @@ -57,3 +57,19 @@ export function savePref(key: string, value: unknown) { /** A shortcut label for this platform: ⌘K on macOS, Ctrl K elsewhere. */ export const keys = (k: string) => (isMac ? `⌘${k}` : `Ctrl ${k}`); + +/** Structural equality for plain JSON-like values (cards, style sheets). Undefined keys count as absent. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + if (Array.isArray(a)) { + const bb = b as unknown[]; + return a.length === bb.length && a.every((x, i) => deepEqual(x, bb[i])); + } + const ra = a as Record; + const rb = b as Record; + const ka = Object.keys(ra).filter((k) => ra[k] !== undefined); + const kb = Object.keys(rb).filter((k) => rb[k] !== undefined); + return ka.length === kb.length && ka.every((k) => Object.prototype.hasOwnProperty.call(rb, k) && deepEqual(ra[k], rb[k])); +} diff --git a/apps/desktop/src/views/outline.tsx b/apps/desktop/src/views/outline.tsx index 3bff079..00e1090 100644 --- a/apps/desktop/src/views/outline.tsx +++ b/apps/desktop/src/views/outline.tsx @@ -5,35 +5,11 @@ import { CSS } from '@dnd-kit/utilities'; import { AlertTriangle, GripVertical, Lock, Plus, ListTree, Hammer, FileText } from 'lucide-react'; import { post, put } from '../lib/api'; import { useApp } from '../lib/app'; -import type { SceneSummary, Snapshot, Structure } from '../lib/types'; +import type { SceneSummary, Snapshot } from '../lib/types'; import { cx, firstName, plural } from '../lib/util'; import { Avatar, Badge, Button, Chip, Empty, Kbd, SyncBadge } from '../components/ui'; -type Row = { key: string; kind: 'act' | 'chapter' | 'scene'; id: string }; - -function flatten(st: Structure): Row[] { - const rows: Row[] = []; - for (const a of st.acts) { - rows.push({ key: `act:${a.id}`, kind: 'act', id: a.id }); - for (const c of a.chapters) { - rows.push({ key: `ch:${c.id}`, kind: 'chapter', id: c.id }); - for (const s of c.scenes) rows.push({ key: s, kind: 'scene', id: s }); - } - } - return rows; -} - -function rebuild(st: Structure, rows: Row[]): Structure { - const next: Structure = structuredClone(st); - const chapters = new Map(next.acts.flatMap((a) => a.chapters).map((c) => [c.id, c])); - for (const c of chapters.values()) c.scenes = []; - let current: Structure['acts'][number]['chapters'][number] | undefined = next.acts[0]?.chapters[0]; - for (const r of rows) { - if (r.kind === 'chapter') current = chapters.get(r.id); - else if (r.kind === 'scene' && current) current.scenes.push(r.id); - } - return next; -} +import { flatten, rebuild, type Row } from '../lib/outline'; export function Outline() { const { project: p, go, toast, refresh, markSaved } = useApp(); diff --git a/package.json b/package.json index 27ec5d0..400cad5 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@quillgon/core": "workspace:*", "@quillgon/server": "workspace:*", "@types/node": "^22.10.0", + "happy-dom": "^20.14.6", "playwright-core": "^1.64.0", "tsx": "^4.19.0", "typescript": "^5.9.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 05a6314..ee93e71 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@types/node': specifier: ^22.10.0 version: 22.20.5 + happy-dom: + specifier: ^20.14.6 + version: 20.14.6 playwright-core: specifier: ^1.64.0 version: 1.64.0 @@ -34,7 +37,7 @@ importers: version: 5.9.3 vitest: specifier: ^4.0.0 - version: 4.1.11(@types/node@22.20.5)(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 4.1.11(@types/node@22.20.5)(happy-dom@20.14.6)(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) apps/desktop: dependencies: @@ -90,6 +93,12 @@ importers: '@quillgon/core': specifier: workspace:* version: link:../../packages/core + '@testing-library/dom': + specifier: ^10.4.2 + version: 10.4.2 + '@testing-library/react': + specifier: ^16.3.3 + version: 16.3.3(@testing-library/dom@10.4.2)(@types/react-dom@19.3.0(@types/react@19.3.0))(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) '@types/react': specifier: ^19.2.0 version: 19.3.0 @@ -124,7 +133,7 @@ importers: version: link:../core openai: specifier: ^7.31.0 - version: 7.31.0(undici@7.30.0)(zod@4.6.5) + version: 7.31.0(undici@7.30.0)(ws@8.22.0)(zod@4.6.5) zod: specifier: ^4.1.0 version: 4.6.5 @@ -198,6 +207,14 @@ packages: zod: optional: true + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/runtime@7.29.10': resolution: {integrity: sha512-NpugTzbKgGrNe45Ui32cN8vCFFMxVtP/9rtfwv0nP5FRDAjvYBxJcLVpYA9mr1jxP6cnIT7x9LUzuDTbWYYNRA==} engines: {node: '>=6.9.0'} @@ -778,6 +795,28 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@testing-library/dom@10.4.2': + resolution: {integrity: sha512-yzr2S9HyAIdhz2/6qHgbs665Q7PKVcDF05vsOlHPxG1mo36gKVesdYVeDLnXgfjJ03CrKRk08knc6+E/9m8v2Q==} + engines: {node: '>=18'} + + '@testing-library/react@16.3.3': + resolution: {integrity: sha512-Uo193NgQbPMz6lrrhtRQQFcMC6Re/ELLFbbuVL30WDlZxlpZf9/lMHTAVxPRLw1q1iu9OJmR1c2BLiENRstdBg==} + engines: {node: '>=18'} + peerDependencies: + '@testing-library/dom': ^10.0.0 + '@types/react': ^18.0.0 || ^19.0.0 + '@types/react-dom': ^18.0.0 || ^19.0.0 + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@types/aria-query@5.0.4': + resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} + '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} @@ -804,6 +843,12 @@ packages: '@types/react@19.3.0': resolution: {integrity: sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==} + '@types/whatwg-mimetype@3.0.2': + resolution: {integrity: sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==} + + '@types/ws@8.18.2': + resolution: {integrity: sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w==} + '@vitejs/plugin-react@6.1.2': resolution: {integrity: sha512-fhJNGUdscQa4HkI0xFxwonWyhnVReCPDbXI7FH7jmzpnVCrFQI7PRrhWecO263610vOrs9tr2Y4xcf8uAdFxbw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -849,10 +894,21 @@ packages: '@vitest/utils@4.1.11': resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + aria-hidden@1.2.6: resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} engines: {node: '>=10'} + aria-query@5.3.0: + resolution: {integrity: sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -870,6 +926,10 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + buffer-image-size@0.6.4: + resolution: {integrity: sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==} + engines: {node: '>=4.0'} + buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} @@ -912,6 +972,10 @@ packages: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -923,6 +987,9 @@ packages: resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==} engines: {node: '>=0.3.1'} + dom-accessibility-api@0.5.16: + resolution: {integrity: sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==} + electron@44.7.0: resolution: {integrity: sha512-aUq9/zLAAFYc+a9gy6cd0C3Y/N/RmovLCD8TV8LhIzNhuND6HEgsu96jJz9cbAxvckSPv+Trf2PKEjktoZgFkw==} engines: {node: '>= 22.12.0'} @@ -931,6 +998,10 @@ packages: end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + entities@7.0.1: + resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} + engines: {node: '>=0.12'} + env-paths@3.0.0: resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -987,6 +1058,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + happy-dom@20.14.6: + resolution: {integrity: sha512-5DDsf4QmEIlRL6NkCv/fxSjYHHkZceL/IaYQdJc2h3G7DRTEai++VTjbfL0LB9geW2VrA3+vrjF3Efx+OTtX0Q==} + engines: {node: '>=20.0.0'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -996,6 +1071,9 @@ packages: ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + json-schema-to-ts@3.1.1: resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} engines: {node: '>=16'} @@ -1079,6 +1157,10 @@ packages: '@types/react': optional: true + lz-string@1.5.0: + resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} + hasBin: true + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -1163,6 +1245,10 @@ packages: deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true + pretty-format@27.5.1: + resolution: {integrity: sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==} + engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} + progress@2.0.3: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} @@ -1179,6 +1265,9 @@ packages: peerDependencies: react: ^19.3.0 + react-is@17.0.2: + resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} + react-remove-scroll-bar@2.3.8: resolution: {integrity: sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==} engines: {node: '>=10'} @@ -1433,6 +1522,10 @@ packages: w3c-keyname@2.2.8: resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + whatwg-mimetype@3.0.0: + resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==} + engines: {node: '>=12'} + why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} @@ -1441,6 +1534,18 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.22.0: + resolution: {integrity: sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + yaml@2.9.1: resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} engines: {node: '>= 14.6'} @@ -1458,6 +1563,14 @@ snapshots: optionalDependencies: zod: 4.6.5 + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/runtime@7.29.10': {} '@codemirror/autocomplete@6.20.3': @@ -1939,6 +2052,29 @@ snapshots: '@standard-schema/spec@1.1.0': {} + '@testing-library/dom@10.4.2': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/runtime': 7.29.10 + '@types/aria-query': 5.0.4 + aria-query: 5.3.0 + dom-accessibility-api: 0.5.16 + lz-string: 1.5.0 + picocolors: 1.1.1 + pretty-format: 27.5.1 + + '@testing-library/react@16.3.3(@testing-library/dom@10.4.2)(@types/react-dom@19.3.0(@types/react@19.3.0))(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)': + dependencies: + '@babel/runtime': 7.29.10 + '@testing-library/dom': 10.4.2 + react: 19.3.0 + react-dom: 19.3.0(react@19.3.0) + optionalDependencies: + '@types/react': 19.3.0 + '@types/react-dom': 19.3.0(@types/react@19.3.0) + + '@types/aria-query@5.0.4': {} + '@types/better-sqlite3@7.6.13': dependencies: '@types/node': 22.20.5 @@ -1968,6 +2104,12 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/whatwg-mimetype@3.0.2': {} + + '@types/ws@8.18.2': + dependencies: + '@types/node': 22.20.5 + '@vitejs/plugin-react@6.1.2(vite@8.3.4(@types/node@24.19.1)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1))': dependencies: '@rolldown/pluginutils': 1.0.1 @@ -2014,10 +2156,18 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.2.0 + ansi-regex@5.0.1: {} + + ansi-styles@5.2.0: {} + aria-hidden@1.2.6: dependencies: tslib: 2.8.1 + aria-query@5.3.0: + dependencies: + dequal: 2.0.3 + assertion-error@2.0.1: {} base64-js@1.5.1: {} @@ -2037,6 +2187,10 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + buffer-image-size@0.6.4: + dependencies: + '@types/node': 22.20.5 + buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -2074,12 +2228,16 @@ snapshots: deep-extend@0.6.0: {} + dequal@2.0.3: {} + detect-libc@2.1.2: {} detect-node-es@1.1.0: {} diff@8.0.4: {} + dom-accessibility-api@0.5.16: {} + electron@44.7.0: dependencies: '@electron-internal/extract-zip': 1.0.5 @@ -2092,6 +2250,8 @@ snapshots: dependencies: once: 1.4.0 + entities@7.0.1: {} + env-paths@3.0.0: {} es-module-lexer@2.3.2: {} @@ -2152,12 +2312,27 @@ snapshots: graceful-fs@4.2.11: {} + happy-dom@20.14.6: + dependencies: + '@types/node': 22.20.5 + '@types/whatwg-mimetype': 3.0.2 + '@types/ws': 8.18.2 + buffer-image-size: 0.6.4 + entities: 7.0.1 + whatwg-mimetype: 3.0.0 + ws: 8.22.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + ieee754@1.2.1: {} inherits@2.0.4: {} ini@1.3.8: {} + js-tokens@4.0.0: {} + json-schema-to-ts@3.1.1: dependencies: '@babel/runtime': 7.29.10 @@ -2218,6 +2393,8 @@ snapshots: optionalDependencies: '@types/react': 19.3.0 + lz-string@1.5.0: {} + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.6.0 @@ -2244,9 +2421,10 @@ snapshots: dependencies: wrappy: 1.0.2 - openai@7.31.0(undici@7.30.0)(zod@4.6.5): + openai@7.31.0(undici@7.30.0)(ws@8.22.0)(zod@4.6.5): optionalDependencies: undici: 7.30.0 + ws: 8.22.0 zod: 4.6.5 pathe@2.0.3: {} @@ -2278,6 +2456,12 @@ snapshots: tar-fs: 2.1.5 tunnel-agent: 0.6.0 + pretty-format@27.5.1: + dependencies: + ansi-regex: 5.0.1 + ansi-styles: 5.2.0 + react-is: 17.0.2 + progress@2.0.3: {} pump@3.0.4: @@ -2297,6 +2481,8 @@ snapshots: react: 19.3.0 scheduler: 0.28.0 + react-is@17.0.2: {} + react-remove-scroll-bar@2.3.8(@types/react@19.3.0)(react@19.3.0): dependencies: react: 19.3.0 @@ -2498,7 +2684,7 @@ snapshots: tsx: 4.23.15 yaml: 2.9.1 - vitest@4.1.11(@types/node@22.20.5)(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)): + vitest@4.1.11(@types/node@22.20.5)(happy-dom@20.14.6)(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)): dependencies: '@vitest/expect': 4.1.11 '@vitest/mocker': 4.1.11(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) @@ -2522,11 +2708,14 @@ snapshots: why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 22.20.5 + happy-dom: 20.14.6 transitivePeerDependencies: - msw w3c-keyname@2.2.8: {} + whatwg-mimetype@3.0.0: {} + why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0 @@ -2534,6 +2723,8 @@ snapshots: wrappy@1.0.2: {} + ws@8.22.0: {} + yaml@2.9.1: {} zod@4.6.5: {} diff --git a/vitest.config.ts b/vitest.config.ts index 3a6e974..35bd53e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,14 @@ export default defineConfig({ testTimeout: 20000, }, }, + { + test: { + name: 'ui', + include: ['apps/desktop/src/**/*.test.{ts,tsx}'], + environment: 'happy-dom', + testTimeout: 20000, + }, + }, { test: { name: 'live', From 1ad7657855639f005c153459bfcadfa632138259 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 19:54:38 +0000 Subject: [PATCH 07/23] Make UI saves trustworthy; report errors; fix Escape, focus and labels Saves - Extract SceneEditor's data logic into hooks: useProseAutosave, useSceneDetail, useStyleCheck (and useDraft for the style settings). - Prose autosave failures are caught: the text is kept, a banner and the status bar say so, a toast offers Retry, and the next edit retries. Saves are serialised so an older PUT never lands after a newer one. - Unmounting or switching scene flushes a pending save instead of dropping it; a failed flush after leaving still toasts with Retry. Closing the page sends a pending save with keepalive. - A refresh (SSE-driven or after a job) no longer overwrites prose typed since the last save, nor a dirty card. Jobs flush prose first. - cardDirty and the style-sheet dirty flag are memoised structural comparisons instead of JSON.stringify on every render; the style settings no longer reset local edits on every project refresh. - Unsaved card and style edits are guarded: in-app navigation (go(), links, back/forward) asks via the app's Dialog; closing or reloading the window is blocked (Electron gets the in-app dialog, then closes). Errors - A global unhandledrejection handler toasts; errors already reported by job() are marked so they are not toasted twice. - Catch the remaining unhandled rejections in welcome, build, shell (branches, voice check, export), history, ripple, style settings and the scene's impact and review dialogs. - The shared EventSource reconnects with backoff after it closes and tells listeners to refresh after a drop (token handling unchanged). Keyboard and accessibility - One Escape stack: only the topmost layer (dialog, drawer, palette, branch menu) closes. - useShortcuts attaches once (stable rail toggle, state via ref); the ripple key handler has deps and ignores keys while focus is in an input, textarea, select, contenteditable or a real button, or while a layer is open. - Dialog and Drawer are labelled by their title, focus their first field on open, trap Tab, and restore focus on close. - Field links its label to a single native control (useId, htmlFor, aria-describedby for hints) and names composite controls as a group. - window.prompt/confirm replaced by the app's Dialog (DialogsProvider: useConfirm/usePrompt) for what-if naming, scene deletion, ripple and history reverts. - The branch menu opens on ArrowDown, moves with Up/Down/Home/End and returns focus to its button; the ripple list uses a roving tabindex. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016zsDekiv5nExWZqocNHWNL --- apps/desktop/src/App.tsx | 9 +- apps/desktop/src/components/confirm.test.tsx | 76 ++++++ apps/desktop/src/components/confirm.tsx | 127 ++++++++++ apps/desktop/src/components/shell.tsx | 143 ++++++++--- apps/desktop/src/components/ui.test.tsx | 133 ++++++++++ apps/desktop/src/components/ui.tsx | 79 +++--- apps/desktop/src/hooks/use-draft.test.ts | 42 ++++ apps/desktop/src/hooks/use-draft.ts | 29 +++ .../src/hooks/use-prose-autosave.test.tsx | 228 ++++++++++++++++++ apps/desktop/src/hooks/use-prose-autosave.ts | 186 ++++++++++++++ .../src/hooks/use-scene-detail.test.tsx | 140 +++++++++++ apps/desktop/src/hooks/use-scene-detail.ts | 84 +++++++ .../desktop/src/hooks/use-style-check.test.ts | 46 ++++ apps/desktop/src/hooks/use-style-check.ts | 40 +++ apps/desktop/src/lib/api.ts | 64 ++++- apps/desktop/src/lib/app.test.tsx | 102 ++++++++ apps/desktop/src/lib/app.tsx | 126 +++++++++- apps/desktop/src/lib/layers.ts | 135 +++++++++++ apps/desktop/src/lib/unsaved.test.ts | 44 ++++ apps/desktop/src/lib/unsaved.ts | 63 +++++ apps/desktop/src/views/build.tsx | 11 +- apps/desktop/src/views/history.tsx | 21 +- apps/desktop/src/views/ripple.tsx | 55 ++++- apps/desktop/src/views/scene.tsx | 191 +++++++++------ apps/desktop/src/views/style-settings.tsx | 62 +++-- apps/desktop/src/views/welcome.tsx | 8 +- 26 files changed, 2038 insertions(+), 206 deletions(-) create mode 100644 apps/desktop/src/components/confirm.test.tsx create mode 100644 apps/desktop/src/components/confirm.tsx create mode 100644 apps/desktop/src/components/ui.test.tsx create mode 100644 apps/desktop/src/hooks/use-draft.test.ts create mode 100644 apps/desktop/src/hooks/use-draft.ts create mode 100644 apps/desktop/src/hooks/use-prose-autosave.test.tsx create mode 100644 apps/desktop/src/hooks/use-prose-autosave.ts create mode 100644 apps/desktop/src/hooks/use-scene-detail.test.tsx create mode 100644 apps/desktop/src/hooks/use-scene-detail.ts create mode 100644 apps/desktop/src/hooks/use-style-check.test.ts create mode 100644 apps/desktop/src/hooks/use-style-check.ts create mode 100644 apps/desktop/src/lib/app.test.tsx create mode 100644 apps/desktop/src/lib/layers.ts create mode 100644 apps/desktop/src/lib/unsaved.test.ts create mode 100644 apps/desktop/src/lib/unsaved.ts diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 1e56076..1a86730 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -1,5 +1,6 @@ import { useEffect } from 'react'; import { AppProvider, useApp } from './lib/app'; +import { DialogsProvider } from './components/confirm'; import { CommandPalette, Sidebar, StatusBar, Toasts, TopBar, useRail, useShortcuts } from './components/shell'; import { Welcome } from './views/welcome'; import { Dashboard } from './views/dashboard'; @@ -77,8 +78,10 @@ function Shell() { export function App() { return ( - - - + + + + + ); } diff --git a/apps/desktop/src/components/confirm.test.tsx b/apps/desktop/src/components/confirm.test.tsx new file mode 100644 index 0000000..5f56aee --- /dev/null +++ b/apps/desktop/src/components/confirm.test.tsx @@ -0,0 +1,76 @@ +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, describe, expect, it } from 'vitest'; +import { DialogsProvider, useConfirm, usePrompt } from './confirm'; + +afterEach(cleanup); + +function Harness({ onApi }: { onApi: (api: { confirm: ReturnType; prompt: ReturnType }) => void }) { + onApi({ confirm: useConfirm(), prompt: usePrompt() }); + return ; +} + +function setup() { + let api!: { confirm: ReturnType; prompt: ReturnType }; + render( + + (api = a)} /> + , + ); + return api; +} + +describe('confirm and prompt dialogs', () => { + it('confirm resolves true on the confirm button', async () => { + const { confirm } = setup(); + let answer: Promise; + act(() => { + answer = confirm({ title: 'Delete s04?', message: 'Gone for good.', confirmLabel: 'Delete scene', danger: true }); + }); + expect(screen.getByRole('dialog', { name: 'Delete s04?' })).toBeTruthy(); + // The safe choice has focus first. + expect(document.activeElement?.textContent).toBe('Cancel'); + act(() => void fireEvent.click(screen.getByText('Delete scene'))); + await expect(answer!).resolves.toBe(true); + expect(screen.queryByRole('dialog')).toBeNull(); + }); + + it('confirm resolves false on Escape or Cancel', async () => { + const { confirm } = setup(); + let first: Promise; + act(() => { + first = confirm({ title: 'Revert?' }); + }); + act(() => void fireEvent.keyDown(document.activeElement!, { key: 'Escape' })); + await expect(first!).resolves.toBe(false); + let second: Promise; + act(() => { + second = confirm({ title: 'Revert?' }); + }); + act(() => void fireEvent.click(screen.getByText('Cancel'))); + await expect(second!).resolves.toBe(false); + }); + + it('prompt returns the trimmed value, validates, and returns null on cancel', async () => { + const { prompt } = setup(); + let name: Promise; + act(() => { + name = prompt({ title: 'New what-if', label: 'Name', initial: 'what-if-', validate: (v) => (v === 'main' ? 'Taken' : null) }); + }); + const input = screen.getByLabelText('Name') as HTMLInputElement; + expect(document.activeElement).toBe(input); + expect(input.value).toBe('what-if-'); + fireEvent.change(input, { target: { value: 'main' } }); + act(() => void fireEvent.submit(input.closest('form')!)); + expect(screen.getByText('Taken')).toBeTruthy(); + fireEvent.change(input, { target: { value: ' storm ' } }); + act(() => void fireEvent.submit(input.closest('form')!)); + await expect(name!).resolves.toBe('storm'); + + let cancelled: Promise; + act(() => { + cancelled = prompt({ title: 'Again', label: 'Name' }); + }); + act(() => void fireEvent.click(screen.getByText('Cancel'))); + await expect(cancelled!).resolves.toBeNull(); + }); +}); diff --git a/apps/desktop/src/components/confirm.tsx b/apps/desktop/src/components/confirm.tsx new file mode 100644 index 0000000..813e3c6 --- /dev/null +++ b/apps/desktop/src/components/confirm.tsx @@ -0,0 +1,127 @@ +import { createContext, useCallback, useContext, useId, useMemo, useState, type ReactNode } from 'react'; +import { Button, Dialog, Field } from './ui'; + +export interface ConfirmOptions { + title: ReactNode; + message?: ReactNode; + confirmLabel?: string; + cancelLabel?: string; + /** Style the confirm button as destructive. */ + danger?: boolean; +} + +export interface PromptOptions { + title: ReactNode; + message?: ReactNode; + label: string; + initial?: string; + placeholder?: string; + confirmLabel?: string; + /** Return an error message to keep the dialog open. */ + validate?: (value: string) => string | null; +} + +type Request = + | { kind: 'confirm'; opts: ConfirmOptions; resolve: (ok: boolean) => void } + | { kind: 'prompt'; opts: PromptOptions; resolve: (value: string | null) => void }; + +interface Dialogs { + confirm: (opts: ConfirmOptions) => Promise; + prompt: (opts: PromptOptions) => Promise; +} + +const Ctx = createContext(null); + +/** The app's replacement for window.confirm/prompt: promise-based, rendered with Dialog, one at a time. */ +export function DialogsProvider({ children }: { children: ReactNode }) { + const [queue, setQueue] = useState([]); + const confirm = useCallback((opts: ConfirmOptions) => new Promise((resolve) => setQueue((q) => [...q, { kind: 'confirm', opts, resolve }])), []); + const prompt = useCallback((opts: PromptOptions) => new Promise((resolve) => setQueue((q) => [...q, { kind: 'prompt', opts, resolve }])), []); + const value = useMemo(() => ({ confirm, prompt }), [confirm, prompt]); + const top = queue[0]; + const settle = (v: boolean | string | null) => { + if (!top) return; + setQueue((q) => q.slice(1)); + (top.resolve as (x: typeof v) => void)(v); + }; + return ( + + {children} + {top?.kind === 'confirm' && } + {top?.kind === 'prompt' && } + + ); +} + +function useDialogs(): Dialogs { + const ctx = useContext(Ctx); + if (!ctx) throw new Error('DialogsProvider is missing'); + return ctx; +} + +export const useConfirm = () => useDialogs().confirm; +export const usePrompt = () => useDialogs().prompt; + +function ConfirmDialog({ opts, onDone }: { opts: ConfirmOptions; onDone: (ok: boolean) => void }) { + return ( + onDone(false)} + footer={ + <> + + + + } + > + {opts.message &&

{opts.message}

} + + ); +} + +function PromptDialog({ opts, onDone }: { opts: PromptOptions; onDone: (value: string | null) => void }) { + const [value, setValue] = useState(opts.initial ?? ''); + const [error, setError] = useState(null); + const formId = useId(); + const submit = () => { + const v = value.trim(); + if (!v) return setError('Enter a name first.'); + const problem = opts.validate?.(v) ?? null; + if (problem) return setError(problem); + onDone(v); + }; + return ( + onDone(null)} + footer={ + <> + + + + } + > +
{ + e.preventDefault(); + submit(); + }} + > + {opts.message &&

{opts.message}

} + + e.target.setSelectionRange(e.target.value.length, e.target.value.length)} onChange={(e) => (setValue(e.target.value), setError(null))} /> + +
+ + ); +} diff --git a/apps/desktop/src/components/shell.tsx b/apps/desktop/src/components/shell.tsx index bf42d08..4d5988d 100644 --- a/apps/desktop/src/components/shell.tsx +++ b/apps/desktop/src/components/shell.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { Command } from 'cmdk'; import { AlertTriangle, @@ -39,6 +39,8 @@ import { useApp, type View } from '../lib/app'; import { get, post } from '../lib/api'; import { ago, cx, isMac, keys, loadPref, mod, plural, savePref } from '../lib/util'; import { Button, Kbd, stageName } from './ui'; +import { usePrompt } from './confirm'; +import { menuKeyDown, useEscape } from '../lib/layers'; const NAV: { view: View; label: string; icon: typeof LayoutDashboard; shortcut: string }[] = [ { view: 'dashboard', label: 'Writing desk', icon: LayoutDashboard, shortcut: '1' }, @@ -127,21 +129,41 @@ const TITLES: Record = { }; export function TopBar() { - const { project, route, go, setPalette, toast, refresh } = useApp(); + const { project, route, go, setPalette, toast, refresh, fail } = useApp(); + const prompt = usePrompt(); const [menu, setMenu] = useState(false); const ref = useRef(null); + const trigger = useRef(null); + const menuRef = useRef(null); useEffect(() => { if (!menu) return; const h = (e: MouseEvent) => !ref.current?.contains(e.target as Node) && setMenu(false); window.addEventListener('mousedown', h); return () => window.removeEventListener('mousedown', h); }, [menu]); + // Opening the menu moves focus to its first item; Escape closes it and returns to the button. + useEffect(() => { + if (menu) menuRef.current?.querySelector('[role="menuitem"]')?.focus(); + }, [menu]); + const closeMenu = useCallback((refocus = true) => { + setMenu(false); + if (refocus) trigger.current?.focus(); + }, []); + useEscape(closeMenu, menu); if (!project) return null; const scene = route.view === 'scene' ? project.scenes.find((s) => s.id === route.id) : null; const branch = project.branches.current; const newWhatIf = async () => { - const name = prompt('Name this what-if (a git branch):', 'what-if-'); + closeMenu(); + const name = await prompt({ + title: 'New what-if', + message: 'A what-if is a git branch: changes made there stay there until you merge them into main.', + label: 'Name', + initial: 'what-if-', + confirmLabel: 'Create what-if', + validate: (v) => (project.branches.all.includes(v.replace(/\s+/g, '-')) ? 'A what-if with that name already exists.' : null), + }); if (!name) return; try { await post('/branches', { name: name.trim().replace(/\s+/g, '-') }); @@ -152,16 +174,24 @@ export function TopBar() { } }; const switchTo = async (b: string) => { - setMenu(false); - await post('/branches/switch', { name: b }); - toast({ kind: 'info', message: `Switched to ${b}` }); + closeMenu(); + try { + await post('/branches/switch', { name: b }); + toast({ kind: 'info', message: `Switched to ${b}` }); + } catch (e) { + fail(`Could not switch to ${b}`)(e); + } refresh(); }; const merge = async () => { - setMenu(false); - await post('/branches/switch', { name: 'main' }); - const r = await post<{ ok: boolean; conflicts: string[] }>('/branches/merge', { from: branch }); - toast(r.ok ? { kind: 'success', message: `Merged “${branch}” into main` } : { kind: 'error', message: 'Merge conflicts', detail: r.conflicts.join(', ') }); + closeMenu(); + try { + await post('/branches/switch', { name: 'main' }); + const r = await post<{ ok: boolean; conflicts: string[] }>('/branches/merge', { from: branch }); + toast(r.ok ? { kind: 'success', message: `Merged “${branch}” into main` } : { kind: 'error', message: 'Merge conflicts', detail: r.conflicts.join(', ') }); + } catch (e) { + fail(`Could not merge “${branch}”`)(e); + } refresh(); }; @@ -185,27 +215,50 @@ export function TopBar() { )}
- {menu && ( -
-
What-ifs
+