diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a871a727c..76502ab35 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ name: release on: push: tags: ['v*'] - # The person, 27 Oct: GitHub CI is a second opinion. When it finds something, that becomes an item; it does not + # GitHub CI is a second opinion. When it finds something, that becomes an item; it does not # hold a release the board already proved. A tag whose Linux jobs failed publishes by hand through publish-proven. workflow_dispatch: inputs: diff --git a/docs/formats.md b/docs/formats.md index 341db43e7..0e64b81e1 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -130,6 +130,33 @@ A signed board uses event-log format 2; older tools refuse it. A builder never picks an engine number (#1101). An item that adds a board move adds its own `src/-engine.js` leaf as `export const NAME_ENGINE = unassignedEngine();` (importing `unassignedEngine` from `./engine-next.js`); on its branch the leaf reads as one above the highest landed engine. The landing merge writes the next number into the leaf, in landing order, inside that item's merge commit, and sets the `move engine` row above to the highest landed number. A landed number never changes, so recorded moves replay as they were written. Moves are recorded under landed numbers only: code with an unnumbered engine is for its branch's own tests, and a move it records on a real board before landing may later read under another item's number, so the item lands before its moves are used. +The unassigned `COMPACT_VERDICT_REVIEW_ENGINE` leaf adds lossless wire-only YAML review-path trees (#14). A `verify` move uses `review.paths: {format: 'pullboard-review-path-yaml-v1', bytes, sha256, yaml}` only when the complete encoded object is smaller than the literal array. `yaml` is a string containing an actual YAML block mapping, not a JSON object relabeled as YAML. `bytes` and `sha256` bind the UTF-8 JSON of the complete original sorted unique path array. The reader bounds expansion to the existing 32 MiB Git-capture ceiling and checks the exact envelope fields, canonical tree, literal path grammar, expanded byte length and SHA-256 before verification. Stored reviews and snapshots still contain exact arrays: there are no globs, omitted paths, changed notes or relaxed freshness checks. This is not a universal guarantee that every review fits the relay request limit. + +After compatible readers are released and adopted, ordinary outbox flushing also compacts an older queued `verify` move's literal paths when that makes its sealed request smaller. It first pulls the ordered journal and settles any existing receipt by the original move ID. Only the unresolved move's wire copy changes: the ID, actor, criterion, source bindings, logical command intent, reverse captures and later queued moves remain intact. A lost acknowledgement is reconciled by the next pull, not by creating a replacement verdict. An unreadable journal leaves the queue unchanged. This is not permission to rewrite an already sealed `pending` send or replay an uncertain effect; those retain their existing custody and recovery rules. + +CLI/device processes that replay executable records, including devices that build presentation snapshots, need the landed feature engine before receiving the new representation; an older reader refuses before receipt or cursor advancement. The relay stores sealed records. Browser clients consume device-built presentation snapshots and stored literal arrays, not the Node executable move codec. Browser presentation compatibility remains a separate qualification obligation. + +The dependency-free reader accepts only the emitted YAML subset: + +- Each nonempty key is one literal path component, double-quoted with canonical `JSON.stringify` escapes. Additionally U+007F–U+009F, U+2028, U+2029, U+FFFE and U+FFFF are escaped as lowercase `\uNNNN` so YAML cannot normalize line breaks or reject non-printable characters. Slashes separate levels, never appear within keys, and punctuation/wildcards have no special meaning. +- Two spaces indent each level; siblings are sorted by JavaScript string order. A branch ends in `:` and has children. An exact leaf ends in `: null`. LF terminates every line; the empty tree is exactly `{}\n` (one mapping and a trailing LF). +- If both `node` and `node/child.txt` are reviewed (for example a file/directory transition across revisions), `node` is a branch with `"": null` as its first child. This empty-key marker means exactly the parent path, not the subtree. A branch with only that marker is noncanonical; use a normal leaf instead. +- No unquoted keys, comments, document markers, aliases, anchors, tags, flow collections (except the empty root), implicit values, duplicate keys, empty branches or alternate escaping/indentation are accepted. Re-encoding must reproduce the exact tree; malformed or ambiguous input is refused rather than repaired. This is not a general YAML parser. + +For `['node', 'node.txt', 'node/child.txt', 'src/[literal]*?.js', 'src/line\nbreak.js']`, the `yaml` string is: + +```yaml +"node": + "": null + "child.txt": null +"node.txt": null +"src": + "[literal]*?.js": null + "line\nbreak.js": null +``` + +Landing assigns this feature's engine number normally. Older CLI engines stop before receipt/cursor changes; the hosted browser currently requires exact engine equality and also stops. Release CLI and hosted reader assets together, with the hosted `/relay/engine.js` derived from the same package; do not serve this Node-only codec (`node:crypto` and the native engine leaf) as a browser module. The browser currently consumes presentation snapshots, not executable review args, so no codec import is added there. Browser runtime and mixed-version rollout still need release qualification. Already queued moves are not rewritten by this change; recovery must preserve their original logical intent and ordered identity. + Engine 130 (`MECHANICAL_MERGE_ENGINE`) gives new automatic conflict returns version-two `computedConflict` receipts. Health, fresh conflict returns and landing share conservative mechanical decisions: structural JSON retains independent additions and unchanged-value removals, jointly raised numeric counters keep the larger value, and add-only text unions must pass a syntax-only check for JavaScript. Health classifies declared generated `docs/lifecycle.md` without executing item code; only landing runs its generator in the merged tree before its gate. A failed generator blocks with `LAND_MERGE`, naming the item, file and generator, rather than reopening accepted work as a conflict. Version-one receipts continue to verify their original plain-Git conflict paths on replay; version-two receipts verify mechanical conflict paths. The receipt version is part of the immutable-object cache identity. Fresh authors cannot reuse version-one semantics, and a version-two receipt requires engine 130 before changing a board or replay cursor. Relay transport capability `apiVersion: 2` is independent of these versions and diff --git a/src/compact-verdict-review-engine.js b/src/compact-verdict-review-engine.js new file mode 100644 index 000000000..0dca29879 --- /dev/null +++ b/src/compact-verdict-review-engine.js @@ -0,0 +1,3 @@ +/** Literal YAML review-path trees. Older engines stop before replay, never misread coverage. */ +import { unassignedEngine } from './engine-next.js'; +export const COMPACT_VERDICT_REVIEW_ENGINE = unassignedEngine(); diff --git a/src/compact-verdict-review.js b/src/compact-verdict-review.js new file mode 100644 index 000000000..60989c289 --- /dev/null +++ b/src/compact-verdict-review.js @@ -0,0 +1,120 @@ +/** Literal YAML block trees at the move boundary; stored review paths remain exact strings. */ +import { createHash } from 'node:crypto'; +import { Refused } from './refused.js'; +import { COMPACT_VERDICT_REVIEW_ENGINE } from './compact-verdict-review-engine.js'; + +const FORMAT = 'pullboard-review-path-yaml-v1'; +// Same maximum as the existing Git path-capture reader, not a worker resource quota. +const MAX_BYTES = 32 * 1024 * 1024; +/** Measure the UTF-8 wire size, including non-ASCII filenames. */ +const bytes = text => new TextEncoder().encode(text).length; +/** Bind the complete expanded JSON array, not just the tree's leaves. */ +const hash = text => createHash('sha256').update(text).digest('hex'); +/** Refuse without guessing coverage or exposing any path. */ +const bad = () => new Refused('BAD_VERDICT_REVIEW', 'the lossless review paths are invalid; retain the original move and restore its exact encoded paths'); + +/** Match the native review validator's literal repository-relative path grammar. */ +function validPath(path) { + return typeof path === 'string' && !path.includes('\0') && path.split('/').every(part => part && part !== '.' && part !== '..'); +} + +/** Quote literal keys; escape YAML line separators and non-printable C1 characters too. */ +function quote(key) { + return JSON.stringify(key).replace(/[\u007f-\u009f\u2028\u2029\ufffe\uffff]/gu, c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`); +} + +/** Emit only a canonical YAML block-mapping subset, without recursive calls or prototype keys. */ +function yamlTree(paths) { + const root = { children: new Map(), leaf: false }; + for (const path of paths) { + let node = root; + for (const part of path.split('/')) { + if (!node.children.has(part)) node.children.set(part, { children: new Map(), leaf: false }); + node = node.children.get(part); + } + node.leaf = true; + } + const lines = [], stack = [...root.children.keys()].sort().reverse().map(key => ({ key, node: root.children.get(key), depth: 0 })); + while (stack.length) { + const { key, node, depth } = stack.pop(), indent = ' '.repeat(depth); + lines.push(`${indent}${quote(key)}:${node.children.size ? '' : ' null'}\n`); + if (node.children.size) { + // Empty component marks this exact path as well as its descendants (file/directory transitions). + if (node.leaf) lines.push(`${indent} "": null\n`); + for (const child of [...node.children.keys()].sort().reverse()) + stack.push({ key: child, node: node.children.get(child), depth: depth + 1 }); + } + } + return lines.join('') || '{}\n'; +} + +/** Encode a sorted unique literal array without dropping or generalizing any path. */ +export function compactReviewPaths(paths) { + if (!Array.isArray(paths) || !paths.every(validPath) || JSON.stringify(paths) !== JSON.stringify([...new Set(paths)].sort())) throw bad(); + const text = JSON.stringify(paths), size = bytes(text); + if (size > MAX_BYTES) throw bad(); + return { format: FORMAT, bytes: size, sha256: hash(text), yaml: yamlTree(paths) }; +} + +/** Read only the emitted subset; never interpret general YAML, aliases, tags or implicit scalars. */ +function treePaths(yaml, limit) { + if (yaml === '{}\n') return []; + if (!yaml.endsWith('\n')) throw bad(); + const paths = [], stack = [{ path: '', previous: null }]; + let size = 2; + for (const line of yaml.slice(0, -1).split('\n')) { + const row = /^( *)("(?:[^"\\]|\\.)*"):( null)?$/u.exec(line); + if (!row || row[1].length % 2) throw bad(); + const depth = row[1].length / 2; + if (depth >= stack.length) throw bad(); + stack.length = depth + 1; + let key; + try { key = JSON.parse(row[2]); } catch { throw bad(); } + const parent = stack[depth], leaf = row[3] !== undefined; + if (quote(key) !== row[2] || parent.previous !== null && parent.previous >= key + || key.includes('/') || key === '' && (!leaf || depth === 0) || key !== '' && !validPath(key)) throw bad(); + parent.previous = key; + const path = key === '' ? parent.path : parent.path ? `${parent.path}/${key}` : key; + if (leaf) { + size += bytes(JSON.stringify(path)) + (paths.length ? 1 : 0); + if (size > limit) throw bad(); + paths.push(path); + } else stack.push({ path, previous: null }); + } + return paths.sort(); +} + +/** Bound expansion and validate canonical shape, length and digest before native verification. */ +export function expandReviewPaths(value) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== 'bytes,format,sha256,yaml' || value.format !== FORMAT + || !Number.isSafeInteger(value.bytes) || value.bytes < 2 || value.bytes > MAX_BYTES + || typeof value.sha256 !== 'string' || !/^[0-9a-f]{64}$/u.test(value.sha256) || typeof value.yaml !== 'string') throw bad(); + const paths = treePaths(value.yaml, value.bytes), text = JSON.stringify(paths); + if (bytes(text) !== value.bytes || hash(text) !== value.sha256 || yamlTree(paths) !== value.yaml) throw bad(); + return paths; +} + +/** Compact only when smaller; preserve the caller's args and every other move field. */ +export function compactReviewMove(move) { + if (move.operation !== 'verify' || !Array.isArray(move.args?.[1]?.review?.paths)) return move; + const paths = compactReviewPaths(move.args[1].review.paths); + if (bytes(JSON.stringify(paths)) >= bytes(JSON.stringify(move.args[1].review.paths))) return move; + return { ...move, engine: Math.max(move.engine, COMPACT_VERDICT_REVIEW_ENGINE), args: [move.args[0], + { ...move.args[1], review: { ...move.args[1].review, paths } }, ...move.args.slice(2)] }; +} + +/** Stop a YAML envelope mislabeled as an older feature engine before receipt or cursor writes. */ +export function requireCompactReviewEngine(move) { + const paths = move?.operation === 'verify' ? move.args?.[1]?.review?.paths : undefined; + if (paths !== undefined && !Array.isArray(paths) && move.engine < COMPACT_VERDICT_REVIEW_ENGINE) + throw new Refused('ENGINE_VERSION', 'compact review paths require the lossless review-path engine; do not replay or discard this move with an older reader'); +} + +/** Expand only wire paths; the board and its freshness projector still receive the original array. */ +export function expandReviewArgs(move, args) { + requireCompactReviewEngine(move); + if (move.operation === 'verify' && args[1]?.review && !Array.isArray(args[1].review.paths)) + args[1].review.paths = expandReviewPaths(args[1].review.paths); + return args; +} diff --git a/src/engine.js b/src/engine.js index 046eefe2d..412e6df11 100644 --- a/src/engine.js +++ b/src/engine.js @@ -47,6 +47,11 @@ import { } from './submit-invariants.js'; import { MERGE_ONLY_VERDICT_ENGINE } from './merge-only-verdict-engine.js'; import { requireCoordinatorHold } from './coordinator-hold.js'; +import { + compactReviewMove, + expandReviewArgs, + requireCompactReviewEngine, +} from './compact-verdict-review.js'; import { MOVE_REGISTRY, @@ -155,12 +160,13 @@ export function prepareEngineMove(board, operation, args, { id = randomUUID(), a if (operation === 'editMilestone' && values[1] && 'target' in values[1]) values[1].target = canonicalMilestoneTarget(values[1].target); const move = { version: 1, engine: ENGINE_VERSION, id, operation, args: values, ...(actor === undefined ? {} : { actor }) }; // JSON is the wire format: optional undefined fields become absent on every replica alike. - return JSON.parse(JSON.stringify(move)); + return JSON.parse(JSON.stringify(compactReviewMove(move))); } /** Stop before interpreting actors or operations that a newer engine may have changed. */ export function requireSupportedEngine(move) { if (Number.isSafeInteger(move?.engine) && move.engine > ENGINE_VERSION) throw new Refused('ENGINE_VERSION', `move engine version ${move.engine} is newer than this pullboard engine version ${ENGINE_VERSION}; upgrade pullboard before applying the relay order`); + requireCompactReviewEngine(move); } /** Validate protocol identity before a future engine or malformed input can alter any replica. */ @@ -174,7 +180,7 @@ function validateMove(move) { /** Restore only the frozen criterion callback; no receiver runs another machine's Git or shell. */ export function executableArgs(move, board) { - const args = structuredClone(move.args); + const args = expandReviewArgs(move, structuredClone(move.args)); if (move.operation === 'rehomeItem' && args[1]?.sourceMoveId !== move.id) throw new Refused('RELAY_MOVE', 'a re-home source MOVEID does not match its sealed move'); if (move.operation === 'appendFact') { if (!args[1] || typeof args[1] !== 'object' || Array.isArray(args[1])) throw new Refused('RELAY_MOVE', 'a fact needs its typed arguments; use the current Pullboard engine'); diff --git a/src/relay-outbox.js b/src/relay-outbox.js index d3687ff8d..ba84a5b23 100644 --- a/src/relay-outbox.js +++ b/src/relay-outbox.js @@ -20,6 +20,7 @@ import { gzipSync, } from 'node:zlib'; import * as store from './board.js'; +import { compactReviewMove } from './compact-verdict-review.js'; import { engineReceipt, executableArgs, @@ -672,9 +673,12 @@ async function sendQueued(link, key) { let sent = 0; for (const entry of outbox?.moves ?? []) { if ((entry.refs ?? []).some((ref) => outbox.moves.some((other) => other.move.id === ref.add))) break; - const { move, lost } = resolveMove(entry, outbox); + const { move: queued, lost } = resolveMove(entry, outbox); if (lost.length) break; - if (localRecords(root, (board) => engineReceipt(board, move.id))) continue; + if (localRecords(root, (board) => engineReceipt(board, queued.id))) continue; + // Older writers queued literal paths. Compact only the unsent wire copy: the original intent and + // reverse captures remain exact, and an acknowledged move is settled by its existing receipt. + const move = compactReviewMove(queued); const sequence = outbox.baseSequence + sent + 1; const sealed = await sealedRecord(key, move, state, 'move', sequence); const registration = seatReceipt(move, sequence, sealed); diff --git a/test/compact-review-outbox.test.js b/test/compact-review-outbox.test.js new file mode 100644 index 000000000..f115587c4 --- /dev/null +++ b/test/compact-review-outbox.test.js @@ -0,0 +1,251 @@ +/** @pullboard-test-input source-tree */ +/** @pullboard-test-loads src/compact-verdict-review.js src/engine.js src/relay-outbox.js src/relay-ordering.js */ +/** Real Git/SQLite and sealed loopback transport; synthetic identities only [H3,H16,V8]. */ +import assert from 'node:assert/strict'; +import { createHash, randomBytes } from 'node:crypto'; +import { createServer } from 'node:http'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import * as store from '../src/board.js'; +import { expandReviewPaths } from '../src/compact-verdict-review.js'; +import { COMPACT_VERDICT_REVIEW_ENGINE } from '../src/compact-verdict-review-engine.js'; +import { + appliedSequence, + applyRelayMove, + engineReceipt, + executableArgs, + prepareEngineMove, +} from '../src/engine.js'; +import { localRecords, request, saveLink } from '../src/relay.js'; +import { storeBoardKey } from '../src/relay-key.js'; +import { locked } from '../src/relay-lock.js'; +import { + flushOutbox, + outboxFile, + queueMove, + readOutbox, +} from '../src/relay-outbox.js'; +import { withOutboxGuard } from '../src/relay-outbox-guard.js'; +import { relayOrdering } from '../src/relay-ordering.js'; +import { operationIntent } from '../src/relay-sender.js'; +import { seal, unseal } from '../src/seal.js'; +import { captureVerdictReview } from '../src/verdict-freshness.js'; +import { submitFixture } from './board-submit-fixture.js'; +import { runFixtureGit } from './fixture-child.js'; + +/** Hash real fixture data without copying any private board content. */ +function sha256(value) { return createHash('sha256').update(value).digest('hex'); } + +/** Build the native outbox an older writer queued, not a pre-compacted replacement. */ +async function queuedReview(t) { + const scratch = mkdtempSync(join(tmpdir(), 'pullboard-compact-outbox-')); + const previousHome = process.env.PULLBOARD_HOME; + process.env.PULLBOARD_HOME = join(scratch, 'home'); + t.after(() => { + if (previousHome === undefined) delete process.env.PULLBOARD_HOME; + else process.env.PULLBOARD_HOME = previousHome; + rmSync(scratch, { recursive: true, force: true }); + }); + const root = join(scratch, 'repo'); + mkdirSync(root); + const git = (...args) => runFixtureGit(args, { cwd: root }); + git('init', '-q', '-b', 'main'); + git('config', 'user.name', 'Outbox fixture'); + git('config', 'user.email', 'fixture@example.invalid'); + writeFileSync(join(root, 'pullboard.json'), JSON.stringify({ lanes: { web: { owns: ['web/'] } } })); + git('add', '.'); + git('commit', '-qm', 'synthetic base'); + const trunk = git('rev-parse', 'HEAD'); + git('switch', '-qc', 'candidate'); + const paths = [...Array.from({ length: 1420 }, (_, n) => + `web/synthetic-review-fixtures/a-long-shared-component-directory/another-shared-directory/case-${String(n).padStart(4, '0')}.txt`), + 'web/[literal]*?.txt'].sort(); + for (const path of paths) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), 'candidate\n'); + } + git('add', '.'); + git('commit', '-qm', 'synthetic candidate'); + const head = git('rev-parse', 'HEAD'), tree = git('rev-parse', 'HEAD^{tree}'); + git('switch', '-q', 'main'); + const boardId = randomBytes(16).toString('hex'), key = randomBytes(32); + storeBoardKey(boardId, key); + const file = join(root, '.git/pullboard/relay.json'); + const journal = [], bodies = [], moves = [], warnings = []; + /** Seal the real executable record with its board/kind/sequence binding. */ + const encode = async (move, sequence) => Buffer.from(await seal(key, Buffer.from(JSON.stringify(move)), + { boardId, kind: 'move', sequence })).toString('base64url'); + /** Decode the actual authenticated fixture record for native replay. */ + const decode = async record => JSON.parse(new TextDecoder().decode(await unseal(key, + Buffer.from(record.sealed, 'base64url'), { boardId, kind: record.kind, sequence: record.event_id }))); + /** Fixture transport identity is bound by native applyRelayMove; no live enrollment is used. */ + const sender = actor => ({ kind: 'agent', userId: 'fixture-owner', agent: actor, root: true, lineage: 'fixture-lineage' }); + let id, digest, review; + const board = store.openBoard(join(dirname(file), 'board.sqlite'), undefined, { isMain: true, gitRoot: root }); + try { + store.ensureCoordinator(board, root); + for (const seat of ['web-1', 'web-2']) { + const move = prepareEngineMove(board, 'register', [{ lane: 'web', id: seat, + path: join(root, seat), createdBy: 'coordinator' }], { actor: 'coordinator' }); + move.engine = COMPACT_VERDICT_REVIEW_ENGINE - 1; + const sequence = journal.length + 1, sealed = await encode(move, sequence), at = new Date().toISOString(); + const registration = { target: seat, creator: 'coordinator', sequence, digest: sha256(sealed) }; + const outcome = applyRelayMove(board, move, { sequence, at, sender: sender('coordinator'), + kind: 'move', digest: registration.digest, registration }); + assert.equal(outcome.error, undefined); + journal.push({ event_id: sequence, kind: 'move', sealed, event_at: at, sender: sender('coordinator'), registration }); + } + id = store.addItem(board, { by: 'coordinator', lane: 'web', title: 'Synthetic queued review' }); + const text = JSON.stringify({ rows: [], policy: { version: 1, commit: trunk, verify: { policy: 'any', family: 'off' } } }); + digest = sha256(text); + store.claim(board, id, { agentId: 'web-1', lane: 'web', head: trunk, + leaseMs: 3_600_000, freeze: () => ({ text, digest }) }); + submitFixture(board, id, { agentId: 'web-1', commit: head, tree, policyCommit: trunk }); + review = captureVerdictReview(root, board, store.getItem(board, id), trunk, head); + assert.deepEqual(review.paths, paths); + } finally { store.closeBoard(board); } + let unavailable = false; + const server = createServer(async (req, res) => { + try { + const url = new URL(req.url, 'http://127.0.0.1'); + if (req.method === 'GET') { + if (unavailable) { res.writeHead(503); res.end(JSON.stringify({ version: 1, + error: { code: 'UNAVAILABLE', message: 'synthetic unreadable journal' } })); return; } + res.end(JSON.stringify({ version: 1, events: journal.filter(record => record.event_id > Number(url.searchParams.get('after'))) })); + return; + } + assert.equal(req.method, 'POST'); + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const bytes = Buffer.concat(chunks); + bodies.push(bytes.length); + if (bytes.length > 100_000) { + res.writeHead(413); + res.end(JSON.stringify({ version: 1, error: { code: 'BAD_REQUEST', message: 'body exceeds 100000 bytes' } })); + return; + } + const body = JSON.parse(bytes); + assert.equal(body.sequence, journal.length + 1); + const record = { event_id: body.sequence, kind: 'move', sealed: body.sealed, event_at: new Date().toISOString() }; + const move = await decode(record); + record.sender = sender(move.actor); + journal.push(record); + moves.push(move); + res.end(JSON.stringify({ version: 1, event: record })); + } catch (error) { + res.writeHead(500); + res.end(JSON.stringify({ version: 1, error: { code: 'FIXTURE_ERROR', message: error.message } })); + } + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); })); + const state = { version: 1, board: boardId, url: `http://127.0.0.1:${server.address().port}`, + mode: 'ordered', baselineAccepted: true, sequence: journal.length, token: null, + machine: 'fixture-machine', computerName: 'fixture' }; + saveLink(file, state); + const io = { err: line => warnings.push(line) }; + const sealedRecord = async (_key, move, _state, _kind, sequence) => encode(move, sequence); + const ordering = relayOrdering({ request, localRecords, decoded: async (_key, record) => decode(record), + replayRelayRecord: async (b, _key, _state, record) => applyRelayMove(b, await decode(record), + { sequence: record.event_id, at: record.event_at, sender: record.sender, kind: record.kind }), + reportRecoveredReply: () => {}, saveLink, sealedRecord, flush: () => { throw new Error('unexpected recursive flush'); } }); + let loseReply = false; + const link = { root, file, state, io, localRecords, saveLink, request, sealedRecord, + applyRecords: ordering.applyRecords, fetchCheckpoint: () => { throw new Error('native reverse captures must remain usable'); }, + postPending: async (_root, _file, _state, pending) => { + const logical = structuredClone(pending.move.args); + if (pending.move.operation === 'verify' && !Array.isArray(logical[1].review.paths)) + logical[1].review.paths = expandReviewPaths(logical[1].review.paths); + assert.equal(pending.intent, operationIntent(pending.move.operation, logical), 'logical command matching is unchanged'); + if (pending.move.operation === 'verify') { + const durable = (await readOutbox(outboxFile(file), key, boardId)).moves.find(entry => entry.move.id === pending.move.id); + assert.ok(Array.isArray(durable.move.args[1].review.paths), 'the older durable record is not amended'); + assert.equal(durable.move.engine, COMPACT_VERDICT_REVIEW_ENGINE - 1); + assert.equal(durable.intent, pending.intent); + } + const reply = await request(state, `/api/v1/boards/${boardId}/moves`, { method: 'POST', + body: { sequence: pending.sequence, sealed: pending.sealed } }, io); + assert.equal(reply.event.sealed, pending.sealed); + if (loseReply) { loseReply = false; throw Object.assign(new Error('synthetic lost acknowledgement'), { code: 'RELAY_UNAVAILABLE' }); } + } }; + const args = [id, { agentId: 'web-2', decision: 'ACCEPT', note: 'Synthetic protocol fixture, not a product review.', + head, digest, policy: 'any', familyPolicy: 'off', review, check: 'green' }]; + const move = localRecords(root, b => prepareEngineMove(b, 'verify', args, { actor: 'web-2' })); + // Reconstruct the previous released writer's array form, retaining its real native bindings. + move.engine = COMPACT_VERDICT_REVIEW_ENGINE - 1; + move.args[1].review.paths = [...paths]; + await withOutboxGuard(file, () => queueMove(link, { operation: 'verify', args, move, quiet: true, + applyLocally: () => localRecords(root, b => store.verify(b, ...executableArgs(move, b))) })); + const factArgs = [id, { agentId: 'web-2', kind: 'note', text: 'later queued fixture fact' }]; + const tail = localRecords(root, b => prepareEngineMove(b, 'appendFact', factArgs, { actor: 'web-2' })); + tail.engine = COMPACT_VERDICT_REVIEW_ENGINE - 1; + await withOutboxGuard(file, () => queueMove(link, { operation: 'appendFact', args: factArgs, move: tail, quiet: true, + applyLocally: () => localRecords(root, b => store.appendFact(b, ...executableArgs(tail, b))) })); + const original = await readOutbox(outboxFile(file), key, boardId); + return { root, file, state, link, key, boardId, journal, bodies, moves, warnings, id, digest, review, move, tail, original, + setUnavailable: value => { unavailable = value; }, loseReply: () => { loseReply = true; }, + flush: () => locked(file, () => flushOutbox(link), io), + remote: async () => { + const remoteArgs = [id, { agentId: 'web-2', kind: 'note', text: 'independent remote fixture fact' }]; + const remote = localRecords(root, b => prepareEngineMove(b, 'appendFact', remoteArgs, { actor: 'web-2' })); + remote.engine = COMPACT_VERDICT_REVIEW_ENGINE - 1; + const sequence = journal.length + 1; + journal.push({ event_id: sequence, kind: 'move', sealed: await encode(remote, sequence), + event_at: new Date().toISOString(), sender: sender(remote.actor) }); + } }; +} + +/** Check real receipt, actor, exact coverage, frozen criterion and once-only tail after native replay. */ +function assertSettled(f) { + assert.equal(existsSync(outboxFile(f.file)), false); + assert.deepEqual(f.moves.map(move => move.id), [f.move.id, f.tail.id]); + const expanded = structuredClone(f.moves[0]); + expanded.engine = f.move.engine; + expanded.args[1].review.paths = expandReviewPaths(expanded.args[1].review.paths); + assert.deepEqual(expanded, f.move, 'only wire coverage representation and its engine floor change'); + assert.ok(f.moves[0].engine >= COMPACT_VERDICT_REVIEW_ENGINE); + assert.deepEqual(f.moves[1], f.tail, 'the later unrelated move is not rewritten'); + localRecords(f.root, board => { + const item = store.getItem(board, f.id), verdict = store.acceptedVerdict(board, item); + assert.equal(item.item_status, 'verified'); + assert.equal(item.item_frozen_digest, f.digest); + assert.equal(verdict.verdict_by, f.move.actor); + assert.deepEqual(verdict.review, f.review); + for (const move of [f.move, f.tail]) assert.equal(engineReceipt(board, move.id).outcome.error, undefined); + assert.equal(appliedSequence(board), f.journal.length); + assert.equal(store.itemThread(board, f.id).filter(event => event.type === 'fact' && event.text === 'later queued fixture fact').length, 1); + }); + assert.equal(f.warnings.some(line => /checkpoint|cannot be taken back/u.test(line)), false); + assert.ok(f.bodies.every(bytes => bytes < 100_000)); +} + +for (const interleaved of [false, true]) { + test(`older queued review uses native compact send/rebuild${interleaved ? ', concurrent remote move and lost ACK' : ''} [H3,H16,V8]`, async t => { + const f = await queuedReview(t); + assert.ok(Array.isArray(f.original.moves[0].move.args[1].review.paths)); + assert.equal(f.original.moves[0].intent, operationIntent('verify', f.move.args)); + if (interleaved) { await f.remote(); f.loseReply(); } + if (interleaved) await assert.rejects(f.flush(), /synthetic lost acknowledgement/u); + await f.flush(); + assertSettled(f); + const count = f.bodies.length; + await f.flush(); + assert.equal(f.bodies.length, count, 'a settled move is never posted again'); + t.diagnostic(`native recovery: ${f.review.paths.length} exact paths, ${f.moves.length} posts, max HTTP JSON ${Math.max(...f.bodies)} bytes`); + }); +} + +test('unreadable remote journal leaves old queue, link and sealed pending untouched [H3,H16]', async t => { + const f = await queuedReview(t); + f.state.pending = { move: structuredClone(f.move), sequence: f.state.sequence + 1, sealed: 'opaque-existing-pending' }; + saveLink(f.file, f.state); + const queue = readFileSync(outboxFile(f.file)), link = readFileSync(f.file); + f.setUnavailable(true); + await assert.rejects(f.flush()); + assert.deepEqual(readFileSync(outboxFile(f.file)), queue); + assert.deepEqual(readFileSync(f.file), link); + assert.deepEqual(await readOutbox(outboxFile(f.file), f.key, f.boardId), f.original); + assert.equal(f.bodies.length, 0); +}); diff --git a/test/compact-verdict-review.test.js b/test/compact-verdict-review.test.js new file mode 100644 index 000000000..876ebc2e5 --- /dev/null +++ b/test/compact-verdict-review.test.js @@ -0,0 +1,188 @@ +/** @pullboard-test-input source-tree */ +/** @pullboard-test-loads src/compact-verdict-review.js src/engine.js src/verdict-freshness.js */ +/** Synthetic wire and real Git/SQLite regressions; no relay, credentials or product gate claims [H3,H16,V8]. */ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import * as store from '../src/board.js'; +import { compactReviewMove, compactReviewPaths, expandReviewPaths } from '../src/compact-verdict-review.js'; +import { COMPACT_VERDICT_REVIEW_ENGINE } from '../src/compact-verdict-review-engine.js'; +import { appliedSequence, applyEngineMove, engineReceipt, executableArgs, prepareEngineMove } from '../src/engine.js'; +import { ENGINE_VERSION } from '../src/machine.js'; +import { generateBoardKey, seal, unseal } from '../src/seal.js'; +import { captureVerdictReview, verdictProjector } from '../src/verdict-freshness.js'; +import { submitFixture } from './board-submit-fixture.js'; +import { runFixtureGit } from './fixture-child.js'; + +/** Generate long shared prefixes, not private repository filenames. */ +function manyPaths() { + return Array.from({ length: 1420 }, (_, n) => + `web/synthetic-review-fixtures/a-long-shared-component-directory/another-shared-directory/case-${String(n).padStart(4, '0')}.txt`); +} + +/** Construct a synthetic move with stable review, actor and logical identity. */ +function reviewMove(paths) { + return { version: 1, engine: ENGINE_VERSION, id: 'synthetic-review', operation: 'verify', actor: 'web-2', args: [1, + { agentId: 'web-2', decision: 'ACCEPT', note: 'Synthetic encoding regression, not a product review.', digest: 'd'.repeat(64), + head: 'a'.repeat(40), review: { commit: 'a'.repeat(40), head: 'a'.repeat(40), trunk: 'b'.repeat(40), paths } }] }; +} + +/** Assert the specific fail-closed coverage refusal, not an arbitrary exception. */ +function invalid(value) { + assert.throws(() => expandReviewPaths(value), { code: 'BAD_VERDICT_REVIEW' }); +} + +test('review paths are an actual readable YAML block tree with explicit literal leaves [H16,V8]', () => { + const value = compactReviewPaths(['node', 'node.txt', 'node/child.txt', 'src/a.js', 'src/b.js']); + assert.equal(value.format, 'pullboard-review-path-yaml-v1'); + assert.equal(value.yaml, '"node":\n "": null\n "child.txt": null\n"node.txt": null\n"src":\n "a.js": null\n "b.js": null\n'); + assert.deepEqual(expandReviewPaths(value), ['node', 'node.txt', 'node/child.txt', 'src/a.js', 'src/b.js']); + assert.equal(compactReviewPaths([]).yaml, '{}\n'); + assert.deepEqual(expandReviewPaths(compactReviewPaths([])), []); +}); + +test('YAML keys retain punctuation, Unicode, line breaks and literal wildcard characters [H16,V8]', () => { + const paths = ['__proto__/constructor', 'null', 'true', 'web/#: &*?![]{}.txt', 'web/[literal]*?.txt', + 'web/line\nbreak.txt', 'web/return\rtab\t.txt', 'web/quote"\\name.txt', 'web/space name.txt', + 'web/\u007f\u0085\u009f\u2028\u2029\ufffe\uffff.txt', 'web/\ud800.txt', 'web/é.txt', 'web/😀.txt', 'web/😁.txt'].sort(); + const encoded = JSON.parse(JSON.stringify(compactReviewPaths(paths))); + assert.deepEqual(expandReviewPaths(encoded), paths); + assert.ok(!/[\u007f-\u009f\u2028\u2029\ufffe\uffff]/u.test(encoded.yaml), 'YAML-sensitive characters are escaped, not normalized'); + for (const bad of [['b', 'a'], ['a', 'a'], ['../a'], ['/a'], ['a//b'], ['a\0b'], ['a/./b'], ['a/../b']]) + assert.throws(() => compactReviewPaths(bad), { code: 'BAD_VERDICT_REVIEW' }); +}); + +test('corrupt, omitted, duplicate, ambiguous or noncanonical YAML coverage is refused [H16,V8]', () => { + const value = compactReviewPaths(['web/component/a', 'web/component/b']); + const mutations = [ + v => { v.format = 'unknown'; }, v => { v.bytes += 1; }, v => { v.bytes = 32 * 1024 * 1024 + 1; }, + v => { v.sha256 = '0'.repeat(64); }, v => { v.sha256 = {}; }, + v => { v.yaml = v.yaml.replace(' "b": null\n', ''); }, + v => { v.yaml = v.yaml.replace(' "b": null', ' "a": null'); }, + v => { v.yaml = v.yaml.replace(' "a": null', ' "a": null'); }, + v => { v.yaml = v.yaml.replace(' "a": null', '\t"a": null'); }, + v => { v.yaml = v.yaml.replace(' "a": null', ' "a": null'); }, + v => { v.yaml = v.yaml.replace('"component":', '"component": null'); }, + v => { v.yaml = v.yaml.replace('"a": null', '"a":'); }, + v => { v.yaml = v.yaml.replace('"a": null', '"a": {}'); }, + v => { v.yaml = v.yaml.replace('"a": null', '"a": *alias'); }, + v => { v.yaml = v.yaml.replace('"a": null', '"a": !!null null'); }, + v => { v.yaml = v.yaml.replace('"a": null', 'a: null'); }, + v => { v.yaml = v.yaml.replace('"a": null', '"\\u0061": null'); }, + v => { v.yaml = v.yaml.replace('"a": null', '"a/b": null'); }, + v => { v.yaml += '"empty":\n'; }, v => { v.yaml += '"": null\n'; }, + v => { v.yaml += '# comment\n'; }, v => { v.yaml = '---\n' + v.yaml; }, + v => { v.yaml = v.yaml.slice(0, -1); }, v => { v.yaml = v.yaml.replaceAll('\n', '\r\n'); }, + v => { v.extra = true; }, v => { v.yaml = null; }, + ]; + for (const mutate of mutations) { const changed = structuredClone(value); mutate(changed); invalid(changed); } + for (const malformed of [null, {}, [], { ...value, yaml: [] }, { ...value, bytes: 1 }]) invalid(malformed); + // Rebind metadata too: grammar/shape rejection cannot depend only on a stale digest. + for (const yaml of ['"a":\n "": null\n', '"a": null\n"a": null\n', '"a": null\n"a":\n "b": null\n', + '"a": null\n"b":\n', '"a": null\n"b": null\n', '"..": null\n']) { + const text = JSON.stringify(['a']); + invalid({ format: value.format, bytes: Buffer.byteLength(text), sha256: createHash('sha256').update(text).digest('hex'), yaml }); + } +}); + +test('wire compaction is smaller-only and leaves caller args and non-path fields exact [H3,H16,V8]', () => { + const original = reviewMove(manyPaths()), before = structuredClone(original); + const compact = compactReviewMove(original); + assert.deepEqual(original, before); + assert.ok(!Array.isArray(compact.args[1].review.paths)); + assert.ok(compact.engine >= COMPACT_VERDICT_REVIEW_ENGINE); + const restored = structuredClone(compact); + restored.args[1].review.paths = expandReviewPaths(restored.args[1].review.paths); + assert.deepEqual(restored, original); + const short = reviewMove(['a']); + assert.equal(compactReviewMove(short), short); + const unrelated = { ...original, operation: 'shout' }; + assert.equal(compactReviewMove(unrelated), unrelated); +}); + +test('a generated oversized review fits the actual sealed JSON request without a larger cap [H3,H16]', async t => { + const original = reviewMove(manyPaths()), compact = compactReviewMove(original); + const key = await generateBoardKey(); + const binding = { boardId: 'synthetic-encoding-board', kind: 'move', sequence: 1 }; + const sizes = []; + for (const move of [original, compact]) { + const plain = new TextEncoder().encode(JSON.stringify(move)); + const sealed = await seal(key, plain, binding); + // Match relayOrdering.postPending's verify request (no registration receipt). + const body = JSON.stringify({ sequence: 1, sealed: Buffer.from(sealed).toString('base64url') }); + sizes.push(Buffer.byteLength(body)); + assert.deepEqual(JSON.parse(new TextDecoder().decode(await unseal(key, sealed, binding))), move); + } + assert.ok(sizes[0] > 100_000); + assert.ok(sizes[1] < 100_000); + t.diagnostic(`synthetic sealed HTTP JSON bytes: ${sizes[0]} -> ${sizes[1]}; paths: ${original.args[1].review.paths.length}`); +}); + +/** Capture a genuine synthetic Git delta and submit native SQLite state without claiming a gate ran. */ +function submitted(t) { + const root = mkdtempSync(join(tmpdir(), 'pullboard-compact-review-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const git = (...args) => runFixtureGit(args, { cwd: root }); + git('init', '-q', '-b', 'main'); + writeFileSync(join(root, 'pullboard.json'), JSON.stringify({ lanes: { web: { owns: ['web/'] } } })); + git('add', '.'); + git('-c', 'user.name=Encoding fixture', '-c', 'user.email=encoding@example.invalid', 'commit', '-qm', 'synthetic base'); + const trunk = git('rev-parse', 'HEAD'); + git('switch', '-qc', 'candidate'); + const paths = [...manyPaths(), 'web/[literal]*?.txt'].sort(); + for (const path of paths) { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), 'candidate\n'); } + git('add', '.'); + git('-c', 'user.name=Encoding fixture', '-c', 'user.email=encoding@example.invalid', 'commit', '-qm', 'synthetic candidate'); + const commit = git('rev-parse', 'HEAD'), tree = git('rev-parse', 'HEAD^{tree}'); + git('switch', '-q', 'main'); + const board = store.openBoard(join(root, 'board.sqlite')); + t.after(() => store.closeBoard(board)); + store.register(board, { lane: 'coordinator', path: root }); + const builder = store.register(board, { lane: 'web', path: join(root, 'builder') }); + const reviewer = store.register(board, { lane: 'web', path: join(root, 'reviewer') }); + const id = store.addItem(board, { by: 'coordinator', lane: 'web', title: 'Synthetic compact review' }); + const text = JSON.stringify({ rows: [], policy: { version: 1, commit: trunk, verify: { policy: 'any', family: 'off' } } }); + const digest = createHash('sha256').update(text).digest('hex'); + store.claim(board, id, { agentId: builder, lane: 'web', leaseMs: 3_600_000, freeze: () => ({ text, digest }) }); + submitFixture(board, id, { agentId: builder, commit, tree, policyCommit: trunk }); + const review = captureVerdictReview(root, board, store.getItem(board, id), trunk, commit); + assert.deepEqual(review.paths, paths); + const options = { ...reviewMove(paths).args[1], agentId: reviewer, digest, head: commit, review }; + return { root, git, board, id, reviewer, options, paths, trunk }; +} + +test('native capture, decode, refusals, stored arrays and literal freshness retain semantics [H3,H16,V8]', t => { + const box = submitted(t); + const { board, id, options, paths } = box; + const move = prepareEngineMove(board, 'verify', [id, options], { id: 'native-compact-review', actor: box.reviewer }); + assert.ok(!Array.isArray(move.args[1].review.paths)); + assert.deepEqual(executableArgs(move, board)[1].review, options.review); + for (const engine of [COMPACT_VERDICT_REVIEW_ENGINE - 1, ENGINE_VERSION + 1]) { + assert.throws(() => applyEngineMove(board, { ...move, engine }, { sequence: 1, at: '2026-10-30T12:00:00Z' }), { code: 'ENGINE_VERSION' }); + assert.equal(appliedSequence(board), 0); + assert.equal(engineReceipt(board, move.id), null); + } + const malformed = structuredClone(move); + malformed.id = 'corrupt-review'; malformed.args[1].review.paths.yaml += '"unexpected": null\n'; + const refused = applyEngineMove(board, malformed, { sequence: 1, at: '2026-10-30T12:00:00Z' }); + assert.equal(refused.error?.code, 'BAD_VERDICT_REVIEW'); + assert.equal(store.verdictsFor(board, id).length, 0); + const outcome = applyEngineMove(board, move, { sequence: 2, at: '2026-10-30T12:00:01Z' }); + assert.equal(outcome.error, undefined, JSON.stringify(outcome)); + assert.deepEqual(store.verdictsFor(board, id)[0].review, options.review); + assert.deepEqual(applyEngineMove(board, move, { sequence: 2, at: '2026-10-30T12:00:01Z' }), outcome); + const unrelated = ['unrelated.txt', 'web/literal-neighbor.txt', paths.at(-1).replace('1419', '1420')]; + for (const path of [...unrelated, paths[0], paths.at(-1), 'web/[literal]*?.txt']) { + box.git('switch', '-qC', 'main', box.trunk); + mkdirSync(dirname(join(box.root, path)), { recursive: true }); writeFileSync(join(box.root, path), 'trunk change\n'); + box.git('add', '--', path); + box.git('-c', 'user.name=Encoding fixture', '-c', 'user.email=encoding@example.invalid', 'commit', '-qm', 'synthetic trunk change'); + const state = verdictProjector(box.root, board).state(store.getItem(board, id)); + if (unrelated.includes(path)) assert.equal(state, null, 'neither directory coverage nor wildcard matching is introduced'); + else { assert.equal(state?.code, 'STALE_VERDICT'); assert.equal(state.file, path); } + } + assert.deepEqual(store.verdictsFor(board, id)[0].review.paths, paths, 'freshness never rewrites the review'); +}); diff --git a/test/resources-lease-fixture.js b/test/resources-lease-fixture.js index 92c814280..404a1cca1 100644 --- a/test/resources-lease-fixture.js +++ b/test/resources-lease-fixture.js @@ -8,6 +8,7 @@ import { DatabaseSync } from 'node:sqlite'; import * as store from '../src/board.js'; import { exportBoard, importBoard } from '../src/exchange.js'; import { appliedSequence, applyEngineMove, prepareEngineMove } from '../src/engine.js'; +import { SUBMIT_ADMISSION_ENGINE } from '../src/submit-admission-engine.js'; import { registerProject } from '../src/projects.js'; import { serveView } from '../src/serve.js'; import { runFixtureChild } from './fixture-child.js'; @@ -237,7 +238,7 @@ function admittedCompletionReplicas(box, id, snapshot) { commit: scenario === 'different commit' ? '1'.repeat(receipt.commit.length) : receipt.commit, tree: scenario === 'different tree' ? '2'.repeat(receipt.tree.length) : receipt.tree, submissionProof: { version: 1, files: [], full: false, deferred: true, reason: 'private replay of actual native proof' } }]); - if (scenario === 'historical engine') move.engine = proof.engine - 1; + if (scenario === 'historical engine') move.engine = SUBMIT_ADMISSION_ENGINE - 1; const result = applyEngineMove(peer, move, { sequence: appliedSequence(peer) + 1, at: clock.now().toISOString() }); assert.equal(result.error?.code, scenario === 'same admission' ? undefined : 'AGENT_GRANT_EXPIRED', scenario); } finally { store.closeBoard(peer); } diff --git a/test/team-engine.test.js b/test/team-engine.test.js index bdae581ff..b3d53857d 100644 --- a/test/team-engine.test.js +++ b/test/team-engine.test.js @@ -133,7 +133,9 @@ test('malformed sealed reviews cannot append proof or bind a different submissio { ...valid, paths: ['web/../feature.js'] }, { ...valid, paths: ['web//feature.js'] }, { ...valid, unexpected: true }]; for (const [index, review] of invalid.entries()) { - const move = prepareEngineMove(one, 'verify', [1, verdict(review)], { id: `invalid-review-${index}` }); + const move = prepareEngineMove(one, 'verify', [1, verdict(valid)], { id: `invalid-review-${index}` }); + // Inject malformed wire input after the sender's path encoder so the receiver is tested. + move.args[1].review = review; const order = { sequence: index + 1, at: AT }; const first = applyEngineMove(one, move, order); assert.equal(first.error?.code, 'BAD_VERDICT_REVIEW', JSON.stringify(first));