From afd231d534d70b3198b3c04b1c05a6f3e7d24734 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Mon, 6 Jul 2026 12:46:33 -0700 Subject: [PATCH 1/3] feat(engine,producer): drawElement fast-capture default-on with runtime self-verification safety net MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip useDrawElement + worker-encode defaults on (HF_DE_BATCH default 4), clamped in resolveConfig to hosts where drawElement can engage (macOS + hardware-GPU browser) so page-side shader compositing is untouched everywhere else; explicit env opt-in keeps attempt-and-gate semantics. Safety net makes default-on safe: the compile/init gates catch predictable incompatibility; this catches the intermittent residue no static analysis can see (stale paints, dropped background images, transient blank frames). - engine: captureDeVerificationFrames — K=4 (HF_DE_VERIFY) ground-truth screenshots at init, after gates + armStaticDedup, BEFORE canvas injection (post-injection screenshots show the canvas bitmap, not the DOM). Runs the video-injection hook per sample; double-captures so rAF-driven text counters settle (a single immediate screenshot captures stale text and false-positives). Skips png, <10 frames, implausible __hf.duration (infinite-repeat GSAP sentinel). - producer: guardFrame on both worker-encode drains — rolling-median blank guard with retry-once at drain (byte-identical retry ⇒ deterministic dark frame, accepted; retry save/restores the static-dedup anchor) + ffmpeg PSNR self-verify vs ground truth (HF_DE_VERIFY_MIN_DB, default 32dB; natural agreement ≥45dB, damage ≤25dB). Breach dumps the frame pair to tmpdir and throws DrawElementVerificationError. - orchestrator: one-shot retry — on verification error the whole render re-runs with forceScreenshot (slower, never wrong); telemetry flag deSelfVerifyFallback. - tooling: de-canary-suite.sh (7-comp release gate with expected verdicts), de-gatecheck.sh (init-only corpus routing classifier), we-render.mjs. Validated: canary suite 7/7; 611-comp routing sample 54% drawelement / 37.5% gated / 8.3% comp-defect; 12/12 risk-band renders clean on bare defaults (48/48 verify samples); engine suite 888 passed; caught two real intermittent damage classes in the wild (background-image drop, root-props offset) that previously shipped silently. Kill switches: PRODUCER_EXPERIMENTAL_FAST_CAPTURE=false, HF_DE_WORKER_ENCODE=false, HF_DE_BATCH=0, HF_DE_VERIFY=0. Co-Authored-By: Claude Fable 5 --- .fallowrc.jsonc | 1 + packages/cli/src/commands/render.ts | 11 +- packages/engine/src/config.test.ts | 18 +- packages/engine/src/config.ts | 44 +++-- packages/engine/src/index.ts | 2 + packages/engine/src/services/frameCapture.ts | 117 ++++++++++- packages/producer/de-canary-suite.sh | 60 ++++++ packages/producer/de-gatecheck.sh | 51 +++++ .../src/services/render/observability.ts | 2 + .../render/stages/captureStreamingStage.ts | 184 +++++++++++++++--- .../src/services/renderOrchestrator.ts | 107 ++++++---- .../css-var-fonts/output/output.mp4 | Bin 71779 -> 130 bytes .../tests/video-hfid-no-id/output/output.mp4 | Bin 1678576 -> 132 bytes .../tests/video-hfid-no-id/src/clip.mp4 | Bin 814218 -> 131 bytes packages/producer/we-render.mjs | 24 +++ 15 files changed, 534 insertions(+), 87 deletions(-) create mode 100644 packages/producer/de-canary-suite.sh create mode 100644 packages/producer/de-gatecheck.sh create mode 100644 packages/producer/we-render.mjs diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index e842514234..480ef20940 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -14,6 +14,7 @@ "packages/producer/src/runtime-conformance.ts", "packages/producer/src/benchmark.ts", "packages/producer/scripts/generate-font-data.ts", + "packages/producer/we-render.mjs", "packages/producer/scripts/validate-fast-video.ts", "packages/cli/scripts/generate-font-data.ts", "packages/engine/scripts/test-fitTextFontSize-browser.ts", diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 979368d379..d5dcd45f8c 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -361,11 +361,12 @@ export default defineCommand({ "experimental-fast-capture": { type: "boolean", description: - "EXPERIMENTAL. Capture frames via Chrome's drawElementImage API " + - "instead of Page.captureScreenshot — reads DOM paint records directly, " + - "~46% faster on GPU. Transparent (PNG) renders on SwiftShader (Docker) " + - "auto-fall back to screenshot capture. Incompatible with page-side " + - "shader compositing. Default: false. Env: PRODUCER_EXPERIMENTAL_FAST_CAPTURE.", + "Capture frames via Chrome's drawElementImage API instead of " + + "Page.captureScreenshot — reads DOM paint records directly, ~2x faster. " + + "Default: on where it can engage (macOS + hardware-GPU browser); " + + "incompatible compositions and self-verification failures fall back to " + + "screenshot capture automatically. Pass =false to disable. " + + "Env: PRODUCER_EXPERIMENTAL_FAST_CAPTURE.", // No `default` — an omitted flag must stay `undefined` so the `!= null` // guard below leaves PRODUCER_EXPERIMENTAL_FAST_CAPTURE untouched and the // env fallback survives (matches the --low-memory-mode idiom). diff --git a/packages/engine/src/config.test.ts b/packages/engine/src/config.test.ts index 738876c251..206cc5f719 100644 --- a/packages/engine/src/config.test.ts +++ b/packages/engine/src/config.test.ts @@ -222,17 +222,31 @@ describe("resolveConfig", () => { }); describe("useDrawElement (PRODUCER_EXPERIMENTAL_FAST_CAPTURE)", () => { - it("defaults to false", () => { + it("default is clamped off on software-GPU hosts (page-side compositing preserved)", () => { const config = resolveConfig(); expect(config.useDrawElement).toBe(false); + expect(config.enablePageSideCompositing).toBe(true); + }); + + it("default engages on macOS with a hardware-GPU browser", () => { + setEnv("PRODUCER_BROWSER_GPU_MODE", "hardware"); + const config = resolveConfig(); + expect(config.useDrawElement).toBe(process.platform === "darwin"); }); - it("enabled when PRODUCER_EXPERIMENTAL_FAST_CAPTURE=true", () => { + it("explicit env opt-in skips the platform clamp", () => { setEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", "true"); const config = resolveConfig(); expect(config.useDrawElement).toBe(true); }); + it("env kill switch wins over the default", () => { + setEnv("PRODUCER_BROWSER_GPU_MODE", "hardware"); + setEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", "false"); + const config = resolveConfig(); + expect(config.useDrawElement).toBe(false); + }); + it("explicit override wins over the env var", () => { setEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", "true"); const config = resolveConfig({ useDrawElement: false }); diff --git a/packages/engine/src/config.ts b/packages/engine/src/config.ts index c8f80e5b6f..479c30c721 100644 --- a/packages/engine/src/config.ts +++ b/packages/engine/src/config.ts @@ -64,18 +64,21 @@ export interface EngineConfig { */ staticFrameDedup: boolean; /** - * EXPERIMENTAL. Use drawElementImage for frame capture (requires the - * CanvasDrawElement Chrome flag, added globally in buildChromeArgs). - * Surfaced via the CLI `--experimental-fast-capture` flag. - * Env fallback: `PRODUCER_EXPERIMENTAL_FAST_CAPTURE`. + * Use drawElementImage for frame capture (requires the CanvasDrawElement + * Chrome flag, added globally in buildChromeArgs). Default ON, clamped in + * `resolveConfig` to hosts where it can actually engage (macOS + hardware-GPU + * browser); compile/init gates and the runtime self-verification net route + * incompatible or damaged renders back to screenshot capture. + * Kill switch: `PRODUCER_EXPERIMENTAL_FAST_CAPTURE=false` (or the CLI + * `--experimental-fast-capture=false`). */ useDrawElement: boolean; /** - * EXPERIMENTAL. Pipeline JPEG encode into an in-page OffscreenCanvas Worker - * for the drawElement fast-capture path (macOS hardware GPU only). The worker - * encodes frame N while the main thread seeks+paints frame N+1, targeting - * ~1.65–1.96× wall-time speedup. No-op unless `useDrawElement` is also true. - * Default: off. Env: `HF_DE_WORKER_ENCODE=true`. + * Pipeline JPEG encode into an in-page OffscreenCanvas Worker for the + * drawElement fast-capture path (macOS hardware GPU only). The worker + * encodes frame N while the main thread seeks+paints frame N+1 + * (~1.65–1.96× wall-time speedup). No-op unless `useDrawElement` is also + * true. Kill switch: `HF_DE_WORKER_ENCODE=false`. */ enableDrawElementWorkerEncode: boolean; /** @@ -250,8 +253,8 @@ export const DEFAULT_CONFIG: EngineConfig = { protocolTimeout: 300_000, forceScreenshot: false, staticFrameDedup: true, - useDrawElement: false, - enableDrawElementWorkerEncode: false, + useDrawElement: true, + enableDrawElementWorkerEncode: true, // Auto-detected per host in `resolveConfig`; defaults off for the raw // DEFAULT_CONFIG (used directly by tests and worker-sizing fallbacks). lowMemoryMode: false, @@ -516,10 +519,27 @@ export function resolveConfig(overrides?: Partial): EngineConfig { ...overrides, }; + // Default-on drawElement is clamped to hosts where it can actually engage + // (macOS + a hardware-GPU browser; SwiftShader drops transparent sub-layers — + // crbug 521434899). Without the clamp, the default would needlessly disable + // page-side shader compositing (below) on Linux/Docker and macOS-software + // hosts where DE never runs. An EXPLICIT opt-in (env or caller override) + // skips the clamp and keeps the old semantics — attempt DE, let the + // init-time gates route away — which debugging relies on. + const explicitDrawElementOptIn = + env("PRODUCER_EXPERIMENTAL_FAST_CAPTURE") === "true" || overrides?.useDrawElement === true; + if ( + merged.useDrawElement && + !explicitDrawElementOptIn && + !(process.platform === "darwin" && merged.browserGpuMode === "hardware") + ) { + merged.useDrawElement = false; + } + // drawElement capture and page-side shader compositing are mutually // incompatible capture strategies (drawElement reads paint records directly // and bypasses the page-side prepare→composite→resolve protocol). When - // experimental fast capture is on, force page-side compositing off so shader + // fast capture is on, force page-side compositing off so shader // transitions fall back to the Node-side layered blend rather than silently // dropping. This keeps the flag self-consistent and avoids a per-session // incompatibility warning on every fast-capture render. diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 20efb4cec7..49b5545015 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -84,6 +84,8 @@ export { captureFrameToBuffer, captureFrameToBufferPipelined, captureFramesBatchPipelined, + DrawElementVerificationError, + isDrawElementVerificationError, writeCapturedFrame, discardWarmupCapture, getCompositionDuration, diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 19fb5016c5..27fef96c4e 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -146,6 +146,42 @@ export interface CaptureSession { * worker-encode path (mirrors `lastFrameBuffer` on the screenshot path). Only set * when static-frame dedup is armed on the drawElement path. */ lastEncodeResult?: Promise; + /** Per-render self-verification ground truth (ungated-release safety net): + * K screenshot frames captured at init BEFORE the drawElement canvas is + * injected (the only window where a page screenshot shows the live DOM, not + * the capture canvas's stale bitmap). The producer drain compares the DE + * frame at each index against these; a breach aborts the render with + * DrawElementVerificationError and the orchestrator re-renders via the + * screenshot path. */ + deVerifyFrames?: Map; +} + +/** + * drawElement self-verification failure — a captured DE frame diverged from its + * pre-injection screenshot ground truth (or a blank frame survived a retry). + * The orchestrator catches this and re-renders the whole job with + * forceScreenshot. Discriminant-based guard (not instanceof) so it survives + * duplicated module instances across package boundaries. + */ +export class DrawElementVerificationError extends Error { + constructor(message: string) { + super(message); + this.name = "DrawElementVerificationError"; + // Discriminant property, assigned dynamically: isDrawElementVerificationError + // reads it structurally so detection survives duplicated module instances + // across package boundaries (where instanceof fails). + (this as unknown as { deVerificationFailure: boolean }).deVerificationFailure = true; + } +} + +export function isDrawElementVerificationError(err: unknown): boolean { + // Walk the cause chain — the producer wraps capture errors (CaptureStageError). + let e: unknown = err; + for (let depth = 0; depth < 5 && typeof e === "object" && e !== null; depth++) { + if ((e as { deVerificationFailure?: boolean }).deVerificationFailure === true) return true; + e = (e as { cause?: unknown }).cause; + } + return false; } // Circular buffer for browser console messages dumped on render failure diagnostics. @@ -581,6 +617,10 @@ async function initDrawElementOrTransparentBackground( // passed) and the DOM is still normal (canvas not yet injected, so the // verification screenshots are valid). drawElement-path only; see armStaticDedup. await armStaticDedup(session, page, logInitPhase); + // Self-verification ground truth: must ALSO run pre-injection — after the + // canvas wraps the root, a page screenshot shows the canvas's last-drawn + // bitmap, not the live DOM (see the Lim 6 boundary-screenshot note). + await captureDeVerificationFrames(session, page, logInitPhase); await injectDrawElementCanvas(page, session.options.width, session.options.height); if (transparent) { await initTransparentBackground(session.page); @@ -2176,7 +2216,7 @@ async function computeTimelineAtRiskFrames( * toggled / detached / freshly-shown at a clip-cut boundary). Per-frame, not * whole-comp — callers fall back to screenshot for the single frame. */ -export function isNoCachedPaintRecordError(err: unknown): boolean { +function isNoCachedPaintRecordError(err: unknown): boolean { const msg = err instanceof Error ? err.message : String(err); return msg.includes("No cached paint record"); } @@ -2740,6 +2780,81 @@ export async function getCompositionDuration(session: CaptureSession): Promise void, +): Promise { + const k = Math.max(0, Math.min(8, Math.floor(Number(process.env.HF_DE_VERIFY ?? "4")) || 0)); + if (k === 0 || process.env.HF_FORCE_DRAWELEMENT === "1") return; + if (session.options.format === "png") return; // worker-encode drain (the consumer) is jpeg-only + const fps = fpsToNumber(session.options.fps); + const duration = await page.evaluate( + () => (window as unknown as { __hf?: { duration?: number } }).__hf?.duration ?? 0, + ); + const totalFrames = Math.floor(duration * fps); + if (totalFrames < 10) return; + if (duration > 3600) { + // Implausible __hf.duration (e.g. infinite-repeat GSAP timelines report a + // huge sentinel; the producer renders the data-duration clamp instead). + // Fraction-based indices computed from it would never be drained. + // ponytail: skip verification here; threading the producer-resolved + // duration into CaptureOptions is the upgrade path if coverage matters. + logInitPhase(`drawElement self-verify skipped: implausible duration ${duration}s`); + return; + } + const boundary = await computeClipBoundaryFrames(page, fps); + const fractions = [0.15, 0.4, 0.65, 0.9, 0.25, 0.55, 0.8, 0.05].slice(0, k); + const frames = new Map(); + for (const f of fractions) { + let idx = Math.min(totalFrames - 1, Math.max(1, Math.round(totalFrames * f))); + // Nudge off clip-cut boundaries (±1-frame desync is legitimate there). + let guard = 0; + while (boundary.has(idx) && guard++ < 6) idx = Math.min(totalFrames - 1, idx + 2); + if (frames.has(idx)) continue; + const t = quantizeTimeToFrame(idx / fps, fps); + await page.evaluate((tt: number) => { + const hf = (window as unknown as { __hf?: { seek?: (x: number) => void } }).__hf; + if (hf && typeof hf.seek === "function") hf.seek(tt); + }, t); + // Video frame injection (same hook the real capture paths run) — without + // it,