From 251add2d0f884f351371563b38f4f92476d3de8c Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Fri, 3 Jul 2026 19:19:00 -0700 Subject: [PATCH] feat(producer): fast-capture render stages + remote bg-image localizer --- .../producer/scripts/validate-fast-video.ts | 79 ++++++++ .../src/generated/hf-early-stub-inline.ts | 2 +- packages/producer/src/regression-harness.ts | 46 ++++- .../src/services/htmlCompiler.test.ts | 46 +++++ .../producer/src/services/htmlCompiler.ts | 93 ++++++++- .../services/render/stages/captureStage.ts | 51 +++-- .../render/stages/captureStreamingStage.ts | 102 ++++++++-- .../services/render/stages/compileStage.ts | 59 +++++- .../src/services/render/stages/probeStage.ts | 73 +++++++ .../src/services/renderOrchestrator.ts | 9 + packages/producer/stubs/hf-early-stub.ts | 191 +++++++++++++++++- .../css-spinner-render-compat/src/index.html | 2 +- .../tests/fast-capture-3d/src/index.html | 64 ++++++ .../tests/fast-capture-gsap/meta.json | 13 ++ .../fast-capture-gsap/output/compiled.html | 42 ++++ .../tests/fast-capture-gsap/output/output.mp4 | 3 + .../tests/fast-capture-gsap/src/index.html | 32 +++ .../render-symlinked-assets/src/index.html | 1 + 18 files changed, 858 insertions(+), 50 deletions(-) create mode 100644 packages/producer/scripts/validate-fast-video.ts create mode 100644 packages/producer/tests/fast-capture-3d/src/index.html create mode 100644 packages/producer/tests/fast-capture-gsap/meta.json create mode 100644 packages/producer/tests/fast-capture-gsap/output/compiled.html create mode 100644 packages/producer/tests/fast-capture-gsap/output/output.mp4 create mode 100644 packages/producer/tests/fast-capture-gsap/src/index.html diff --git a/packages/producer/scripts/validate-fast-video.ts b/packages/producer/scripts/validate-fast-video.ts new file mode 100644 index 0000000000..e178b79c91 --- /dev/null +++ b/packages/producer/scripts/validate-fast-video.ts @@ -0,0 +1,79 @@ +/** + * Validate the fast-capture (drawElementImage) VIDEO path on real Linux. + * + * drawElementImage draws a snapshot taken at the paint event; capturing video + * needs a fresh per-frame paint. On Linux headless-shell that paint comes from + * the per-frame HeadlessExperimental.beginFrame — so video should capture + * correctly there (see docs/fast-capture-limitations.md, Limitation 2). This + * could not be validated under Docker-on-rosetta (renders hung); this script is + * meant to run on a native amd64 Linux runner inside Dockerfile.test. + * + * Renders a video composition twice — baseline (screenshot) and fast + * (drawElement) — and asserts the fast output matches the baseline (PSNR above + * threshold), proving the video was captured and not dropped to black. + * + * PRODUCER_VALIDATE_COMP=sub-composition-video \ + * bunx tsx scripts/validate-fast-video.ts + * + * Exit 0 = fast video matches baseline; exit 1 = regression (black/stale video). + */ +import { execFileSync } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { createRenderJob, executeRenderJob } from "../src/index.js"; + +// `||` not `??` — the workflow passes empty strings on a push trigger (inputs +// are only populated for workflow_dispatch), and "" must fall through to the default. +const COMP = process.env.PRODUCER_VALIDATE_COMP || "sub-composition-video"; +const MIN_PSNR = Number.parseFloat(process.env.PRODUCER_VALIDATE_MIN_PSNR || "25"); +const work = mkdtempSync(join(tmpdir(), "fastvideo-")); + +process.env.PRODUCER_ENABLE_BROWSER_POOL = "false"; + +async function render(mode: "baseline" | "fast", out: string): Promise { + process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = mode === "fast" ? "true" : "false"; + const job = createRenderJob({ + fps: 30, + quality: "high", + format: "mp4", + workers: 1, + useGpu: false, + hdrMode: "force-sdr", + }); + await executeRenderJob(job, resolve("tests", COMP, "src"), out); +} + +function psnr(a: string, b: string): number { + const out = execFileSync( + "bash", + ["-c", `ffmpeg -y -i "${a}" -i "${b}" -lavfi psnr -f null - 2>&1`], + { encoding: "utf8" }, + ); + const m = out.match(/average:(\S+)/); + if (!m) throw new Error(`ffmpeg psnr produced no average:\n${out}`); + return m[1] === "inf" ? Number.POSITIVE_INFINITY : Number.parseFloat(m[1]); +} + +async function main(): Promise { + const baseline = join(work, "baseline.mp4"); + const fast = join(work, "fast.mp4"); + console.log(`[validate-fast-video] comp=${COMP} minPsnr=${MIN_PSNR}`); + await render("baseline", baseline); + await render("fast", fast); + const db = psnr(baseline, fast); + console.log(`[validate-fast-video] fast-vs-baseline PSNR = ${db} dB`); + if (db < MIN_PSNR) { + console.error( + `[validate-fast-video] FAIL — ${db} dB < ${MIN_PSNR} dB. Fast capture dropped video ` + + `(stale/black snapshot). The Linux BeginFrame paint path is not capturing video.`, + ); + process.exit(1); + } + console.log("[validate-fast-video] PASS — fast video matches baseline."); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/producer/src/generated/hf-early-stub-inline.ts b/packages/producer/src/generated/hf-early-stub-inline.ts index 8a938fbe8c..9e05bd0a4d 100644 --- a/packages/producer/src/generated/hf-early-stub-inline.ts +++ b/packages/producer/src/generated/hf-early-stub-inline.ts @@ -1,6 +1,6 @@ // AUTO-GENERATED by scripts/build-hf-early-stub.ts — do not edit const HF_EARLY_STUB_IIFE: string = - '"use strict";(()=>{var T=100,_=[],u=[],l=!1,s=!1;function m(n){let i=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof i=="function"?i(n):requestAnimationFrame(n)}function y(n){let i=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof i=="function"){i(n,0);return}setTimeout(n,0)}function g(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function c(n){let i=n.proxy.__hfReal,e=i[n.method];if(typeof e=="function"){let o=n.method==="add"?n.args.map(g):n.args;e.call(i,...o)}}function r(n,i,e){let o={proxy:n,method:i,args:e};return n.__hfQueue.push(o),u.push(o),P(),n}function d(n){let i=n.proxy.__hfQueue.indexOf(n);i>=0&&n.proxy.__hfQueue.splice(i,1)}function t(){for(;u.length>0;){let n=u.shift();n&&(d(n),c(n))}x()}function f(){s=!1,window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function x(){s||(s=!0,y(()=>{u.length===0?f():s=!1}))}function k(){l=!1;let n=u.splice(0,T);for(let i of n)d(i),c(i);u.length>0?(l=!0,m(k)):f()}function P(){l||(l=!0,window.__hfTimelinesBuilding=!0,m(k))}var O=new Set(["to","from","fromTo","set","add"]);function b(n,i){let e=i;for(;e!==null&&e!==Object.prototype;){for(let o of Object.getOwnPropertyNames(e)){if(o==="constructor"||o==="then"||o in n||O.has(o)||o.charAt(0)==="_")continue;let a=Object.getOwnPropertyDescriptor(e,o);if(!a||typeof a.value!="function")continue;let h=a.value;n[o]=function(...p){t();let w=h.call(i,...p);return w===i?n:w}}e=Object.getPrototypeOf(e)}}function v(n){let i={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...e){return r(i,"to",e)},from(...e){return r(i,"from",e)},fromTo(...e){return r(i,"fromTo",e)},set(...e){return r(i,"set",e)},add(...e){return r(i,"add",e)},pause(...e){return t(),n.pause(...e),i},play(...e){return t(),n.play(...e),i},seek(...e){return t(),n.seek(...e),i},totalTime(...e){return t(),e.length>0?(n.totalTime(...e),i):n.totalTime()},time(...e){return t(),e.length>0?(n.time(...e),i):n.time()},duration(...e){return t(),e.length>0?(n.duration(...e),i):n.duration()},getChildren(...e){t();let o=n.getChildren(...e);return Array.isArray(o)?o:[]},paused(...e){return t(),e.length>0?(n.paused(...e),i):n.paused()},timeScale(...e){return t(),e.length>0?(n.timeScale(...e),i):n.timeScale()},kill(){t(),n.kill()}};return b(i,n),_.push(i),i}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{t(),u.length===0&&window.__hfTimelinesBuilding&&f()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(i){if(n=i,!i||typeof i.timeline!="function")return;let e=i.timeline.bind(i);i.timeline=o=>v(e(o))}})}catch{}}})();\n'; + '"use strict";(()=>{var b=100,O=[],u=[],w=!1,f=!1,c=new Set,p=new Set;function y(n){let e=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof e=="function"?e(n):requestAnimationFrame(n)}function x(n){let e=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof e=="function"){e(n,0);return}setTimeout(n,0)}function S(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function _(n){if(n===null||typeof n!="object"||Array.isArray(n))return n;let e=n;if(!("opacity"in e)||"autoAlpha"in e||"visibility"in e)return n;let o={};for(let t of Object.keys(e))t==="opacity"?o.autoAlpha=e[t]:o[t]=e[t];return o}function T(n,e){if(n!=="add"&&v(e),window.__HF_FAST_CAPTURE_AUTOALPHA__!==!0||n==="add")return e;let o=e.slice(),t=!1;if(o.length>1){let i=_(o[1]);i!==o[1]&&(t=!0),o[1]=i}if(n==="fromTo"&&o.length>2){let i=_(o[2]);i!==o[2]&&(t=!0),o[2]=i}return t&&o[0]!==null&&o[0]!==void 0&&c.add(o[0]),o}function h(n){if(n===null||typeof n!="object"||Array.isArray(n))return!1;let e=n;return"rotationX"in e||"rotationY"in e||"transformPerspective"in e}var m=new Set;function v(n){let e=n[0];if(e==null)return;let o=window;m.has(e)||(m.add(e),o.__hfAllTweenTargets=Array.from(m)),(h(n[1])||h(n[2]))&&(p.add(e),o.__hf3dTweenTargets=Array.from(p))}function R(n){if(typeof n=="string")try{return Array.from(document.querySelectorAll(n))}catch{return[]}if(n instanceof Element)return[n];if(Array.isArray(n)||typeof NodeList<"u"&&n instanceof NodeList){let e=[];for(let o of n)o instanceof Element&&e.push(o);return e}return[]}function F(){if(window.__HF_FAST_CAPTURE_AUTOALPHA__!==!0||c.size===0)return;let n=Array.from(c);c.clear();for(let e of n)for(let o of R(e)){let t=o;if(!t.style||t.style.visibility!=="")continue;let i="";try{i=getComputedStyle(o).opacity}catch{continue}i==="0"&&(t.style.visibility="hidden")}}function g(n){let e=n.proxy.__hfReal,o=e[n.method];if(typeof o=="function"){let t=n.method==="add"?n.args.map(S):n.args;o.call(e,...t)}}function l(n,e,o){let t={proxy:n,method:e,args:T(e,o)};return n.__hfQueue.push(t),u.push(t),I(),n}function A(n){let e=n.proxy.__hfQueue.indexOf(n);e>=0&&n.proxy.__hfQueue.splice(e,1)}function r(){for(;u.length>0;){let n=u.shift();n&&(A(n),g(n))}E()}function k(){f=!1,F(),window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function E(){f||(f=!0,x(()=>{u.length===0?k():f=!1}))}function P(){w=!1;let n=u.splice(0,b);for(let e of n)A(e),g(e);u.length>0?(w=!0,y(P)):k()}function I(){w||(w=!0,window.__hfTimelinesBuilding=!0,y(P))}var C=new Set(["to","from","fromTo","set","add"]);function G(n,e){let o=e;for(;o!==null&&o!==Object.prototype;){for(let t of Object.getOwnPropertyNames(o)){if(t==="constructor"||t==="then"||t in n||C.has(t)||t.charAt(0)==="_")continue;let i=Object.getOwnPropertyDescriptor(o,t);if(!i||typeof i.value!="function")continue;let s=i.value;n[t]=function(...d){r();let a=s.call(e,...d);return a===e?n:a}}o=Object.getPrototypeOf(o)}}function H(n){let e={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...o){return l(e,"to",o)},from(...o){return l(e,"from",o)},fromTo(...o){return l(e,"fromTo",o)},set(...o){return l(e,"set",o)},add(...o){return l(e,"add",o)},pause(...o){return r(),n.pause(...o),e},play(...o){return r(),n.play(...o),e},seek(...o){return r(),n.seek(...o),e},totalTime(...o){return r(),o.length>0?(n.totalTime(...o),e):n.totalTime()},time(...o){return r(),o.length>0?(n.time(...o),e):n.time()},duration(...o){return r(),o.length>0?(n.duration(...o),e):n.duration()},getChildren(...o){r();let t=n.getChildren(...o);return Array.isArray(t)?t:[]},paused(...o){return r(),o.length>0?(n.paused(...o),e):n.paused()},timeScale(...o){return r(),o.length>0?(n.timeScale(...o),e):n.timeScale()},kill(){r(),n.kill()}};return G(e,n),O.push(e),e}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{r(),u.length===0&&window.__hfTimelinesBuilding&&k()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(e){if(n=e,!e||typeof e.timeline!="function")return;let o=e.timeline.bind(e);e.timeline=i=>H(o(i));for(let i of["to","from","set"]){let s=e[i];if(typeof s!="function")continue;let d=s.bind(e);e[i]=(...a)=>d(...T(i,a))}let t=e.fromTo;if(typeof t=="function"){let i=t.bind(e);e.fromTo=(...s)=>i(...T("fromTo",s))}}})}catch{}}})();\n'; /** * Returns the pre-built HyperFrames early stub IIFE as a string constant. diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index 6e98872226..3f1101c619 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -113,6 +113,13 @@ type TestMetadata = { workers?: number; // Optional: auto-calculates if omitted /** Force HDR in the harness; omitted/false preserves historical SDR-only test behavior. */ hdr?: boolean; + /** + * Render this suite with the experimental fast-capture path + * (drawElementImage, `--experimental-fast-capture`). The golden must be + * regenerated with the flag on. Used by the `fast-capture` regression + * guard; omit for the default screenshot/BeginFrame capture. + */ + experimentalFastCapture?: boolean; /** * Render-time variable overrides, equivalent to `hyperframes render * --variables ''`. Injected as `window.__hfVariables` before any @@ -374,6 +381,11 @@ function validateMetadata(meta: unknown): TestMetadata { if (rc.hdr !== undefined && typeof rc.hdr !== "boolean") { throw new Error("meta.json: 'renderConfig.hdr' must be a boolean (or omit for false)"); } + if (rc.experimentalFastCapture !== undefined && typeof rc.experimentalFastCapture !== "boolean") { + throw new Error( + "meta.json: 'renderConfig.experimentalFastCapture' must be a boolean (or omit for false)", + ); + } if ( rc.variables !== undefined && (rc.variables === null || typeof rc.variables !== "object" || Array.isArray(rc.variables)) @@ -1017,18 +1029,30 @@ async function runTestSuite( await runDistributedSimulatedRender(distributedInput); } } else { - const job = createRenderJob({ - fps: suite.meta.renderConfig.fps, - quality: "high", // Always use max quality for tests - format: outputFormat, - workers: suite.meta.renderConfig.workers, - useGpu: false, - debug: false, - hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr", - variables: suite.meta.renderConfig.variables, - }); + // Opt-in fast capture (drawElementImage): drives resolveConfig via the env + // var, scoped to this suite's render so it never leaks to other suites. + const useFast = suite.meta.renderConfig.experimentalFastCapture === true; + const prevFast = process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE; + if (useFast) process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = "true"; + try { + const job = createRenderJob({ + fps: suite.meta.renderConfig.fps, + quality: "high", // Always use max quality for tests + format: outputFormat, + workers: suite.meta.renderConfig.workers, + useGpu: false, + debug: false, + hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr", + variables: suite.meta.renderConfig.variables, + }); - await executeRenderJob(job, tempSrcDir, renderedOutputPath); + await executeRenderJob(job, tempSrcDir, renderedOutputPath); + } finally { + if (useFast) { + if (prevFast === undefined) delete process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE; + else process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = prevFast; + } + } } console.log(JSON.stringify({ event: "rendering_complete", suite: suite.id })); diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index c2c368a823..a545eb4e63 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { describe, expect, it, mock, beforeAll } from "bun:test"; import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -8,6 +9,7 @@ import { compileForRender, detectRenderModeHints, detectShaderTransitionUsage, + detectThreeDTransformUsage, discoverAudioVolumeAutomationFromTimeline, inlineExternalScripts, localizeRemoteMediaSources, @@ -649,6 +651,50 @@ describe("detectRenderModeHints", () => { }); }); +describe("detectThreeDTransformUsage", () => { + it("detects CSS perspective property", () => { + expect(detectThreeDTransformUsage("")).toBe(true); + }); + + it("detects transform-style preserve-3d", () => { + expect(detectThreeDTransformUsage("")).toBe( + true, + ); + }); + + it("detects backface-visibility", () => { + expect(detectThreeDTransformUsage("")).toBe( + true, + ); + }); + + it("detects perspective() transform function", () => { + expect(detectThreeDTransformUsage('
')).toBe( + true, + ); + }); + + it("detects GSAP transformPerspective", () => { + expect( + detectThreeDTransformUsage(""), + ).toBe(true); + }); + + it("does not match flat GSAP rotationX without a perspective context", () => { + expect(detectThreeDTransformUsage("")).toBe( + false, + ); + }); + + it("does not match translateZ(0) promotion hack", () => { + expect(detectThreeDTransformUsage('
')).toBe(false); + }); + + it("does not match perspective: none", () => { + expect(detectThreeDTransformUsage("")).toBe(false); + }); +}); + describe("detectShaderTransitionUsage", () => { it("detects authored HyperShader initialization", () => { const html = ` diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index b0dd795080..39a10a974e 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication complexity /** * HTML Compiler for Producer * @@ -67,6 +68,10 @@ export interface CompiledComposition { staticDuration: number; renderModeHints: RenderModeHints; hasShaderTransitions: boolean; + /** Author HTML/CSS/scripts use a CSS 3D rendering context (pre-CDN-inline scan). */ + usesThreeDTransforms: boolean; + /** Author HTML/CSS use mix-blend-mode (pre-CDN-inline scan). */ + usesMixBlendMode: boolean; } /** Adapts linkedom's `parseHTML` to the `checkSubCompositionUsability` contract. */ @@ -273,6 +278,35 @@ export function detectRenderModeHints(html: string): RenderModeHints { }; } +/** + * 3D rendering-context signals. drawElementImage paints elements inside a + * CSS 3D rendering context incorrectly: backface-visibility:hidden is + * ignored (mid-flip elements show their mirrored backface), sibling content + * of the 3D context can drop out of the capture, and the context's + * background is lost. Observed on real-world gen_os comps (flip-card and + * rotationX scene-entrance patterns) on macOS hardware GPU — this is a + * drawElementImage limitation, not a SwiftShader artifact. + * + * Only genuine 3D-context signals are matched: `perspective` (property or + * transform function), `transform-style: preserve-3d`, `backface-visibility`, + * `matrix3d(` / `rotate3d(`, and GSAP's `transformPerspective`. Flat + * rotationX/Y tweens without a perspective context render as 2D and are + * deliberately NOT matched, nor is the ubiquitous `translateZ(0)` promotion + * hack. + */ +const THREE_D_CONTEXT_PATTERN = + /transform-style\s*:\s*preserve-3d|backface-visibility\s*:|perspective\s*:\s*[0-9]|perspective\s*\(|matrix3d\s*\(|rotate3d\s*\(|\btransformPerspective\b/i; + +export function detectThreeDTransformUsage(html: string): boolean { + return THREE_D_CONTEXT_PATTERN.test(html); +} + +const MIX_BLEND_MODE_PATTERN = /mix-blend-mode\s*:/i; + +function detectMixBlendModeUsage(html: string): boolean { + return MIX_BLEND_MODE_PATTERN.test(html); +} + const SHADER_TRANSITION_USAGE_PATTERN = /\b(?:(?:window|globalThis)\s*\.\s*)?HyperShader\s*\.\s*init\s*\(|\b__hf\s*\.\s*transitions\s*=/; @@ -1176,6 +1210,47 @@ export async function localizeRemoteImageSources( ); } +// Match a remote url() inside a `background` / `background-image` CSS declaration +// (style blocks or inline style attrs). `[^;}"']*?` lets position/color tokens +// precede the url() in the shorthand while stopping at the declaration boundary. +const REMOTE_BG_URL_RE = + /background(?:-image)?\s*:\s*[^;}"']*?url\(\s*["']?(https?:\/\/[^"')]+)["']?\s*\)/gi; + +/** + * Download remote CSS `background-image: url(https://...)` references and rewrite + * them to local same-origin paths. + * + * Why: `drawElementImage` (fast capture) OMITS cross-origin content, so a remote + * background image renders BLACK on the drawElement path while the screenshot + * baseline captures it (origin-agnostic) — a whole-region mismatch (e.g. 10f79c0b + * picsum.photos backgrounds, 9.3 dB). ``/`