From 4f126a05e8474faec20d9da8098e12ebdf9b1f5f Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 02:20:48 -0700 Subject: [PATCH 01/28] feat: add 3D world effect pack and multipass host --- apps/slides/examples/3d/world-tour.mdx | 88 +++++++++ apps/slides/src/TypeGPUShader.tsx | 71 ++++--- .../src/__tests__/effectsSettings.test.tsx | 4 +- .../src/__tests__/gpuRenderGraph.test.ts | 136 ++++++++++++++ apps/slides/src/__tests__/gpuTexture.test.ts | 4 +- .../src/__tests__/typeGPUShader.test.ts | 53 +++++- apps/slides/src/gpuRenderGraph.ts | 171 +++++++++++++++++ packages/presentation/src/GPUShader.tsx | 32 +++- .../src/__tests__/effectInstallation.test.ts | 6 +- .../src/__tests__/effectPacks.test.ts | 10 +- .../presentation/src/effectPacks/schema.ts | 4 +- packages/presentation/src/index.ts | 2 +- packages/slides-effects/packs/world/README.md | 103 +++++++++++ packages/slides-effects/packs/world/World.tsx | 124 +++++++++++++ packages/slides-effects/packs/world/camera.ts | 81 ++++++++ packages/slides-effects/packs/world/graph.ts | 63 +++++++ packages/slides-effects/packs/world/pack.json | 7 + packages/slides-effects/packs/world/render.ts | 169 +++++++++++++++++ packages/slides-effects/packs/world/scene.ts | 173 ++++++++++++++++++ .../scripts/renderGraphBenchmark.ts | 119 ++++++++++++ .../scripts/world-benchmark.mjs | 16 ++ .../slides-effects/scripts/worldBenchmark.ts | 30 +++ packages/slides-effects/tests/world.test.ts | 157 ++++++++++++++++ .../tests/worldPlayback.test.tsx | 169 +++++++++++++++++ packages/slides-web/export.ts | 2 +- packages/slides-web/tests/export.test.mjs | 1 + 26 files changed, 1752 insertions(+), 43 deletions(-) create mode 100644 apps/slides/examples/3d/world-tour.mdx create mode 100644 apps/slides/src/__tests__/gpuRenderGraph.test.ts create mode 100644 apps/slides/src/gpuRenderGraph.ts create mode 100644 packages/slides-effects/packs/world/README.md create mode 100644 packages/slides-effects/packs/world/World.tsx create mode 100644 packages/slides-effects/packs/world/camera.ts create mode 100644 packages/slides-effects/packs/world/graph.ts create mode 100644 packages/slides-effects/packs/world/pack.json create mode 100644 packages/slides-effects/packs/world/render.ts create mode 100644 packages/slides-effects/packs/world/scene.ts create mode 100644 packages/slides-effects/scripts/renderGraphBenchmark.ts create mode 100644 packages/slides-effects/scripts/world-benchmark.mjs create mode 100644 packages/slides-effects/scripts/worldBenchmark.ts create mode 100644 packages/slides-effects/tests/world.test.ts create mode 100644 packages/slides-effects/tests/worldPlayback.test.tsx diff --git a/apps/slides/examples/3d/world-tour.mdx b/apps/slides/examples/3d/world-tour.mdx new file mode 100644 index 000000000..5b6f82fb3 --- /dev/null +++ b/apps/slides/examples/3d/world-tour.mdx @@ -0,0 +1,88 @@ +--- +title: World Tour +aspectRatio: 16:9 +width: 1920 +height: 1080 +transition: fade +theme: + backgroundColor: "#050811" + color: "#f8fafc" +--- + +import { Text } from "react-native" +import { World, Camera, Graph, tree, wave, subset, Orbs, Sphere, Box, Tube, Ring, Fog, Bloom, Vignette, Grain } from "slides:world/World" + +export const network = tree("root", [ + ["north", [["n1", ["n11", "n12", "n13"]], ["n2", ["n21", "n22", "n23"]]]], + ["east", [["e1", ["e11", "e12", "e13"]], ["e2", ["e21", "e22", "e23"]]]], + ["west", [["w1", ["w11", "w12", "w13"]], ["w2", ["w21", "w22", "w23"]]]], +]) + +# Into the world + + + + + + + + + + +--- + +# A moving viewpoint + + + + + + + + + +Orbit +Dolly + +--- + +# Across the branches + + + + + + + + + +Cascade +One branch + +--- + +# Light in the fog + + + + + + + + + Room for a thought + + +--- + +# Simple shapes + + + + + + + + + + diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 196ded1ed..78cb28a0c 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -3,10 +3,13 @@ import { AlphaType, ColorType, Skia } from "@shopify/react-native-skia"; import { PixelRatio, Platform, View } from "react-native"; import { Canvas, type CanvasRef, type RNCanvasContext } from "react-native-webgpu"; import { makeMutable, runOnJS, runOnUI, useFrameCallback, useSharedValue, type SharedValue } from "react-native-reanimated"; -import { PresentationCanvasScaleContext, useBackgroundIntensity, useBackgroundSize, usePresentationValue, usePlayback, samplePlayback, usePreparationTask, type GPUShaderProps } from "@legend-apps/presentation"; +import { PresentationCanvasScaleContext, useBackgroundIntensity, useBackgroundSize, usePresentationValue, usePlayback, samplePlayback, usePreparationTask, type GPURenderGraphDefinition, type GPUShaderProps } from "@legend-apps/presentation"; +import { prepareRenderGraph, encodeRenderGraph, type RenderGraphResources } from "./gpuRenderGraph"; import { reportSlideError } from "./slidesStore"; +let nextGPUResourceId = 0; type Resources = { + id: number; alive: SharedValue; disposed: SharedValue; device: GPUDevice; @@ -15,7 +18,8 @@ type Resources = { states: GPUBuffer[]; groups: GPUBindGroup[]; compute?: GPUComputePipeline; - render: GPURenderPipeline; + render?: GPURenderPipeline; + graph?: SharedValue; particles?: GPURenderPipeline; frameData?: GPUBuffer; composite?: { pipeline: GPURenderPipeline; group: GPUBindGroup; texture: GPUTexture }; @@ -25,7 +29,7 @@ type Resources = { export function TypeGPUShader({ code, width: configuredWidth = 1696, height: configuredHeight = 650, background = false, particles = false, particleCount = 256, particleVertexCount = 6, particleBlend = "additive", previewTime = 4, clock: clockMode = "step", image, composite = false, frameData, frameDataSize = 0, borderRadius, - accessibilityLabel, active = true, children, maxTime, frameTimeIndex }: GPUShaderProps) { + accessibilityLabel, active = true, children, maxTime, frameTimeIndex, renderGraph }: GPUShaderProps) { const size = useBackgroundSize(); const intensity = useBackgroundIntensity(); const displayScale = useContext(PresentationCanvasScaleContext); @@ -55,15 +59,15 @@ export function TypeGPUShader({ code, width: configuredWidth = 1696, height: con }, [width, height, displayScale, resources]); useEffect(() => { if (!supported) return; - return startShaderCanvas({ canvas, code, particles, particleCount, particleBlend, image, composite, width, height, displayScale, frameDataSize, + return startShaderCanvas({ canvas, code, particles, particleCount, particleBlend, image, composite, width, height, displayScale, frameDataSize, renderGraph, reportError: caught => reportSlideError(caught, errorContext.current.slideIndex, errorContext.current.isPreview), setResources, setError }); - }, [supported, code, particles, particleCount, particleBlend, image, composite, frameDataSize]); + }, [supported, code, particles, particleCount, particleBlend, image, composite, frameDataSize, renderGraph]); if (!supported) throw new Error("WebGPU is required to render this effect"); if (error) throw new Error(error); return - {resources && setPreparedResources(resources)} />} + {resources && { setError(message); reportSlideError(message, errorContext.current.slideIndex, errorContext.current.isPreview); }} onReady={() => setPreparedResources(resources)} />} {resources && children} ; } @@ -87,8 +91,8 @@ function requestShaderDevice(adapter: GPUAdapter): Promise { } // Setup stays outside React compilation; the UI renderer owns transferred handles. -function startShaderCanvas({ canvas, code, particles, particleCount, particleBlend = "additive", image, composite = false, width = 1696, height = 650, displayScale = 1, frameDataSize = 0, reportError, setResources, setError }: { - canvas: RefObject; code: string; +function startShaderCanvas({ canvas, code, particles, particleCount, particleBlend = "additive", image, composite = false, width = 1696, height = 650, displayScale = 1, frameDataSize = 0, renderGraph, reportError, setResources, setError }: { + canvas: RefObject; code: string; renderGraph?: GPURenderGraphDefinition; particles: boolean; particleCount: number; particleBlend?: "additive" | "over"; image?: string; composite?: boolean; width?: number; height?: number; displayScale?: number; frameDataSize?: number; reportError(error: unknown): void; setResources(resources: Resources | undefined): void; setError(error: string | undefined): void; @@ -123,6 +127,13 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle if (Platform.OS === "web") resizeBrowserSurface(context, width, height, displayScale); const format = navigator.gpu.getPreferredCanvasFormat(); context.configure({ device, format, alphaMode: "premultiplied" }); + if (renderGraph) { + const clock = device.createBuffer({ size: 32, usage: 64 | 8 }); + const frameData = device.createBuffer({ size: Math.max(16, frameDataSize * 4), usage: 128 | 8 }); + const graph = await prepareRenderGraph(device, renderGraph, format, clock, frameData); + await transfer({ device, context, clock, frameData, graph, states: [], groups: [] }); + return; + } const fieldFormat = composite ? "rgba16float" : format; const module = device.createShaderModule({ code }); const diagnostics = await module.getCompilationInfo(); @@ -184,16 +195,7 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle }); compositeResources = { pipeline, group, texture }; } - validationScope = false; - const validationError = await device.popErrorScope(); - if (validationError) throw new Error(validationError.message); - if (cancelled) { device.destroy(); return; } - const alive = makeMutable(true); - const disposed = makeMutable(false); - const ownedDevice = device; - release = () => runOnUI(destroyGPU)(ownedDevice, alive, disposed); - transferred = true; - setResources({ alive, disposed, device, context, clock, states, groups, compute, render, particles: particlePipeline, frameData, composite: compositeResources }); + await transfer({ device, context, clock, states, groups, compute, render, particles: particlePipeline, frameData, composite: compositeResources }); } catch (caught) { if (validationScope) { validationScope = false; @@ -203,11 +205,21 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle if (!cancelled) { setError(caught instanceof Error ? caught.message : String(caught)); reportError(caught); } } } + async function transfer(program: Omit & { graph?: RenderGraphResources }) { + validationScope = false; + const validationError = await program.device.popErrorScope(); + if (validationError) throw new Error(validationError.message); + if (cancelled) { program.device.destroy(); return; } + const alive = makeMutable(true), disposed = makeMutable(false); + release = () => runOnUI(destroyGPU)(program.device, alive, disposed); + transferred = true; + setResources({ ...program, graph: program.graph ? makeMutable(program.graph) : undefined, alive, disposed, id: ++nextGPUResourceId }); + } void setup(); return () => { cancelled = true; if (release) release(); else if (!transferred) device?.destroy(); }; } -function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, width, height, aspect, brightness, particleCount, particleVertexCount = 6, clockMode = "step", frameData, previewTime, onError, onReady }: { resources: Resources; effectActive?: boolean; maxTime?:number; frameTimeIndex?:number; width: number; height: number; aspect: number; brightness: number; particleCount: number; particleVertexCount?: number; clockMode?: "slide" | "step"; frameData?: SharedValue; previewTime: number; onError(message: string): void; onReady(): void }) { +function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, displayScale = 1, width, height, aspect, brightness, particleCount, particleVertexCount = 6, clockMode = "step", frameData, previewTime, onError, onReady }: { resources: Resources; effectActive?: boolean; maxTime?:number; frameTimeIndex?:number; displayScale?:number; width: number; height: number; aspect: number; brightness: number; particleCount: number; particleVertexCount?: number; clockMode?: "slide" | "step"; frameData?: SharedValue; previewTime: number; onError(message: string): void; onReady(): void }) { const active = usePresentationValue("isActive"); const preview = usePresentationValue("isPreview"); const preparing = usePresentationValue("isPreparing"); @@ -219,7 +231,7 @@ function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, const finished = useSharedValue(false); const simulationEpoch = useSharedValue({ phase: "", slideKey: "", stepKey: "" }); const { device, context, clock, groups, compute, render, particles } = r; - const draw = (dt: number) => { + const draw = (dt: number, frozen = false) => { "worklet"; if (!alive.value) return; const state = playback.value; @@ -230,11 +242,12 @@ function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, count.value = 0; } simulationEpoch.value = { phase: state.phase, slideKey: state.slideKey, stepKey: state.stepKey }; + const frameIndex = frozen ? Math.max(0, count.value - 1) : count.value; try { const time = samplePlayback(state, previewTime, clockMode); - device.queue.writeBuffer(clock, 0, new Float32Array([time, dt, count.value, state.stepIndex, aspect, brightness, samplePlayback(state, previewTime, "slide"), samplePlayback(state, previewTime, "step")])); - submitGPUFrame(r, count.value, particleCount, particleVertexCount, frameData); - count.value += 1; + device.queue.writeBuffer(clock, 0, new Float32Array([time, dt, frameIndex, state.stepIndex, aspect, brightness, samplePlayback(state, previewTime, "slide"), samplePlayback(state, previewTime, "step")])); + submitGPUFrame(r, frameIndex, particleCount, particleVertexCount, frameData); + if (!frozen) count.value += 1; if (!rendered.value) { rendered.value = true; runOnJS(onReady)(); } } catch (caught) { alive.set(false); @@ -259,9 +272,10 @@ function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, if (phase === "playing") finished.value=false; if (phase === "preview" || phase === "preparing") { finished.value=false; count.value = 0; draw(0); } else if (phase === "playing" && (!rendered.value || !effectActive)) draw(0); + else if (r.graph && rendered.value && (phase === "paused" || phase === "outgoing")) draw(0, true); })(); return () => frame.setActive(false); - }, [active, effectActive, maxTime, frameTimeIndex, preview, preparing, previewTime, step, frame, rendered, finished, brightness, playback, width, height, clockMode, frameData, particleVertexCount]); + }, [active, effectActive, maxTime, frameTimeIndex, preview, preparing, previewTime, step, frame, rendered, finished, brightness, playback, displayScale, width, height, clockMode, frameData, particleVertexCount]); useEffect(() => () => { // Queue destruction behind the final UI frame, never race a submitted callback. runOnUI(destroyGPU)(device, alive, disposed); @@ -301,7 +315,7 @@ async function loadImageTexture(device: GPUDevice, uri: string) { const pixels = image.readPixels(0, 0, { width, height, colorType: ColorType.RGBA_8888, alphaType: AlphaType.Premul }); if (!(pixels instanceof Uint8Array)) throw new Error("Could not read GPU image pixels"); const texture = device.createTexture({ size: [width, height], format: "rgba8unorm", usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST }); - device.queue.writeTexture({ texture }, pixels, { bytesPerRow: width * 4 }, [width, height]); + device.queue.writeTexture({ texture }, pixels.buffer, { offset: pixels.byteOffset, bytesPerRow: width * 4 }, [width, height]); return texture; } finally { image.dispose(); } } finally { data.dispose(); } @@ -313,6 +327,11 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part const data = frameData?.value; if (r.frameData && data) device.queue.writeBuffer(r.frameData, 0, new Float32Array(data!)); const encoder = device.createCommandEncoder(); + if (r.graph) { + encodeRenderGraph(r.graph.value, device, encoder, context.getCurrentTexture()); + device.queue.submit([encoder.finish()]); context.present(); + return; + } const source = compute ? frame % 2 : 0; const target = compute ? 1-source : 0; if (compute) { @@ -325,7 +344,7 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part loadOp: "clear", storeOp: "store", }] }); pass.setBindGroup(0, groups[target]); - pass.setPipeline(render); pass.draw(3); + pass.setPipeline(render!); pass.draw(3); if (particles) { pass.setPipeline(particles); pass.draw(particleVertexCount, particleCount); } pass.end(); if (r.composite) { diff --git a/apps/slides/src/__tests__/effectsSettings.test.tsx b/apps/slides/src/__tests__/effectsSettings.test.tsx index 2f84de360..74faf4b7d 100644 --- a/apps/slides/src/__tests__/effectsSettings.test.tsx +++ b/apps/slides/src/__tests__/effectsSettings.test.tsx @@ -215,7 +215,7 @@ test("failed compiler output does not expose progress JSON as an error", async ( test("incompatible releases and dependencies stay visible without install actions", async () => { const incompatible = structuredClone(catalog); - incompatible.packs.backgrounds.versions["2.0.0"].hostApiVersion = 2; + incompatible.packs.backgrounds.versions["2.0.0"].hostApiVersion = 3; incompatible.packs.media.versions["1.0.0"].capabilities = ["future-engine"]; incompatible.packs.charts.versions["1.0.0"].dependencies = { media: "1.0.0" }; respond = () => ok({ catalog: incompatible, url }); @@ -227,7 +227,7 @@ test("incompatible releases and dependencies stay visible without install action expect(button(row, "Install")).toBeUndefined(); expect(button(row, "Update to 2.0.0")).toBeUndefined(); } - expect(section(renderer, "Backgrounds").props.description).toContain("host API 2"); + expect(section(renderer, "Backgrounds").props.description).toContain("host API 3"); expect(section(renderer, "Charts").props.description).toContain("future-engine"); expect(button(section(renderer, "Backgrounds"), "Remove")).toBeDefined(); expect(commands).toEqual([["--effect-catalog"]]); diff --git a/apps/slides/src/__tests__/gpuRenderGraph.test.ts b/apps/slides/src/__tests__/gpuRenderGraph.test.ts new file mode 100644 index 000000000..21cb2af05 --- /dev/null +++ b/apps/slides/src/__tests__/gpuRenderGraph.test.ts @@ -0,0 +1,136 @@ +// @ts-nocheck GPU descriptors are captured at the real setup/encode boundary. +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { transformSync } from "@babel/core"; +import { prepareRenderGraph, encodeRenderGraph } from "../gpuRenderGraph"; + +function gpu() { + const textures = [], pipelines = [], groups = [], commands = [], writes = []; + const device = { + queue: { writeBuffer: (...args) => writes.push(args) }, + createBuffer: descriptor => ({ descriptor }), createSampler: () => ({}), + createShaderModule: descriptor => ({ descriptor, getCompilationInfo: async () => ({ messages: [] }) }), + createBindGroupLayout: descriptor => descriptor, createPipelineLayout: descriptor => descriptor, + createComputePipeline: descriptor => { pipelines.push(descriptor); return descriptor; }, + createRenderPipeline: descriptor => { pipelines.push(descriptor); return descriptor; }, + createTexture: descriptor => { + const texture = { descriptor, width: descriptor.size[0], height: descriptor.size[1], destroyed: false, createView: view => ({ texture, view }), destroy() { this.destroyed = true; } }; + textures.push(texture); return texture; + }, + createBindGroup: descriptor => { groups.push(descriptor); return descriptor; }, + }; + const encoder = { + beginComputePass: () => ({ setPipeline() {}, setBindGroup() {}, dispatchWorkgroups: (...args) => commands.push(["compute", ...args]), end() {} }), + beginRenderPass: descriptor => { + commands.push(["render", descriptor]); + return { setPipeline() {}, setBindGroup() {}, draw: (...args) => commands.push(["draw", ...args]), end() {} }; + }, + }; + return { device, encoder, textures, pipelines, groups, commands, writes }; +} +const definition = { + targets: [{ id: "hdr", format: "rgba16float" }, { id: "depth", format: "depth32float" }, { id: "bloom", format: "rgba16float", scale: .5, mips: 3 }], + buffers: [{ id: "instances", size: 16, init: new Float32Array([99, 1, 2, 3, 4, 99]).subarray(1, 5) }], + passes: [ + { kind: "compute", id: "positions", code: "", entry: "main", reads: ["instances"], writes: ["hdr"], workgroups: "perPixel" }, + { kind: "render", id: "scene", code: "", vertex: "vertex", fragment: "fragment", color: ["hdr"], depth: "depth", vertexCount: 24, instanceCount: 40 }, + { kind: "fullscreen", id: "bloom", code: "", fragment: "bloom", reads: ["hdr"], output: "bloom", mip: 1 }, + { kind: "fullscreen", id: "screen", code: "", fragment: "final", reads: ["bloom@1", "depth"], output: "screen" }, + ], +}; + +test("multipass setup binds declared resources, initializes buffers, and submits ordered compute/render/mip passes", async () => { + const g = gpu(), clock = {}, uniforms = {}; + const r = await prepareRenderGraph(g.device, definition, "bgra8unorm", clock, uniforms); + expect(g.pipelines).toHaveLength(4); + expect(g.writes[0][2]).toBe(definition.buffers[0].init.buffer); + expect(g.writes[0].slice(3)).toEqual([4, 16]); + expect(new Float32Array(g.writes[0][2], g.writes[0][3], g.writes[0][4] / 4)).toEqual(new Float32Array([1, 2, 3, 4])); + const screen = { width: 1920, height: 1080, createView: () => "screen" }; + encodeRenderGraph(r, g.device, g.encoder, screen); + expect(g.textures.map(t => t.descriptor.size)).toEqual([[1920, 1080], [1920, 1080], [960, 540]]); + expect(g.textures[2].descriptor.mipLevelCount).toBe(3); + expect(g.groups[0].entries.map(e => e.binding)).toEqual([0, 1, 2, 3, 4]); + expect(g.groups[0].entries[0].resource.buffer).toBe(clock); + expect(g.groups[0].entries[1].resource.buffer).toBe(uniforms); + expect(g.groups[3].entries[3].resource.view).toEqual({ baseMipLevel: 1, mipLevelCount: 1 }); + expect(g.commands.map(c => c[0])).toEqual(["compute", "render", "draw", "render", "draw", "render", "draw"]); + expect(g.commands[0]).toEqual(["compute", 240, 135]); + expect(g.commands[2]).toEqual(["draw", 24, 40]); + expect(g.commands[1][1].depthStencilAttachment.depthClearValue).toBe(1); + expect(g.commands.at(-2)[1].colorAttachments[0].view).toBe("screen"); + encodeRenderGraph(r, g.device, g.encoder, screen); + expect(g.groups).toHaveLength(4); expect(g.textures).toHaveLength(3); + encodeRenderGraph(r, g.device, g.encoder, { ...screen, width: 3840, height: 2160 }); + expect(g.pipelines).toHaveLength(4); expect(g.groups).toHaveLength(8); + expect(g.textures.slice(0, 3).every(t => t.destroyed)).toBe(true); + expect(g.textures.slice(3).map(t => t.descriptor.size)).toEqual([[3840, 2160], [3840, 2160], [1920, 1080]]); +}); + +test("graph rejects invalid references and shader diagnostics before a frame can receive pipelines", async () => { + const g = gpu(); + await expect(prepareRenderGraph(g.device, { ...definition, passes: [{ ...definition.passes[3], reads: ["bloom@3"] }] }, "bgra8unorm", {}, {})).rejects.toThrow("Invalid GPU mip"); + await expect(prepareRenderGraph(g.device, { ...definition, passes: [{ ...definition.passes[0], writes: ["depth"] }] }, "bgra8unorm", {}, {})).rejects.toThrow("Depth textures"); + await expect(prepareRenderGraph(g.device, { ...definition, passes: [{ ...definition.passes[0], reads: ["hdr@0"] }] }, "bgra8unorm", {}, {})).rejects.toThrow("cannot read and write"); + g.device.createShaderModule = () => ({ getCompilationInfo: async () => ({ messages: [{ type: "error", message: "invalid fragment" }] }) }); + await expect(prepareRenderGraph(g.device, definition, "bgra8unorm", {}, {})).rejects.toThrow("positions: invalid fragment"); + expect(g.pipelines).toHaveLength(0); +}); + + +test("multipass encoding compiles for the host UI runtime", () => { + const filename = fileURLToPath(new URL("../gpuRenderGraph.ts", import.meta.url)); + const require = createRequire(filename); + const result = transformSync(readFileSync(filename, "utf8"), { + filename, babelrc: false, configFile: false, + presets: [require.resolve("@react-native/babel-preset")], + plugins: [require.resolve("react-native-worklets/plugin")], + }); + expect(result.code).toContain("__workletHash"); +}); + +test("native callback copies retain one mutable graph across draws and resizing", async () => { + const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); + const require = createRequire(filename); + const { code } = transformSync(readFileSync(filename, "utf8"), { + filename, babelrc: false, configFile: false, presets: [require.resolve("@react-native/babel-preset")], + }); + const g = gpu(); + Object.assign(g.device, { __gpuHandle: true, addEventListener() {}, pushErrorScope() {}, popErrorScope: async () => null, + lost: new Promise(() => {}), destroy() {}, createCommandEncoder: () => g.encoder }); + g.device.queue.submit = () => {}; + g.encoder.finish = () => ({}); + const screen = { width: 1920, height: 1080, createView: () => "screen" }; + const context = { __gpuHandle: true, configure() {}, getCurrentTexture: () => screen, present() {} }; + const makeMutable = value => ({ __shared: true, value }); + const mocks = { + react: {}, "react-native": { Platform: { OS: "macos" } }, "@shopify/react-native-skia": {}, "react-native-webgpu": {}, + "@legend-apps/presentation": {}, "./slidesStore": {}, + "./gpuRenderGraph": { prepareRenderGraph, encodeRenderGraph }, + "react-native-reanimated": { makeMutable, runOnUI: fn => fn }, + }; + const { startShaderCanvas, submitGPUFrame } = new Function("require", "exports", "navigator", code + "\nreturn { startShaderCanvas, submitGPUFrame };")( + name => mocks[name] ?? require(name), {}, + { gpu: { requestAdapter: async () => ({ requestDevice: async () => g.device }), getPreferredCanvasFormat: () => "bgra8unorm" } }, + ); + let stop; + const resources = await new Promise((resolve, reject) => { + stop = startShaderCanvas({ canvas: { current: { getContext: () => context } }, renderGraph: definition, code: "", particles: false, particleCount: 1, + setResources: value => { if (value) resolve(value); }, setError: error => { if (error) reject(new Error(error)); }, reportError: reject }); + }); + // Worklet closures copy plain objects; shared values and native GPU handles retain identity. + const copy = value => { + if (!value || typeof value !== "object" || value.__shared || value.__gpuHandle) return value; + return Array.isArray(value) ? value.map(copy) : Object.fromEntries(Object.entries(value).map(([key, field]) => [key, copy(field)])); + }; + expect(resources.graph.__shared).toBe(true); + for (let frame = 0; frame < 6; frame++) submitGPUFrame(copy(resources), frame, 1, 6); + expect(g.textures).toHaveLength(3); expect(g.groups).toHaveLength(4); + screen.width = 3840; screen.height = 2160; + for (let frame = 0; frame < 6; frame++) submitGPUFrame(copy(resources), frame, 1, 6); + expect(g.textures).toHaveLength(6); expect(g.groups).toHaveLength(8); + expect(g.textures.slice(0, 3).every(texture => texture.destroyed)).toBe(true); + stop(); +}); diff --git a/apps/slides/src/__tests__/gpuTexture.test.ts b/apps/slides/src/__tests__/gpuTexture.test.ts index 47a82307d..a809d0206 100644 --- a/apps/slides/src/__tests__/gpuTexture.test.ts +++ b/apps/slides/src/__tests__/gpuTexture.test.ts @@ -25,11 +25,11 @@ test("browser images upload premultiplied pixels and release decoding handles ev expect(events).toEqual(["closed","closed"]); }); test("native file URLs decode once, upload pixels, and release Skia handles",async()=>{ - const events=[],texture={},pixels=new Uint8Array(48); + const events=[],texture={},pixels=Uint8Array.from({length:56},(_,index)=>index).subarray(4,52); const image={width:()=>4,height:()=>3,readPixels:(_,__,options)=>{expect(options.alphaType).toBe("premul");return pixels;},dispose:()=>events.push("image")}; const skia={AlphaType:{Premul:"premul"},ColorType:{RGBA_8888:"rgba"},Skia:{Data:{fromURI:async uri=>{expect(uri).toBe("file:///deck/image.png");return {dispose:()=>events.push("data")};}},Image:{MakeImageFromEncoded:()=>image}}}; const load=loader("macos",skia,null,events); - const device={createTexture:d=>{expect(d.usage).toBe(6);return texture;},queue:{writeTexture:(to,data,layout,size)=>{expect(to.texture).toBe(texture);expect(data).toBe(pixels);expect(layout.bytesPerRow).toBe(16);expect(size).toEqual([4,3]);}}}; + const device={createTexture:d=>{expect(d.usage).toBe(6);return texture;},queue:{writeTexture:(to,data,layout,size)=>{expect(to.texture).toBe(texture);expect(data).toBe(pixels.buffer);expect(layout).toEqual({offset:4,bytesPerRow:16});expect(new Uint8Array(data,layout.offset,48)).toEqual(pixels);expect(size).toEqual([4,3]);}}}; expect(await load(device,"file:///deck/image.png")).toBe(texture); expect(events).toEqual(["image","data"]); }); diff --git a/apps/slides/src/__tests__/typeGPUShader.test.ts b/apps/slides/src/__tests__/typeGPUShader.test.ts index 816db4ade..ee01c331b 100644 --- a/apps/slides/src/__tests__/typeGPUShader.test.ts +++ b/apps/slides/src/__tests__/typeGPUShader.test.ts @@ -5,7 +5,7 @@ import { createRequire } from "node:module"; import { transformSync } from "@babel/core"; import { fileURLToPath } from "node:url"; -function frames({ preview = false, active = true, effectActive = true, maxTime, frameTimeIndex, preparing = false, composite = false, data = false, clockMode = "step", simulation = true } = {}) { +function frames({ preview = false, active = true, effectActive = true, maxTime, frameTimeIndex, preparing = false, composite = false, data = false, clockMode = "step", simulation = true, graph = false } = {}) { const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); const require = createRequire(filename); const { code } = transformSync(readFileSync(filename, "utf8"), { @@ -45,17 +45,18 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, runOnJS: fn => fn, runOnUI: fn => fn, }, "./slidesStore": {}, + "./gpuRenderGraph": { encodeRenderGraph: () => calls.push(["graph"]) }, }; const UIFrames = new Function("require", "exports", code + "\nreturn UIFrames;")( name => mocks[name] ?? require(name), {}, ); UIFrames({ resources: { alive: { value: true, set(value) { this.value = value; } }, disposed: { value: false }, device, context: { getCurrentTexture: () => ({ createView() {} }), present() {} }, - clock: {}, states: [], groups: simulation ? ["A", "B"] : ["A"], compute: simulation ? {} : undefined, render: {}, particles: {}, frameData: data ? {} : undefined, + clock: {}, graph: graph ? { value: {} } : undefined, states: [], groups: simulation ? ["A", "B"] : ["A"], compute: simulation ? {} : undefined, render: {}, particles: {}, frameData: data ? {} : undefined, composite: composite ? { pipeline: {}, group: "lens", texture: { createView() {} } } : undefined }, effectActive, maxTime, frameTimeIndex, clockMode, frameData: data ? frameData : undefined, aspect: 2.61, brightness: 1, particleCount: 2048, previewTime: 7, onError: error => errors.push(error), onReady: () => ready++ }); const cleanup = effects.map(effect => effect()); return { calls, uniforms, errors, ready: () => ready, enabled: () => enabled, - playback, frameData, + playback, frameData, redraw: () => effects[0](), tick: () => { playback.value = { ...playback.value, stepTime: playback.value.stepTime + 0.016 }; callback({ timeSincePreviousFrame: 16 }); }, stop: () => cleanup.forEach(fn => fn?.()), device }; @@ -285,3 +286,49 @@ test("geometry can retain slide time across steps while uploading UI data and co f.tick(); expect(f.uniforms).toHaveLength(count); f.stop(); }); + + +test("multipass surfaces redraw a frozen frame after paused and outgoing resizing", () => { + const f = frames({ graph: true, data: true }); + f.tick(); + const lastFrame = f.uniforms.at(-2); + expect(f.calls.filter(call => call[0] === "graph")).toHaveLength(2); + for (const phase of ["paused", "outgoing"]) { + f.playback.value = { ...f.playback.value, phase, stepTime: lastFrame[0] }; + f.redraw(); + expect(f.uniforms.at(-2).slice(0, 4)).toEqual([lastFrame[0], 0, lastFrame[2], lastFrame[3]]); + const count = f.uniforms.length; + f.tick(); expect(f.uniforms).toHaveLength(count); + } + f.stop(); + expect(f.calls.filter(call => call[0] === "destroy")).toHaveLength(1); +}); + +test("frame failures report the latest slide context as well as stopping the surface", () => { + const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); + const require = createRequire(filename); + const { code } = transformSync(readFileSync(filename, "utf8"), { + filename, babelrc: false, configFile: false, presets: [require.resolve("@react-native/babel-preset")], + }); + let state = 0, refIndex = 0, slideIndex = 1, preview = false; + const refs = [], errors = [], reports = []; + const mocks = { + react: { useContext: () => 1, useRef: value => refs[refIndex++] ??= { current: value }, useEffect() {}, useLayoutEffect: effect => effect(), + useState: () => [state++ % 3 === 0 ? { id: 1 } : undefined, message => errors.push(message)] }, + "react-native": { View: "View", Platform: { OS: "web" } }, "react-native-webgpu": { Canvas: "Canvas" }, "@shopify/react-native-skia": {}, + "react-native-reanimated": {}, "./gpuRenderGraph": {}, + "@legend-apps/presentation": { useBackgroundSize: () => ({ width: 1920, height: 1080 }), useBackgroundIntensity: () => 1, + usePresentationValue: field => field === "slideIndex" ? slideIndex : preview, usePreparationTask() {} }, + "./slidesStore": { reportSlideError: (...args) => reports.push(args) }, + }; + const renderer = new Function("require", "exports", "navigator", code + "\nreturn TypeGPUShader;")( + name => mocks[name] ?? require(name), {}, { gpu: {} }, + ); + const first = renderer({ code: "" }); + const onError = first.props.children[1].props.onError; + slideIndex = 4; preview = true; state = 0; refIndex = 0; + renderer({ code: "" }); + onError("draw failed"); + expect(errors).toEqual(["draw failed"]); + expect(reports).toEqual([["draw failed", 4, true]]); +}); diff --git a/apps/slides/src/gpuRenderGraph.ts b/apps/slides/src/gpuRenderGraph.ts new file mode 100644 index 000000000..8b2f6f051 --- /dev/null +++ b/apps/slides/src/gpuRenderGraph.ts @@ -0,0 +1,171 @@ +import type { GPURenderGraphDefinition, GPUPass, GPUTarget } from "@legend-apps/presentation"; + +export const graphVertex = ` +struct GraphVertex { @builtin(position) position: vec4f, @location(0) uv: vec2f }; +@vertex fn graphVertex(@builtin(vertex_index) id: u32) -> GraphVertex { + let p = vec2f(f32((id << 1u) & 2u), f32(id & 2u)); + return GraphVertex(vec4f(p * 2.0 - 1.0, 0.0, 1.0), vec2f(p.x, 1.0 - p.y)); +}`; + +type PassResources = { pass: GPUPass; pipeline: GPUComputePipeline | GPURenderPipeline; layout: GPUBindGroupLayout; group?: GPUBindGroup }; +export type RenderGraphResources = { + targets: GPUTarget[]; textures: Record; buffers: Record; + passes: PassResources[]; clock: GPUBuffer; uniforms: GPUBuffer; sampler: GPUSampler; + width: number; height: number; +}; + +function reference(value: string): [string, number] { + "worklet"; + const parts = value.split("@"); + return [parts[0], parts.length === 1 ? 0 : parts.length === 2 && parts[1] !== "" ? Number(parts[1]) : NaN]; +} +function targetFor(targets: GPUTarget[], value: string) { + "worklet"; + const [id] = reference(value); + const target = targets.find(target => target.id === id); + if (!target) throw new Error("Unknown GPU target: " + value); + return target; +} +function view(r: RenderGraphResources, value: string) { + "worklet"; + const [id, mip] = reference(value); + return r.textures[id].createView({ baseMipLevel: mip, mipLevelCount: 1 }); +} +function output(pass: GPUPass) { + "worklet"; + if (pass.kind === "fullscreen") return pass.mip === undefined ? pass.output : pass.output + "@" + pass.mip; + return pass.kind === "render" ? pass.color[0] ?? pass.depth ?? "screen" : pass.writes?.[0] ?? "screen"; +} +function dimensions(r: RenderGraphResources, value: string, width: number, height: number): [number, number] { + "worklet"; + const [id, mip] = reference(value); + if (value === "screen" || r.buffers[id]) return [width, height]; + const target = targetFor(r.targets, value); + if (r.textures[id] && r.width === width && r.height === height) return [Math.max(1, r.textures[id].width >> mip), Math.max(1, r.textures[id].height >> mip)]; + const scale = (target.scale ?? 1) / Math.pow(2, mip); + return [Math.max(1, Math.floor(width * scale)), Math.max(1, Math.floor(height * scale))]; +} + +/** Setup is shared by native/web hosts and standalone browser measurements. */ +export async function prepareRenderGraph(device: GPUDevice, definition: GPURenderGraphDefinition, format: GPUTextureFormat, + clock: GPUBuffer, uniforms: GPUBuffer): Promise { + const { targets, buffers = [], passes } = definition; + const ids = new Set(); + for (const resource of [...targets, ...buffers]) { + if (!resource.id || resource.id === "screen" || resource.id.includes("@") || resource.id in Object.prototype || ids.has(resource.id)) throw new Error("Invalid or duplicate GPU resource: " + resource.id); + ids.add(resource.id); + } + for (const target of targets) { + if (!Number.isFinite(target.scale ?? 1) || (target.scale ?? 1) <= 0 || !Number.isInteger(target.mips ?? 1) || (target.mips ?? 1) < 1) throw new Error("Invalid GPU target dimensions: " + target.id); + } + const r: RenderGraphResources = { targets, textures: {}, buffers: {}, passes: [], clock, uniforms, + sampler: device.createSampler({ minFilter: "linear", magFilter: "linear", mipmapFilter: "linear" }), width: 0, height: 0 }; + for (const buffer of buffers) { + if (!Number.isInteger(buffer.size) || buffer.size <= 0 || buffer.size % 4 || (buffer.init && buffer.init.byteLength > buffer.size)) throw new Error("Invalid GPU buffer size: " + buffer.id); + r.buffers[buffer.id] = device.createBuffer({ size: buffer.size, usage: 128 | 8 }); + if (buffer.init) device.queue.writeBuffer(r.buffers[buffer.id], 0, buffer.init.buffer, buffer.init.byteOffset, buffer.init.byteLength); + } + const checkReference = (value: string) => { + const [id, mip] = reference(value); + if (id === "screen") { if (value !== "screen") throw new Error("Screen has no mip levels"); return; } + if (!ids.has(id)) throw new Error("Unknown GPU resource: " + value); + if (!Number.isInteger(mip) || mip < 0 || (r.buffers[id] ? mip !== 0 : mip >= (targetFor(targets, value).mips ?? 1))) throw new Error("Invalid GPU mip: " + value); + }; + for (const pass of passes) { + for (const value of [...(pass.reads ?? []), ...(pass.kind === "compute" ? pass.writes ?? [] : pass.kind === "render" ? [...pass.color, ...(pass.depth ? [pass.depth] : [])] : [output(pass)])]) checkReference(value); + const reads = new Set((pass.reads ?? []).map(value => { const [id, mip] = reference(value); return id + "@" + mip; })); + const writes = pass.kind === "compute" ? pass.writes ?? [] : pass.kind === "render" ? [...pass.color, ...(pass.depth ? [pass.depth] : [])] : [output(pass)]; + if (writes.some(value => { const [id, mip] = reference(value); return reads.has(id + "@" + mip); })) throw new Error("GPU pass cannot read and write the same resource: " + pass.id); + const entries: GPUBindGroupLayoutEntry[] = [ + { binding: 0, visibility: 7, buffer: { type: "uniform" } }, + { binding: 1, visibility: 7, buffer: { type: "read-only-storage" } }, + { binding: 2, visibility: 7, sampler: { type: "filtering" } }, + ]; + for (const [index, value] of [...(pass.reads ?? []), ...(pass.kind === "compute" ? pass.writes ?? [] : [])].entries()) { + const writing = index >= (pass.reads?.length ?? 0), [id] = reference(value); + if (value === "screen") throw new Error("Screen cannot be bound as a GPU resource"); + const target = r.buffers[id] ? undefined : targetFor(targets, value); + if (writing && target?.format === "depth32float") throw new Error("Depth textures cannot be storage outputs"); + entries.push({ binding: index + 3, visibility: writing ? 4 : 7, + ...(r.buffers[id] ? { buffer: { type: writing ? "storage" as const : "read-only-storage" as const } } + : writing ? { storageTexture: { access: "write-only" as const, format: target!.format } } + : { texture: { sampleType: target!.format === "depth32float" ? "depth" as const : target!.format === "r32float" ? "unfilterable-float" as const : "float" as const } }) }); + } + const module = device.createShaderModule({ code: pass.code + (pass.kind === "fullscreen" ? graphVertex : "") }); + const diagnostics = await module.getCompilationInfo(); + const errors = diagnostics.messages.filter(message => message.type === "error"); + if (errors.length) throw new Error(pass.id + ": " + errors.map(message => message.message).join("\n")); + const layout = device.createBindGroupLayout({ entries }); + const pipelineLayout = device.createPipelineLayout({ bindGroupLayouts: [layout] }); + let pipeline: GPUComputePipeline | GPURenderPipeline; + if (pass.kind === "compute") { + pipeline = device.createComputePipeline({ layout: pipelineLayout, compute: { module, entryPoint: pass.entry } }); + } else { + const colors = pass.kind === "render" ? pass.color : [output(pass)]; + const blend = pass.kind === "render" ? pass.blend : "none"; + pipeline = device.createRenderPipeline({ layout: pipelineLayout, + vertex: { module, entryPoint: pass.kind === "render" ? pass.vertex : "graphVertex" }, + fragment: { module, entryPoint: pass.fragment, targets: colors.map(color => ({ format: color === "screen" ? format : targetFor(targets, color).format, + ...(blend && blend !== "none" ? { blend: { color: { srcFactor: "one" as const, dstFactor: blend === "over" ? "one-minus-src-alpha" as const : "one" as const, operation: "add" as const }, + alpha: { srcFactor: "one" as const, dstFactor: "one-minus-src-alpha" as const, operation: "add" as const } } } : {}) })) }, + ...(pass.kind === "render" && pass.depth ? { depthStencil: { format: targetFor(targets, pass.depth).format, depthWriteEnabled: true, depthCompare: pass.depthCompare ?? "less" } } : {}), + }); + } + r.passes.push({ pass, pipeline, layout }); + } + return r; +} + +/** Recreate only texture views/bind groups when the drawing-buffer size changes. */ +export function resizeRenderGraph(r: RenderGraphResources, device: GPUDevice, width: number, height: number) { + "worklet"; + if (r.width === width && r.height === height) return; + for (const target of r.targets) { + const old = r.textures[target.id]; + const [w, h] = dimensions(r, target.id, width, height); + const mips = target.mips ?? 1; + // Allocate enough base pixels for all declared mip levels, even in tiny previews. + const minSize = Math.pow(2, mips - 1); + r.textures[target.id] = device.createTexture({ size: [Math.max(minSize, w), Math.max(minSize, h)], format: target.format, mipLevelCount: mips, + usage: 16 | 4 | (target.format === "depth32float" ? 0 : 8) }); + old?.destroy(); + } + for (const resource of r.passes) { + const pass = resource.pass; + resource.group = device.createBindGroup({ layout: resource.layout, entries: [ + { binding: 0, resource: { buffer: r.clock } }, { binding: 1, resource: { buffer: r.uniforms } }, { binding: 2, resource: r.sampler }, + ...[...(pass.reads ?? []), ...(pass.kind === "compute" ? pass.writes ?? [] : [])].map((value, index) => { + const [id] = reference(value); + return { binding: index + 3, resource: r.buffers[id] ? { buffer: r.buffers[id] } : view(r, value) }; + }), + ] }); + } + r.width = width; r.height = height; +} + +/** One encoder for every pass; the caller submits and presents the final screen. */ +export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, encoder: GPUCommandEncoder, screen: GPUTexture) { + "worklet"; + resizeRenderGraph(r, device, screen.width, screen.height); + for (const resource of r.passes) { + const pass = resource.pass; + if (pass.kind === "compute") { + const command = encoder.beginComputePass(); + command.setPipeline(resource.pipeline as GPUComputePipeline); command.setBindGroup(0, resource.group!); + const [width, height] = dimensions(r, output(pass), r.width, r.height); + const groups = pass.workgroups; + if (groups === "perPixel") command.dispatchWorkgroups(Math.ceil(width / 8), Math.ceil(height / 8)); + else if (Array.isArray(groups)) command.dispatchWorkgroups(groups[0], groups[1] ?? 1, groups[2] ?? 1); + else command.dispatchWorkgroups(Math.ceil(groups.perInstance / 64)); + command.end(); + } else { + const colors = pass.kind === "render" ? pass.color : [output(pass)]; + const clear = pass.kind === "render" ? pass.clear : undefined; + const command = encoder.beginRenderPass({ colorAttachments: colors.map(color => ({ view: color === "screen" ? screen.createView() : view(r, color), + clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" : "clear", storeOp: "store" })), + ...(pass.kind === "render" && pass.depth ? { depthStencilAttachment: { view: view(r, pass.depth), depthClearValue: 1, depthLoadOp: clear === false ? "load" as const : "clear" as const, depthStoreOp: "store" as const } } : {}) }); + command.setPipeline(resource.pipeline as GPURenderPipeline); command.setBindGroup(0, resource.group!); + command.draw(pass.kind === "render" ? pass.vertexCount : 3, pass.kind === "render" ? pass.instanceCount ?? 1 : 1); command.end(); + } + } +} diff --git a/packages/presentation/src/GPUShader.tsx b/packages/presentation/src/GPUShader.tsx index 262b0f5eb..03fe741f7 100644 --- a/packages/presentation/src/GPUShader.tsx +++ b/packages/presentation/src/GPUShader.tsx @@ -1,4 +1,4 @@ -import { createContext, useContext, type ComponentType, type ReactNode } from "react"; +import { createContext, useContext, useMemo, type ComponentType, type ReactNode } from "react"; import type { SharedValue } from "react-native-reanimated"; export type GPUShaderProps = { @@ -9,6 +9,7 @@ export type GPUShaderProps = { frameData?: SharedValue; frameDataSize?: number; active?: boolean; children?: ReactNode; maxTime?: number; frameTimeIndex?: number; + renderGraph?: GPURenderGraphDefinition; borderRadius?: number; accessibilityLabel?: string; }; const GPUShaderContext = createContext | null>(null); @@ -21,3 +22,32 @@ export function GPUShader(props: GPUShaderProps) { if (!Renderer) throw new Error("GPUShader requires a GPUShaderProvider"); return ; } + +export type GPUTarget = { + id: string; format: "rgba16float" | "rgba8unorm" | "depth32float" | "r32float"; + scale?: number; mips?: number; +}; +export type GPUBufferTarget = { id: string; size: number; init?: Float32Array }; +/** Resource references accept `id@N` to select mip N; bindings follow reads, then writes. */ +export type GPUPass = + | { kind: "compute"; id: string; code: string; entry: string; + /** perPixel assumes an 8x8 workgroup; perInstance assumes 64 threads. */ + workgroups: [number, number?, number?] | "perPixel" | { perInstance: number }; + reads?: string[]; writes?: string[] } + | { kind: "render"; id: string; code: string; vertex: string; fragment: string; + color: string[]; depth?: string; vertexCount: number; instanceCount?: number; + blend?: "none" | "additive" | "over"; reads?: string[]; + clear?: [number, number, number, number] | false; + depthCompare?: "never" | "less" | "equal" | "less-equal" | "greater" | "not-equal" | "greater-equal" | "always" } + | { kind: "fullscreen"; id: string; code: string; fragment: string; + output: string; reads?: string[]; mip?: number }; +export type GPURenderGraphDefinition = { targets: GPUTarget[]; buffers?: GPUBufferTarget[]; passes: GPUPass[] }; +export type GPURenderGraphProps = Omit & + GPURenderGraphDefinition & { uniforms?: SharedValue; /** Float capacity, defaults to 4096. */ uniformsSize?: number }; +/** Group 0: Frame at 0, read-only float uniforms at 1, linear sampler at 2, + * then reads and compute writes. Fullscreen vertex output has position and location-0 UV. */ +export function GPURenderGraph({ targets, buffers, passes, uniforms, uniformsSize = 4096, ...props }: GPURenderGraphProps) { + const renderGraph = useMemo(() => ({ targets, buffers, passes }), [targets, buffers, passes]); + return ; +} diff --git a/packages/presentation/src/__tests__/effectInstallation.test.ts b/packages/presentation/src/__tests__/effectInstallation.test.ts index f547c02e1..8c8c889a4 100644 --- a/packages/presentation/src/__tests__/effectInstallation.test.ts +++ b/packages/presentation/src/__tests__/effectInstallation.test.ts @@ -203,12 +203,12 @@ test("the compiler CLI lists, installs with progress, compiles from Application test("future host versions and capabilities are isolated before any archive download", async () => { const f = fixture(); - f.catalog.packs.motion.versions["2.0.0"].hostApiVersion = 2; - expect((await loadEffectCatalog(f.options.catalogURL)).packs.motion.versions["2.0.0"].hostApiVersion).toBe(2); + f.catalog.packs.motion.versions["2.0.0"].hostApiVersion = 3; + expect((await loadEffectCatalog(f.options.catalogURL)).packs.motion.versions["2.0.0"].hostApiVersion).toBe(3); await installEffectPacks(["motion"], f.options); const lock = fs.readFileSync(effectLockPath(f.deck), "utf8"); const active = fs.readFileSync(path.join(f.directory, "motion/active.json"), "utf8"); - for (const incompatible of [{ hostApiVersion: 2, capabilities: [] }, { hostApiVersion: 1, capabilities: ["future-engine"] }]) { + for (const incompatible of [{ hostApiVersion: 3, capabilities: [] }, { hostApiVersion: 1, capabilities: ["future-engine"] }]) { Object.assign(f.catalog.packs.motion.versions["2.0.0"], incompatible); f.catalog.packs.motion.latest = "2.0.0"; const before = f.requests(); diff --git a/packages/presentation/src/__tests__/effectPacks.test.ts b/packages/presentation/src/__tests__/effectPacks.test.ts index fc73ef366..9dcd863a8 100644 --- a/packages/presentation/src/__tests__/effectPacks.test.ts +++ b/packages/presentation/src/__tests__/effectPacks.test.ts @@ -62,7 +62,7 @@ test("manifests reject missing, incompatible, unexported and escaping entries", expect(() => resolveEffectPack(ref, packs)).toThrow(); } const manifest = path.join(packs, "motion/pack.json"); - fs.writeFileSync(manifest, JSON.stringify({ id: "motion", version: "1.0.0", hostApiVersion: 2, exports: { Pose: "./Pose.ts" } })); + fs.writeFileSync(manifest, JSON.stringify({ id: "motion", version: "1.0.0", hostApiVersion: 3, exports: { Pose: "./Pose.ts" } })); expect(() => resolveEffectPack("slides:motion/Pose", packs)).toThrow("incompatible"); fs.writeFileSync(manifest, JSON.stringify({ id: "motion", version: "1.0.0", hostApiVersion: 1, exports: { Pose: "../../outside.ts" } })); expect(() => resolveEffectPack("slides:motion/Pose", packs)).toThrow(); @@ -93,7 +93,7 @@ test("cross-pack imports require declared dependency versions and track both roo }); test("local pack manifests reject capabilities absent from this host before evaluation", async () => { - for (const requirements of [{ hostApiVersion: 2, capabilities: [] }, { hostApiVersion: 1, capabilities: ["unknown-engine"] }]) { + for (const requirements of [{ hostApiVersion: 3, capabilities: [] }, { hostApiVersion: 1, capabilities: ["unknown-engine"] }]) { const { deck, packs } = fixture({ "packs/motion/pack.json": JSON.stringify({ id: "motion", version: "1.0.0", ...requirements, exports: { Pose: "./Pose.ts" } }), "packs/motion/Pose.ts": 'throw new Error("must never evaluate"); export const sample = 1;', @@ -112,3 +112,9 @@ test("the signed host retains every capability required by the current source pa expect(effectPackIncompatibility(manifest)).toBeUndefined(); } }); + +test("host API 2 accepts multipass packs while retaining API 1", () => { + expect(effectPackIncompatibility({ hostApiVersion: 2, capabilities: ["webgpu", "webgpu-multipass", "reanimated"] })).toBeUndefined(); + expect(effectPackIncompatibility({ hostApiVersion: 1, capabilities: ["webgpu"] })).toBeUndefined(); + for (const hostApiVersion of [0, 1.5, 3]) expect(effectPackIncompatibility({ hostApiVersion })).toContain("this app provides 2"); +}); diff --git a/packages/presentation/src/effectPacks/schema.ts b/packages/presentation/src/effectPacks/schema.ts index 18319a388..398b29bda 100644 --- a/packages/presentation/src/effectPacks/schema.ts +++ b/packages/presentation/src/effectPacks/schema.ts @@ -48,9 +48,9 @@ export function validateCatalog(catalog: EffectCatalog) { } /** Capabilities exposed by the signed Slides host; packs cannot add native engines. */ -const effectHostCapabilities = new Set(["reanimated", "skia", "webgpu", "lottie", "native-video"]); +const effectHostCapabilities = new Set(["reanimated", "skia", "webgpu", "webgpu-multipass", "lottie", "native-video"]); export function effectPackIncompatibility(pack: { hostApiVersion: number; capabilities?: string[] }) { - if (pack.hostApiVersion !== 1) return `Requires host API ${pack.hostApiVersion}; this app provides 1.`; + if (!Number.isSafeInteger(pack.hostApiVersion) || pack.hostApiVersion < 1 || pack.hostApiVersion > 2) return `Requires host API ${pack.hostApiVersion}; this app provides 2.`; if (pack.capabilities !== undefined && !strings(pack.capabilities)) return "Invalid host capabilities."; const missing = (pack.capabilities ?? []).filter(capability => !effectHostCapabilities.has(capability)); if (missing.length) return `Requires unavailable host capabilities: ${missing.join(", ")}.`; diff --git a/packages/presentation/src/index.ts b/packages/presentation/src/index.ts index 3ec877309..0bd6e9766 100644 --- a/packages/presentation/src/index.ts +++ b/packages/presentation/src/index.ts @@ -65,4 +65,4 @@ export { NativeVideoView } from "./NativeVideoView"; export { createTransitionProgress, interpolateTransition, subtractTransition, divideTransition, sampleTransitionValue, resolveTransitionStyle, useTransitionStyle, type TransitionProgress, type TransitionStyle } from "./transitionValues"; -export { GPUShader, GPUShaderProvider, type GPUShaderProps } from "./GPUShader"; +export { GPUShader, GPUShaderProvider, GPURenderGraph, type GPUShaderProps, type GPURenderGraphProps, type GPURenderGraphDefinition, type GPUPass, type GPUTarget, type GPUBufferTarget } from "./GPUShader"; diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md new file mode 100644 index 000000000..c5f9e6b49 --- /dev/null +++ b/packages/slides-effects/packs/world/README.md @@ -0,0 +1,103 @@ +# World + +A deterministic instanced 3D scene for macOS and offline WebGPU exports. It uses +the presentation clock on the UI thread and a generic multipass GPU host. Import +only this pack's entrypoint: + +```tsx +import { World, Camera, Graph, tree, wave, subset, Orbs, Ring, Fog, Bloom, Vignette, Grain } from "slides:world/World"; + +const network = tree("root", [["a", ["a1", "a2"]], ["b", ["b1", "b2"]]]); + + + + + + + + + +``` + +`World` accepts `background`, `backgroundColor`, `quality`, `exposure`, `scrim`, +`previewTime`, `active`, `borderRadius` and `fallback`. Normal React Native +children are overlaid above the canvas; keep text in those children or in normal +slide layers. A local GPU boundary displays `fallback` on unsupported adapters; +the default is the dark background. GPU errors produce a diagnostic. A background +world fills the background host's dimensions. + +`Camera.shots` are ordered keyframes with either a `step` or an `at` time in +seconds. Times and step numbers should increase within their respective clock. +Each shot inherits unspecified values from the pose sampled when it starts; +moving prior shots freeze at that boundary. `position`, `target`, `fov`, `roll`, +`orbit`, `follow` and `dolly` are supported. Orbit speed is radians per second; +dolly is the distance from the target. Easing is `linear`, `cubic`, `smoothstep`, +`expo` or `spring`; durations are seconds. `follow` uses instance ids, including +`node:` and `orb:`. Reuse a world `id` and set +`enter={{ from: "previous", duration: 1 }}` on a later slide to ease from its last +live camera pose. Preparation and previews never overwrite retained live poses. + +`Sphere`, `Box`, `Tube` and `Ring` accept `position`, `size`, `scale`, `color`, +`emissive` and `id`. Tube adds `end`, `curve` and `thickness`; ring adds `radius` +and `thickness`. `Orbs` accepts `count`, `colors`, `radius`, `spread`, `position`, +`seed`, `drift`, `emissive` and an `id` prefix (default `orb`). Use distinct prefixes +for multiple orb fields; all scene instance ids must be unique. Its seeded +distribution and drifting motion are repeatable at any sampled time. No frame +integration is used. + +`Graph` accepts a tree, `radius`, `levelHeight`, `position`, `node` and `edge`. +Node defaults are `color`, `dim`, `size`, `emissive` and `scale`. Use the `nodes` +map for per-id animated position, color, scale, size or emissive overrides. +Edges attach to the evaluated node positions. Edge options are `color`, +`curve`, `thickness` and `glow`. `wave.speed` is seconds per depth level, and +`subset.branches` selects roots plus descendants. The latest activation eases +from the previous state; other branches become dim. Duplicate ids are rejected. + +Primitive position, scale, size, color and emissive, exposure, fog settings, +bloom settings, vignette and grain accept constants, step maps or keyframes: + +```tsx + +``` + +The first value is the baseline. Default scalar/vector transitions last 0.65s. +Colors use `#RGB` or `#RRGGBB` and interpolate in linear light. Direct entry into +a later step settles missing earlier steps and animates the current step from +that state. Scrubbing and reverse navigation do not depend on prior GPU frames. +Backward navigation restarts the selected step's scripted transition. +Keyframe tables and graph activation are sampled in a UI worklet into one storage +uniform array. This keeps frame work off JavaScript, but uniform evaluation and +upload still scale with the instance count; the graph API targets 20–60 nodes. + +The pipeline is an instanced HDR scene with depth, a half-resolution bloom mip +chain, exponential height fog, ACES exposure/tonemapping, sRGB encoding, +vignette and grain. `high` and `auto` use full-resolution scene targets and five +bloom levels. `medium` uses 0.85 scene scale and five levels; `low` uses 0.65 scene +scale and three levels. Output resolution stays unchanged. `auto` currently +selects the high preset; it does not change quality based on frame timing. + +For a raw WGSL escape hatch, use `GPURenderGraph` from +`@legend-apps/presentation`. `buildWorldGraph` and the pure `scene.ts` helpers +expose the same description and deterministic uniform samples for measurement. +The scene-specific custom pass, material, reflection and depth-of-field APIs are +not part of this initial pack. + +Generate a standalone browser measurement page from the repository root: + +```sh +bun packages/slides-effects/scripts/world-benchmark.mjs --output /tmp/world-harness.html +``` + +Open that file in Chrome with WebGPU and run +`await benchmarkWorld({ width: 1920, height: 1080, quality: "high" })` in its console. +Repeat at 3840×2160 and with each preset. Results report completed-submit frame +times and per-pass GPU timestamps when available. Throughput measurements do not +measure display refresh rate or prove a particular presentation frame rate. diff --git a/packages/slides-effects/packs/world/World.tsx b/packages/slides-effects/packs/world/World.tsx new file mode 100644 index 000000000..dce091006 --- /dev/null +++ b/packages/slides-effects/packs/world/World.tsx @@ -0,0 +1,124 @@ +import { GPURenderGraph, samplePlayback, useBackgroundSize, usePlayback, usePresentationValue } from "@legend-apps/presentation"; +import { Children, Component, Fragment, isValidElement, useMemo, useState, type ReactNode } from "react"; +import { View } from "react-native"; +import { makeMutable, useAnimatedReaction, useDerivedValue, useSharedValue, type SharedValue } from "react-native-reanimated"; +import type { CameraOptions, CameraPose } from "./camera"; +import { buildWorldGraph, worldGraphTopology } from "./render"; +import { graphInstances, orbInstances, prepareWorldScene, primitive, sampleWorldCamera, sampleWorldUniforms, + type Animated, type GraphOptions, type OrbsOptions, type PrimitiveOptions, type WorldScene } from "./scene"; + +export { tree, wave, subset, layoutTree } from "./graph"; +export { evaluateCamera } from "./camera"; +export { buildWorldGraph } from "./render"; +export { orbInstances, graphInstances, primitive, prepareWorldScene, sampleWorldUniforms } from "./scene"; +export type { Shot, CameraPose, Vec3 } from "./camera"; +export type { Animated, GraphOptions, OrbsOptions, PrimitiveOptions, WorldScene } from "./scene"; + +export function Camera(_: CameraOptions) { return null; } +export function Graph(_: GraphOptions) { return null; } +export function Orbs(_: OrbsOptions) { return null; } +export function Sphere(_: PrimitiveOptions) { return null; } +export function Box(_: PrimitiveOptions) { return null; } +export function Tube(_: PrimitiveOptions) { return null; } +export function Ring(_: PrimitiveOptions & { radius?: Animated }) { return null; } +export function Fog(_: NonNullable) { return null; } +export function Bloom(_: NonNullable) { return null; } +export function Vignette(_: { amount?: Animated }) { return null; } +export function Grain(_: { amount?: Animated }) { return null; } + +const poses = new Map>(); +function retainedPose(id: string) { + let pose = poses.get(id); + if (!pose) { pose = makeMutable(null); poses.set(id, pose); } + return pose; +} + +class WorldGPU extends Component<{ fallback: ReactNode; children: ReactNode }, { failed: boolean }> { + state = { failed: false }; + static getDerivedStateFromError() { return { failed: true }; } + componentDidCatch(error: Error) { console.warn("World could not initialize WebGPU:", error.message); } + render() { return this.state.failed ? this.props.fallback : this.props.children; } +} + +export type WorldProps = { + id?: string; width?: number; height?: number; background?: boolean; backgroundColor?: string; + quality?: WorldScene["quality"]; exposure?: Animated; previewTime?: number; borderRadius?: number; + enter?: { from: "previous"; duration?: number }; scrim?: boolean | number; fallback?: ReactNode; active?: boolean; + children?: ReactNode; accessibilityLabel?: string; +}; +export function World(props: WorldProps) { + // id defines a continuity boundary; changing it creates a fresh sampled origin. + return ; +} +function worldContent(children: ReactNode, quality: WorldScene["quality"], exposure: Animated, backgroundColor: string, scrim: boolean | number) { + const scene: WorldScene = { instances: [], quality, exposure, backgroundColor, scrim: typeof scrim === "number" ? scrim : scrim ? 0.4 : 0 }; + const overlay: ReactNode[] = []; + function collect(content: ReactNode) { + Children.forEach(content, child => { + if (!isValidElement>(child)) { + if (child && (typeof child !== "string" || child.trim())) overlay.push(child); + return; + } + const options = child.props; + if (child.type === Fragment) collect(options.children as ReactNode); + else if (child.type === Camera) scene.camera = options as CameraOptions; + else if (child.type === Graph) scene.instances.push(...graphInstances(options as GraphOptions)); + else if (child.type === Orbs) scene.instances.push(...orbInstances(options)); + else if (child.type === Sphere) scene.instances.push(primitive("sphere", options)); + else if (child.type === Box) scene.instances.push(primitive("box", options)); + else if (child.type === Tube) scene.instances.push(primitive("tube", options)); + else if (child.type === Ring) scene.instances.push(primitive("ring", { ...options, size: options.radius ?? options.size } as PrimitiveOptions)); + else if (child.type === Fog) scene.fog = options; + else if (child.type === Bloom) scene.bloom = options; + else if (child.type === Vignette) scene.vignette = options.amount as Animated | undefined; + else if (child.type === Grain) scene.grain = options.amount as Animated | undefined; + else overlay.push(child); + }); + } + collect(children); + return { prepared: prepareWorldScene(scene), overlay }; +} +function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHeight = 1080, background = false, backgroundColor = "#05060a", quality = "auto", + exposure = 1, previewTime = 4, borderRadius = 0, enter, scrim = false, fallback, active = true, children, accessibilityLabel }: WorldProps) { + const size = useBackgroundSize(); + const width = background ? size.width : configuredWidth; + const height = background ? size.height : configuredHeight; + // GPU pipelines keep their identity across clock navigation and surface resizes. + const { prepared, overlay } = useMemo(() => worldContent(children, quality, exposure, backgroundColor, scrim), [children, quality, exposure, backgroundColor, scrim]); + const { counts: { box, ring, sphere, tube }, backgroundVector: [red, green, blue] } = worldGraphTopology(prepared); + const graph = useMemo(() => buildWorldGraph({ quality, counts: { box, ring, sphere, tube }, backgroundVector: [red, green, blue] }), + [quality, box, ring, sphere, tube, red, green, blue]); + const playback = usePlayback(); + const [continuity] = useState(() => id ? retainedPose(id) : undefined); + const origin = useSharedValue<{ key: string; pose: CameraPose | null }>({ key: "", pose: null }); + const sampleClock = samplePlayback; + const cameraEvaluator = sampleWorldCamera; + const sampleUniforms = sampleWorldUniforms; + const enterDuration = enter?.duration ?? 1; + const continuePrevious = enter?.from === "previous"; + const phase = usePresentationValue("playbackPhase"); + const entryPose = continuePrevious && active && (phase === undefined || phase === "playing") ? continuity : undefined; + useAnimatedReaction(() => playback.value, state => { + "worklet"; + if (!continuity || !active || state.phase !== "playing") return; + if (origin.value.key !== state.slideKey) origin.value = { key: state.slideKey, pose: continuePrevious ? continuity.value : null }; + const clock = { ...state, slideTime: sampleClock(state, previewTime, "slide"), stepTime: sampleClock(state, previewTime, "step") }; + continuity.value = cameraEvaluator(prepared, clock, origin.value.pose, enterDuration); + }, [prepared, continuePrevious, enterDuration, previewTime, active]); + const uniforms = useDerivedValue(() => { + "worklet"; + const state = playback.value; + const clock = { ...state, slideTime: sampleClock(state, previewTime, "slide"), stepTime: sampleClock(state, previewTime, "step") }; + const previous = state.phase === "preview" || state.phase === "preparing" ? null + : origin.value.key === state.slideKey ? origin.value.pose : state.phase === "playing" ? entryPose?.value : null; + return sampleUniforms(prepared, clock, previous, enterDuration); + }, [prepared, previewTime, enterDuration, entryPose]); + return + + + + {overlay.length > 0 && {overlay}} + ; +} diff --git a/packages/slides-effects/packs/world/camera.ts b/packages/slides-effects/packs/world/camera.ts new file mode 100644 index 000000000..5cfeb1301 --- /dev/null +++ b/packages/slides-effects/packs/world/camera.ts @@ -0,0 +1,81 @@ +export type Vec3 = [number, number, number]; +export type Ease = "linear" | "cubic" | "smoothstep" | "expo" | "spring"; +export type ClockSample = { slideTime: number; stepTime: number; stepIndex: number; stepOrigins: Record }; +export type CameraPose = { position: Vec3; target: Vec3; fov: number; roll: number }; +export type Shot = { + step?: number; at?: number; position?: Vec3; target?: Vec3; fov?: number; roll?: number; + orbit?: { center: Vec3; radius: number; speed?: number; height?: number; phase?: number }; + dolly?: number; follow?: string; duration?: number; ease?: Ease; +}; +export type CameraOptions = { shots?: Shot[]; duration?: number; ease?: Ease }; +type ObjectPositions = Record | ((id: string, clock: ClockSample) => Vec3 | undefined); +export const defaultPose: CameraPose = { position: [0, 5, 18], target: [0, 0, 0], fov: 42, roll: 0 }; + +export function easeProgress(value: number, ease: Ease = "cubic") { + "worklet"; + const p = Math.max(0, Math.min(1, value)); + if (ease === "linear") return p; + if (ease === "smoothstep") return p * p * (3 - 2 * p); + if (ease === "expo") return p === 0 || p === 1 ? p : p < 0.5 ? Math.pow(2, 20 * p - 10) / 2 : (2 - Math.pow(2, -20 * p + 10)) / 2; + if (ease === "spring") return p === 1 ? 1 : 1 - Math.exp(-7 * p) * Math.cos(p * 10); + return p < 0.5 ? 4 * p * p * p : 1 - Math.pow(-2 * p + 2, 3) / 2; +} +export function mixVector(a: Vec3, b: Vec3, t: number): Vec3 { + "worklet"; + return [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t]; +} +export function shotOrigin(shot: { step?: number; at?: number }, clock: ClockSample) { + "worklet"; + return shot.at ?? clock.stepOrigins[shot.step ?? 0] ?? ((shot.step ?? 0) === 0 ? 0 : clock.slideTime - clock.stepTime); +} +export function clockAtTime(clock: ClockSample, slideTime: number): ClockSample { + "worklet"; + return { ...clock, slideTime, stepTime: clock.stepTime - (clock.slideTime - slideTime) }; +} +function destination(shot: Shot, base: CameraPose, elapsed: number, objects: ObjectPositions, clock: ClockSample): CameraPose { + "worklet"; + let position = shot.position ?? base.position; + const followed = shot.follow ? typeof objects === "function" ? objects(shot.follow, clock) : objects[shot.follow] : undefined; + let target = followed ?? shot.target ?? base.target; + if (shot.orbit) { + const orbit = shot.orbit; + const angle = (orbit.phase ?? 0) + elapsed * (orbit.speed ?? 0.12); + position = [orbit.center[0] + Math.sin(angle) * orbit.radius, orbit.center[1] + (orbit.height ?? 4), orbit.center[2] + Math.cos(angle) * orbit.radius]; + target = followed ?? orbit.center; + } + if (shot.dolly !== undefined) { + const dx = position[0] - target[0], dy = position[1] - target[1], dz = position[2] - target[2]; + const scale = Math.max(0.05, shot.dolly) / Math.max(0.001, Math.sqrt(dx * dx + dy * dy + dz * dz)); + position = [target[0] + dx * scale, target[1] + dy * scale, target[2] + dz * scale]; + } + return { position, target, fov: shot.fov ?? base.fov, roll: shot.roll ?? base.roll }; +} + +function evaluateShots(options: CameraOptions, clock: ClockSample, objects: ObjectPositions, count: number): CameraPose { + "worklet"; + const shots = options.shots ?? []; + let index = -1; + for (let i = 0; i < count; i++) if (shots[i].at !== undefined ? clock.slideTime >= shots[i].at! : clock.stepIndex >= (shots[i].step ?? 0)) index = i; + if (index < 0) return defaultPose; + const shot = shots[index]; + const start = shotOrigin(shot, clock); + const elapsed = Math.max(0, clock.slideTime - start); + const from = index === 0 ? defaultPose : evaluateShots(options, clockAtTime(clock, start), objects, index); + const end = destination(shot, from, elapsed, objects, clock); + const duration = shot.duration ?? options.duration ?? 1.2; + const completed = shot.at === undefined && (shot.step ?? 0) < clock.stepIndex && clock.stepOrigins[shot.step ?? 0] === undefined; + const t = index === 0 || completed ? 1 : easeProgress(duration <= 0 ? 1 : elapsed / duration, shot.ease ?? options.ease); + return { position: mixVector(from.position, end.position, t), target: mixVector(from.target, end.target, t), + fov: from.fov + (end.fov - from.fov) * t, roll: from.roll + (end.roll - from.roll) * t }; +} +/** Stateless evaluation also handles direct step jumps and backwards scrubbing. */ +export function evaluateCamera(options: CameraOptions, clock: ClockSample, objects: ObjectPositions = {}, previous?: CameraPose | null, enterDuration = 1): CameraPose { + "worklet"; + let pose = evaluateShots(options, clock, objects, options.shots?.length ?? 0); + if (previous) { + const t = easeProgress(enterDuration <= 0 ? 1 : clock.slideTime / enterDuration, options.ease); + pose = { position: mixVector(previous.position, pose.position, t), target: mixVector(previous.target, pose.target, t), + fov: previous.fov + (pose.fov - previous.fov) * t, roll: previous.roll + (pose.roll - previous.roll) * t }; + } + return pose; +} diff --git a/packages/slides-effects/packs/world/graph.ts b/packages/slides-effects/packs/world/graph.ts new file mode 100644 index 000000000..ed77cb42a --- /dev/null +++ b/packages/slides-effects/packs/world/graph.ts @@ -0,0 +1,63 @@ +import { clockAtTime, easeProgress, type ClockSample, type Vec3 } from "./camera"; + +export type TreeNode = { id: string; children?: TreeNode[] }; +export type TreeBranch = string | [string, TreeBranch[]]; +export function tree(id: string, children: TreeBranch[] = []): TreeNode { + return { id, children: children.map(child => typeof child === "string" ? { id: child } : tree(child[0], child[1])) }; +} +export type LayoutNode = { id: string; parent?: string; depth: number; position: Vec3; ancestors: string[] }; +export type GraphLayout = { nodes: LayoutNode[]; edges: { source: string; target: string }[] }; +export type Activation = { kind: "wave"; from: string; step: number; speed: number; intensity?: number; duration?: number } + | { kind: "subset"; branches: string[]; step: number; intensity?: number; duration?: number }; +export function wave({ from, step, speed = 0.35, intensity = 1, duration = 0.5 }: { from: string; step: number; speed?: number; intensity?: number; duration?: number }): Activation { + return { kind: "wave", from, step, speed, intensity, duration }; +} +export function subset({ branches, step, intensity = 1, duration = 0.5 }: { branches: string[]; step: number; intensity?: number; duration?: number }): Activation { + return { kind: "subset", branches, step, intensity, duration }; +} + +export function layoutTree(data: TreeNode, { radius = 7, levelHeight = 2.4, position = [0, 3, 0] as Vec3 } = {}): GraphLayout { + const nodes: LayoutNode[] = [], edges: GraphLayout["edges"] = []; + const ids = new Set(); + function visit(node: TreeNode, parent: LayoutNode | undefined, start: number, end: number) { + if (ids.has(node.id)) throw new Error(`World graph has duplicate node id: ${node.id}`); + ids.add(node.id); + const depth = parent ? parent.depth + 1 : 0; + const angle = (start + end) / 2; + const r = depth === 0 ? 0 : radius * (1 - Math.exp(-depth * 0.55)); + const current: LayoutNode = { id: node.id, parent: parent?.id, depth, + position: [position[0] + Math.sin(angle) * r, position[1] - depth * levelHeight, position[2] + Math.cos(angle) * r], + ancestors: parent ? [...parent.ancestors, parent.id] : [] }; + nodes.push(current); + if (parent) edges.push({ source: parent.id, target: node.id }); + const children = node.children ?? []; + children.forEach((child, i) => visit(child, current, start + (end - start) * i / children.length, start + (end - start) * (i + 1) / children.length)); + } + visit(data, undefined, 0, Math.PI * 2); + return { nodes, edges }; +} + +function activationFrames(node: LayoutNode, nodes: LayoutNode[], activations: Activation[], clock: ClockSample, count: number): number { + "worklet"; + let index = -1; + for (let i = 0; i < count; i++) if (activations[i].step <= clock.stepIndex) index = i; + if (index < 0) return 0; + const activation = activations[index]; + let selected = false, delay = 0; + if (activation.kind === "subset") selected = activation.branches.some(branch => node.id === branch || node.ancestors.includes(branch)); + else { + const source = nodes.find(item => item.id === activation.from); + selected = !!source && (node.id === source.id || node.ancestors.includes(source.id)); + if (selected && source) delay = Math.max(0, node.depth - source.depth) * activation.speed; + } + const origin = clock.stepOrigins[activation.step] ?? clock.slideTime - clock.stepTime; + const previous = activationFrames(node, nodes, activations, clockAtTime(clock, origin), index); + const elapsed = clock.slideTime - origin - delay; + const completed = activation.step < clock.stepIndex && clock.stepOrigins[activation.step] === undefined; + const t = completed ? 1 : easeProgress((activation.duration ?? 0.5) <= 0 ? elapsed >= 0 ? 1 : 0 : elapsed / (activation.duration ?? 0.5)); + return previous + ((selected ? activation.intensity ?? 1 : 0) - previous) * t; +} +export function activationAt(node: LayoutNode, nodes: LayoutNode[], activations: Activation[], clock: ClockSample) { + "worklet"; + return activationFrames(node, nodes, activations, clock, activations.length); +} diff --git a/packages/slides-effects/packs/world/pack.json b/packages/slides-effects/packs/world/pack.json new file mode 100644 index 000000000..aadb3c666 --- /dev/null +++ b/packages/slides-effects/packs/world/pack.json @@ -0,0 +1,7 @@ +{ + "id": "world", + "version": "1.0.0", + "hostApiVersion": 2, + "exports": { "World": "./World.tsx" }, + "capabilities": ["webgpu", "webgpu-multipass", "reanimated"] +} diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts new file mode 100644 index 000000000..6b8244e1f --- /dev/null +++ b/packages/slides-effects/packs/world/render.ts @@ -0,0 +1,169 @@ +import type { GPUPass, GPURenderGraphDefinition } from "@legend-apps/presentation"; +import { worldHeaderSize, worldInstanceSize, type PreparedWorldScene, type PrimitiveKind, type WorldScene } from "./scene"; +import type { Vec3 } from "./camera"; + +export const worldBindings = ` +struct Frame { time:f32, dt:f32, frame:f32, step:f32, aspect:f32, brightness:f32, slideTime:f32, stepTime:f32 } +@group(0) @binding(0) var frame:Frame; +@group(0) @binding(1) var uniforms:array; +@group(0) @binding(2) var linearSampler:sampler; +fn vector(i:u32)->vec3f { return vec3f(uniforms[i],uniforms[i+1u],uniforms[i+2u]); } +fn cameraForward()->vec3f { return normalize(vector(4u)-vector(0u)); } +fn cameraRight()->vec3f { + let forward=cameraForward(); + let axis=select(vec3f(0.,1.,0.),vec3f(0.,0.,1.),abs(forward.y)>.99); + let right=normalize(cross(forward,axis)); let up=cross(right,forward); + return right*cos(uniforms[7])+up*sin(uniforms[7]); +} +fn cameraUp()->vec3f { return cross(cameraRight(),cameraForward()); } +fn project(p:vec3f)->vec4f { + let delta=p-vector(0u); let z=dot(delta,cameraForward()); let near=.05; let far=500.; + return vec4f(dot(delta,cameraRight())/(uniforms[3]*frame.aspect),dot(delta,cameraUp())/uniforms[3],far/(far-near)*z-far*near/(far-near),z); +} +`; +const triangleCorner = ` +fn corner(id:u32)->vec2f { + let corners=array(vec2f(0.,0.),vec2f(1.,0.),vec2f(0.,1.),vec2f(0.,1.),vec2f(1.,0.),vec2f(1.,1.)); + return corners[id%6u]; +} +`; +const mesh: Record = { + sphere: { count: 12 * 8 * 6, code: ` + let cell=vid/6u; let uv=(vec2f(f32(cell%12u),f32(cell/12u))+corner(vid))/vec2f(12.,8.); + let longitude=uv.x*6.2831853; let latitude=uv.y*3.14159265; + normal=vec3f(sin(latitude)*cos(longitude),cos(latitude),sin(latitude)*sin(longitude)); + point=normal*radius; +` }, + box: { count: 36, code: ` + let face=vid/6u; let uv=corner(vid)*2.-1.; + if(face==0u) { normal=vec3f(1.,0.,0.); point=vec3f(1.,uv.y,uv.x); } + if(face==1u) { normal=vec3f(-1.,0.,0.); point=vec3f(-1.,uv.y,-uv.x); } + if(face==2u) { normal=vec3f(0.,1.,0.); point=vec3f(uv.x,1.,uv.y); } + if(face==3u) { normal=vec3f(0.,-1.,0.); point=vec3f(uv.x,-1.,-uv.y); } + if(face==4u) { normal=vec3f(0.,0.,1.); point=vec3f(uv.x,uv.y,1.); } + if(face==5u) { normal=vec3f(0.,0.,-1.); point=vec3f(-uv.x,uv.y,-1.); } + point*=radius; +` }, + tube: { count: 12 * 6 * 6, code: ` + let cell=vid/6u; let uv=(vec2f(f32(cell%12u),f32(cell/12u))+corner(vid))/vec2f(12.,6.); + let endpoint=vector(base+12u)-origin; + let control=endpoint*.5+vec3f(0.,length(endpoint)*uniforms[base+15u],0.); + let t=uv.x; let tangent=normalize(2.*(1.-t)*control+2.*t*(endpoint-control)); + let axis=select(vec3f(0.,1.,0.),vec3f(1.,0.,0.),abs(tangent.y)>.95); + let right=normalize(cross(tangent,axis)); let up=cross(right,tangent); + normal=right*cos(uv.y*6.2831853)+up*sin(uv.y*6.2831853); + point=2.*(1.-t)*t*control+t*t*endpoint+normal*uniforms[base+19u]; +` }, + ring: { count: 32 * 6 * 6, code: ` + let cell=vid/6u; let uv=(vec2f(f32(cell%32u),f32(cell/32u))+corner(vid))/vec2f(32.,6.); + let a=uv.x*6.2831853; let b=uv.y*6.2831853; let radial=vec3f(cos(a),0.,sin(a)); + normal=radial*cos(b)+vec3f(0.,sin(b),0.); + point=radial*radius+normal*uniforms[base+19u]; +` }, +}; + +function geometryCode(kind: PrimitiveKind, offset: number) { + return `${worldBindings}${triangleCorner} +struct Vertex { @builtin(position) position:vec4f, @location(0) world:vec3f, @location(1) normal:vec3f, @location(2) color:vec3f, @location(3) emissive:f32 } +@vertex fn worldVertex(@builtin(vertex_index) vid:u32,@builtin(instance_index) iid:u32)->Vertex { + let base=${worldHeaderSize}u+(iid+${offset}u)*${worldInstanceSize}u; + let origin=vector(base); let radius=uniforms[base+3u]; + var point=vec3f(0.); var normal=vec3f(0.,1.,0.); + ${mesh[kind].code} + let scale=max(vector(base+8u),vec3f(.0001)); + let world=origin+point*scale; + var result:Vertex; result.position=project(world); result.world=world; result.normal=normalize(normal/scale); + result.color=vector(base+4u); result.emissive=uniforms[base+7u]; return result; +} +@fragment fn worldFragment(in:Vertex)->@location(0) vec4f { + let normal=normalize(in.normal); let view=normalize(vector(0u)-in.world); + let diffuse=max(0.,dot(normal,normalize(vec3f(-.4,1.,.6)))); + let rim=pow(1.-abs(dot(normal,view)),3.); + let color=in.color*(.12+diffuse*.55+in.emissive)+in.color*rim*(.25+in.emissive*.12); + return vec4f(color,1.); +}`; +} + +const filterCode = `${worldBindings} +@group(0) @binding(3) var source:texture_2d; +fn tentFilter(uv:vec2f)->vec3f { + let pixel=1./vec2f(textureDimensions(source)); + return textureSampleLevel(source,linearSampler,uv,0.).rgb*.25 + +(textureSampleLevel(source,linearSampler,uv+pixel*vec2f(-1.,-1.),0.).rgb + +textureSampleLevel(source,linearSampler,uv+pixel*vec2f(1.,-1.),0.).rgb + +textureSampleLevel(source,linearSampler,uv+pixel*vec2f(-1.,1.),0.).rgb + +textureSampleLevel(source,linearSampler,uv+pixel*vec2f(1.,1.),0.).rgb)*.1875; +} +`; +const downsample = `${filterCode} +@fragment fn downsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(tentFilter(uv),1.); } +`; +const threshold = `${filterCode} +@fragment fn threshold(@location(0) uv:vec2f)->@location(0) vec4f { + let color=tentFilter(uv); let brightness=max(color.r,max(color.g,color.b)); + let contribution=max(0.,brightness-uniforms[10])/max(.0001,brightness); + return vec4f(color*contribution,1.); +}`; +const upsample = `${filterCode} +@group(0) @binding(4) var high:texture_2d; +@fragment fn upsample(@location(0) uv:vec2f)->@location(0) vec4f { + let color=textureSampleLevel(high,linearSampler,uv,0.).rgb; + return vec4f(color+tentFilter(uv)*uniforms[11],1.); +}`; +const finalCode = `${worldBindings} +@group(0) @binding(3) var hdr:texture_2d; +@group(0) @binding(4) var depth:texture_depth_2d; +@group(0) @binding(5) var bloom:texture_2d; +fn aces(x:vec3f)->vec3f { return clamp((x*(2.51*x+.03))/(x*(2.43*x+.59)+.14),vec3f(0.),vec3f(1.)); } +fn srgb(x:vec3f)->vec3f { return select(x*12.92,1.055*pow(x,vec3f(1./2.4))-.055,x>vec3f(.0031308)); } +@fragment fn tonemap(@location(0) uv:vec2f)->@location(0) vec4f { + let pixel=clamp(vec2i(uv*vec2f(textureDimensions(depth))),vec2i(0),vec2i(textureDimensions(depth))-vec2i(1)); + let z=textureLoad(depth,pixel,0); let near=.05; let far=500.; + let distance=near*far/(far-z*(far-near)); + let ray=cameraForward()+cameraRight()*(uv.x*2.-1.)*frame.aspect*uniforms[3]+cameraUp()*(1.-uv.y*2.)*uniforms[3]; + let world=vector(0u)+ray*distance; + let heightDensity=exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); + let fog=1.-exp(-distance*uniforms[12]*heightDensity); + var color=mix(textureSampleLevel(hdr,linearSampler,uv,0.).rgb,vector(20u),clamp(fog,0.,1.)); + color+=textureSampleLevel(bloom,linearSampler,uv,0.).rgb*uniforms[9]; + color=srgb(aces(color*uniforms[8]*frame.brightness)); + let vignette=1.-uniforms[15]*smoothstep(.1,.8,length(uv-.5)); + color*=vignette*(1.-uniforms[17]*.65); + let noise=fract(sin(dot(uv*vec2f(textureDimensions(hdr)),vec2f(12.9898,78.233))+floor(frame.time*24.))*43758.5453)-.5; + color+=noise*uniforms[16]; + return vec4f(clamp(color,vec3f(0.),vec3f(1.)),1.); +}`; + +export type WorldGraphTopology = { quality: WorldScene["quality"]; counts: Record; backgroundVector: Vec3 }; +export function worldGraphTopology(scene: PreparedWorldScene): WorldGraphTopology { + const counts: Record = { box: 0, ring: 0, sphere: 0, tube: 0 }; + for (const instance of scene.instances) counts[instance.kind]++; + return { quality: scene.quality, counts, backgroundVector: scene.backgroundVector }; +} +/** Pure graph description, shared by decks and the browser performance harness. */ +export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): GPURenderGraphDefinition & { uniformsSize: number } { + const topology = "instances" in scene ? worldGraphTopology(scene) : scene; + const scale = topology.quality === "low" ? 0.65 : topology.quality === "medium" ? 0.85 : 1; + const levels = topology.quality === "low" ? 3 : 5; + const instanceCount = Object.values(topology.counts).reduce((a, b) => a + b, 0); + const passes: GPUPass[] = []; + let offset = 0; + const background = topology.backgroundVector; + for (const kind of ["box", "ring", "sphere", "tube"] as const) { + const count = topology.counts[kind]; + if (count || passes.length === 0 && instanceCount === 0) { + passes.push({ kind: "render", id: `scene-${kind}`, code: geometryCode(kind, offset), vertex: "worldVertex", fragment: "worldFragment", + color: ["hdr"], depth: "depth", vertexCount: mesh[kind].count, instanceCount: count, + clear: passes.length === 0 ? [background[0], background[1], background[2], 1] : false }); + } + offset += count; + } + passes.push({ kind: "fullscreen", id: "bloom-threshold", code: threshold, fragment: "threshold", reads: ["hdr"], output: "bloom@0" }); + for (let mip = 1; mip < levels; mip++) passes.push({ kind: "fullscreen", id: `bloom-down-${mip}`, code: downsample, fragment: "downsample", reads: [`bloom@${mip - 1}`], output: `bloom@${mip}` }); + passes.push({ kind: "fullscreen", id: `bloom-up-${levels - 1}`, code: downsample, fragment: "downsample", reads: [`bloom@${levels - 1}`], output: `bloom-up@${levels - 1}` }); + for (let mip = levels - 2; mip >= 0; mip--) passes.push({ kind: "fullscreen", id: `bloom-up-${mip}`, code: upsample, fragment: "upsample", reads: [`bloom-up@${mip + 1}`, `bloom@${mip}`], output: `bloom-up@${mip}` }); + passes.push({ kind: "fullscreen", id: "tonemap", code: finalCode, fragment: "tonemap", reads: ["hdr", "depth", "bloom-up@0"], output: "screen" }); + return { targets: [{ id: "hdr", format: "rgba16float", scale }, { id: "depth", format: "depth32float", scale }, + { id: "bloom", format: "rgba16float", scale: scale * 0.5, mips: levels }, { id: "bloom-up", format: "rgba16float", scale: scale * 0.5, mips: levels }], + passes, uniformsSize: worldHeaderSize + instanceCount * worldInstanceSize }; +} diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts new file mode 100644 index 000000000..996bde1e7 --- /dev/null +++ b/packages/slides-effects/packs/world/scene.ts @@ -0,0 +1,173 @@ +import { clockAtTime, easeProgress, evaluateCamera, mixVector, type CameraOptions, type CameraPose, type ClockSample, type Ease, type Vec3 } from "./camera"; +import { activationAt, layoutTree, type Activation, type LayoutNode, type TreeNode } from "./graph"; + +export type Keyframe = { step?: number; at?: number; value: T; duration?: number; ease?: Ease }; +export type Animated = T | Record | Keyframe[]; +export type PrimitiveKind = "sphere" | "box" | "tube" | "ring"; +type Track = { initial: T; frames: Keyframe[] }; +export type PrimitiveOptions = { + id?: string; position?: Animated; size?: Animated; color?: Animated; + emissive?: Animated; scale?: Animated; end?: Vec3; curve?: number; thickness?: number; +}; +export type Instance = { + id?: string; kind: PrimitiveKind; position: Track; size: Track; color: Track; + emissive: Track; scale: Track; end: Vec3; curve: number; thickness: number; + drift?: number; seed?: number; + endpoints?: [string, string]; + graph?: { node: LayoutNode; nodes: LayoutNode[]; activate: Activation[]; dim: Vec3 }; +}; +export type GraphOptions = { + data: TreeNode; layout?: "cone"; radius?: number; levelHeight?: number; position?: Vec3; + node?: { color?: Animated; dim?: string; size?: Animated; emissive?: Animated; scale?: Animated }; + nodes?: Record; + edge?: { color?: string; curve?: number; glow?: number; thickness?: number }; + activate?: Activation[]; +}; +export type OrbsOptions = { + id?: string; count?: number; colors?: string[]; radius?: number; spread?: Vec3; position?: Vec3; seed?: number; drift?: number; emissive?: number; +}; +export type WorldScene = { + camera?: CameraOptions; instances: Instance[]; exposure?: Animated; quality?: "auto" | "low" | "medium" | "high"; + backgroundColor?: string; fog?: { density?: Animated; color?: Animated; height?: Animated; falloff?: Animated }; + bloom?: { intensity?: Animated; threshold?: Animated; radius?: Animated }; vignette?: Animated; grain?: Animated; scrim?: number; +}; + +export function colorVector(color: string): Vec3 { + const hex = color.startsWith("#") ? color.slice(1) : color; + const full = hex.length === 3 ? hex.split("").map(c => c + c).join("") : hex; + if (!/^[0-9a-f]{6}$/i.test(full)) throw new Error(`World colors require #RGB or #RRGGBB: ${color}`); + return [0, 2, 4].map(index => { + const value = parseInt(full.slice(index, index + 2), 16) / 255; + return value <= 0.04045 ? value / 12.92 : Math.pow((value + 0.055) / 1.055, 2.4); + }) as Vec3; +} +export function compileTrack(value: Animated, transform: (value: T) => U = (v => v as unknown as U)): Track { + let frames: Keyframe[]; + if (Array.isArray(value) && value.length > 0 && typeof value[0] === "object" && value[0] !== null && "value" in value[0]) frames = value as Keyframe[]; + else if (typeof value === "object" && !Array.isArray(value) && value !== null) frames = Object.entries(value).map(([step, v]) => ({ step: Number(step), value: v })).sort((a, b) => a.step! - b.step!); + else return { initial: transform(value as T), frames: [] }; + if (frames.length === 0) throw new Error("World keyframes must not be empty"); + return { initial: transform(frames[0].value), frames: frames.map(frame => ({ ...frame, value: transform(frame.value) })) }; +} +function sampleTrackFrames(track: Track, clock: ClockSample, count: number): T { + "worklet"; + let index = -1; + for (let i = 0; i < count; i++) if (track.frames[i].at !== undefined ? clock.slideTime >= track.frames[i].at! : clock.stepIndex >= (track.frames[i].step ?? 0)) index = i; + if (index < 0) return track.initial; + const frame = track.frames[index]; + const origin = frame.at ?? clock.stepOrigins[frame.step ?? 0] ?? clock.slideTime - clock.stepTime; + const value = sampleTrackFrames(track, clockAtTime(clock, origin), index); + const elapsed = Math.max(0, clock.slideTime - origin); + const completed = frame.at === undefined && (frame.step ?? 0) < clock.stepIndex && clock.stepOrigins[frame.step ?? 0] === undefined; + const t = completed ? 1 : easeProgress((frame.duration ?? 0.65) <= 0 ? 1 : elapsed / (frame.duration ?? 0.65), frame.ease); + return (typeof value === "number" ? value + ((frame.value as number) - value) * t : mixVector(value, frame.value as Vec3, t)) as T; +} +export function sampleTrack(track: Track, clock: ClockSample): T { + "worklet"; + return sampleTrackFrames(track, clock, track.frames.length); +} +export function primitive(kind: PrimitiveKind, options: PrimitiveOptions = {}): Instance { + return { id: options.id, kind, position: compileTrack(options.position ?? [0, 0, 0]), size: compileTrack(options.size ?? 0.5), + color: compileTrack(options.color ?? "#22d3ee", colorVector), emissive: compileTrack(options.emissive ?? 1.5), + scale: compileTrack(options.scale ?? [1, 1, 1]), end: options.end ?? [0, 2, 0], curve: options.curve ?? 0, + thickness: options.thickness ?? (kind === "ring" ? 0.04 : 0.025) }; +} +export function graphInstances(options: GraphOptions): Instance[] { + const layout = layoutTree(options.data, options); + const nodeOptions = options.node ?? {}, edgeOptions = options.edge ?? {}; + const dim = colorVector(nodeOptions.dim ?? "#101c38"); + const nodeById = new Map(layout.nodes.map(node => [node.id, node])); + const nodes = layout.nodes.map(node => { + const instance = primitive("sphere", { position: node.position, + color: nodeOptions.color ?? "#b56bff", size: nodeOptions.size ?? 0.3, scale: nodeOptions.scale, + emissive: nodeOptions.emissive ?? 5, ...options.nodes?.[node.id], id: `node:${node.id}` }); + instance.graph = { node, nodes: layout.nodes, activate: options.activate ?? [], dim }; + return instance; + }); + const edges = layout.edges.map(edge => { + const source = nodeById.get(edge.source)!, target = nodeById.get(edge.target)!; + const instance = primitive("tube", { id: `edge:${edge.source}:${edge.target}`, position: source.position, end: target.position, + color: edgeOptions.color ?? (typeof nodeOptions.color === "string" ? nodeOptions.color : "#b56bff"), + emissive: edgeOptions.glow ?? 1.8, curve: edgeOptions.curve ?? 0.4, thickness: edgeOptions.thickness ?? 0.025 }); + instance.graph = { node: target, nodes: layout.nodes, activate: options.activate ?? [], dim }; + instance.endpoints = [`node:${source.id}`, `node:${target.id}`]; + return instance; + }); + return [...nodes, ...edges]; +} +export function orbInstances({ id = "orb", count = 48, colors = ["#22d3ee", "#b56bff"], radius = 0.22, spread = [12, 7, 9], position = [0, 0, 0], seed = 1, drift = 0.4, emissive = 5 }: OrbsOptions = {}): Instance[] { + if (!Number.isInteger(count) || count < 0 || count > 10000) throw new Error("World orb count must be between 0 and 10000"); + if (colors.length === 0) throw new Error("World orbs require at least one color"); + let state = seed >>> 0; + function random() { state = (Math.imul(state, 1664525) + 1013904223) >>> 0; return state / 4294967296; } + return Array.from({ length: count }, (_, index) => { + const location: Vec3 = [position[0] + (random() - 0.5) * spread[0], position[1] + (random() - 0.5) * spread[1], position[2] + (random() - 0.5) * spread[2]]; + return { ...primitive("sphere", { id: `${id}:${index}`, position: location, size: radius * (0.6 + random() * 0.8), color: colors[index % colors.length], emissive }), drift, seed: random() * 100 }; + }); +} + +export const worldHeaderSize = 32; +export const worldInstanceSize = 24; +export function instancePosition(instance: Instance, clock: ClockSample): Vec3 { + "worklet"; + const position = sampleTrack(instance.position, clock); + const drift = instance.drift ?? 0, seed = instance.seed ?? 0; + return [position[0] + Math.sin(clock.slideTime * 0.27 + seed) * drift, + position[1] + Math.cos(clock.slideTime * 0.31 + seed * 2) * drift, + position[2] + Math.sin(clock.slideTime * 0.19 + seed * 3) * drift]; +} +export function sampleWorldCamera(scene: WorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1): CameraPose { + "worklet"; + return evaluateCamera(scene.camera ?? {}, clock, (id, sample) => { + const instance = scene.instances.find(instance => instance.id === id); + return instance ? instancePosition(instance, sample) : undefined; + }, previous, enterDuration); +} +export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1): number[] { + "worklet"; + const values = new Array(worldHeaderSize + scene.instances.length * worldInstanceSize).fill(0); + const objects: Record = {}; + for (const instance of scene.instances) if (instance.id) objects[instance.id] = instancePosition(instance, clock); + const camera = sampleWorldCamera(scene, clock, previous, enterDuration); + values[0] = camera.position[0]; values[1] = camera.position[1]; values[2] = camera.position[2]; values[3] = Math.tan(camera.fov * Math.PI / 360); + values[4] = camera.target[0]; values[5] = camera.target[1]; values[6] = camera.target[2]; values[7] = camera.roll; + values[8] = sampleTrack(scene.exposureTrack, clock); + for (const field of scene.postTracks) values[field.index] = sampleTrack(field.track, clock); + values[17] = scene.scrim ?? 0; + const fog = sampleTrack(scene.fogTrack, clock); + values[20] = fog[0]; values[21] = fog[1]; values[22] = fog[2]; + for (let i = 0; i < scene.instances.length; i++) { + const instance = scene.instances[i]; + const index = worldHeaderSize + i * worldInstanceSize; + const position = instance.endpoints ? objects[instance.endpoints[0]] : instance.id ? objects[instance.id] : instancePosition(instance, clock); + const endpoint = instance.endpoints ? objects[instance.endpoints[1]] : instance.end; + const scale = sampleTrack(instance.scale, clock); + const color = sampleTrack(instance.color, clock); + const activation = instance.graph ? activationAt(instance.graph.node, instance.graph.nodes, instance.graph.activate, clock) : 1; + const dim = instance.graph?.dim ?? color; + values[index] = position[0]; values[index + 1] = position[1]; values[index + 2] = position[2]; + values[index + 3] = sampleTrack(instance.size, clock); + values[index + 4] = dim[0] + (color[0] - dim[0]) * activation; + values[index + 5] = dim[1] + (color[1] - dim[1]) * activation; + values[index + 6] = dim[2] + (color[2] - dim[2]) * activation; + values[index + 7] = sampleTrack(instance.emissive, clock) * activation; + values[index + 8] = scale[0]; values[index + 9] = scale[1]; values[index + 10] = scale[2]; + values[index + 12] = endpoint[0]; values[index + 13] = endpoint[1]; values[index + 14] = endpoint[2]; + values[index + 15] = instance.curve; values[index + 19] = instance.thickness; + } + return values; +} + +export type PreparedWorldScene = WorldScene & { exposureTrack: Track; fogTrack: Track; backgroundVector: Vec3; postTracks: { index: number; track: Track }[] }; +export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { + const ids = new Set(); + for (const instance of scene.instances) if (instance.id) { + if (ids.has(instance.id)) throw new Error(`World instance id is duplicated: ${instance.id}. Give each instance or orb field a unique id.`); + ids.add(instance.id); + } + const instances = [...scene.instances].sort((a, b) => a.kind.localeCompare(b.kind)); + const fields: [number, Animated][] = [[9, scene.bloom?.intensity ?? 0], [10, scene.bloom?.threshold ?? 1], [11, scene.bloom?.radius ?? 0.8], + [12, scene.fog?.density ?? 0], [13, scene.fog?.height ?? 0], [14, scene.fog?.falloff ?? 0.15], [15, scene.vignette ?? 0.25], [16, scene.grain ?? 0.015]]; + return { ...scene, instances, exposureTrack: compileTrack(scene.exposure ?? 1), fogTrack: compileTrack(scene.fog?.color ?? scene.backgroundColor ?? "#05060a", colorVector), + backgroundVector: colorVector(scene.backgroundColor ?? "#05060a"), postTracks: fields.map(([index, value]) => ({ index, track: compileTrack(value) })) }; +} diff --git a/packages/slides-effects/scripts/renderGraphBenchmark.ts b/packages/slides-effects/scripts/renderGraphBenchmark.ts new file mode 100644 index 000000000..0eb08a537 --- /dev/null +++ b/packages/slides-effects/scripts/renderGraphBenchmark.ts @@ -0,0 +1,119 @@ +import type { GPURenderGraphDefinition } from "@legend-apps/presentation"; +import { encodeRenderGraph, prepareRenderGraph } from "../../../apps/slides/src/gpuRenderGraph"; + +export type BenchmarkOptions = { + definition: GPURenderGraphDefinition; + uniformsAt: (seconds: number) => number[]; + width: number; + height: number; + warmup?: number; + frames?: number; + repetitions?: number; +}; + +function summarize(values: number[]) { + const sorted = [...values].sort((a, b) => a - b); + const median = (sorted[Math.floor(sorted.length / 2)] + sorted[Math.floor((sorted.length - 1) / 2)]) / 2; + return { medianMs: median, p95Ms: sorted[Math.ceil(sorted.length * 0.95) - 1], + minMs: sorted[0], maxMs: sorted[sorted.length - 1], throughputFps: 1000 / median }; +} + +/** Fixed clock samples measure rendering without introducing a playback driver. */ +export async function benchmarkRenderGraph({ definition, uniformsAt, width, height, warmup = 30, frames = 120, repetitions = 3 }: BenchmarkOptions) { + if (![width, height, warmup, frames, repetitions].every(Number.isInteger) || width < 1 || height < 1 || warmup < 0 || frames < 1 || repetitions < 1) { + throw new Error("Benchmark dimensions and sample counts must be positive integers"); + } + if (!navigator.gpu) throw new Error("WebGPU unavailable"); + const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); + if (!adapter) throw new Error("No WebGPU adapter"); + const timestamps = adapter.features.has("timestamp-query"); + const device = await adapter.requestDevice({ requiredFeatures: timestamps ? ["timestamp-query"] : [] }); + const failures: string[] = []; + device.addEventListener("uncapturederror", event => failures.push(event.error.message)); + const canvas = document.createElement("canvas"); + canvas.width = width; canvas.height = height; + canvas.style.cssText = "width:100%;height:auto;display:block"; + document.body.replaceChildren(canvas); + const context = canvas.getContext("webgpu"); + if (!context) { device.destroy(); throw new Error("No WebGPU canvas context"); } + const format = navigator.gpu.getPreferredCanvasFormat(); + context.configure({ device, format, alphaMode: "opaque" }); + const initial = uniformsAt(0); + const clock = device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }); + const uniforms = device.createBuffer({ size: Math.max(16, initial.length * 4), usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST }); + const passCount = definition.passes.length; + const query = timestamps ? device.createQuerySet({ type: "timestamp", count: passCount * 2 }) : undefined; + const queryBytes = passCount * 16; + const resolve = query ? device.createBuffer({ size: queryBytes, usage: GPUBufferUsage.QUERY_RESOLVE | GPUBufferUsage.COPY_SRC }) : undefined; + const readback = query ? device.createBuffer({ size: queryBytes, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ }) : undefined; + const samples: { repetition: number; frame: number; seconds: number; completedMs: number; gpuMs?: number; passes?: Record }[] = []; + try { + device.pushErrorScope("validation"); + const graph = await prepareRenderGraph(device, definition, format, clock, uniforms); + const setupError = await device.popErrorScope(); + if (setupError) throw new Error(setupError.message); + async function render(frame: number, timed: boolean) { + const seconds = 2 + frame / 60; + const start = performance.now(); + const values = uniformsAt(seconds); + if (values.length !== initial.length) throw new Error("Benchmark uniforms changed size"); + device.queue.writeBuffer(clock, 0, new Float32Array([seconds, 1 / 60, frame, 1, width / height, 1, seconds, seconds])); + device.queue.writeBuffer(uniforms, 0, new Float32Array(values)); + const encoder = device.createCommandEncoder(); + let index = 0; + // This proxy adds timers around the production encoder's unmodified passes. + const timedEncoder = query && timed ? { + beginRenderPass(descriptor: GPURenderPassDescriptor) { + const offset = index++ * 2; + return encoder.beginRenderPass({ ...descriptor, timestampWrites: { querySet: query, beginningOfPassWriteIndex: offset, endOfPassWriteIndex: offset + 1 } }); + }, + beginComputePass(descriptor?: GPUComputePassDescriptor) { + const offset = index++ * 2; + return encoder.beginComputePass({ ...descriptor, timestampWrites: { querySet: query, beginningOfPassWriteIndex: offset, endOfPassWriteIndex: offset + 1 } }); + }, + } as GPUCommandEncoder : encoder; + encodeRenderGraph(graph, device, timedEncoder, context!.getCurrentTexture()); + if (query && timed) { + encoder.resolveQuerySet(query, 0, passCount * 2, resolve!, 0); + encoder.copyBufferToBuffer(resolve!, 0, readback!, 0, queryBytes); + } + device.queue.submit([encoder.finish()]); + await device.queue.onSubmittedWorkDone(); + const completedMs = performance.now() - start; + let gpuMs: number | undefined, passes: Record | undefined; + if (readback && timed) { + await readback.mapAsync(GPUMapMode.READ); + const ticks = new BigUint64Array(readback.getMappedRange()); + passes = Object.fromEntries(definition.passes.map((pass, i) => [pass.id, Number(ticks[i * 2 + 1] - ticks[i * 2]) / 1e6])); + gpuMs = Number(ticks[passCount * 2 - 1] - ticks[0]) / 1e6; + readback.unmap(); + } + if (failures.length) throw new Error(failures.join("\n")); + return { seconds, completedMs, ...(gpuMs === undefined ? {} : { gpuMs, passes }) }; + } + for (let frame = 0; frame < warmup; frame++) await render(frame, false); + for (let repetition = 0; repetition < repetitions; repetition++) { + for (let frame = 0; frame < frames; frame++) samples.push({ repetition, frame, ...await render(frame, true) }); + } + // Refresh after the promise chain so the exported still has a live texture. + await render(90, false); + const png = canvas.toDataURL("image/png"); + const info = adapter.info; + return { + environment: { browser: navigator.userAgent, adapter: { vendor: info.vendor, architecture: info.architecture, device: info.device, description: info.description }, + timestampQuery: timestamps, width, height, warmup, frames, repetitions, devicePixelRatio, visibility: document.visibilityState }, + methodology: "Unpaced fixed-clock samples; completedMs includes uniform evaluation, writes, encode, submit and queue completion. GPU frame span is the last pass end minus the first pass beginning; pass scopes may overlap and are non-additive. GPU timestamps exclude CPU and presentation. Throughput is not display-paced FPS.", + pipeline: { targets: definition.targets, passes: definition.passes.map(pass => ({ id: pass.id, kind: pass.kind })) }, + completed: summarize(samples.map(sample => sample.completedMs)), + gpu: timestamps ? summarize(samples.map(sample => sample.gpuMs!)) : null, + passes: timestamps ? Object.fromEntries(definition.passes.map(pass => [pass.id, summarize(samples.map(sample => sample.passes![pass.id]))])) : null, + repetitions: Array.from({ length: repetitions }, (_, repetition) => ({ repetition, + completed: summarize(samples.filter(sample => sample.repetition === repetition).map(sample => sample.completedMs)), + gpu: timestamps ? summarize(samples.filter(sample => sample.repetition === repetition).map(sample => sample.gpuMs!)) : null })), + samples, png, + }; + } finally { + context.unconfigure(); + device.destroy(); + } +} diff --git a/packages/slides-effects/scripts/world-benchmark.mjs b/packages/slides-effects/scripts/world-benchmark.mjs new file mode 100644 index 000000000..b58649341 --- /dev/null +++ b/packages/slides-effects/scripts/world-benchmark.mjs @@ -0,0 +1,16 @@ +import { mkdir } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const args = process.argv.slice(2); +if (args.includes("--help") || args.length !== 2 || args[0] !== "--output") { + console.log("Usage: bun packages/slides-effects/scripts/world-benchmark.mjs --output /absolute/path/harness.html\nGenerates a standalone offline WebGPU measurement page. In Chrome, call await benchmarkWorld({width:1920,height:1080,quality:'high'}).\nResult includes raw samples, per-pass GPU timestamps when available, completed-submit wall times and a PNG data URL.\nGenerate and launch the browser through the heavy gate; call the measurement through the fps gate. Keep generated pages/results outside the repository."); + process.exit(args.includes("--help") ? 0 : 2); +} +const output = resolve(args[1]); +const result = await Bun.build({ entrypoints: [fileURLToPath(new URL("./worldBenchmark.ts", import.meta.url))], target: "browser" }); +if (!result.success) throw new AggregateError(result.logs, "World benchmark bundling failed"); +const code = (await result.outputs[0].text()).replaceAll("World render benchmark

Ready. Call benchmarkWorld({width:1920,height:1080,quality:"high"}) from DevTools.

`); +console.log(output); diff --git a/packages/slides-effects/scripts/worldBenchmark.ts b/packages/slides-effects/scripts/worldBenchmark.ts new file mode 100644 index 000000000..83a80775b --- /dev/null +++ b/packages/slides-effects/scripts/worldBenchmark.ts @@ -0,0 +1,30 @@ +import { graphInstances, orbInstances, prepareWorldScene, primitive, sampleWorldUniforms } from "../packs/world/scene"; +import { tree, wave } from "../packs/world/graph"; +import { buildWorldGraph } from "../packs/world/render"; +import { benchmarkRenderGraph } from "./renderGraphBenchmark"; + +type Options = { width: number; height: number; quality: "low" | "medium" | "high"; frames?: number; repetitions?: number; warmup?: number }; + +export async function benchmarkWorld(options: Options) { + const data = tree("root", Array.from({ length: 3 }, (_, branch) => [ + `branch-${branch}`, + Array.from({ length: 3 }, (_, group) => [`group-${branch}-${group}`, Array.from({ length: 3 }, (_, leaf) => `leaf-${branch}-${group}-${leaf}`)]), + ]) as Parameters[1]); + const scene = prepareWorldScene({ + quality: options.quality, + camera: { shots: [{ at: 0, orbit: { center: [0, -0.5, 0], radius: 22, height: 8, speed: 0.06 }, fov: 42 }] }, + instances: [ + ...orbInstances({ count: 64, seed: 7, spread: [18, 9, 10], colors: ["#22d3ee", "#b56bff"], emissive: 5 }), + ...graphInstances({ data, node: { color: "#b56bff", size: 0.27 }, edge: { color: "#22d3ee", curve: 0.4, glow: 2 }, activate: [wave({ from: "root", step: 1, speed: 0.35 })] }), + primitive("ring", { size: 8, position: [0, -5, 0], color: "#22d3ee", emissive: 5 }), + ], + bloom: { intensity: 1.4, threshold: 1, radius: 0.8 }, fog: { density: 0.025, color: "#05060a" }, + vignette: 0.3, grain: 0.015, exposure: 1.1, + }); + const result = await benchmarkRenderGraph({ ...options, definition: buildWorldGraph(scene), + uniformsAt: seconds => sampleWorldUniforms(scene, { slideTime: seconds, stepTime: seconds, stepIndex: 1, stepOrigins: { 0: 0, 1: 0 } }) }); + return { ...result, workload: { quality: options.quality, orbs: 64, graphNodes: 40, graphEdges: 39, rings: 1, + clock: "2 + frame / 60 seconds, step 1 at time zero; each repetition replays the same samples", seed: 7 } }; +} + +Object.assign(window, { benchmarkWorld }); diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts new file mode 100644 index 000000000..01d645e69 --- /dev/null +++ b/packages/slides-effects/tests/world.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, test } from "bun:test"; +import { evaluateCamera, type CameraOptions, type ClockSample } from "../packs/world/camera"; +import { activationAt, layoutTree, subset, tree, wave } from "../packs/world/graph"; +import { buildWorldGraph } from "../packs/world/render"; +import { compileTrack, graphInstances, instancePosition, orbInstances, prepareWorldScene, primitive, sampleTrack, sampleWorldCamera, sampleWorldUniforms, worldHeaderSize, worldInstanceSize } from "../packs/world/scene"; + +function clock(time: number, step = 0, origins: Record = { 0: 0 }): ClockSample { + return { slideTime: time, stepTime: time - (origins[step] ?? 0), stepIndex: step, stepOrigins: origins }; +} +describe("world camera", () => { + test("samples step changes and scrubs without integration history", () => { + const options: CameraOptions = { duration: 2, ease: "linear", shots: [ + { step: 0, position: [0, 0, 20], target: [0, 0, 0], fov: 40 }, + { step: 1, position: [8, 4, 12], fov: 60 }, + ] }; + expect(evaluateCamera(options, clock(4, 1, { 0: 0, 1: 4 })).position).toEqual([0, 0, 20]); + expect(evaluateCamera(options, clock(5, 1, { 0: 0, 1: 4 })).position).toEqual([4, 2, 16]); + expect(evaluateCamera(options, clock(6, 1, { 0: 0, 1: 4 })).fov).toBe(60); + expect(evaluateCamera(options, clock(5, 1, { 0: 0, 1: 4 })).position).toEqual([4, 2, 16]); + expect(evaluateCamera(options, clock(1))).toEqual(evaluateCamera(options, clock(0))); + // A directly entered step still has an explicit initial pose. + expect(evaluateCamera(options, clock(0, 1, { 1: 0 })).position).toEqual([0, 0, 20]); + }); + test("freezes the previous moving shot at the next keyframe's origin", () => { + const first: CameraOptions = { ease: "linear", shots: [{ at: 0, orbit: { center: [0, 0, 0], radius: 10, speed: 0.2, height: 3 } }] }; + const options: CameraOptions = { ...first, duration: 2, shots: [...first.shots!, { at: 4, position: [0, 3, 20] }] }; + const boundary = evaluateCamera(first, clock(4)); + expect(evaluateCamera(options, clock(4)).position).toEqual(boundary.position); + const halfway = evaluateCamera(options, clock(5)); + expect(halfway.position[0]).toBeCloseTo(boundary.position[0] * 0.5); + expect(halfway.position[2]).toBeCloseTo((boundary.position[2] + 20) * 0.5); + }); + test("evaluates follow, dolly, orbit and continuity from the same clock", () => { + const follow: CameraOptions = { shots: [{ position: [0, 0, 20], follow: "node:a", dolly: 5 }] }; + const pose = evaluateCamera(follow, clock(4), { "node:a": [0, 0, 10] }); + expect(pose.target).toEqual([0, 0, 10]); + expect(pose.position).toEqual([0, 0, 15]); + const previous = { position: [10, 0, 10] as [number, number, number], target: [0, 0, 0] as [number, number, number], fov: 40, roll: 0 }; + expect(evaluateCamera(follow, clock(0), { "node:a": [0, 0, 10] }, previous, 1)).toEqual(previous); + expect(evaluateCamera(follow, clock(1), { "node:a": [0, 0, 10] }, previous, 1)).toEqual(pose); + const orbit: CameraOptions = { shots: [{ orbit: { center: [1, 2, 3], radius: 10, height: 4, speed: Math.PI / 2 } }] }; + expect(evaluateCamera(orbit, clock(1)).position[0]).toBeCloseTo(11); + expect(evaluateCamera(orbit, clock(1)).position[1]).toBe(6); + }); + test("a direct later-step entry completes missing earlier shots before starting its own", () => { + const options: CameraOptions = { ease: "linear", duration: 1, shots: [ + { step: 0, position: [0, 0, 20] }, { step: 1, position: [5, 0, 15] }, + { step: 2, position: [10, 0, 10] }, { step: 3, position: [15, 0, 5] }, + ] }; + expect(evaluateCamera(options, clock(0, 3, { 3: 0 })).position).toEqual([10, 0, 10]); + expect(evaluateCamera(options, clock(0.5, 3, { 3: 0 })).position).toEqual([12.5, 0, 7.5]); + expect(evaluateCamera(options, clock(4, 3, { 3: 0 })).position).toEqual([15, 0, 5]); + expect(evaluateCamera(options, clock(0, 1, { 1: 0 })).position).toEqual([0, 0, 20]); + const orbit: CameraOptions = { ease: "linear", duration: 1, shots: [ + { step: 0, position: [0, 0, 20] }, { step: 1, orbit: { center: [0, 0, 0], radius: 10, height: 3, speed: 0.5 } }, + { step: 2, target: [0, 0, 0], dolly: 5 }, + ] }; + expect(evaluateCamera(orbit, clock(0, 2, { 2: 0 })).position).toEqual([0, 3, 10]); + const settled = evaluateCamera(orbit, clock(1, 2, { 2: 0 })).position; + expect(settled[0]).toBe(0); + expect(Math.hypot(...settled)).toBeCloseTo(5); + expect(evaluateCamera(orbit, clock(4, 2, { 2: 0 })).position).toEqual(settled); + }); + test("leaving a moving follow target samples its exact keyframe-boundary position", () => { + const options: CameraOptions = { ease: "linear", duration: 2, shots: [ + { at: 0, position: [0, 0, 20], follow: "moving" }, { at: 4, position: [10, 0, 10], target: [0, 0, 0] }, + ] }; + const objects = (_id: string, sample: ClockSample): [number, number, number] => [sample.slideTime * 2, 0, 0]; + expect(evaluateCamera(options, clock(4), objects).target).toEqual([8, 0, 0]); + expect(evaluateCamera(options, clock(5), objects).target).toEqual([4, 0, 0]); + const moving = orbInstances({ id: "moving", count: 1, drift: 1 })[0]; + const scene = prepareWorldScene({ instances: [moving], camera: { ...options, shots: [ + { at: 0, follow: "moving:0" }, options.shots![1], + ] } }); + const boundary = instancePosition(moving, clock(4)); + expect(sampleWorldCamera(scene, clock(4)).target).toEqual(boundary); + const halfway = sampleWorldCamera(scene, clock(5)).target; + halfway.forEach((value, index) => expect(value).toBeCloseTo(boundary[index] / 2)); + }); +}); + +describe("world graph", () => { + const data = tree("root", [["a", ["a1", "a2"]], ["b", ["b1", "b2"]]]); + test("lays out unique finite nodes on widening cone levels", () => { + const layout = layoutTree(data); + expect(layout.nodes).toHaveLength(7); + expect(layout.edges).toHaveLength(6); + expect(layout.nodes.find(node => node.id === "a1")?.ancestors).toEqual(["root", "a"]); + expect(layout.nodes.every(node => node.position.every(Number.isFinite))).toBe(true); + const a = layout.nodes.find(node => node.id === "a")!, a1 = layout.nodes.find(node => node.id === "a1")!; + expect(a1.position[1]).toBeLessThan(a.position[1]); + expect(Math.hypot(a1.position[0], a1.position[2])).toBeGreaterThan(Math.hypot(a.position[0], a.position[2])); + expect(() => layoutTree(tree("root", ["root"]))).toThrow("duplicate node id"); + }); + test("wave cascades through descendants; a later subset dims untouched branches", () => { + const { nodes } = layoutTree(data); + const root = nodes[0], leaf = nodes.find(node => node.id === "a1")!, b = nodes.find(node => node.id === "b")!; + const activate = [wave({ from: "root", step: 1, speed: 0.5, duration: 0.25 }), subset({ branches: ["b"], step: 2, duration: 0.5 })]; + expect(activationAt(root, nodes, activate, clock(1, 0))).toBe(0); + expect(activationAt(root, nodes, activate, clock(2.25, 1, { 0: 0, 1: 2 }))).toBe(1); + expect(activationAt(leaf, nodes, activate, clock(2.25, 1, { 0: 0, 1: 2 }))).toBe(0); + expect(activationAt(leaf, nodes, activate, clock(3.25, 1, { 0: 0, 1: 2 }))).toBe(1); + expect(activationAt(leaf, nodes, activate, clock(5, 2, { 0: 0, 1: 2, 2: 4 }))).toBe(0); + expect(activationAt(b, nodes, activate, clock(5, 2, { 0: 0, 1: 2, 2: 4 }))).toBe(1); + // Backwards samples produce the same activation, independent of previous samples. + expect(activationAt(leaf, nodes, activate, clock(3.25, 1, { 0: 0, 1: 2 }))).toBe(1); + }); + test("animated node positions keep attached curved tube endpoints in sync", () => { + const scene = prepareWorldScene({ instances: graphInstances({ data, nodes: { a: { position: { 0: [1, 2, 3], 1: [5, 6, 7] } } } }) }); + const values = sampleWorldUniforms(scene, clock(3, 1, { 0: 0, 1: 2 })); + const nodeIndex = worldHeaderSize + scene.instances.findIndex(instance => instance.id === "node:a") * worldInstanceSize; + const edgeIndex = worldHeaderSize + scene.instances.findIndex(instance => instance.id === "edge:root:a") * worldInstanceSize; + expect(values.slice(nodeIndex, nodeIndex + 3)).toEqual([5, 6, 7]); + expect(values.slice(edgeIndex + 12, edgeIndex + 15)).toEqual([5, 6, 7]); + }); + test("missing historical steps are settled during direct-entry preparation and previews", () => { + const { nodes } = layoutTree(data); + const leaf = nodes.find(node => node.id === "a1")!; + const activate = [wave({ from: "root", step: 1 }), subset({ branches: ["b"], step: 3 })]; + expect(activationAt(leaf, nodes, activate, clock(0, 3, { 3: 0 }))).toBe(1); + expect(activationAt(leaf, nodes, activate, clock(4, 3, { 3: 0 }))).toBe(0); + expect(activationAt(leaf, nodes, activate, clock(0, 1, { 1: 0 }))).toBe(0); + const track = compileTrack({ 0: 0, 1: 10, 2: 20, 3: 30 }); + expect(sampleTrack(track, clock(0, 3, { 3: 0 }))).toBe(20); + expect(sampleTrack(track, clock(4, 3, { 3: 0 }))).toBe(30); + }); +}); + +test("scalar/vector keyframes and post settings are clock-driven", () => { + const track = compileTrack({ 0: 1, 1: 3 }); + expect(sampleTrack(track, clock(1.325, 1, { 0: 0, 1: 1 }))).toBeCloseTo(2); + const scene = prepareWorldScene({ instances: [], exposure: { 0: 1, 1: 2 }, fog: { density: { 0: 0.01, 1: 0.1 }, color: { 0: "#000000", 1: "#ffffff" } }, bloom: { intensity: { 0: 1, 1: 2 } } }); + const values = sampleWorldUniforms(scene, clock(2, 1, { 0: 0, 1: 1 })); + expect(values[8]).toBe(2); expect(values[9]).toBe(2); expect(values[12]).toBe(0.1); + expect(values.slice(20, 23)).toEqual([1, 1, 1]); + const overlapping = compileTrack([{ at: 0, value: 0 }, { at: 1, value: 10, duration: 2, ease: "linear" }, { at: 2, value: 20, duration: 2, ease: "linear" }]); + expect(sampleTrack(overlapping, clock(2))).toBe(5); + expect(sampleTrack(overlapping, clock(3))).toBe(12.5); +}); + +test("HDR scene and five-level half-resolution bloom describe independent attachments", () => { + const scene = prepareWorldScene({ instances: [...orbInstances({ count: 3 }), primitive("box"), primitive("ring"), primitive("tube")], bloom: { intensity: 1.5 } }); + const graph = buildWorldGraph(scene); + expect(graph.targets.find(target => target.id === "hdr")?.format).toBe("rgba16float"); + expect(graph.targets.find(target => target.id === "depth")?.format).toBe("depth32float"); + expect(graph.targets.find(target => target.id === "bloom")).toMatchObject({ scale: 0.5, mips: 5 }); + const render = graph.passes.filter(pass => pass.kind === "render"); + expect(render.map(pass => pass.instanceCount)).toEqual([1, 1, 3, 1]); + expect(render[0].clear).toBeArray(); + expect(render.slice(1).every(pass => pass.clear === false)).toBe(true); + expect(graph.passes.at(-1)).toMatchObject({ id: "tonemap", output: "screen" }); + expect(graph.uniformsSize).toBe(sampleWorldUniforms(scene, clock(0)).length); + expect(orbInstances({ count: 3, seed: 7 })).toEqual(orbInstances({ count: 3, seed: 7 })); + expect(orbInstances({ count: 3, seed: 7 })).not.toEqual(orbInstances({ count: 3, seed: 8 })); + expect(() => prepareWorldScene({ instances: [...orbInstances({ count: 1 }), ...orbInstances({ count: 1 })] })).toThrow("id is duplicated"); + expect(() => prepareWorldScene({ instances: [...orbInstances({ id: "near", count: 1 }), ...orbInstances({ id: "far", count: 1 })] })).not.toThrow(); +}); diff --git a/packages/slides-effects/tests/worldPlayback.test.tsx b/packages/slides-effects/tests/worldPlayback.test.tsx new file mode 100644 index 000000000..5c7c1a02e --- /dev/null +++ b/packages/slides-effects/tests/worldPlayback.test.tsx @@ -0,0 +1,169 @@ +// @ts-nocheck Exercise the pack's UI-clock boundary without a native GPU. +import "../../../apps/slides/src/__tests__/nativeMock"; +import { reactUI } from "../../../apps/slides/src/__tests__/uiClockMock"; +import { expect, mock, test } from "bun:test"; +import React, { Profiler } from "react"; +import { act, create } from "react-test-renderer"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { createPlaybackState, advancePlayback } from "../../presentation/src/playbackState"; +import { samplePlayback } from "../../presentation/src/playback"; + +const playback = { value: createPlaybackState({ phase: "playing", slideKey: "a", stepKey: "0", stepIndex: 0 }) }; +mock.module("@legend-apps/presentation", () => ({ + GPURenderGraph: props => React.createElement("render-graph", props), + usePlayback: () => playback, samplePlayback, useBackgroundSize: () => ({ width: 1280, height: 720 }), + usePresentationValue: () => playback.value.phase, +})); +const { World, Camera, Orbs, Sphere } = await import("../packs/world/World"); + +test("world uniforms sample preparation/preview and retain live continuity without React frames", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + let tree, commits = 0; + const scene = <>; + const next = ; + const draw = content => commits++}>{content}; + const mount = async (content, settle = true) => { await act(() => { tree = create(draw(content)); }); if (settle) reactUI(); }; + const unmount = async () => { await act(() => tree.unmount()); }; + const values = () => tree.root.findByType("render-graph").props.uniforms.value; + try { + await mount({scene}); + expect(tree.root.findByType("render-graph").props.width).toBe(1280); + const initialCommits = commits; + playback.value = advancePlayback(advancePlayback(playback.value, 0), 2000); + reactUI(); + const lastLive = values().slice(0, 3); + expect(lastLive[0]).toBeCloseTo(Math.sin(0.4) * 10); + expect(commits).toBe(initialCommits); + await unmount(); + + // A thumbnail of the next world must not replace the last live camera. + playback.value = createPlaybackState({ phase: "preview", slideKey: "b-preview", stepKey: "0", stepIndex: 0 }); + await mount({next}); + expect(values().slice(0, 3)).toEqual([0, 5, 30]); + await unmount(); + + playback.value = createPlaybackState({ phase: "preparing", slideKey: "b-prepare", stepKey: "0", stepIndex: 0 }); + await mount({next}); + expect(values().slice(0, 3)).toEqual([0, 5, 30]); + await unmount(); + + // Explicitly inactive worlds also leave the retained live pose alone. + playback.value = createPlaybackState({ phase: "playing", slideKey: "b-inactive", stepKey: "0", stepIndex: 0 }); + await mount({next}); + await unmount(); + + playback.value = createPlaybackState({ phase: "playing", slideKey: "b", stepKey: "0", stepIndex: 0 }); + await mount({next}, false); + expect(values().slice(0, 3)).toEqual(lastLive); + reactUI(); + expect(values().slice(0, 3)).toEqual(lastLive); + playback.value = advancePlayback(advancePlayback(playback.value, 3000), 5000); + reactUI(); + expect(values().slice(0, 3)).toEqual([0, 5, 30]); + } finally { await act(() => tree?.unmount()); } +}); + +test("equal-content parent renders and resize retain pipeline definitions while uniforms change", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + playback.value = createPlaybackState({ phase: "playing", slideKey: "identity", stepKey: "0", stepIndex: 0 }); + let tree; + const content = (width = 800, x = 1, quality = "high") => + {"\n "} + {"\n"} + ; + try { + await act(() => { tree = create(content()); }); + const graph = () => tree.root.findByType("render-graph").props; + const passes = graph().passes, targets = graph().targets; + expect(tree.root.findAllByType("view")).toHaveLength(1); + await act(() => tree.update(content())); + expect(graph().passes).toBe(passes); + expect(graph().targets).toBe(targets); + await act(() => tree.update(content(1000, 4))); + expect(graph().passes).toBe(passes); + expect(graph().uniforms.value[32]).toBe(4); + expect(graph().width).toBe(1000); + await act(() => tree.update(content(1000, 4, "low"))); + expect(graph().passes).not.toBe(passes); + } finally { await act(() => tree?.unmount()); } +}); + +test("compiled recursive world evaluators survive an isolated UI-worklet closure", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "world-worklets-")); + try { + const { compileDeck } = await import("../../presentation/src/compiler"); + const deck = path.join(directory, "world.mdx"); + fs.writeFileSync(deck, 'export { World, sampleWorldUniforms, prepareWorldScene, graphInstances, tree, wave } from "slides:world/World"\n\n# World'); + const compiled = await compileDeck(deck, { effectPacksDirectory: path.resolve(import.meta.dirname, "../packs") }); + if (!compiled.success) throw new Error(compiled.errors.join("\n")); + const require = createRequire(import.meta.url); + const module = { exports: {} }; + const callbacks = [], mutables = []; + const presentation = { GPURenderGraph: props => React.createElement("compiled-graph", props), + usePlayback: () => playback, samplePlayback, useBackgroundSize: () => ({ width: 800, height: 500 }), + usePresentationValue: () => playback.value.phase }; + const reanimated = { + makeMutable: value => { const shared = { value }; mutables.push(shared); return shared; }, + useSharedValue: value => React.useRef({ value }).current, + useAnimatedReaction: () => {}, + useDerivedValue: fn => { callbacks.push(fn); return { get value() { return fn(); } }; }, + }; + Function("require", "module", "exports", compiled.code)(name => { + if (name === "react" || name === "react/compiler-runtime" || name === "react/jsx-runtime") return require(name); + if (name === "react-native") return { View: "view" }; + if (name === "@legend-apps/presentation") return presentation; + if (name === "react-native-reanimated") return reanimated; + return {}; + }, module, module.exports); + const cache = new Map(); + function restore(fn) { + if (!fn.__workletHash) return fn; + if (cache.has(fn)) return cache.get(fn); + const context = { __closure: {}, _recur: null }; + const restored = Function(`return (${fn.__initData.code});`)().bind(context); + context._recur = restored; + cache.set(fn, restored); + for (const [key, value] of Object.entries(fn.__closure)) { + expect(value).not.toBe(presentation); expect(value).not.toBe(reanimated); + context.__closure[key] = typeof value === "function" ? restore(value) : value; + } + return restored; + } + const api = module.exports; + expect(api.sampleWorldUniforms.__workletHash).toBeNumber(); + const sample = restore(api.sampleWorldUniforms); + const scene = api.prepareWorldScene({ + instances: api.graphInstances({ data: api.tree("root", [["a", ["a1", "a2"]]]), activate: [api.wave({ from: "root", step: 1 })] }), + camera: { shots: [{ step: 0, position: [0, 4, 20] }, { step: 1, orbit: { center: [0, 0, 0], radius: 10 } }, { step: 2, follow: "node:a", dolly: 5 }] }, + exposure: { 0: 1, 1: 1.5, 2: 0.8 }, bloom: { intensity: 1.4 }, + }); + for (const clock of [ + { slideTime: 0, stepTime: 0, stepIndex: 0, stepOrigins: { 0: 0 } }, + { slideTime: 3, stepTime: 1, stepIndex: 1, stepOrigins: { 0: 0, 1: 2 } }, + { slideTime: 1, stepTime: 1, stepIndex: 2, stepOrigins: { 2: 0 } }, + ]) expect(sample(scene, clock)).toEqual(api.sampleWorldUniforms(scene, clock)); + // Mapper inputs come from the serialized closure, including unread values. + let renderer; + try { + playback.value = createPlaybackState({ phase: "playing", slideKey: "closure", stepKey: "0", stepIndex: 0 }); + await act(() => { renderer = create(React.createElement(api.World, { id: "closure" })); }); + const retained = mutables[0]; + expect(Object.values(callbacks.at(-1).__closure)).not.toContain(retained); + await act(() => renderer.unmount()); + playback.value = createPlaybackState({ phase: "preview", slideKey: "closure-preview", stepKey: "0", stepIndex: 0 }); + await act(() => { renderer = create(React.createElement(api.World, { id: "closure", enter: { from: "previous" } })); }); + expect(Object.values(callbacks.at(-1).__closure)).not.toContain(retained); + await act(() => renderer.unmount()); + playback.value = createPlaybackState({ phase: "playing", slideKey: "closure-next", stepKey: "0", stepIndex: 0 }); + await act(() => { renderer = create(React.createElement(api.World, { id: "closure", enter: { from: "previous" } })); }); + expect(Object.values(callbacks.at(-1).__closure)).toContain(retained); + await act(() => renderer.unmount()); + const count = mutables.length; + await act(() => { renderer = create(React.createElement(api.World)); }); + expect(mutables.length).toBe(count); + } finally { await act(() => renderer?.unmount()); } + } finally { fs.rmSync(directory, { recursive: true, force: true }); } +}, 30000); diff --git a/packages/slides-web/export.ts b/packages/slides-web/export.ts index 820d89479..6004cafa0 100644 --- a/packages/slides-web/export.ts +++ b/packages/slides-web/export.ts @@ -20,7 +20,7 @@ const workletsPlugin = require("react-native-worklets/plugin"); const src = path.join(webRoot, "src"); // How the web player renders each effect pack capability; any other capability is unsupported on web. const webCapabilities: Record = { - reanimated: "Reanimated web", skia: "CanvasKit", lottie: "Skottie", "native-video": "HTML video", webgpu: "WebGPU (requires a WebGPU browser)", + reanimated: "Reanimated web", skia: "CanvasKit", lottie: "Skottie", "native-video": "HTML video", webgpu: "WebGPU (requires a WebGPU browser)", "webgpu-multipass": "WebGPU multipass (requires a WebGPU browser)", }; export function webEffectSupport(found: { id: string; capabilities?: unknown }[]) { diff --git a/packages/slides-web/tests/export.test.mjs b/packages/slides-web/tests/export.test.mjs index 20771ef7c..43cabbe4f 100644 --- a/packages/slides-web/tests/export.test.mjs +++ b/packages/slides-web/tests/export.test.mjs @@ -19,6 +19,7 @@ test("reports how each effect pack runs on web and flags native-only capabilitie expect(effects).toEqual({ animation: { reanimated: "Reanimated web", lottie: "Skottie" }, camera: { skia: "CanvasKit", arkit: "unsupported" } }); expect(unsupported).toEqual(['Effect pack "camera" is unsupported on web: it requires the native "arkit" capability.']); expect(webEffectSupport([{ id: "local" }])).toEqual({ effects: { local: {} }, unsupported: [] }); + expect(webEffectSupport([{ id: "world", capabilities: ["webgpu-multipass"] }])).toEqual({ effects: { world: { "webgpu-multipass": "WebGPU multipass (requires a WebGPU browser)" } }, unsupported: [] }); }); test("Skia detection follows compiled host imports and built-in effects, not slide prose", () => { From 3d85dfffd5a95b010e70995beea67b9b72ca3221 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 03:32:34 -0700 Subject: [PATCH 02/28] feat: add world lighting and animated data scenes --- apps/slides/examples/3d/data-flow.mdx | 102 ++++++ apps/slides/examples/3d/world-tour.mdx | 154 +++++++- .../slides-effects/examples/world/example.mdx | 32 ++ packages/slides-effects/packs/world/README.md | 179 ++++++++- packages/slides-effects/packs/world/World.tsx | 83 ++++- packages/slides-effects/packs/world/camera.ts | 26 +- packages/slides-effects/packs/world/post.ts | 138 +++++++ packages/slides-effects/packs/world/render.ts | 286 +++++++++++---- packages/slides-effects/packs/world/scene.ts | 344 ++++++++++++++++-- .../scripts/renderGraphBenchmark.ts | 26 +- .../scripts/world-benchmark.mjs | 2 +- .../slides-effects/scripts/worldBenchmark.ts | 42 ++- packages/slides-effects/tests/world.test.ts | 154 +++++++- .../tests/worldPlayback.test.tsx | 19 +- .../slides-effects/tests/worldRender.test.ts | 94 +++++ 15 files changed, 1545 insertions(+), 136 deletions(-) create mode 100644 apps/slides/examples/3d/data-flow.mdx create mode 100644 packages/slides-effects/examples/world/example.mdx create mode 100644 packages/slides-effects/packs/world/post.ts create mode 100644 packages/slides-effects/tests/worldRender.test.ts diff --git a/apps/slides/examples/3d/data-flow.mdx b/apps/slides/examples/3d/data-flow.mdx new file mode 100644 index 000000000..71ff2b784 --- /dev/null +++ b/apps/slides/examples/3d/data-flow.mdx @@ -0,0 +1,102 @@ +--- +title: Data Flow +width: 1920 +height: 1080 +transition: fade +theme: + backgroundColor: "#050811" + color: "#f8fafc" +--- + +import { Text } from "react-native" +import { World, Camera, Graph, tree, wave, subset, PathPulse, Rows, Voxels, WorldCounter, Stage, flyTo, Box, Ring, Floor, PointLight, RimLight, Bloom, Fog, DOF, Vignette } from "slides:world/World" + +export const network = tree("root", [ + ["north", [["n1", ["n11", "n12", "n13"]], ["n2", ["n21", "n22", "n23"]]]], + ["east", [["e1", ["e11", "e12", "e13"]], ["e2", ["e21", "e22", "e23"]]]], + ["west", [["w1", ["w11", "w12", "w13"]], ["w2", ["w21", "w22", "w23"]]]], +]) + +# Follow one branch + + + + + + + + +Across the tree +Only this branch + +--- + +# A single path + + + + + + + + + +Follow the pulse + +--- + +# Change in a moving column + + + + + + + + + + +One changed row +Focus follows the next change + +--- + +# Three places to land + + + + + + + + + + + + + + + + + + + + +A softer glow +A quiet landing + +--- + +# Less to carry + + + + + + Active cells + + + + + +About one fifth remains diff --git a/apps/slides/examples/3d/world-tour.mdx b/apps/slides/examples/3d/world-tour.mdx index 5b6f82fb3..ceaad65e9 100644 --- a/apps/slides/examples/3d/world-tour.mdx +++ b/apps/slides/examples/3d/world-tour.mdx @@ -10,7 +10,30 @@ theme: --- import { Text } from "react-native" -import { World, Camera, Graph, tree, wave, subset, Orbs, Sphere, Box, Tube, Ring, Fog, Bloom, Vignette, Grain } from "slides:world/World" +import { World, Camera, Graph, tree, wave, subset, Orbs, Sphere, Box, Tube, Ring, Spray, Material, PointLight, SpotLight, RimLight, Floor, Fog, Shafts, Bloom, DOF, Chromatic, Vignette, Grain, Pass, Custom, SDF } from "slides:world/World" + +export const scanlines = ` +fn customPass(uv:vec2f, color:vec4f, ctx:PassContext)->vec4f { + let line=.96+.04*sin(uv.y*800.+ctx.slideTime*.3); + return vec4f(color.rgb*line,color.a); +}` + +export const ribbons = ` +fn shape(id:u32,vid:u32,ctx:ShapeContext)->ShapeVertex { + let corners=array(vec2f(-1.,-1.),vec2f(1.,-1.),vec2f(-1.,1.),vec2f(-1.,1.),vec2f(1.,-1.),vec2f(1.,1.)); + let p=corners[vid%6u]; + let x=p.x+(f32(id)-1.)*3.; + return ShapeVertex(vec3f(x,p.y+sin(ctx.slideTime+f32(id))*.4,0.),vec3f(0.,0.,1.)); +} +fn shade(id:u32,world:vec3f,normal:vec3f,ctx:ShapeContext)->Material { + return Material(vec3f(.03,.3,.4),vec3f(.03,.7,.9)*2.,.4,.1); +}` + +export const roundedBlock = ` +fn sdf(p:vec3f)->f32 { + let q=abs(p)-vec3f(.8,1.3,.8); + return length(max(q,vec3f(0.)))+min(max(q.x,max(q.y,q.z)),0.)-.15; +}` export const network = tree("root", [ ["north", [["n1", ["n11", "n12", "n13"]], ["n2", ["n21", "n22", "n23"]]]], @@ -46,6 +69,17 @@ export const network = tree("root", [ --- +# One continuous camera + + + + + + + + +--- + # Across the branches @@ -86,3 +120,121 @@ export const network = tree("root", [ + + +--- + +# A point of focus + + + + + + + + + + +Near +Far + +--- + +# On a mirror floor + + + + + + + + + + +--- + +# Shape the light + + + + + + + + + + + +--- + +# Light through the air + + + + + + + + + + +--- + +# A drifting spray + + + + + + + + +--- + +# A small color shift + + + + + + + + + + +Separate the edges + + +--- + +# A pass of your own + + + + + + + + +--- + +# Geometry in a few lines + + + + + + + +--- + +# A bounded distance field + + + + + + + + + diff --git a/packages/slides-effects/examples/world/example.mdx b/packages/slides-effects/examples/world/example.mdx new file mode 100644 index 000000000..a43315ab2 --- /dev/null +++ b/packages/slides-effects/examples/world/example.mdx @@ -0,0 +1,32 @@ +--- +title: World +width: 1920 +height: 1080 +theme: + backgroundColor: "#050811" + color: "#f8fafc" +--- + +import { World, Camera, Orbs, Graph, tree, wave, subset, Bloom, Fog } from "slides:world/World" + +export const branches = tree("root", [["a", [["a1", ["a11", "a12"]], ["a2", ["a21", "a22"]]]], ["b", [["b1", ["b11", "b12"]], ["b2", ["b21", "b22"]]]], ["c", [["c1", ["c11", "c12"]], ["c2", ["c21", "c22"]]]]]) + +# A world of light + + + + + + + +--- +steps: 3 +--- + +# A selective cascade + + + + + + diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index c5f9e6b49..be7e8b33a 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -77,9 +77,10 @@ Keyframe tables and graph activation are sampled in a UI worklet into one storag uniform array. This keeps frame work off JavaScript, but uniform evaluation and upload still scale with the instance count; the graph API targets 20–60 nodes. -The pipeline is an instanced HDR scene with depth, a half-resolution bloom mip -chain, exponential height fog, ACES exposure/tonemapping, sRGB encoding, -vignette and grain. `high` and `auto` use full-resolution scene targets and five +The pipeline is an instanced HDR scene with depth, optional planar reflections, +height fog and volumetric lights, depth of field, a half-resolution bloom mip +chain, chromatic aberration, ACES exposure/tonemapping, sRGB encoding, vignette +and grain. `high` and `auto` use full-resolution scene targets and five bloom levels. `medium` uses 0.85 scene scale and five levels; `low` uses 0.65 scene scale and three levels. Output resolution stays unchanged. `auto` currently selects the high preset; it does not change quality based on frame timing. @@ -87,8 +88,8 @@ selects the high preset; it does not change quality based on frame timing. For a raw WGSL escape hatch, use `GPURenderGraph` from `@legend-apps/presentation`. `buildWorldGraph` and the pure `scene.ts` helpers expose the same description and deterministic uniform samples for measurement. -The scene-specific custom pass, material, reflection and depth-of-field APIs are -not part of this initial pack. +The scene hooks below share that pipeline. All animation samples the same clock; +no frame history or per-frame React render is required. Generate a standalone browser measurement page from the repository root: @@ -101,3 +102,171 @@ Open that file in Chrome with WebGPU and run Repeat at 3840×2160 and with each preset. Results report completed-submit frame times and per-pass GPU timestamps when available. Throughput measurements do not measure display refresh rate or prove a particular presentation frame rate. + + +## Focus, floor and lighting + +```tsx +import { DOF, Floor, Material, PointLight, SpotLight, RimLight, Shafts, Chromatic } from "slides:world/World"; + + + + + + + + + + + + + +``` + +Focus accepts an animated distance or object id. Camera shots also accept `focus`. +An explicit DOF focus track takes precedence; otherwise the active shot supplies +focus, falling back to the camera target. +Object focus resolves the same sampled position as camera follow, then uses +view-space depth to match the depth buffer. A pulse, logical row or stage can be +both a camera target and a focus target. + +Materials can be declared by id or passed inline through a primitive's `material` +prop. Albedo, emissive intensity, roughness, metal and rim strength support tracks. +Point and spot lights support animated position, target, color, intensity, range +and spot angle (radians). At most four local lights are supported. `RimLight` +sets the shared rim color/intensity. The mirror redraws the scene about the floor +plane into a half-resolution reflection target and clips geometry below it. +Roughness blurs that reflection; reflectivity controls its strength. + +DOF uses a half-resolution gather and depth-aware composite. Shafts integrate +point/spot scattering along the visible camera ray at quarter resolution. +Static `scale` overrides are available on DOF, Floor (reflection resolution) and +Shafts; DOF and Shafts also accept `samples`. These change pipeline topology. +Animated focus, aperture, blur radius, lighting and effect strengths update uniforms. + +## Paths, recycled rows and stages + +```tsx +import { PathPulse, Spray, Rows, Stage, flyTo } from "slides:world/World"; + + + + + + + + +``` + +Pulse paths accept positions or ids; bare graph ids resolve as `node:`. +The head follows the graph's curved edges. Its short straight trail joins two +sampled path positions and can cut across a bend. Ambient Spray accepts +`position`, `spread`, `velocity` and `seed`; `under="leaves:lit"` emits beneath +active leaves. Motion, birth and lifetime are analytic samples of absolute time. + +Rows keep a fixed instance pool while logical row ids advance with the clock. +Only listed changes flash; recycled slots do not inherit a previous row's flash. +Use `rows:row:4` to follow or focus logical row 4 while it is visible. +`width`, `depth`, `start`, `flashDuration` and `position` configure the column. +Stages offset nested primitives and multiply their glow. Stage positions and glow +can be animated. Lights and the floor remain world-space settings even when +nested in a Stage. `flyTo` targets `stage:` with a deterministic eased transition; +use that explicit prefix when following a stage directly. An eased flight is the +default; a positive `arc` adds vertical curvature to the transition. + +## Voxel reduction and counter + +```tsx +import { Voxels, WorldCounter } from "slides:world/World"; + + + +``` + +`to` is the surviving count fraction. Surviving boxes compact and other boxes +shrink away. The counter samples the same reduction and draws numeric digits +after the built-in post effects, so focus, fog and bloom do not obscure them. +Custom hooks after tonemap also affect the digits. Position and +height use normalized canvas coordinates. Put a static label in a normal text +overlay. The counter does not schedule React updates. + +## Post order + +The default HDR order is fog, DOF, then bloom. Set +`` to reorder these stages. Supply each +name exactly once; unknown, missing or duplicate stages are errors. A disabled +effect is a no-op stage, so custom hooks retain a predictable attachment point. +Tonemapping remains after the HDR stages. Custom hooks can also run after it. + +## WGSL hooks + +A custom post fragment receives the previous stage's color, linear depth and the +shared clock. `sampleHDR(uv)` reads the original scene HDR target; +`sampleDepth(uv)` returns linear view depth. + +```tsx +import { Pass } from "slides:world/World"; + +const lines = ` +fn customPass(uv:vec2f, color:vec4f, ctx:PassContext)->vec4f { + let line=.96+.04*sin(uv.y*800.+ctx.slideTime*.3); + return vec4f(color.rgb*line,color.a); +}`; + +``` + +`PassContext` fields are `time`, `slideTime`, `stepTime`, `step` and `depth`. +Use the function name `customPass`; `pass` is reserved in WGSL. Placement accepts +`scene`, `fog`, `dof`, `bloom` or `tonemap` through one `before` or `after` prop. +Before tonemapping, colors are linear HDR. After tonemapping, colors are sRGB LDR; +output then copies to the screen without a second tonemap. `before="scene"` is +invalid. Additional `reads` follow the fixed previous/depth/HDR texture bindings +at group 0 binding 6 onward; declare their WGSL bindings explicitly. Raw +`GPURenderGraph` exposes the full resource/pass contract. + +Custom instanced geometry supplies local positions/normals and a material. The +pack handles camera projection, depth, lighting, fog and the post chain. + +```tsx +import { Custom } from "slides:world/World"; + +const quad = ` +fn shape(id:u32,vid:u32,ctx:ShapeContext)->ShapeVertex { + let corners=array(vec2f(-1.,-1.),vec2f(1.,-1.),vec2f(-1.,1.), + vec2f(-1.,1.),vec2f(1.,-1.),vec2f(1.,1.)); + let p=corners[vid%6u]; + return ShapeVertex(vec3f(p.x+(f32(id)-1.)*3.,p.y,0.),vec3f(0.,0.,1.)); +} +fn shade(id:u32,world:vec3f,normal:vec3f,ctx:ShapeContext)->Material { + return Material(vec3f(.03,.3,.4),vec3f(.03,.7,.9)*2.,.4,.1); +}`; + +``` + +`ShapeContext` contains `time`, `slideTime`, `stepTime` and `step`. `ShapeVertex` +contains `position` and `normal`. WGSL `Material` contains `albedo`, `emissive` +(RGB), `roughness` and `metal`. Primitive position, scale and material props are +also available on Custom. + +An SDF runs inside a local bounding-box proxy and writes the depth of the actual +hit. Keep the shape inside its declared bounds; `steps` controls march cost. + +```tsx +import { SDF } from "slides:world/World"; + +const roundedBox = ` +fn sdf(p:vec3f)->f32 { + let q=abs(p)-vec3f(.8,1.3,.8); + return length(max(q,vec3f(0.)))+min(max(q.x,max(q.y,q.z)),0.)-.15; +}`; + +``` + +WGSL compilation/validation errors use the normal slide diagnostic path. The +`world-tour.mdx` deck executes all three hook examples; `data-flow.mdx` shows +pulse following, changed rows, stage flights and voxel reduction. diff --git a/packages/slides-effects/packs/world/World.tsx b/packages/slides-effects/packs/world/World.tsx index dce091006..382dc5dca 100644 --- a/packages/slides-effects/packs/world/World.tsx +++ b/packages/slides-effects/packs/world/World.tsx @@ -4,15 +4,17 @@ import { View } from "react-native"; import { makeMutable, useAnimatedReaction, useDerivedValue, useSharedValue, type SharedValue } from "react-native-reanimated"; import type { CameraOptions, CameraPose } from "./camera"; import { buildWorldGraph, worldGraphTopology } from "./render"; -import { graphInstances, orbInstances, prepareWorldScene, primitive, sampleWorldCamera, sampleWorldUniforms, - type Animated, type GraphOptions, type OrbsOptions, type PrimitiveOptions, type WorldScene } from "./scene"; +import { compileTrack, customInstances, graphInstances, orbInstances, prepareWorldScene, primitive, pulseInstances, rowInstances, + sampleWorldCamera, sampleWorldUniforms, sdfInstance, sprayInstances, voxelInstances, + type Animated, type CustomOptions, type GraphOptions, type Instance, type LightOptions, type MaterialOptions, type OrbsOptions, + type PostOptions, type PrimitiveOptions, type PulseOptions, type RowsOptions, type SDFOptions, type SprayOptions, type StageMotion, type StageOptions, type VoxelsOptions, type WorldScene } from "./scene"; export { tree, wave, subset, layoutTree } from "./graph"; -export { evaluateCamera } from "./camera"; +export { evaluateCamera, flyTo } from "./camera"; export { buildWorldGraph } from "./render"; -export { orbInstances, graphInstances, primitive, prepareWorldScene, sampleWorldUniforms } from "./scene"; +export { orbInstances, graphInstances, primitive, prepareWorldScene, pulseInstances, rowInstances, sprayInstances, voxelInstances, sampleWorldUniforms } from "./scene"; export type { Shot, CameraPose, Vec3 } from "./camera"; -export type { Animated, GraphOptions, OrbsOptions, PrimitiveOptions, WorldScene } from "./scene"; +export type { Animated, GraphOptions, OrbsOptions, PrimitiveOptions, WorldScene, MaterialOptions, LightOptions, PulseOptions, RowsOptions, VoxelsOptions } from "./scene"; export function Camera(_: CameraOptions) { return null; } export function Graph(_: GraphOptions) { return null; } @@ -25,6 +27,23 @@ export function Fog(_: NonNullable) { return null; } export function Bloom(_: NonNullable) { return null; } export function Vignette(_: { amount?: Animated }) { return null; } export function Grain(_: { amount?: Animated }) { return null; } +export function PathPulse(_: PulseOptions) { return null; } +export function Spray(_: SprayOptions) { return null; } +export function Rows(_: RowsOptions) { return null; } +export function Voxels(_: VoxelsOptions) { return null; } +export function Stage(_: StageOptions & { children?: ReactNode }) { return null; } +export function WorldCounter(_: NonNullable) { return null; } +export function DOF(_: NonNullable) { return null; } +export function Floor(_: NonNullable) { return null; } +export function Shafts(_: NonNullable) { return null; } +export function Chromatic(_: { amount?: Animated }) { return null; } +export function RimLight(_: NonNullable) { return null; } +export function PointLight(_: Omit) { return null; } +export function SpotLight(_: Omit) { return null; } +export function Material(_: MaterialOptions & { id: string }) { return null; } +export function Pass(_: PostOptions) { return null; } +export function Custom(_: CustomOptions) { return null; } +export function SDF(_: SDFOptions) { return null; } const poses = new Map>(); function retainedPose(id: string) { @@ -45,29 +64,54 @@ export type WorldProps = { quality?: WorldScene["quality"]; exposure?: Animated; previewTime?: number; borderRadius?: number; enter?: { from: "previous"; duration?: number }; scrim?: boolean | number; fallback?: ReactNode; active?: boolean; children?: ReactNode; accessibilityLabel?: string; + postOrder?: WorldScene["postOrder"]; }; export function World(props: WorldProps) { // id defines a continuity boundary; changing it creates a fresh sampled origin. return ; } -function worldContent(children: ReactNode, quality: WorldScene["quality"], exposure: Animated, backgroundColor: string, scrim: boolean | number) { - const scene: WorldScene = { instances: [], quality, exposure, backgroundColor, scrim: typeof scrim === "number" ? scrim : scrim ? 0.4 : 0 }; +function worldContent(children: ReactNode, quality: WorldScene["quality"], exposure: Animated, backgroundColor: string, scrim: boolean | number, postOrder: WorldScene["postOrder"]) { + const scene: WorldScene = { instances: [], stages: [], materials: {}, lights: [], passes: [], quality, exposure, backgroundColor, postOrder, + scrim: typeof scrim === "number" ? scrim : scrim ? 0.4 : 0 }; const overlay: ReactNode[] = []; - function collect(content: ReactNode) { + function add(instances: Instance[], stages: StageMotion[]) { for (const instance of instances) { if (stages.length) instance.stages = stages; scene.instances.push(instance); } } + function collect(content: ReactNode, stages: StageMotion[] = []) { Children.forEach(content, child => { if (!isValidElement>(child)) { if (child && (typeof child !== "string" || child.trim())) overlay.push(child); return; } const options = child.props; - if (child.type === Fragment) collect(options.children as ReactNode); + if (child.type === Fragment) collect(options.children as ReactNode, stages); + else if (child.type === Stage) { + const stageOptions = options as StageOptions; + if (scene.stages!.some(item => item.stage.id === stageOptions.id)) throw new Error(`World stage id is duplicated: ${stageOptions.id}`); + const stage: StageMotion = { id: stageOptions.id, position: compileTrack(stageOptions.position ?? [0, 0, 0]), glow: compileTrack(stageOptions.glow ?? 1) }; + scene.stages!.push({ stage, parents: stages }); + collect(options.children as ReactNode, [...stages, stage]); + } else if (child.type === Camera) scene.camera = options as CameraOptions; - else if (child.type === Graph) scene.instances.push(...graphInstances(options as GraphOptions)); - else if (child.type === Orbs) scene.instances.push(...orbInstances(options)); - else if (child.type === Sphere) scene.instances.push(primitive("sphere", options)); - else if (child.type === Box) scene.instances.push(primitive("box", options)); - else if (child.type === Tube) scene.instances.push(primitive("tube", options)); - else if (child.type === Ring) scene.instances.push(primitive("ring", { ...options, size: options.radius ?? options.size } as PrimitiveOptions)); + else if (child.type === Graph) add(graphInstances(options as GraphOptions), stages); + else if (child.type === Orbs) add(orbInstances(options), stages); + else if (child.type === Sphere) add([primitive("sphere", options)], stages); + else if (child.type === Box) add([primitive("box", options)], stages); + else if (child.type === Tube) add([primitive("tube", options)], stages); + else if (child.type === Ring) add([primitive("ring", { ...options, size: options.radius ?? options.size } as PrimitiveOptions)], stages); + else if (child.type === PathPulse) add(pulseInstances(options as PulseOptions), stages); + else if (child.type === Spray) add(sprayInstances(options), stages); + else if (child.type === Rows) add(rowInstances(options as RowsOptions), stages); + else if (child.type === Voxels) add(voxelInstances(options as VoxelsOptions), stages); + else if (child.type === Custom) add(customInstances(options as CustomOptions), stages); + else if (child.type === SDF) add([sdfInstance(options as SDFOptions)], stages); + else if (child.type === Material) { const material = options as MaterialOptions & { id: string }; scene.materials![material.id] = material; } + else if (child.type === WorldCounter) scene.counter = options as NonNullable; + else if (child.type === DOF) scene.dof = options; + else if (child.type === Floor) scene.floor = options; + else if (child.type === Shafts) scene.shafts = options; + else if (child.type === Chromatic) scene.chromatic = options.amount as Animated | undefined; + else if (child.type === RimLight) scene.rim = options; + else if (child.type === PointLight || child.type === SpotLight) scene.lights!.push({ ...options, kind: child.type === PointLight ? "point" : "spot" }); + else if (child.type === Pass) scene.passes!.push(options as PostOptions); else if (child.type === Fog) scene.fog = options; else if (child.type === Bloom) scene.bloom = options; else if (child.type === Vignette) scene.vignette = options.amount as Animated | undefined; @@ -79,15 +123,14 @@ function worldContent(children: ReactNode, quality: WorldScene["quality"], expos return { prepared: prepareWorldScene(scene), overlay }; } function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHeight = 1080, background = false, backgroundColor = "#05060a", quality = "auto", - exposure = 1, previewTime = 4, borderRadius = 0, enter, scrim = false, fallback, active = true, children, accessibilityLabel }: WorldProps) { + exposure = 1, previewTime = 4, borderRadius = 0, enter, scrim = false, fallback, active = true, children, accessibilityLabel, postOrder }: WorldProps) { const size = useBackgroundSize(); const width = background ? size.width : configuredWidth; const height = background ? size.height : configuredHeight; // GPU pipelines keep their identity across clock navigation and surface resizes. - const { prepared, overlay } = useMemo(() => worldContent(children, quality, exposure, backgroundColor, scrim), [children, quality, exposure, backgroundColor, scrim]); - const { counts: { box, ring, sphere, tube }, backgroundVector: [red, green, blue] } = worldGraphTopology(prepared); - const graph = useMemo(() => buildWorldGraph({ quality, counts: { box, ring, sphere, tube }, backgroundVector: [red, green, blue] }), - [quality, box, ring, sphere, tube, red, green, blue]); + const { prepared, overlay } = useMemo(() => worldContent(children, quality, exposure, backgroundColor, scrim, postOrder), [children, quality, exposure, backgroundColor, scrim, postOrder]); + const topology = JSON.stringify(worldGraphTopology(prepared)); + const graph = useMemo(() => buildWorldGraph(JSON.parse(topology)), [topology]); const playback = usePlayback(); const [continuity] = useState(() => id ? retainedPose(id) : undefined); const origin = useSharedValue<{ key: string; pose: CameraPose | null }>({ key: "", pose: null }); diff --git a/packages/slides-effects/packs/world/camera.ts b/packages/slides-effects/packs/world/camera.ts index 5cfeb1301..f2863a9db 100644 --- a/packages/slides-effects/packs/world/camera.ts +++ b/packages/slides-effects/packs/world/camera.ts @@ -5,11 +5,23 @@ export type CameraPose = { position: Vec3; target: Vec3; fov: number; roll: numb export type Shot = { step?: number; at?: number; position?: Vec3; target?: Vec3; fov?: number; roll?: number; orbit?: { center: Vec3; radius: number; speed?: number; height?: number; phase?: number }; - dolly?: number; follow?: string; duration?: number; ease?: Ease; + dolly?: number; follow?: string; duration?: number; ease?: Ease; focus?: number | string; + stage?: string; offset?: Vec3; arc?: number; }; export type CameraOptions = { shots?: Shot[]; duration?: number; ease?: Ease }; type ObjectPositions = Record | ((id: string, clock: ClockSample) => Vec3 | undefined); export const defaultPose: CameraPose = { position: [0, 5, 18], target: [0, 0, 0], fov: 42, roll: 0 }; +export function flyTo(stage: string, options: Omit = {}): Shot { + return { ...options, stage }; +} +export function cameraFocus(options: CameraOptions, clock: ClockSample) { + "worklet"; + let focus: number | string | undefined; + for (const shot of options.shots ?? []) if (shot.at !== undefined ? clock.slideTime >= shot.at : clock.stepIndex >= (shot.step ?? 0)) { + if (shot.focus !== undefined) focus = shot.focus; + } + return focus; +} export function easeProgress(value: number, ease: Ease = "cubic") { "worklet"; @@ -37,6 +49,14 @@ function destination(shot: Shot, base: CameraPose, elapsed: number, objects: Obj let position = shot.position ?? base.position; const followed = shot.follow ? typeof objects === "function" ? objects(shot.follow, clock) : objects[shot.follow] : undefined; let target = followed ?? shot.target ?? base.target; + if (shot.stage) { + const anchor = typeof objects === "function" ? objects(`stage:${shot.stage}`, clock) : objects[`stage:${shot.stage}`]; + if (anchor) { + target = anchor; + const offset = shot.offset ?? [0, 4, 10]; + position = [anchor[0] + offset[0], anchor[1] + offset[1], anchor[2] + offset[2]]; + } + } if (shot.orbit) { const orbit = shot.orbit; const angle = (orbit.phase ?? 0) + elapsed * (orbit.speed ?? 0.12); @@ -65,7 +85,9 @@ function evaluateShots(options: CameraOptions, clock: ClockSample, objects: Obje const duration = shot.duration ?? options.duration ?? 1.2; const completed = shot.at === undefined && (shot.step ?? 0) < clock.stepIndex && clock.stepOrigins[shot.step ?? 0] === undefined; const t = index === 0 || completed ? 1 : easeProgress(duration <= 0 ? 1 : elapsed / duration, shot.ease ?? options.ease); - return { position: mixVector(from.position, end.position, t), target: mixVector(from.target, end.target, t), + const position = mixVector(from.position, end.position, t); + if (shot.arc && t > 0 && t < 1) position[1] += Math.sin(t * Math.PI) * shot.arc; + return { position, target: mixVector(from.target, end.target, t), fov: from.fov + (end.fov - from.fov) * t, roll: from.roll + (end.roll - from.roll) * t }; } /** Stateless evaluation also handles direct step jumps and backwards scrubbing. */ diff --git a/packages/slides-effects/packs/world/post.ts b/packages/slides-effects/packs/world/post.ts new file mode 100644 index 000000000..98e0de53c --- /dev/null +++ b/packages/slides-effects/packs/world/post.ts @@ -0,0 +1,138 @@ +import type { Vec3 } from "./camera"; + +export type CustomGeometry = { wgsl: string; vertexCount: number }; +export type SDFGeometry = { wgsl: string; bounds: [Vec3, Vec3]; steps?: number }; +export type WorldPostPass = { id?: string; wgsl: string; before?: string; after?: string; reads?: string[] }; + +export function depthCode() { + return ` +fn linearDepth(z:f32)->f32 { return .05*500./(500.-z*(500.-.05)); } +fn depthAt(uv:vec2f)->f32 { + let pixel=clamp(vec2i(uv*vec2f(textureDimensions(depth))),vec2i(0),vec2i(textureDimensions(depth))-vec2i(1)); + return linearDepth(textureLoad(depth,pixel,0)); +} +fn worldRay(uv:vec2f)->vec3f { return cameraForward()+cameraRight()*(uv.x*2.-1.)*frame.aspect*uniforms[3]+cameraUp()*(1.-uv.y*2.)*uniforms[3]; } +`; +} +export function shaftCode(bindings: string, lights: number, steps: number) { + return `${bindings} +@group(0) @binding(3) var depth:texture_depth_2d; +${depthCode()} +@fragment fn shafts(@location(0) uv:vec2f)->@location(0) vec4f { + let ray=worldRay(uv); let direction=normalize(ray); + let distance=min(depthAt(uv),100.); let stride=distance/${steps}.; + let density=max(uniforms[46],.0001); let anisotropy=clamp(uniforms[47],-.8,.8); + var scattering=vec3f(0.); var opticalDepth=0.; + for(var sampleIndex=0u;sampleIndex<${steps}u;sampleIndex++) { + let z=(f32(sampleIndex)+.5)*stride; let world=vector(0u)+ray*z; + let localDensity=density*exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); + let transmission=exp(-opticalDepth); opticalDepth+=localDensity*stride*length(ray); + for(var light=0u;light<${lights}u;light++) { + let base=64u+light*12u; let delta=vector(base)-world; let lightDistance=max(length(delta),.001); + let toward=delta/lightDistance; let radius=max(uniforms[base+7u],.001); + let attenuation=pow(max(0.,1.-lightDistance/radius),2.)/(1.+lightDistance*lightDistance*.06); + var cone=1.; let cosine=uniforms[base+11u]; + if(cosine>-.99) { cone=smoothstep(cosine,min(.9999,cosine+.08),dot(-toward,normalize(vector(base+8u)))); } + let angle=dot(direction,toward); let phase=(1.-anisotropy*anisotropy)/pow(max(.01,1.+anisotropy*anisotropy-2.*anisotropy*angle),1.5); + scattering+=vector(base+4u)*uniforms[base+3u]*attenuation*cone*phase*transmission*localDensity*stride*length(ray); + } + } + return vec4f(scattering*uniforms[45],1.); +}`; +} +export function fogCode(bindings: string, shafts: boolean) { + return `${bindings} +@group(0) @binding(3) var source:texture_2d; +@group(0) @binding(4) var depth:texture_depth_2d; +${shafts ? "@group(0) @binding(5) var shaftField:texture_2d;" : ""} +${depthCode()} +@fragment fn fog(@location(0) uv:vec2f)->@location(0) vec4f { + let distance=depthAt(uv); let world=vector(0u)+worldRay(uv)*distance; + let heightDensity=exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); + let fogAmount=1.-exp(-distance*uniforms[12]*heightDensity); + var color=mix(textureSampleLevel(source,linearSampler,uv,0.).rgb,vector(20u),clamp(fogAmount,0.,1.)); + ${shafts ? "color+=textureSampleLevel(shaftField,linearSampler,uv,0.).rgb;" : ""} + return vec4f(color,1.); +}`; +} +export function dofCode(bindings: string, samples: number) { + const common = `${bindings} +@group(0) @binding(3) var source:texture_2d; +@group(0) @binding(4) var depth:texture_depth_2d; +${depthCode()} +fn coc(z:f32)->f32 { return clamp(abs(z-uniforms[32])/max(z,.05)*uniforms[33],0.,1.); } +`; + return { + gather: `${common} +@fragment fn gather(@location(0) uv:vec2f)->@location(0) vec4f { + let centerDepth=depthAt(uv); let radius=coc(centerDepth)*uniforms[34]; + let pixel=1./vec2f(textureDimensions(depth)); var color=vec3f(0.); var weight=0.; + for(var i=0u;i<${samples}u;i++) { + let angle=f32(i)*2.39996323; let disk=sqrt((f32(i)+.5)/${samples}.); + let sampleUV=uv+vec2f(cos(angle),sin(angle))*disk*radius*pixel; + let sampleDepth=depthAt(sampleUV); let sampleCoc=coc(sampleDepth); + // Reject distant background crossing a nearer silhouette; let foreground bokeh spread. + let foreground=sampleDepth<=centerDepth+.1; + let contribution=select(exp(-abs(sampleDepth-centerDepth)/max(.1,centerDepth*.025)),1.,foreground); + let w=contribution*max(.05,sampleCoc); color+=textureSampleLevel(source,linearSampler,sampleUV,0.).rgb*w; weight+=w; + } + return vec4f(color/max(weight,.0001),1.); +}`, + composite: `${common} +@group(0) @binding(5) var blurred:texture_2d; +@fragment fn focusComposite(@location(0) uv:vec2f)->@location(0) vec4f { + let amount=smoothstep(0.,.15,coc(depthAt(uv))); + return vec4f(mix(textureSampleLevel(source,linearSampler,uv,0.).rgb,textureSampleLevel(blurred,linearSampler,uv,0.).rgb,amount),1.); +}`, + }; +} +export function customPostCode(bindings: string, wgsl: string) { + return `${bindings} +@group(0) @binding(3) var previous:texture_2d; +@group(0) @binding(4) var depth:texture_depth_2d; +@group(0) @binding(5) var sceneHDR:texture_2d; +${depthCode()} +struct PassContext { time:f32, slideTime:f32, stepTime:f32, step:f32, depth:f32 } +fn sampleHDR(uv:vec2f)->vec4f { return textureSampleLevel(sceneHDR,linearSampler,uv,0.); } +fn sampleDepth(uv:vec2f)->f32 { return depthAt(uv); } +${wgsl} +@fragment fn customPost(@location(0) uv:vec2f)->@location(0) vec4f { + return customPass(uv,textureSampleLevel(previous,linearSampler,uv,0.),PassContext(frame.time,frame.slideTime,frame.stepTime,frame.step,depthAt(uv))); +}`; +} +export function finalCode(bindings: string) { + return `${bindings} +@group(0) @binding(3) var source:texture_2d; +fn aces(x:vec3f)->vec3f { return clamp((x*(2.51*x+.03))/(x*(2.43*x+.59)+.14),vec3f(0.),vec3f(1.)); } +fn srgb(x:vec3f)->vec3f { return select(x*12.92,1.055*pow(x,vec3f(1./2.4))-.055,x>vec3f(.0031308)); } +fn rectangle(p:vec2f,center:vec2f,halfSize:vec2f)->f32 { let q=abs(p-center)-halfSize; return length(max(q,vec2f(0.)))+min(max(q.x,q.y),0.); } +fn digit(p:vec2f,value:u32)->f32 { + let masks=array(119u,36u,93u,109u,46u,107u,123u,37u,127u,111u); let mask=masks[min(value,9u)]; + var distance=100.; + if((mask&1u)!=0u) { distance=min(distance,rectangle(p,vec2f(.5,.08),vec2f(.32,.055))); } + if((mask&2u)!=0u) { distance=min(distance,rectangle(p,vec2f(.13,.28),vec2f(.055,.16))); } + if((mask&4u)!=0u) { distance=min(distance,rectangle(p,vec2f(.87,.28),vec2f(.055,.16))); } + if((mask&8u)!=0u) { distance=min(distance,rectangle(p,vec2f(.5,.5),vec2f(.32,.055))); } + if((mask&16u)!=0u) { distance=min(distance,rectangle(p,vec2f(.13,.72),vec2f(.055,.16))); } + if((mask&32u)!=0u) { distance=min(distance,rectangle(p,vec2f(.87,.72),vec2f(.055,.16))); } + if((mask&64u)!=0u) { distance=min(distance,rectangle(p,vec2f(.5,.92),vec2f(.32,.055))); } + return 1.-smoothstep(0.,.025,distance); +} +@fragment fn tonemap(@location(0) uv:vec2f)->@location(0) vec4f { + let shift=(uv-.5)*uniforms[44]; + var color=vec3f(textureSampleLevel(source,linearSampler,uv+shift,0.).r,textureSampleLevel(source,linearSampler,uv,0.).g,textureSampleLevel(source,linearSampler,uv-shift,0.).b); + color=srgb(aces(color*uniforms[8]*frame.brightness)); + color*=1.-uniforms[15]*smoothstep(.1,.8,length(uv-.5)); color*=1.-uniforms[17]*.65; + let noise=fract(sin(dot(uv*vec2f(textureDimensions(source)),vec2f(12.9898,78.233))+floor(frame.time*24.))*43758.5453)-.5; + color+=noise*uniforms[16]; + if(uniforms[53]>.5) { + let value=floor(clamp(uniforms[56],0.,999999.)); var digits=1.; var remaining=u32(value); + for(var place=0u;place<5u;place++) { if(remaining>=10u) { digits+=1.; remaining/=10u; } } + let height=max(.005,uniforms[59]); let width=height*.55/frame.aspect; let p=(uv-vec2f(uniforms[57],uniforms[58]))/vec2f(width,height); + var amount=0.; + for(var i=0u;i<6u;i++) { if(f32(i) = { ` }, }; -function geometryCode(kind: PrimitiveKind, offset: number) { - return `${worldBindings}${triangleCorner} -struct Vertex { @builtin(position) position:vec4f, @location(0) world:vec3f, @location(1) normal:vec3f, @location(2) color:vec3f, @location(3) emissive:f32 } -@vertex fn worldVertex(@builtin(vertex_index) vid:u32,@builtin(instance_index) iid:u32)->Vertex { +function lightingCode(lightCount: number, reflection: boolean) { + return ` +fn eye()->vec3f { let camera=vector(0u); return ${reflection ? "vec3f(camera.x,2.*uniforms[36]-camera.y,camera.z)" : "camera"}; } +fn projected(world:vec3f)->vec4f { return project(${reflection ? "vec3f(world.x,2.*uniforms[36]-world.y,world.z)" : "world"}); } +struct Material { albedo:vec3f, emissive:vec3f, roughness:f32, metal:f32 } +struct ShapeContext { time:f32, slideTime:f32, stepTime:f32, step:f32 } +struct ShapeVertex { position:vec3f, normal:vec3f } +fn shapeContext()->ShapeContext { return ShapeContext(frame.time,frame.slideTime,frame.stepTime,frame.step); } +fn illuminate(world:vec3f,normal:vec3f,material:Material,rimColor:vec3f,rimIntensity:f32,activation:f32)->vec3f { + let view=normalize(eye()-world); var color=material.albedo*uniforms[52]+material.emissive; + let roughness=clamp(material.roughness,.04,1.); let metal=clamp(material.metal,0.,1.); + ${lightCount === 0 ? "color+=material.albedo*max(0.,dot(normal,normalize(vec3f(-.4,1.,.6))))*.55;" : ""} + for(var light=0u;light<${lightCount}u;light++) { + let base=64u+light*12u; let delta=vector(base)-world; let distance=max(length(delta),.001); let toward=delta/distance; + let range=max(.001,uniforms[base+7u]); let attenuation=pow(max(0.,1.-distance/range),2.)/(1.+distance*distance*.06); + var cone=1.; let cosine=uniforms[base+11u]; + if(cosine>-.99) { cone=smoothstep(cosine,min(.9999,cosine+.08),dot(-toward,normalize(vector(base+8u)))); } + let halfVector=normalize(toward+view+vec3f(.00001)); + let specular=pow(max(0.,dot(normal,halfVector)),mix(128.,3.,roughness))*mix(1.,.12,roughness); + let f0=mix(vec3f(.04),material.albedo,metal); + let diffuse=material.albedo*max(0.,dot(normal,toward))*(1.-metal); + color+=(diffuse+f0*specular)*vector(base+4u)*uniforms[base+3u]*attenuation*cone; + } + let rim=pow(1.-abs(dot(normal,view)),3.); + return color+rim*(rimColor*rimIntensity+vector(48u)*uniforms[51]*activation); +} +`; +} +const vertexFields = ` +struct WorldVertex { @builtin(position) position:vec4f, @location(0) world:vec3f, @location(1) normal:vec3f, + @location(2) color:vec3f, @location(3) emissive:f32, @location(4) material:vec4f, @location(5) albedo:vec3f, + @location(6) opacity:f32, @location(7) @interpolate(flat) instance:u32 } +`; +function geometryCode(kind: PrimitiveKind, offset: number, lights: number, reflection: boolean, custom?: CustomGeometry, sdf?: SDFGeometry) { + const shape = custom ? `let shapeVertex=shape(iid,vid,shapeContext()); point=shapeVertex.position*radius; normal=shapeVertex.normal;` : mesh[kind].code; + const bounds = sdf ? sdf.bounds.map(v => `vec3f(${v.map(n => Number.isInteger(n) ? n + "." : n).join(",")})`) : []; + return `${worldBindings}${triangleCorner}${lightingCode(lights, reflection)}${vertexFields} +${custom?.wgsl ?? sdf?.wgsl ?? ""} +@vertex fn worldVertex(@builtin(vertex_index) vid:u32,@builtin(instance_index) iid:u32)->WorldVertex { let base=${worldHeaderSize}u+(iid+${offset}u)*${worldInstanceSize}u; - let origin=vector(base); let radius=uniforms[base+3u]; + let origin=vector(base); let radius=max(uniforms[base+3u],.0001); var point=vec3f(0.); var normal=vec3f(0.,1.,0.); - ${mesh[kind].code} - let scale=max(vector(base+8u),vec3f(.0001)); - let world=origin+point*scale; - var result:Vertex; result.position=project(world); result.world=world; result.normal=normalize(normal/scale); - result.color=vector(base+4u); result.emissive=uniforms[base+7u]; return result; + ${shape} + ${sdf ? `point=mix(${bounds[0]},${bounds[1]},point/radius*.5+.5)*radius;` : ""} + let scale=max(vector(base+8u),vec3f(.0001)); let world=origin+point*scale; + var result:WorldVertex; result.position=projected(world); result.world=world; result.normal=normalize(normal/scale); + result.color=vector(base+4u); result.emissive=uniforms[base+7u]; result.material=vec4f(uniforms[base+27u],uniforms[base+28u],uniforms[base+29u],uniforms[base+30u]); + result.albedo=vector(base+24u); result.opacity=uniforms[base+23u]; result.instance=iid; return result; } -@fragment fn worldFragment(in:Vertex)->@location(0) vec4f { - let normal=normalize(in.normal); let view=normalize(vector(0u)-in.world); - let diffuse=max(0.,dot(normal,normalize(vec3f(-.4,1.,.6)))); - let rim=pow(1.-abs(dot(normal,view)),3.); - let color=in.color*(.12+diffuse*.55+in.emissive)+in.color*rim*(.25+in.emissive*.12); - return vec4f(color,1.); +${sdf ? ` +struct SDFResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } +@fragment fn worldFragment(in:WorldVertex)->SDFResult { + if(in.opacity<=.001) { discard; } + let base=${worldHeaderSize}u+(in.instance+${offset}u)*${worldInstanceSize}u; + let origin=vector(base); let scale=max(vector(base+8u)*uniforms[base+3u],vec3f(.0001)); + let rayOrigin=(eye()-origin)/scale; let rayDirection=normalize((in.world-eye())/scale); + let direction=select(rayDirection,vec3f(.000001),abs(rayDirection)exitDistance) { break; } local=rayOrigin+rayDirection*distance; let field=sdf(local); + if(abs(field)<.001) { found=true; break; } distance+=max(.0005,abs(field)); + } + if(!found) { discard; } + let hit=origin+local*scale; + ${reflection ? "if(hit.y@location(0) vec4f { + if(in.opacity<=.001) { discard; } + ${reflection ? "if(in.world.y;" : ""} +struct FloorVertex { @builtin(position) position:vec4f, @location(0) uv:vec2f } +@vertex fn floorVertex(@builtin(vertex_index) id:u32)->FloorVertex { + let p=vec2f(f32((id<<1u)&2u),f32(id&2u)); return FloorVertex(vec4f(p*2.-1.,0.,1.),vec2f(p.x,1.-p.y)); +} +struct FloorResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } +@fragment fn floorFragment(in:FloorVertex)->FloorResult { + let ray=cameraForward()+cameraRight()*(in.uv.x*2.-1.)*frame.aspect*uniforms[3]+cameraUp()*(1.-in.uv.y*2.)*uniforms[3]; + if(abs(ray.y)<.0001) { discard; } let distance=(uniforms[36]-vector(0u).y)/ray.y; + let world=vector(0u)+ray*distance; + if(distance<=.05 || distance>=500. || abs(world.x)>uniforms[39] || abs(world.z)>uniforms[39]) { discard; } + let normal=vec3f(0.,1.,0.); let view=normalize(vector(0u)-world); let material=Material(vector(40u),vec3f(0.),uniforms[37],uniforms[43]); + var color=illuminate(world,normal,material,vec3f(0.),0.,1.); + ${mirror ? `let spread=uniforms[37]*.025; let pixel=vec2f(spread,spread*frame.aspect); + var reflected=textureSampleLevel(reflection,linearSampler,in.uv,0.).rgb*.4; + reflected+=(textureSampleLevel(reflection,linearSampler,in.uv+pixel*vec2f(1.,0.),0.).rgb+textureSampleLevel(reflection,linearSampler,in.uv-pixel*vec2f(1.,0.),0.).rgb+textureSampleLevel(reflection,linearSampler,in.uv+pixel*vec2f(0.,1.),0.).rgb+textureSampleLevel(reflection,linearSampler,in.uv-pixel*vec2f(0.,1.),0.).rgb)*.15; + let fresnel=.04+.96*pow(1.-clamp(dot(normal,view),0.,1.),5.); + color=mix(color,reflected,clamp(uniforms[38]*mix(fresnel,1.,uniforms[43]),0.,1.));` : ""} + let clip=project(world); return FloorResult(vec4f(color,1.),clamp(clip.z/clip.w,0.,1.)); }`; } - const filterCode = `${worldBindings} @group(0) @binding(3) var source:texture_2d; fn tentFilter(uv:vec2f)->vec3f { @@ -96,74 +179,137 @@ fn tentFilter(uv:vec2f)->vec3f { } `; const downsample = `${filterCode} -@fragment fn downsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(tentFilter(uv),1.); } -`; +@fragment fn downsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(tentFilter(uv),1.); }`; const threshold = `${filterCode} @fragment fn threshold(@location(0) uv:vec2f)->@location(0) vec4f { let color=tentFilter(uv); let brightness=max(color.r,max(color.g,color.b)); - let contribution=max(0.,brightness-uniforms[10])/max(.0001,brightness); - return vec4f(color*contribution,1.); + return vec4f(color*max(0.,brightness-uniforms[10])/max(.0001,brightness),1.); }`; const upsample = `${filterCode} @group(0) @binding(4) var high:texture_2d; -@fragment fn upsample(@location(0) uv:vec2f)->@location(0) vec4f { - let color=textureSampleLevel(high,linearSampler,uv,0.).rgb; - return vec4f(color+tentFilter(uv)*uniforms[11],1.); -}`; -const finalCode = `${worldBindings} -@group(0) @binding(3) var hdr:texture_2d; -@group(0) @binding(4) var depth:texture_depth_2d; -@group(0) @binding(5) var bloom:texture_2d; -fn aces(x:vec3f)->vec3f { return clamp((x*(2.51*x+.03))/(x*(2.43*x+.59)+.14),vec3f(0.),vec3f(1.)); } -fn srgb(x:vec3f)->vec3f { return select(x*12.92,1.055*pow(x,vec3f(1./2.4))-.055,x>vec3f(.0031308)); } -@fragment fn tonemap(@location(0) uv:vec2f)->@location(0) vec4f { - let pixel=clamp(vec2i(uv*vec2f(textureDimensions(depth))),vec2i(0),vec2i(textureDimensions(depth))-vec2i(1)); - let z=textureLoad(depth,pixel,0); let near=.05; let far=500.; - let distance=near*far/(far-z*(far-near)); - let ray=cameraForward()+cameraRight()*(uv.x*2.-1.)*frame.aspect*uniforms[3]+cameraUp()*(1.-uv.y*2.)*uniforms[3]; - let world=vector(0u)+ray*distance; - let heightDensity=exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); - let fog=1.-exp(-distance*uniforms[12]*heightDensity); - var color=mix(textureSampleLevel(hdr,linearSampler,uv,0.).rgb,vector(20u),clamp(fog,0.,1.)); - color+=textureSampleLevel(bloom,linearSampler,uv,0.).rgb*uniforms[9]; - color=srgb(aces(color*uniforms[8]*frame.brightness)); - let vignette=1.-uniforms[15]*smoothstep(.1,.8,length(uv-.5)); - color*=vignette*(1.-uniforms[17]*.65); - let noise=fract(sin(dot(uv*vec2f(textureDimensions(hdr)),vec2f(12.9898,78.233))+floor(frame.time*24.))*43758.5453)-.5; - color+=noise*uniforms[16]; - return vec4f(clamp(color,vec3f(0.),vec3f(1.)),1.); -}`; +@fragment fn upsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(textureSampleLevel(high,linearSampler,uv,0.).rgb+tentFilter(uv)*uniforms[11],1.); }`; +const bloomComposite = `${worldBindings} +@group(0) @binding(3) var source:texture_2d; +@group(0) @binding(4) var bloom:texture_2d; +@fragment fn bloomComposite(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(textureSampleLevel(source,linearSampler,uv,0.).rgb+textureSampleLevel(bloom,linearSampler,uv,0.).rgb*uniforms[9],1.); }`; +const copyCode = `${worldBindings} +@group(0) @binding(3) var source:texture_2d; +@fragment fn copyColor(@location(0) uv:vec2f)->@location(0) vec4f { return textureSampleLevel(source,linearSampler,uv,0.); }`; -export type WorldGraphTopology = { quality: WorldScene["quality"]; counts: Record; backgroundVector: Vec3 }; +type CustomDraw = CustomGeometry & { offset: number; count: number }; +type SDFDraw = SDFGeometry & { offset: number; count: number }; +export type WorldGraphTopology = { + quality: WorldScene["quality"]; counts: Record; backgroundVector: Vec3; + floor?: boolean; mirror?: boolean; dof?: boolean; shafts?: boolean; lightCount?: number; + dofScale?: number; dofSamples?: number; reflectionScale?: number; shaftScale?: number; shaftSamples?: number; + customs?: CustomDraw[]; sdfs?: SDFDraw[]; passes?: WorldPostPass[]; postOrder?: string[]; +}; export function worldGraphTopology(scene: PreparedWorldScene): WorldGraphTopology { const counts: Record = { box: 0, ring: 0, sphere: 0, tube: 0 }; - for (const instance of scene.instances) counts[instance.kind]++; - return { quality: scene.quality, counts, backgroundVector: scene.backgroundVector }; + const customs: CustomDraw[] = [], sdfs: SDFDraw[] = []; + let previousCustom: CustomGeometry | undefined, previousSDF: SDFGeometry | undefined; + for (let offset = 0; offset < scene.instances.length; offset++) { + const instance = scene.instances[offset]; + if (instance.custom) { + const last = customs.at(-1); + if (last && last.offset + last.count === offset && previousCustom === instance.custom) last.count++; + else customs.push({ ...instance.custom, offset, count: 1 }); + previousCustom = instance.custom; + } else if (instance.sdf) { + const last = sdfs.at(-1); + if (last && last.offset + last.count === offset && previousSDF === instance.sdf) last.count++; + else sdfs.push({ ...instance.sdf, offset, count: 1 }); + previousSDF = instance.sdf; + } else counts[instance.kind]++; + } + return { quality: scene.quality, counts, backgroundVector: scene.backgroundVector, + floor: !!scene.floor, mirror: !!scene.floor?.mirror, dof: !!scene.dof, shafts: !!scene.shafts, lightCount: scene.lights?.length ?? 0, + dofScale: scene.dof?.scale, dofSamples: scene.dof?.samples, reflectionScale: scene.floor?.scale, shaftScale: scene.shafts?.scale, shaftSamples: scene.shafts?.samples, + customs, sdfs, passes: scene.passes, postOrder: scene.postOrder }; } -/** Pure graph description, shared by decks and the browser performance harness. */ +/** Pure topology: animated values travel only through the uniform snapshot. */ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): GPURenderGraphDefinition & { uniformsSize: number } { const topology = "instances" in scene ? worldGraphTopology(scene) : scene; - const scale = topology.quality === "low" ? 0.65 : topology.quality === "medium" ? 0.85 : 1; + const scale = topology.quality === "low" ? .65 : topology.quality === "medium" ? .85 : 1; const levels = topology.quality === "low" ? 3 : 5; - const instanceCount = Object.values(topology.counts).reduce((a, b) => a + b, 0); + const samples = topology.quality === "low" ? 8 : topology.quality === "medium" ? 12 : 16; const passes: GPUPass[] = []; - let offset = 0; + const targets: GPURenderGraphDefinition["targets"] = [{ id: "hdr", format: "rgba16float", scale }, { id: "depth", format: "depth32float", scale }, + { id: "bloom", format: "rgba16float", scale: scale * .5, mips: levels }, { id: "bloom-up", format: "rgba16float", scale: scale * .5, mips: levels }]; const background = topology.backgroundVector; - for (const kind of ["box", "ring", "sphere", "tube"] as const) { - const count = topology.counts[kind]; - if (count || passes.length === 0 && instanceCount === 0) { - passes.push({ kind: "render", id: `scene-${kind}`, code: geometryCode(kind, offset), vertex: "worldVertex", fragment: "worldFragment", - color: ["hdr"], depth: "depth", vertexCount: mesh[kind].count, instanceCount: count, - clear: passes.length === 0 ? [background[0], background[1], background[2], 1] : false }); + const clear: [number, number, number, number] = [background[0], background[1], background[2], 1]; + const customDraws = topology.customs ?? [], sdfDraws = topology.sdfs ?? []; + const lights = topology.lightCount ?? 0; + const instanceCount = Object.values(topology.counts).reduce((a, b) => a + b, 0) + [...customDraws, ...sdfDraws].reduce((sum, draw) => sum + draw.count, 0); + const renderScene = (reflection: boolean, cleared = false) => { + const color = reflection ? "reflect" : "hdr", depth = reflection ? "reflectDepth" : "depth"; + let offset = 0, draws = 0; + const add = (id: string, kind: PrimitiveKind, count: number, start: number, custom?: CustomGeometry, sdf?: SDFGeometry) => { + passes.push({ kind: "render", id: `${reflection ? "reflection" : "scene"}-${id}`, code: geometryCode(kind, start, lights, reflection, custom, sdf), vertex: "worldVertex", fragment: "worldFragment", + color: [color], depth, vertexCount: custom?.vertexCount ?? mesh[kind].count, instanceCount: count, blend: "over", clear: draws++ === 0 && !cleared ? clear : false }); + }; + for (const kind of ["box", "ring", "sphere", "tube"] as const) { if (topology.counts[kind]) add(kind, kind, topology.counts[kind], offset); offset += topology.counts[kind]; } + for (const [index, draw] of customDraws.entries()) add(`custom-${index}`, "box", draw.count, draw.offset, draw); + for (const [index, draw] of sdfDraws.entries()) add(`sdf-${index}`, "box", draw.count, draw.offset, undefined, draw); + if (!draws && !cleared) add("empty", "box", 0, 0); + }; + if (topology.mirror) { + const reflectionScale = scale * (topology.reflectionScale ?? .5); + targets.push({ id: "reflect", format: "rgba16float", scale: reflectionScale }, { id: "reflectDepth", format: "depth32float", scale: reflectionScale }); + renderScene(true); + } + if (topology.floor) passes.push({ kind: "render", id: "floor", code: floorCode(lights, !!topology.mirror), vertex: "floorVertex", fragment: "floorFragment", color: ["hdr"], depth: "depth", vertexCount: 3, clear, reads: topology.mirror ? ["reflect"] : [] }); + renderScene(false, !!topology.floor); + if (topology.shafts) { + targets.push({ id: "shafts", format: "rgba16float", scale: scale * (topology.shaftScale ?? .25) }); + passes.push({ kind: "fullscreen", id: "shafts", code: shaftCode(worldBindings, lights, topology.shaftSamples ?? samples), fragment: "shafts", reads: ["depth"], output: "shafts" }); + } + let color = "hdr", customIndex = 0; + const fullscreen = (id: string, code: string, fragment: string, reads: string[], output = id, targetScale = scale) => { + if (output !== "screen") targets.push({ id: output, format: "rgba16float", scale: targetScale }); + passes.push({ kind: "fullscreen", id, code, fragment, reads, output }); color = output; + }; + const stages = ["scene", "fog", "dof", "bloom", "tonemap"]; + for (const item of topology.passes ?? []) { + if (item.before && item.after || !stages.includes(item.before ?? item.after ?? "bloom") || item.before === "scene") throw new Error("Invalid World post-pass placement"); + } + const custom = (stage: string, before: boolean) => { + for (const item of topology.passes ?? []) { + if ((before ? item.before : item.after ?? (!item.before ? "bloom" : undefined)) !== stage) continue; + const id = `custom-post-${customIndex++}`; + fullscreen(item.id ?? id, customPostCode(worldBindings, item.wgsl), "customPost", [color, "depth", "hdr", ...(item.reads ?? [])], id); } - offset += count; + }; + const order = topology.postOrder ?? ["fog", "dof", "bloom"]; + if (order.length !== 3 || new Set(order).size !== 3 || order.some(stage => !["fog", "dof", "bloom"].includes(stage))) throw new Error("World postOrder must contain fog, dof and bloom exactly once"); + custom("scene", false); + for (const stage of order) { + custom(stage, true); + if (stage === "fog") fullscreen("fog", fogCode(worldBindings, !!topology.shafts), "fog", [color, "depth", ...(topology.shafts ? ["shafts"] : [])]); + if (stage === "dof" && topology.dof) { + const source = color, dof = dofCode(worldBindings, topology.dofSamples ?? samples); + fullscreen("dof-gather", dof.gather, "gather", [source, "depth"], "dof", scale * (topology.dofScale ?? .5)); + fullscreen("dof-composite", dof.composite, "focusComposite", [source, "depth", "dof"]); + } + if (stage === "bloom") { + const bloomSource = color; + passes.push({ kind: "fullscreen", id: "bloom-threshold", code: threshold, fragment: "threshold", reads: [color], output: "bloom@0" }); + for (let mip = 1; mip < levels; mip++) passes.push({ kind: "fullscreen", id: `bloom-down-${mip}`, code: downsample, fragment: "downsample", reads: [`bloom@${mip - 1}`], output: `bloom@${mip}` }); + passes.push({ kind: "fullscreen", id: `bloom-up-${levels - 1}`, code: downsample, fragment: "downsample", reads: [`bloom@${levels - 1}`], output: `bloom-up@${levels - 1}` }); + for (let mip = levels - 2; mip >= 0; mip--) passes.push({ kind: "fullscreen", id: `bloom-up-${mip}`, code: upsample, fragment: "upsample", reads: [`bloom-up@${mip + 1}`, `bloom@${mip}`], output: `bloom-up@${mip}` }); + fullscreen("bloom-composite", bloomComposite, "bloomComposite", [bloomSource, "bloom-up@0"]); + } + custom(stage, false); + } + custom("tonemap", true); + const postTone = (topology.passes ?? []).some(item => item.after === "tonemap"); + fullscreen("tonemap", finalCode(worldBindings), "tonemap", [color], postTone ? "tonemapped" : "screen"); + custom("tonemap", false); + if (postTone) fullscreen("screen", copyCode, "copyColor", [color], "screen"); + const passIds = new Set(); + for (const pass of passes) { + if (passIds.has(pass.id)) throw new Error("World pass id is duplicated: " + pass.id); + passIds.add(pass.id); } - passes.push({ kind: "fullscreen", id: "bloom-threshold", code: threshold, fragment: "threshold", reads: ["hdr"], output: "bloom@0" }); - for (let mip = 1; mip < levels; mip++) passes.push({ kind: "fullscreen", id: `bloom-down-${mip}`, code: downsample, fragment: "downsample", reads: [`bloom@${mip - 1}`], output: `bloom@${mip}` }); - passes.push({ kind: "fullscreen", id: `bloom-up-${levels - 1}`, code: downsample, fragment: "downsample", reads: [`bloom@${levels - 1}`], output: `bloom-up@${levels - 1}` }); - for (let mip = levels - 2; mip >= 0; mip--) passes.push({ kind: "fullscreen", id: `bloom-up-${mip}`, code: upsample, fragment: "upsample", reads: [`bloom-up@${mip + 1}`, `bloom@${mip}`], output: `bloom-up@${mip}` }); - passes.push({ kind: "fullscreen", id: "tonemap", code: finalCode, fragment: "tonemap", reads: ["hdr", "depth", "bloom-up@0"], output: "screen" }); - return { targets: [{ id: "hdr", format: "rgba16float", scale }, { id: "depth", format: "depth32float", scale }, - { id: "bloom", format: "rgba16float", scale: scale * 0.5, mips: levels }, { id: "bloom-up", format: "rgba16float", scale: scale * 0.5, mips: levels }], - passes, uniformsSize: worldHeaderSize + instanceCount * worldInstanceSize }; + return { targets, passes, uniformsSize: worldHeaderSize + instanceCount * worldInstanceSize }; } diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index 996bde1e7..aede7f47f 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -1,19 +1,39 @@ -import { clockAtTime, easeProgress, evaluateCamera, mixVector, type CameraOptions, type CameraPose, type ClockSample, type Ease, type Vec3 } from "./camera"; +import { cameraFocus, clockAtTime, easeProgress, evaluateCamera, mixVector, shotOrigin, type CameraOptions, type CameraPose, type ClockSample, type Ease, type Vec3 } from "./camera"; import { activationAt, layoutTree, type Activation, type LayoutNode, type TreeNode } from "./graph"; export type Keyframe = { step?: number; at?: number; value: T; duration?: number; ease?: Ease }; export type Animated = T | Record | Keyframe[]; export type PrimitiveKind = "sphere" | "box" | "tube" | "ring"; -type Track = { initial: T; frames: Keyframe[] }; +export type Track = { initial: T; frames: Keyframe[] }; +export type MaterialOptions = { id?: string; albedo?: Animated; emissive?: Animated; roughness?: Animated; metal?: Animated; rim?: Animated }; +export type LightOptions = { kind?: "point" | "spot"; position?: Animated; target?: Animated; color?: Animated; intensity?: Animated; range?: Animated; angle?: Animated }; +export type StageOptions = { id: string; position?: Animated; glow?: Animated }; +export type StageMotion = { id: string; position: Track; glow: Track }; +export type Collapse = { step: number; to?: number; duration?: number; ease?: Ease }; +export type PulseOptions = { id: string; path: (string | Vec3)[]; step?: number; duration?: number; color?: Animated; size?: number; trail?: number; emissive?: Animated }; +export type SprayOptions = { id?: string; under?: string; position?: Vec3; rate?: number; lifetime?: number; color?: string; size?: number; spread?: Vec3; velocity?: Vec3; seed?: number; step?: number }; +export type RowsOptions = { id: string; count?: number; position?: Vec3; spacing?: number; speed?: number; start?: number; width?: number; depth?: number; color?: string; flashColor?: string; changes?: { row: number; step: number }[]; flashDuration?: number }; +export type VoxelsOptions = { id: string; grid?: Vec3; spacing?: number; size?: number; position?: Vec3; color?: Animated; emissive?: Animated; collapse?: Collapse }; +export type CustomOptions = PrimitiveOptions & { wgsl: string; count?: number; vertexCount: number }; +export type SDFOptions = PrimitiveOptions & { wgsl: string; bounds: [Vec3, Vec3]; steps?: number }; +export type PostOptions = { id?: string; wgsl: string; before?: string; after?: string; reads?: string[] }; export type PrimitiveOptions = { id?: string; position?: Animated; size?: Animated; color?: Animated; emissive?: Animated; scale?: Animated; end?: Vec3; curve?: number; thickness?: number; + material?: string | MaterialOptions; }; export type Instance = { id?: string; kind: PrimitiveKind; position: Track; size: Track; color: Track; emissive: Track; scale: Track; end: Vec3; curve: number; thickness: number; drift?: number; seed?: number; endpoints?: [string, string]; + stages?: StageMotion[]; material?: string | MaterialOptions; + materialTracks?: { albedo: Track; roughness: Track; metal: Track; rim: Track }; + pulse?: { options: PulseOptions; anchors: Instance[]; curves: number[]; tail: boolean }; + spray?: { options: SprayOptions; sources: Instance[]; phase: number }; + rows?: { options: RowsOptions; slot: number; logical?: number; flashColor: Vec3 }; + voxel?: { options: VoxelsOptions; index: number; count: number; target: Vec3 }; + custom?: { wgsl: string; vertexCount: number }; sdf?: { wgsl: string; bounds: [Vec3, Vec3]; steps?: number }; graph?: { node: LayoutNode; nodes: LayoutNode[]; activate: Activation[]; dim: Vec3 }; }; export type GraphOptions = { @@ -30,6 +50,14 @@ export type WorldScene = { camera?: CameraOptions; instances: Instance[]; exposure?: Animated; quality?: "auto" | "low" | "medium" | "high"; backgroundColor?: string; fog?: { density?: Animated; color?: Animated; height?: Animated; falloff?: Animated }; bloom?: { intensity?: Animated; threshold?: Animated; radius?: Animated }; vignette?: Animated; grain?: Animated; scrim?: number; + stages?: { stage: StageMotion; parents: StageMotion[] }[]; materials?: Record; + dof?: { focus?: Animated; aperture?: Animated; maxBlur?: Animated; scale?: number; samples?: number }; + floor?: { mirror?: boolean; y?: Animated; roughness?: Animated; reflectivity?: Animated; size?: Animated; color?: Animated; metal?: Animated; scale?: number }; + shafts?: { intensity?: Animated; density?: Animated; anisotropy?: Animated; scale?: number; samples?: number }; + chromatic?: Animated; rim?: { color?: Animated; intensity?: Animated }; ambient?: Animated; + lights?: LightOptions[]; passes?: PostOptions[]; + postOrder?: ("fog" | "dof" | "bloom")[]; + counter?: { source: string; position?: [number, number]; size?: number; color?: string }; }; export function colorVector(color: string): Vec3 { @@ -70,7 +98,7 @@ export function primitive(kind: PrimitiveKind, options: PrimitiveOptions = {}): return { id: options.id, kind, position: compileTrack(options.position ?? [0, 0, 0]), size: compileTrack(options.size ?? 0.5), color: compileTrack(options.color ?? "#22d3ee", colorVector), emissive: compileTrack(options.emissive ?? 1.5), scale: compileTrack(options.scale ?? [1, 1, 1]), end: options.end ?? [0, 2, 0], curve: options.curve ?? 0, - thickness: options.thickness ?? (kind === "ring" ? 0.04 : 0.025) }; + thickness: options.thickness ?? (kind === "ring" ? 0.04 : 0.025), material: options.material }; } export function graphInstances(options: GraphOptions): Instance[] { const layout = layoutTree(options.data, options); @@ -106,22 +134,205 @@ export function orbInstances({ id = "orb", count = 48, colors = ["#22d3ee", "#b5 }); } -export const worldHeaderSize = 32; -export const worldInstanceSize = 24; +export function pulseInstances(options: PulseOptions): Instance[] { + if (options.path.length < 2 || (options.duration ?? 2) <= 0) throw new Error("A world pulse needs at least two anchors and a positive duration"); + return [false, true].map(tail => ({ ...primitive(tail ? "tube" : "sphere", { id: tail ? `${options.id}:trail` : options.id, + size: options.size ?? 0.08, thickness: (options.size ?? 0.08) * 0.45, color: options.color ?? "#22d3ee", emissive: options.emissive ?? 8 }), + pulse: { options, anchors: [], curves: [], tail } })); +} +export function sprayInstances(options: SprayOptions = {}): Instance[] { + const count = Math.ceil((options.rate ?? 40) * (options.lifetime ?? 2)); + if (!Number.isFinite(count) || count < 0 || count > 10000 || (options.lifetime ?? 2) <= 0) throw new Error("World spray requires a positive lifetime and at most 10000 particles"); + return orbInstances({ id: options.id ?? "spray", count, colors: [options.color ?? "#22d3ee"], radius: options.size ?? 0.025, + spread: [0, 0, 0], position: options.position, seed: options.seed, drift: 0, emissive: 5 }).map((instance, index) => ({ + ...instance, spray: { options, sources: [], phase: ((index + 1) * 0.61803398875) % 1 }, + })); +} +export function rowInstances(options: RowsOptions): Instance[] { + const count = options.count ?? 9; + if (!Number.isInteger(count) || count < 1 || count > 1000) throw new Error("World rows need an integer pool size between 1 and 1000"); + return Array.from({ length: count }, (_, slot) => ({ ...primitive("box", { id: `${options.id}:slot:${slot}`, position: options.position ?? [0, 0, 0], + scale: [options.width ?? 3, 0.35, options.depth ?? 0.35], size: 0.5, color: options.color ?? "#163047", emissive: 0.1 }), + rows: { options, slot, flashColor: colorVector(options.flashColor ?? "#22d3ee") } })); +} +export function voxelInstances(options: VoxelsOptions): Instance[] { + const grid = options.grid ?? [8, 8, 8]; + if (grid.some(value => !Number.isInteger(value) || value < 1) || grid[0] * grid[1] * grid[2] > 10000 || + options.collapse && ((options.collapse.to ?? 0.2) < 0 || (options.collapse.to ?? 0.2) > 1)) throw new Error("World voxels require a positive grid up to 10000 cells and collapse fraction between zero and one"); + const count = grid[0] * grid[1] * grid[2], remaining = Math.round(count * (options.collapse?.to ?? 0.2)); + const compact = Math.max(1, Math.ceil(Math.cbrt(remaining))); + return Array.from({ length: count }, (_, index) => { + const x = index % grid[0], y = Math.floor(index / grid[0]) % grid[1], z = Math.floor(index / (grid[0] * grid[1])); + const spacing = options.spacing ?? 0.35; + const offset: Vec3 = [(x - (grid[0] - 1) / 2) * spacing, (y - (grid[1] - 1) / 2) * spacing, (z - (grid[2] - 1) / 2) * spacing]; + const center = options.position ?? [0, 0, 0]; + const target: Vec3 = [((index % compact) - (compact - 1) / 2) * spacing, + ((Math.floor(index / compact) % compact) - (compact - 1) / 2) * spacing, + (Math.floor(index / (compact * compact)) - (Math.ceil(remaining / (compact * compact)) - 1) / 2) * spacing]; + return { ...primitive("box", { id: `${options.id}:${index}`, position: [center[0] + offset[0], center[1] + offset[1], center[2] + offset[2]], + size: options.size ?? 0.13, color: options.color ?? "#22d3ee", emissive: options.emissive ?? 2 }), voxel: { options, index, count, target } }; + }); +} +export function customInstances(options: CustomOptions): Instance[] { + const count = options.count ?? 1; + if (!Number.isInteger(count) || count < 1 || count > 10000 || !Number.isInteger(options.vertexCount) || options.vertexCount < 1) throw new Error("World custom geometry needs count between one and 10000 and positive integer vertexCount"); + const custom = { wgsl: options.wgsl, vertexCount: options.vertexCount }; + return Array.from({ length: count }, (_, index) => ({ ...primitive("box", { ...options, size: options.size ?? 1, id: options.id ? `${options.id}:${index}` : undefined }), custom })); +} +export function sdfInstance(options: SDFOptions): Instance { + if (options.bounds.some(bound => bound.some(value => !Number.isFinite(value))) || options.bounds[0].some((value, index) => value >= options.bounds[1][index])) throw new Error("World SDF bounds must be finite and have positive extent"); + if (options.steps !== undefined && (!Number.isInteger(options.steps) || options.steps < 1 || options.steps > 256)) throw new Error("World SDF steps must be an integer between one and 256"); + return { ...primitive("box", { ...options, size: options.size ?? 1 }), sdf: { wgsl: options.wgsl, bounds: options.bounds, steps: options.steps } }; +} + +export const worldHeaderSize = 128; +export const worldInstanceSize = 32; +export function triggerElapsed(step: number, clock: ClockSample) { + "worklet"; + if (clock.stepIndex < step) return -1; + if (clock.stepIndex > step && clock.stepOrigins[step] === undefined) return 1000000; + return clock.slideTime - shotOrigin({ step }, clock); +} +export function collapseProgress(collapse: Collapse | undefined, clock: ClockSample) { + "worklet"; + if (!collapse) return 0; + const elapsed = triggerElapsed(collapse.step, clock); + return elapsed < 0 ? 0 : easeProgress((collapse.duration ?? 2) <= 0 ? 1 : elapsed / (collapse.duration ?? 2), collapse.ease); +} +export function voxelRemaining(instance: Instance, clock: ClockSample): number { + "worklet"; + const voxel = instance.voxel!; + const target = Math.round(voxel.count * (voxel.options.collapse?.to ?? 0.2)); + return voxel.count + (target - voxel.count) * collapseProgress(voxel.options.collapse, clock); +} +function stageOffset(stages: StageMotion[] | undefined, clock: ClockSample): Vec3 { + "worklet"; + const offset: Vec3 = [0, 0, 0]; + for (const stage of stages ?? []) { + const position = sampleTrack(stage.position, clock); + for (let axis = 0; axis < 3; axis++) offset[axis] += position[axis]; + } + return offset; +} +export function rowLogicalIndex(instance: Instance, clock: ClockSample) { + "worklet"; + const rows = instance.rows!; + return rows.logical ?? Math.floor((rows.options.start ?? 0) + clock.slideTime * (rows.options.speed ?? 0.3)) + rows.slot; +} +function rowFlash(instance: Instance, clock: ClockSample) { + "worklet"; + const rows = instance.rows!; + const logical = rowLogicalIndex(instance, clock); + let flash = 0; + for (const change of rows.options.changes ?? []) if (change.row === logical) { + const elapsed = triggerElapsed(change.step, clock), duration = rows.options.flashDuration ?? 0.8; + if (elapsed >= 0 && elapsed < duration) flash = Math.max(flash, Math.pow(1 - elapsed / duration, 2)); + } + return flash; +} +function quadraticPoint(from: Vec3, to: Vec3, curve: number, t: number): Vec3 { + "worklet"; + const dx = to[0] - from[0], dy = to[1] - from[1], dz = to[2] - from[2]; + const lift = Math.sqrt(dx * dx + dy * dy + dz * dz) * curve; + return [from[0] + dx * t, from[1] + dy * t + 2 * (1 - t) * t * lift, from[2] + dz * t]; +} export function instancePosition(instance: Instance, clock: ClockSample): Vec3 { "worklet"; - const position = sampleTrack(instance.position, clock); + let position = sampleTrack(instance.position, clock); + if (instance.pulse) { + const pulse = instance.pulse; + const progress = Math.max(0, Math.min(1, triggerElapsed(pulse.options.step ?? 0, clock) / (pulse.options.duration ?? 2))); + const travel = Math.max(0, progress - (pulse.tail ? pulse.options.trail ?? 0.18 : 0)) * (pulse.anchors.length - 1); + const segment = Math.min(pulse.anchors.length - 2, Math.floor(travel)); + return quadraticPoint(instancePosition(pulse.anchors[segment], clock), instancePosition(pulse.anchors[segment + 1], clock), pulse.curves[segment], travel - segment); + } + if (instance.spray) { + const spray = instance.spray, life = spray.options.lifetime ?? 2; + const elapsed = spray.options.step === undefined ? clock.slideTime : triggerElapsed(spray.options.step, clock); + const age = ((Math.max(0, elapsed) / life + spray.phase) % 1) * life; + if (spray.sources.length) position = instancePosition(spray.sources[Math.floor(spray.phase * spray.sources.length) % spray.sources.length], clock); + else { + const offset = stageOffset(instance.stages, clock); + position = [position[0] + offset[0], position[1] + offset[1], position[2] + offset[2]]; + } + const spread = spray.options.spread ?? [1, 0.3, 1], velocity = spray.options.velocity ?? [0, -1, 0]; + return [position[0] + Math.sin(spray.phase * 97) * spread[0] * age + velocity[0] * age, + position[1] + Math.cos(spray.phase * 53) * spread[1] * age + velocity[1] * age, + position[2] + Math.sin(spray.phase * 71) * spread[2] * age + velocity[2] * age]; + } + if (instance.rows) { + const rows = instance.rows, scroll = (rows.options.start ?? 0) + clock.slideTime * (rows.options.speed ?? 0.3); + position = [position[0], position[1] + (((rows.options.count ?? 9) - 1) / 2 + scroll - rowLogicalIndex(instance, clock)) * (rows.options.spacing ?? 1.2), position[2]]; + } + if (instance.voxel) { + const voxel = instance.voxel, center = voxel.options.position ?? [0, 0, 0]; + if (voxel.index < Math.round(voxel.count * (voxel.options.collapse?.to ?? 0.2))) { + position = mixVector(position, [center[0] + voxel.target[0], center[1] + voxel.target[1], center[2] + voxel.target[2]], collapseProgress(voxel.options.collapse, clock)); + } + } const drift = instance.drift ?? 0, seed = instance.seed ?? 0; - return [position[0] + Math.sin(clock.slideTime * 0.27 + seed) * drift, - position[1] + Math.cos(clock.slideTime * 0.31 + seed * 2) * drift, - position[2] + Math.sin(clock.slideTime * 0.19 + seed * 3) * drift]; + const offset = stageOffset(instance.stages, clock); + return [position[0] + offset[0] + Math.sin(clock.slideTime * 0.27 + seed) * drift, + position[1] + offset[1] + Math.cos(clock.slideTime * 0.31 + seed * 2) * drift, + position[2] + offset[2] + Math.sin(clock.slideTime * 0.19 + seed * 3) * drift]; +} +export function instanceOpacity(instance: Instance, clock: ClockSample): number { + "worklet"; + if (instance.rows) { + const rows = instance.rows, scroll = (rows.options.start ?? 0) + clock.slideTime * (rows.options.speed ?? 0.3); + const fraction = scroll - Math.floor(scroll); + return (rows.slot === 0 ? 1 - fraction : 1) * (rows.slot === (rows.options.count ?? 9) - 1 ? fraction : 1); + } + if (instance.pulse) { + const elapsed = triggerElapsed(instance.pulse.options.step ?? 0, clock), duration = instance.pulse.options.duration ?? 2; + return elapsed < 0 ? 0 : Math.max(0, Math.min(1, (duration + 0.25 - elapsed) / 0.25)); + } + if (instance.voxel) { + return Math.max(0, Math.min(1, voxelRemaining(instance, clock) - instance.voxel.index)); + } + if (instance.spray) { + const spray = instance.spray, elapsed = spray.options.step === undefined ? clock.slideTime : triggerElapsed(spray.options.step, clock); + if (elapsed < 0) return 0; + const phase = (elapsed / (spray.options.lifetime ?? 2) + spray.phase) % 1; + const source = spray.sources[Math.floor(spray.phase * spray.sources.length) % spray.sources.length]; + const lit = source?.graph ? activationAt(source.graph.node, source.graph.nodes, source.graph.activate, clock) : 1; + return Math.sin(phase * Math.PI) * lit; + } + return 1; +} +export function resolveAnchor(scene: WorldScene, id: string, clock: ClockSample): Vec3 | undefined { + "worklet"; + const stage = scene.stages?.find(item => `stage:${item.stage.id}` === id); + if (stage) return stageOffset([...stage.parents, stage.stage], clock); + const row = scene.instances.find(instance => instance.rows && id.startsWith(`${instance.rows.options.id}:row:`)); + if (row) return instancePosition({ ...row, rows: { ...row.rows!, logical: Number(id.slice(id.lastIndexOf(":") + 1)) } }, clock); + const instance = scene.instances.find(instance => instance.id === id || instance.id === `node:${id}`); + if (instance) return instancePosition(instance, clock); + const voxel = scene.instances.find(instance => instance.voxel?.options.id === id); + if (voxel) { + const center = voxel.voxel!.options.position ?? [0, 0, 0], offset = stageOffset(voxel.stages, clock); + return [center[0] + offset[0], center[1] + offset[1], center[2] + offset[2]]; + } } export function sampleWorldCamera(scene: WorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1): CameraPose { "worklet"; - return evaluateCamera(scene.camera ?? {}, clock, (id, sample) => { - const instance = scene.instances.find(instance => instance.id === id); - return instance ? instancePosition(instance, sample) : undefined; - }, previous, enterDuration); + return evaluateCamera(scene.camera ?? {}, clock, (id, sample) => resolveAnchor(scene, id, sample), previous, enterDuration); +} +function focusDepth(value: number | string | undefined, scene: WorldScene, clock: ClockSample, camera: CameraPose): number { + "worklet"; + if (typeof value === "number") return Math.max(0.05, value); + const anchor = value ? resolveAnchor(scene, value, clock) : camera.target; + if (!anchor) return 18; + const forward: Vec3 = [camera.target[0] - camera.position[0], camera.target[1] - camera.position[1], camera.target[2] - camera.position[2]]; + const length = Math.max(0.001, Math.hypot(...forward)); + return Math.max(0.05, ((anchor[0] - camera.position[0]) * forward[0] + (anchor[1] - camera.position[1]) * forward[1] + (anchor[2] - camera.position[2]) * forward[2]) / length); +} +export function sampleFocus(scene: PreparedWorldScene, clock: ClockSample, camera: CameraPose): number { + "worklet"; + const track = scene.focusTrack; + if (!track) return focusDepth(cameraFocus(scene.camera ?? {}, clock), scene, clock, camera); + return sampleTrack({ initial: focusDepth(track.initial, scene, clock, camera), + frames: track.frames.map(frame => ({ ...frame, value: focusDepth(frame.value, scene, clock, camera) })) }, clock); } export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1): number[] { "worklet"; @@ -136,38 +347,129 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl values[17] = scene.scrim ?? 0; const fog = sampleTrack(scene.fogTrack, clock); values[20] = fog[0]; values[21] = fog[1]; values[22] = fog[2]; + values[32] = sampleFocus(scene, clock, camera); + for (const field of scene.colorTracks) { + const color = sampleTrack(field.track, clock); + values[field.index] = color[0]; values[field.index + 1] = color[1]; values[field.index + 2] = color[2]; + } + scene.lightTracks.forEach((light, index) => { + const base = 64 + index * 12, position = sampleTrack(light.position, clock), color = sampleTrack(light.color, clock), target = sampleTrack(light.target, clock); + const direction: Vec3 = [target[0] - position[0], target[1] - position[1], target[2] - position[2]]; + const length = Math.max(0.001, Math.hypot(...direction)); + for (let axis = 0; axis < 3; axis++) { values[base + axis] = position[axis]; values[base + 4 + axis] = color[axis]; values[base + 8 + axis] = direction[axis] / length; } + values[base + 3] = sampleTrack(light.intensity, clock); values[base + 7] = sampleTrack(light.range, clock); + values[base + 11] = light.spot ? Math.cos(sampleTrack(light.angle, clock)) : -1; + }); + if (scene.counter) { + const voxel = scene.instances.find(instance => instance.voxel?.options.id === scene.counter!.source)!; + values[53] = 1; + values[56] = Math.round(voxelRemaining(voxel, clock)); + values[57] = scene.counter.position?.[0] ?? 0.8; values[58] = scene.counter.position?.[1] ?? 0.12; values[59] = scene.counter.size ?? 0.07; + const color = scene.counterColor!; values[60] = color[0]; values[61] = color[1]; values[62] = color[2]; + } for (let i = 0; i < scene.instances.length; i++) { const instance = scene.instances[i]; const index = worldHeaderSize + i * worldInstanceSize; const position = instance.endpoints ? objects[instance.endpoints[0]] : instance.id ? objects[instance.id] : instancePosition(instance, clock); - const endpoint = instance.endpoints ? objects[instance.endpoints[1]] : instance.end; + const offset = stageOffset(instance.stages, clock); + const endpoint = instance.pulse?.tail ? objects[instance.pulse.options.id] : instance.endpoints ? objects[instance.endpoints[1]] + : [instance.end[0] + offset[0], instance.end[1] + offset[1], instance.end[2] + offset[2]]; const scale = sampleTrack(instance.scale, clock); - const color = sampleTrack(instance.color, clock); + let color = sampleTrack(instance.color, clock); const activation = instance.graph ? activationAt(instance.graph.node, instance.graph.nodes, instance.graph.activate, clock) : 1; const dim = instance.graph?.dim ?? color; values[index] = position[0]; values[index + 1] = position[1]; values[index + 2] = position[2]; - values[index + 3] = sampleTrack(instance.size, clock); + values[index + 3] = sampleTrack(instance.size, clock) * (instance.voxel ? instanceOpacity(instance, clock) : 1); + let emissive = sampleTrack(instance.emissive, clock) * activation; + if (instance.rows) { const flash = rowFlash(instance, clock); color = mixVector(color, instance.rows.flashColor, flash); emissive += flash * 8; } + for (const stage of instance.stages ?? []) emissive *= sampleTrack(stage.glow, clock); values[index + 4] = dim[0] + (color[0] - dim[0]) * activation; values[index + 5] = dim[1] + (color[1] - dim[1]) * activation; values[index + 6] = dim[2] + (color[2] - dim[2]) * activation; - values[index + 7] = sampleTrack(instance.emissive, clock) * activation; + values[index + 7] = emissive; values[index + 8] = scale[0]; values[index + 9] = scale[1]; values[index + 10] = scale[2]; values[index + 12] = endpoint[0]; values[index + 13] = endpoint[1]; values[index + 14] = endpoint[2]; values[index + 15] = instance.curve; values[index + 19] = instance.thickness; + values[index + 23] = instanceOpacity(instance, clock); + const material = instance.materialTracks!, sampledAlbedo = sampleTrack(material.albedo, clock); + const albedo = instance.material ? instance.graph ? mixVector(dim, sampledAlbedo, activation) : sampledAlbedo : [values[index + 4], values[index + 5], values[index + 6]]; + values[index + 24] = albedo[0]; values[index + 25] = albedo[1]; values[index + 26] = albedo[2]; + values[index + 27] = sampleTrack(material.roughness, clock); values[index + 28] = sampleTrack(material.metal, clock); values[index + 29] = sampleTrack(material.rim, clock) * activation; + values[index + 30] = activation; } return values; } -export type PreparedWorldScene = WorldScene & { exposureTrack: Track; fogTrack: Track; backgroundVector: Vec3; postTracks: { index: number; track: Track }[] }; +export type PreparedWorldScene = WorldScene & { exposureTrack: Track; fogTrack: Track; backgroundVector: Vec3; + postTracks: { index: number; track: Track }[]; colorTracks: { index: number; track: Track }[]; + focusTrack?: Track; counterColor?: Vec3; + lightTracks: { position: Track; target: Track; color: Track; intensity: Track; range: Track; angle: Track; spot: boolean }[] }; export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { + if ((scene.lights?.length ?? 0) > 4) throw new Error("World supports at most four point or spot lights"); + for (const effect of [scene.dof, scene.floor, scene.shafts]) if (effect) { + if (effect.scale !== undefined && (!Number.isFinite(effect.scale) || effect.scale <= 0 || effect.scale > 1)) throw new Error("World effect scale must be greater than zero and at most one"); + if ("samples" in effect && effect.samples !== undefined && (!Number.isInteger(effect.samples) || effect.samples < 1 || effect.samples > 64)) throw new Error("World effect samples must be an integer between one and 64"); + } const ids = new Set(); for (const instance of scene.instances) if (instance.id) { if (ids.has(instance.id)) throw new Error(`World instance id is duplicated: ${instance.id}. Give each instance or orb field a unique id.`); ids.add(instance.id); } - const instances = [...scene.instances].sort((a, b) => a.kind.localeCompare(b.kind)); + const instances: Instance[] = scene.instances.map(instance => { + const material = typeof instance.material === "string" ? scene.materials?.[instance.material] : instance.material; + if (typeof instance.material === "string" && !material) throw new Error(`Unknown world material: ${instance.material}`); + return { ...instance, emissive: material?.emissive === undefined ? instance.emissive : compileTrack(material.emissive), materialTracks: { + albedo: material?.albedo === undefined ? instance.color : compileTrack(material.albedo, colorVector), roughness: compileTrack(material?.roughness ?? 0.5), + metal: compileTrack(material?.metal ?? 0), rim: compileTrack(material?.rim ?? 1), + } }; + }); + const byId = new Map(instances.filter(instance => instance.id).map(instance => [instance.id!, instance])); + for (const instance of instances) { + if (instance.pulse) { + const anchors = instance.pulse.options.path.map(point => { + if (typeof point !== "string") return { ...primitive("sphere", { position: point }), stages: instance.stages }; + const anchor = byId.get(point) ?? byId.get(`node:${point}`); + if (!anchor) throw new Error(`Unknown world pulse anchor: ${point}`); + return anchor; + }); + const curves = anchors.slice(1).map((target, index) => instances.find(edge => + edge.endpoints?.[0] === anchors[index].id && edge.endpoints?.[1] === target.id || + edge.endpoints?.[1] === anchors[index].id && edge.endpoints?.[0] === target.id)?.curve ?? 0); + instance.pulse = { ...instance.pulse, anchors, curves }; + } + if (instance.spray) { + const under = instance.spray.options.under; + const sources = under === "leaves:lit" ? instances.filter(item => item.kind === "sphere" && item.graph && + !item.graph.nodes.some(node => node.parent === item.graph!.node.id)) : under ? [byId.get(under) ?? byId.get(`node:${under}`)].filter((item): item is typeof instances[number] => !!item) : []; + if (under && sources.length === 0) throw new Error(`Unknown world spray source: ${under}`); + instance.spray = { ...instance.spray, sources }; + } + } + const visiting = new Set(), visited = new Set(); + function verifyAnchors(instance: Instance) { + if (visiting.has(instance)) throw new Error("World motion anchors cannot form a cycle"); + if (visited.has(instance)) return; + visiting.add(instance); + for (const anchor of instance.pulse?.anchors ?? instance.spray?.sources ?? []) verifyAnchors(anchor); + visiting.delete(instance); visited.add(instance); + } + instances.forEach(verifyAnchors); + if (scene.counter && !instances.some(instance => instance.voxel?.options.id === scene.counter!.source)) throw new Error(`Unknown world counter source: ${scene.counter.source}`); + const order = (instance: Instance) => instance.custom ? `1:${instance.custom.wgsl}:${instance.custom.vertexCount}` + : instance.sdf ? `2:${instance.sdf.wgsl}` : `0:${instance.kind}`; + instances.sort((a, b) => order(a).localeCompare(order(b))); const fields: [number, Animated][] = [[9, scene.bloom?.intensity ?? 0], [10, scene.bloom?.threshold ?? 1], [11, scene.bloom?.radius ?? 0.8], - [12, scene.fog?.density ?? 0], [13, scene.fog?.height ?? 0], [14, scene.fog?.falloff ?? 0.15], [15, scene.vignette ?? 0.25], [16, scene.grain ?? 0.015]]; + [12, scene.fog?.density ?? 0], [13, scene.fog?.height ?? 0], [14, scene.fog?.falloff ?? 0.15], [15, scene.vignette ?? 0.25], [16, scene.grain ?? 0.015], + [33, scene.dof ? scene.dof.aperture ?? 0.6 : 0], [34, scene.dof?.maxBlur ?? 12], [36, scene.floor?.y ?? -5], [37, scene.floor?.roughness ?? 0.06], + [38, scene.floor?.reflectivity ?? 0.8], [39, scene.floor?.size ?? 40], [43, scene.floor?.metal ?? 0.8], [44, scene.chromatic ?? 0], + [45, scene.shafts?.intensity ?? 0], [46, scene.shafts?.density ?? 0.03], [47, scene.shafts?.anisotropy ?? 0.3], [51, scene.rim ? scene.rim.intensity ?? 0.8 : 0], [52, scene.ambient ?? 0.12]]; return { ...scene, instances, exposureTrack: compileTrack(scene.exposure ?? 1), fogTrack: compileTrack(scene.fog?.color ?? scene.backgroundColor ?? "#05060a", colorVector), - backgroundVector: colorVector(scene.backgroundColor ?? "#05060a"), postTracks: fields.map(([index, value]) => ({ index, track: compileTrack(value) })) }; + backgroundVector: colorVector(scene.backgroundColor ?? "#05060a"), postTracks: fields.map(([index, value]) => ({ index, track: compileTrack(value) })), + focusTrack: scene.dof?.focus === undefined ? undefined : compileTrack(scene.dof.focus), + colorTracks: [{ index: 40, track: compileTrack(scene.floor?.color ?? "#101827", colorVector) }, { index: 48, track: compileTrack(scene.rim?.color ?? "#ffffff", colorVector) }], + counterColor: scene.counter ? colorVector(scene.counter.color ?? "#ffffff") : undefined, + lightTracks: (scene.lights ?? []).map(light => ({ position: compileTrack(light.position ?? [0, 8, 0]), target: compileTrack(light.target ?? [0, 0, 0]), + color: compileTrack(light.color ?? "#ffffff", colorVector), intensity: compileTrack(light.intensity ?? 4), range: compileTrack(light.range ?? 25), + angle: compileTrack(light.angle ?? 0.45), spot: light.kind === "spot" })), + }; } diff --git a/packages/slides-effects/scripts/renderGraphBenchmark.ts b/packages/slides-effects/scripts/renderGraphBenchmark.ts index 0eb08a537..0cb3b88ce 100644 --- a/packages/slides-effects/scripts/renderGraphBenchmark.ts +++ b/packages/slides-effects/scripts/renderGraphBenchmark.ts @@ -9,6 +9,7 @@ export type BenchmarkOptions = { warmup?: number; frames?: number; repetitions?: number; + paced?: boolean; }; function summarize(values: number[]) { @@ -18,15 +19,16 @@ function summarize(values: number[]) { minMs: sorted[0], maxMs: sorted[sorted.length - 1], throughputFps: 1000 / median }; } -/** Fixed clock samples measure rendering without introducing a playback driver. */ -export async function benchmarkRenderGraph({ definition, uniformsAt, width, height, warmup = 30, frames = 120, repetitions = 3 }: BenchmarkOptions) { +/** Replay fixed clock samples, with optional browser cadence measurement. */ +export async function benchmarkRenderGraph({ definition, uniformsAt, width, height, warmup = 30, frames = 120, repetitions = 3, paced = false }: BenchmarkOptions) { if (![width, height, warmup, frames, repetitions].every(Number.isInteger) || width < 1 || height < 1 || warmup < 0 || frames < 1 || repetitions < 1) { throw new Error("Benchmark dimensions and sample counts must be positive integers"); } if (!navigator.gpu) throw new Error("WebGPU unavailable"); const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); if (!adapter) throw new Error("No WebGPU adapter"); - const timestamps = adapter.features.has("timestamp-query"); + const timestampSupported = adapter.features.has("timestamp-query"); + const timestamps = timestampSupported && !paced; const device = await adapter.requestDevice({ requiredFeatures: timestamps ? ["timestamp-query"] : [] }); const failures: string[] = []; device.addEventListener("uncapturederror", event => failures.push(event.error.message)); @@ -46,13 +48,17 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig const queryBytes = passCount * 16; const resolve = query ? device.createBuffer({ size: queryBytes, usage: GPUBufferUsage.QUERY_RESOLVE | GPUBufferUsage.COPY_SRC }) : undefined; const readback = query ? device.createBuffer({ size: queryBytes, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ }) : undefined; - const samples: { repetition: number; frame: number; seconds: number; completedMs: number; gpuMs?: number; passes?: Record }[] = []; + const samples: { repetition: number; frame: number; seconds: number; completedMs: number; intervalMs?: number; gpuMs?: number; passes?: Record }[] = []; try { device.pushErrorScope("validation"); const graph = await prepareRenderGraph(device, definition, format, clock, uniforms); const setupError = await device.popErrorScope(); if (setupError) throw new Error(setupError.message); + let previousFrame: number | undefined; async function render(frame: number, timed: boolean) { + const now = paced ? await new Promise(resolve => requestAnimationFrame(resolve)) : undefined; + const intervalMs = now === undefined || previousFrame === undefined ? undefined : now - previousFrame; + previousFrame = now; const seconds = 2 + frame / 60; const start = performance.now(); const values = uniformsAt(seconds); @@ -89,7 +95,7 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig readback.unmap(); } if (failures.length) throw new Error(failures.join("\n")); - return { seconds, completedMs, ...(gpuMs === undefined ? {} : { gpuMs, passes }) }; + return { seconds, completedMs, ...(intervalMs === undefined ? {} : { intervalMs }), ...(gpuMs === undefined ? {} : { gpuMs, passes }) }; } for (let frame = 0; frame < warmup; frame++) await render(frame, false); for (let repetition = 0; repetition < repetitions; repetition++) { @@ -99,14 +105,20 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig await render(90, false); const png = canvas.toDataURL("image/png"); const info = adapter.info; + const intervals = samples.flatMap(sample => sample.intervalMs === undefined ? [] : [sample.intervalMs]); return { environment: { browser: navigator.userAgent, adapter: { vendor: info.vendor, architecture: info.architecture, device: info.device, description: info.description }, - timestampQuery: timestamps, width, height, warmup, frames, repetitions, devicePixelRatio, visibility: document.visibilityState }, - methodology: "Unpaced fixed-clock samples; completedMs includes uniform evaluation, writes, encode, submit and queue completion. GPU frame span is the last pass end minus the first pass beginning; pass scopes may overlap and are non-additive. GPU timestamps exclude CPU and presentation. Throughput is not display-paced FPS.", + timestampQuery: timestamps, timestampQuerySupported: timestampSupported, paced, width, height, warmup, frames, repetitions, devicePixelRatio, + viewport: { width: innerWidth, height: innerHeight }, visibility: document.visibilityState }, + methodology: `${paced ? "rAF-paced" : "Unpaced"} fixed-clock samples; completedMs includes uniform evaluation, writes, encode, submit and queue completion. GPU frame span is the last pass end minus the first pass beginning; pass scopes may overlap and are non-additive. GPU timestamps exclude CPU and presentation and are disabled in paced runs to avoid readback overhead. rAF cadence is browser callback cadence, not evidence of physical display presentation.`, pipeline: { targets: definition.targets, passes: definition.passes.map(pass => ({ id: pass.id, kind: pass.kind })) }, completed: summarize(samples.map(sample => sample.completedMs)), gpu: timestamps ? summarize(samples.map(sample => sample.gpuMs!)) : null, passes: timestamps ? Object.fromEntries(definition.passes.map(pass => [pass.id, summarize(samples.map(sample => sample.passes![pass.id]))])) : null, + cadence: intervals.length ? { ...summarize(intervals), intervalCount: intervals.length, + observedFps: intervals.length * 1000 / intervals.reduce((sum, ms) => sum + ms, 0), + longIntervalsOver25Ms: intervals.filter(ms => ms > 25).length, + estimatedMissed60HzFrames: intervals.reduce((sum, ms) => sum + Math.max(0, Math.round(ms / (1000 / 60)) - 1), 0) } : null, repetitions: Array.from({ length: repetitions }, (_, repetition) => ({ repetition, completed: summarize(samples.filter(sample => sample.repetition === repetition).map(sample => sample.completedMs)), gpu: timestamps ? summarize(samples.filter(sample => sample.repetition === repetition).map(sample => sample.gpuMs!)) : null })), diff --git a/packages/slides-effects/scripts/world-benchmark.mjs b/packages/slides-effects/scripts/world-benchmark.mjs index b58649341..86ebf8853 100644 --- a/packages/slides-effects/scripts/world-benchmark.mjs +++ b/packages/slides-effects/scripts/world-benchmark.mjs @@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url"; const args = process.argv.slice(2); if (args.includes("--help") || args.length !== 2 || args[0] !== "--output") { - console.log("Usage: bun packages/slides-effects/scripts/world-benchmark.mjs --output /absolute/path/harness.html\nGenerates a standalone offline WebGPU measurement page. In Chrome, call await benchmarkWorld({width:1920,height:1080,quality:'high'}).\nResult includes raw samples, per-pass GPU timestamps when available, completed-submit wall times and a PNG data URL.\nGenerate and launch the browser through the heavy gate; call the measurement through the fps gate. Keep generated pages/results outside the repository."); + console.log("Usage: bun packages/slides-effects/scripts/world-benchmark.mjs --output /absolute/path/harness.html\nGenerates a standalone offline WebGPU measurement page. In Chrome, call await benchmarkWorld({width:1920,height:1080,quality:'high'}). Add allEffects:true for the full scene or paced:true for rAF interval measurements (without GPU timestamp readback).\nResult includes raw samples, per-pass GPU timestamps when available, completed-submit wall times and a PNG data URL.\nGenerate and launch the browser through the heavy gate; call the measurement through the fps gate. Keep generated pages/results outside the repository."); process.exit(args.includes("--help") ? 0 : 2); } const output = resolve(args[1]); diff --git a/packages/slides-effects/scripts/worldBenchmark.ts b/packages/slides-effects/scripts/worldBenchmark.ts index 83a80775b..59cc92fb8 100644 --- a/packages/slides-effects/scripts/worldBenchmark.ts +++ b/packages/slides-effects/scripts/worldBenchmark.ts @@ -1,22 +1,54 @@ -import { graphInstances, orbInstances, prepareWorldScene, primitive, sampleWorldUniforms } from "../packs/world/scene"; +import { customInstances, graphInstances, orbInstances, prepareWorldScene, primitive, pulseInstances, rowInstances, sampleWorldUniforms, sdfInstance, sprayInstances, voxelInstances, type WorldScene } from "../packs/world/scene"; import { tree, wave } from "../packs/world/graph"; import { buildWorldGraph } from "../packs/world/render"; -import { benchmarkRenderGraph } from "./renderGraphBenchmark"; +import { benchmarkRenderGraph, type BenchmarkOptions } from "./renderGraphBenchmark"; -type Options = { width: number; height: number; quality: "low" | "medium" | "high"; frames?: number; repetitions?: number; warmup?: number }; +type Options = Omit & { quality: NonNullable; allEffects?: boolean }; export async function benchmarkWorld(options: Options) { const data = tree("root", Array.from({ length: 3 }, (_, branch) => [ `branch-${branch}`, Array.from({ length: 3 }, (_, group) => [`group-${branch}-${group}`, Array.from({ length: 3 }, (_, leaf) => `leaf-${branch}-${group}-${leaf}`)]), ]) as Parameters[1]); + const effectSamples = options.quality === "low" ? 8 : options.quality === "medium" ? 12 : 16; + const effects: Partial = options.allEffects ? { + floor: { mirror: true, y: -5.5, roughness: 0.08, reflectivity: 0.8, size: 40, scale: 0.5 }, + dof: { focus: "bench-pulse", aperture: 0.75, maxBlur: 12, scale: 0.5, samples: effectSamples }, + shafts: { intensity: 0.6, density: 0.03, anisotropy: 0.3, scale: 0.25, samples: effectSamples }, chromatic: 0.0025, + rim: { color: "#b56bff", intensity: 1.1 }, + lights: [ + { kind: "spot", position: [-4, 7, 3], target: [0, -4, 0], color: "#22d3ee", intensity: 18, range: 40, angle: 0.6 }, + { kind: "point", position: [5, 2, 2], color: "#b56bff", intensity: 8, range: 28 }, + ], + counter: { source: "bench-voxels", position: [0.86, 0.14], size: 0.065, color: "#22d3ee" }, + passes: [{ after: "bloom", wgsl: `fn customPass(uv:vec2f,color:vec4f,ctx:PassContext)->vec4f { + return vec4f(color.rgb*(.98+.02*sin(uv.y*800.+ctx.slideTime*.3)),color.a); + }` }], + } : {}; const scene = prepareWorldScene({ + ...effects, quality: options.quality, camera: { shots: [{ at: 0, orbit: { center: [0, -0.5, 0], radius: 22, height: 8, speed: 0.06 }, fov: 42 }] }, instances: [ ...orbInstances({ count: 64, seed: 7, spread: [18, 9, 10], colors: ["#22d3ee", "#b56bff"], emissive: 5 }), ...graphInstances({ data, node: { color: "#b56bff", size: 0.27 }, edge: { color: "#22d3ee", curve: 0.4, glow: 2 }, activate: [wave({ from: "root", step: 1, speed: 0.35 })] }), primitive("ring", { size: 8, position: [0, -5, 0], color: "#22d3ee", emissive: 5 }), + ...(options.allEffects ? [ + ...pulseInstances({ id: "bench-pulse", path: ["root", "branch-0", "group-0-0", "leaf-0-0-0"], step: 1, duration: 8, trail: 0.5, color: "#22d3ee", size: 0.12 }), + ...sprayInstances({ id: "bench-spray", under: "leaves:lit", rate: 24, lifetime: 2, seed: 11, color: "#b56bff" }), + ...rowInstances({ id: "bench-rows", count: 8, position: [-7, -1, 0], width: 2.4, spacing: 0.6, speed: 0.8, changes: [{ row: 2, step: 1 }] }), + ...voxelInstances({ id: "bench-voxels", grid: [4, 4, 4], position: [7, -1, 0], spacing: 0.45, size: 0.18, color: "#b56bff", collapse: { step: 1, to: 0.2, duration: 6 } }), + ...customInstances({ id: "bench-custom", count: 2, vertexCount: 6, position: [0, -1, 3], size: 0.5, wgsl: ` + fn shape(id:u32,vid:u32,ctx:ShapeContext)->ShapeVertex { + let corners=array(vec2f(-1.,-1.),vec2f(1.,-1.),vec2f(-1.,1.),vec2f(-1.,1.),vec2f(1.,-1.),vec2f(1.,1.)); + let p=corners[vid%6u]; + return ShapeVertex(vec3f(p.x+(f32(id)-.5)*3.,p.y+sin(ctx.slideTime)*.4,0.),vec3f(0.,0.,1.)); + } + fn shade(id:u32,world:vec3f,normal:vec3f,ctx:ShapeContext)->Material { + return Material(vec3f(.03,.3,.4),vec3f(.03,.7,.9)*2.,.4,.1); + }` }), + sdfInstance({ id: "bench-sdf", position: [-3, 0, 3], size: 0.7, color: "#22d3ee", bounds: [[-1, -1, -1], [1, 1, 1]], wgsl: "fn sdf(p:vec3f)->f32 { return length(p)-.7; }" }), + ] : []), ], bloom: { intensity: 1.4, threshold: 1, radius: 0.8 }, fog: { density: 0.025, color: "#05060a" }, vignette: 0.3, grain: 0.015, exposure: 1.1, @@ -24,6 +56,10 @@ export async function benchmarkWorld(options: Options) { const result = await benchmarkRenderGraph({ ...options, definition: buildWorldGraph(scene), uniformsAt: seconds => sampleWorldUniforms(scene, { slideTime: seconds, stepTime: seconds, stepIndex: 1, stepOrigins: { 0: 0, 1: 0 } }) }); return { ...result, workload: { quality: options.quality, orbs: 64, graphNodes: 40, graphEdges: 39, rings: 1, + allEffects: !!options.allEffects, instanceCount: scene.instances.length, + effects: options.allEffects ? { reflection: { sceneScaleFactor: scene.floor?.scale }, dof: { sceneScaleFactor: scene.dof?.scale, samples: scene.dof?.samples }, + shafts: { sceneScaleFactor: scene.shafts?.scale, samples: scene.shafts?.samples }, chromatic: 0.0025, lights: 2, + pulseInstances: 2, sprayParticles: 48, rows: 8, voxels: 64, customQuads: 2, sdfObjects: 1, customPost: true } : null, clock: "2 + frame / 60 seconds, step 1 at time zero; each repetition replays the same samples", seed: 7 } }; } diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index 01d645e69..a4239675a 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -1,8 +1,10 @@ import { describe, expect, test } from "bun:test"; -import { evaluateCamera, type CameraOptions, type ClockSample } from "../packs/world/camera"; +import { evaluateCamera, flyTo, type CameraOptions, type ClockSample } from "../packs/world/camera"; import { activationAt, layoutTree, subset, tree, wave } from "../packs/world/graph"; import { buildWorldGraph } from "../packs/world/render"; -import { compileTrack, graphInstances, instancePosition, orbInstances, prepareWorldScene, primitive, sampleTrack, sampleWorldCamera, sampleWorldUniforms, worldHeaderSize, worldInstanceSize } from "../packs/world/scene"; +import { compileTrack, customInstances, graphInstances, instanceOpacity, instancePosition, orbInstances, prepareWorldScene, primitive, pulseInstances, + resolveAnchor, rowInstances, rowLogicalIndex, sampleTrack, sampleWorldCamera, sampleWorldUniforms, sdfInstance, sprayInstances, voxelInstances, worldHeaderSize, worldInstanceSize } from "../packs/world/scene"; +import { advancePlayback, createPlaybackState, transitionPlayback } from "../../presentation/src/playbackState"; function clock(time: number, step = 0, origins: Record = { 0: 0 }): ClockSample { return { slideTime: time, stepTime: time - (origins[step] ?? 0), stepIndex: step, stepOrigins: origins }; @@ -155,3 +157,151 @@ test("HDR scene and five-level half-resolution bloom describe independent attach expect(() => prepareWorldScene({ instances: [...orbInstances({ count: 1 }), ...orbInstances({ count: 1 })] })).toThrow("id is duplicated"); expect(() => prepareWorldScene({ instances: [...orbInstances({ id: "near", count: 1 }), ...orbInstances({ id: "far", count: 1 })] })).not.toThrow(); }); + +test("pulse geometry, camera follow and focus share the exact curved path across clock transitions", () => { + const scene = prepareWorldScene({ instances: [...graphInstances({ data: tree("root", [["a", ["leaf"]]]) }), + ...pulseInstances({ id: "pulse", path: ["root", "a", "leaf"], step: 1, duration: 4 })], + camera: { shots: [{ step: 0, position: [0, 4, 20] }, { step: 1, follow: "pulse", dolly: 6, duration: 0 }] }, dof: { focus: "pulse" } }); + const pulse = scene.instances.find(instance => instance.id === "pulse")!; + let state = createPlaybackState({ phase: "playing", slideKey: "pulse", stepKey: "1", stepIndex: 1 }); + state = advancePlayback(advancePlayback(state, 0), 1000); + const position = instancePosition(pulse, state), camera = sampleWorldCamera(scene, state), uniforms = sampleWorldUniforms(scene, state); + expect(camera.target).toEqual(position); + expect(uniforms[32]).toBeCloseTo(6); + const source = resolveAnchor(scene, "root", state)!, target = resolveAnchor(scene, "a", state)!; + expect(position[0]).toBeCloseTo((source[0] + target[0]) / 2); + expect(position[1]).toBeGreaterThan((source[1] + target[1]) / 2); + const trail = scene.instances.find(instance => instance.id === "pulse:trail")!; + const index = worldHeaderSize + scene.instances.indexOf(trail) * worldInstanceSize; + expect(uniforms.slice(index + 12, index + 15)).toEqual(position); + state = transitionPlayback(state, { ...state, phase: "paused" }); + state = advancePlayback(state, 10000); + expect(instancePosition(pulse, state)).toEqual(position); + const jump = createPlaybackState({ phase: "playing", slideKey: "jump", stepKey: "3", stepIndex: 3 }); + expect(instanceOpacity(pulse, jump)).toBe(0); + expect(instancePosition(pulse, jump)).toEqual(resolveAnchor(scene, "leaf", jump)!); + expect(() => prepareWorldScene({ instances: pulseInstances({ id: "bad", path: ["bad", [0, 0, 0]] }) })).toThrow("cycle"); +}); + +test("recycling transfers a flash with logical row identity and never to a reused slot", () => { + const scene = prepareWorldScene({ instances: rowInstances({ id: "rows", count: 4, speed: 1, spacing: 1, changes: [{ row: 1, step: 1 }], flashDuration: 2 }), + dof: { focus: "rows:row:1" }, camera: { shots: [{ position: [0, 0, 10], target: [0, 0, 0] }] } }); + const before = clock(0.9, 1, { 1: 0.8 }), after = clock(1.1, 1, { 1: 0.8 }); + const first = scene.instances[0], second = scene.instances[1]; + expect(rowLogicalIndex(first, before)).toBe(0); expect(rowLogicalIndex(second, before)).toBe(1); + expect(rowLogicalIndex(first, after)).toBe(1); expect(rowLogicalIndex(second, after)).toBe(2); + const brightness = (slot: number, sample: ClockSample) => sampleWorldUniforms(scene, sample)[worldHeaderSize + slot * worldInstanceSize + 7]; + expect(brightness(1, before)).toBeGreaterThan(1); + expect(brightness(0, before)).toBe(0.1); + expect(brightness(0, after)).toBeGreaterThan(1); + expect(brightness(1, after)).toBe(0.1); + expect(resolveAnchor(scene, "rows:row:1", after)).toEqual(instancePosition(first, after)); + expect(sampleWorldUniforms(scene, after)[32]).toBe(10); + expect(brightness(0, clock(0, 3, { 3: 0 }))).toBe(0.1); + let state = createPlaybackState({ phase: "playing", slideKey: "rows", stepKey: "1", stepIndex: 1 }); + state = advancePlayback(advancePlayback(state, 0), 990); + const departing = instanceOpacity(first, state), retained = instanceOpacity(second, state); + const retainedPosition = instancePosition(second, state); + state = advancePlayback(state, 1010); + expect(departing).toBeCloseTo(0.01); + expect(retained).toBe(1); + expect(instanceOpacity(first, state)).toBeCloseTo(0.99); + expect(instanceOpacity(scene.instances.at(-1)!, state)).toBeCloseTo(0.01); + expect(instancePosition(first, state)[1] - retainedPosition[1]).toBeCloseTo(0.02); + expect(brightness(0, state)).toBeGreaterThan(1); +}); + +test("voxel compaction keeps topology fixed and its counter matches the surviving cells", () => { + const scene = prepareWorldScene({ instances: voxelInstances({ id: "cells", grid: [8, 8, 8], collapse: { step: 1, to: 0.2, duration: 2 } }), counter: { source: "cells" } }); + const graph = buildWorldGraph(scene); + const initial = sampleWorldUniforms(scene, clock(0)), middle = sampleWorldUniforms(scene, clock(1, 1, { 1: 0 })), final = sampleWorldUniforms(scene, clock(2, 1, { 1: 0 })); + expect(initial[56]).toBe(512); expect(middle[56]).toBe(307); expect(final[56]).toBe(102); + expect(scene.instances.reduce((sum, instance) => sum + instanceOpacity(instance, clock(2, 1, { 1: 0 })), 0)).toBe(102); + expect(graph.passes.filter(pass => pass.kind === "render")[0].instanceCount).toBe(512); + expect(final.length).toBe(graph.uniformsSize); + expect(sampleWorldUniforms(scene, clock(0, 3, { 3: 0 }))[56]).toBe(102); + expect(sampleWorldUniforms(scene, clock(0))[56]).toBe(512); +}); + +test("removed voxels shrink in place, including collapse to zero and backwards sampling", () => { + const scene = prepareWorldScene({ instances: voxelInstances({ id: "cells", grid: [5, 1, 1], collapse: { step: 1, to: 0.2, duration: 2, ease: "linear" } }), counter: { source: "cells" } }); + let state = createPlaybackState({ phase: "playing", slideKey: "shrink", stepKey: "1", stepIndex: 1 }); + const removed = scene.instances[3], position = instancePosition(removed, state); + state = advancePlayback(advancePlayback(state, 0), 750); + expect(instancePosition(removed, state)).toEqual(position); + expect(instanceOpacity(removed, state)).toBeCloseTo(0.5); + expect(sampleWorldUniforms(scene, state)[worldHeaderSize + 3 * worldInstanceSize + 3]).toBeCloseTo(0.065); + state = advancePlayback(state, 2000); + expect(instancePosition(removed, state)).toEqual(position); + expect(instanceOpacity(removed, state)).toBe(0); + expect(instanceOpacity(removed, clock(0))).toBe(1); + const empty = prepareWorldScene({ instances: voxelInstances({ id: "empty", grid: [2, 2, 2], collapse: { step: 1, to: 0, duration: 1 } }), counter: { source: "empty" } }); + const initial = empty.instances.map(instance => instancePosition(instance, clock(0))); + const finished = clock(1, 1, { 1: 0 }); + expect(sampleWorldUniforms(empty, finished)[56]).toBe(0); + expect(empty.instances.every(instance => instanceOpacity(instance, finished) === 0)).toBe(true); + expect(empty.instances.map(instance => instancePosition(instance, finished))).toEqual(initial); +}); + +test("stages, material references, light direction and HDR post settings share animated uniforms", () => { + const stage = { id: "b", position: compileTrack<[number, number, number]>({ 0: [10, 0, 0], 1: [12, 0, 0] }), glow: compileTrack({ 0: 1, 1: 0.5 }) }; + const scene = prepareWorldScene({ instances: [{ ...primitive("tube", { id: "tube", position: [0, 0, 0], end: [2, 0, 0], material: "stone", emissive: 4 }), stages: [stage] }], + stages: [{ stage, parents: [] }], materials: { stone: { albedo: "#ffffff", roughness: 0.7, metal: 0.2, rim: 0.9 } }, + camera: { shots: [flyTo("b", { step: 0, offset: [0, 0, 10] })] }, + dof: { focus: "stage:b", aperture: 0.8 }, floor: { mirror: true, y: -3 }, shafts: { intensity: 0.7 }, chromatic: 0.003, + lights: [{ kind: "spot", position: [0, 5, 0], target: [0, 0, 0], angle: 0.4 }], rim: { color: "#22d3ee", intensity: 2 } }); + const sample = clock(2, 1, { 0: 0, 1: 1 }), values = sampleWorldUniforms(scene, sample); + expect(resolveAnchor(scene, "stage:b", sample)).toEqual([12, 0, 0]); + expect(sampleWorldCamera(scene, sample).position).toEqual([12, 0, 10]); + expect(values.slice(worldHeaderSize + 12, worldHeaderSize + 15)).toEqual([14, 0, 0]); + expect(values[worldHeaderSize + 7]).toBe(2); + expect(values.slice(worldHeaderSize + 24, worldHeaderSize + 27)).toEqual([1, 1, 1]); + expect(values[worldHeaderSize + 27]).toBe(0.7); expect(values[worldHeaderSize + 28]).toBe(0.2); + expect(values[32]).toBe(10); expect(values[33]).toBe(0.8); expect(values[36]).toBe(-3); expect(values[44]).toBe(0.003); + expect(values[51]).toBe(2); + expect(sampleWorldUniforms(prepareWorldScene({ instances: [] }), sample)[51]).toBe(0); + expect(values.slice(72, 75)).toEqual([0, -1, 0]); expect(values[75]).toBeCloseTo(Math.cos(0.4)); + const named = prepareWorldScene({ instances: [primitive("sphere", { id: "b", position: [1, 2, 3] })], stages: scene.stages }); + expect(resolveAnchor(named, "b", sample)).toEqual([1, 2, 3]); + expect(resolveAnchor(named, "stage:b", sample)).toEqual([12, 0, 0]); +}); + +test("spray sampling is stateless and gates against lit leaves", () => { + const scene = prepareWorldScene({ instances: [...graphInstances({ data: tree("root", ["leaf"]), activate: [wave({ from: "root", step: 1 })] }), + ...sprayInstances({ id: "spray", under: "leaves:lit", rate: 4, lifetime: 2 })] }); + const particles = scene.instances.filter(instance => instance.spray); + expect(particles).toHaveLength(8); + expect(particles.every(instance => instanceOpacity(instance, clock(0)) === 0)).toBe(true); + expect(particles.some(instance => instanceOpacity(instance, clock(2, 1, { 1: 0 })) > 0)).toBe(true); + const positions = particles.map(instance => instancePosition(instance, clock(2, 1, { 1: 0 }))); + particles.forEach(instance => instancePosition(instance, clock(10, 3, { 3: 0 }))); + expect(particles.map(instance => instancePosition(instance, clock(2, 1, { 1: 0 })))).toEqual(positions); +}); + +test("explicit materials and rim lights preserve dim graph nodes until activation", () => { + const scene = prepareWorldScene({ instances: graphInstances({ data: tree("root"), nodes: { root: { material: "bright" } }, activate: [wave({ from: "root", step: 1 })] }), + materials: { bright: { albedo: "#ffffff", rim: 2 } }, rim: { intensity: 3 } }); + const dim = scene.instances[0].graph!.dim; + let state = createPlaybackState({ phase: "playing", slideKey: "material", stepKey: "0", stepIndex: 0 }); + const dark = sampleWorldUniforms(scene, state); + expect(dark.slice(worldHeaderSize + 24, worldHeaderSize + 27)).toEqual(dim); + expect(dark[worldHeaderSize + 7]).toBe(0); + expect(dark[worldHeaderSize + 29]).toBe(0); expect(dark[worldHeaderSize + 30]).toBe(0); + state = transitionPlayback(state, { ...state, stepKey: "1", stepIndex: 1 }); + state = advancePlayback(advancePlayback(state, 0), 1000); + const lit = sampleWorldUniforms(scene, state); + expect(lit.slice(worldHeaderSize + 24, worldHeaderSize + 27)).toEqual([1, 1, 1]); + expect(lit[worldHeaderSize + 29]).toBe(2); expect(lit[worldHeaderSize + 30]).toBe(1); + expect(sampleWorldUniforms(scene, clock(0)).slice(worldHeaderSize + 24, worldHeaderSize + 27)).toEqual(dim); +}); + +test("custom and bounded geometry keep their own draw groups and validate resources before setup", () => { + const a = customInstances({ wgsl: "shape-a", count: 2, vertexCount: 6 }), b = customInstances({ wgsl: "shape-a", count: 3, vertexCount: 6 }); + expect(a[0].custom).toBe(a[1].custom); expect(a[0].custom).not.toBe(b[0].custom); + const scene = prepareWorldScene({ instances: [...a, primitive("sphere"), ...b, sdfInstance({ wgsl: "distance", bounds: [[-1, -1, -1], [1, 1, 1]] })] }); + expect(scene.instances[0].kind).toBe("sphere"); + expect(scene.instances.at(-1)?.sdf).toBeDefined(); + expect(() => prepareWorldScene({ instances: [primitive("box", { material: "missing" })] })).toThrow("material"); + expect(() => prepareWorldScene({ instances: [], dof: { scale: 0 } })).toThrow("scale"); + expect(() => prepareWorldScene({ instances: [], lights: Array.from({ length: 5 }, () => ({})) })).toThrow("four"); +}); diff --git a/packages/slides-effects/tests/worldPlayback.test.tsx b/packages/slides-effects/tests/worldPlayback.test.tsx index 5c7c1a02e..6c4ab3af5 100644 --- a/packages/slides-effects/tests/worldPlayback.test.tsx +++ b/packages/slides-effects/tests/worldPlayback.test.tsx @@ -10,6 +10,7 @@ import path from "node:path"; import { createRequire } from "node:module"; import { createPlaybackState, advancePlayback } from "../../presentation/src/playbackState"; import { samplePlayback } from "../../presentation/src/playback"; +import { worldHeaderSize } from "../packs/world/scene"; const playback = { value: createPlaybackState({ phase: "playing", slideKey: "a", stepKey: "0", stepIndex: 0 }) }; mock.module("@legend-apps/presentation", () => ({ @@ -84,7 +85,7 @@ test("equal-content parent renders and resize retain pipeline definitions while expect(graph().targets).toBe(targets); await act(() => tree.update(content(1000, 4))); expect(graph().passes).toBe(passes); - expect(graph().uniforms.value[32]).toBe(4); + expect(graph().uniforms.value[worldHeaderSize]).toBe(4); expect(graph().width).toBe(1000); await act(() => tree.update(content(1000, 4, "low"))); expect(graph().passes).not.toBe(passes); @@ -96,7 +97,7 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure try { const { compileDeck } = await import("../../presentation/src/compiler"); const deck = path.join(directory, "world.mdx"); - fs.writeFileSync(deck, 'export { World, sampleWorldUniforms, prepareWorldScene, graphInstances, tree, wave } from "slides:world/World"\n\n# World'); + fs.writeFileSync(deck, 'export { World, sampleWorldUniforms, prepareWorldScene, graphInstances, pulseInstances, sprayInstances, rowInstances, voxelInstances, flyTo, tree, wave } from "slides:world/World"\n\n# World'); const compiled = await compileDeck(deck, { effectPacksDirectory: path.resolve(import.meta.dirname, "../packs") }); if (!compiled.success) throw new Error(compiled.errors.join("\n")); const require = createRequire(import.meta.url); @@ -135,15 +136,25 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure const api = module.exports; expect(api.sampleWorldUniforms.__workletHash).toBeNumber(); const sample = restore(api.sampleWorldUniforms); + const stage = { id: "zone", position: { initial: [5, 0, 0], frames: [] }, glow: { initial: 0.5, frames: [] } }; const scene = api.prepareWorldScene({ - instances: api.graphInstances({ data: api.tree("root", [["a", ["a1", "a2"]]]), activate: [api.wave({ from: "root", step: 1 })] }), - camera: { shots: [{ step: 0, position: [0, 4, 20] }, { step: 1, orbit: { center: [0, 0, 0], radius: 10 } }, { step: 2, follow: "node:a", dolly: 5 }] }, + instances: [ + ...api.graphInstances({ data: api.tree("root", [["a", ["a1", "a2"]]]), activate: [api.wave({ from: "root", step: 1 })] }).map(instance => ({ ...instance, stages: [stage] })), + ...api.pulseInstances({ id: "pulse", path: ["root", "a", "a1"], step: 1, duration: 4 }), + ...api.sprayInstances({ id: "mist", under: "leaves:lit", rate: 4, lifetime: 2 }), + ...api.rowInstances({ id: "rows", count: 4, changes: [{ row: 2, step: 2 }], speed: 1 }).map(instance => ({ ...instance, stages: [stage] })), + ...api.voxelInstances({ id: "voxels", grid: [2, 2, 2], collapse: { step: 1, to: 0.2, duration: 2 } }), + ], + stages: [{ stage, parents: [] }], counter: { source: "voxels" }, dof: {}, + camera: { shots: [{ step: 0, position: [0, 4, 20] }, { step: 1, follow: "pulse", dolly: 5, focus: "pulse" }, + { step: 2, follow: "rows:row:2", dolly: 7, focus: "rows:row:2" }, api.flyTo("zone", { step: 3, arc: 3 })] }, exposure: { 0: 1, 1: 1.5, 2: 0.8 }, bloom: { intensity: 1.4 }, }); for (const clock of [ { slideTime: 0, stepTime: 0, stepIndex: 0, stepOrigins: { 0: 0 } }, { slideTime: 3, stepTime: 1, stepIndex: 1, stepOrigins: { 0: 0, 1: 2 } }, { slideTime: 1, stepTime: 1, stepIndex: 2, stepOrigins: { 2: 0 } }, + { slideTime: 0.5, stepTime: 0.5, stepIndex: 3, stepOrigins: { 3: 0 } }, ]) expect(sample(scene, clock)).toEqual(api.sampleWorldUniforms(scene, clock)); // Mapper inputs come from the serialized closure, including unread values. let renderer; diff --git a/packages/slides-effects/tests/worldRender.test.ts b/packages/slides-effects/tests/worldRender.test.ts new file mode 100644 index 000000000..3bebe1482 --- /dev/null +++ b/packages/slides-effects/tests/worldRender.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from "bun:test"; +import { buildWorldGraph, worldGraphTopology } from "../packs/world/render"; +import { customInstances, prepareWorldScene, primitive, rowInstances, sdfInstance, worldHeaderSize, worldInstanceSize, type WorldScene } from "../packs/world/scene"; + +const quad = ` +fn shape(id:u32,vid:u32,ctx:ShapeContext)->ShapeVertex { + let p=array(vec2f(-1.,-1.),vec2f(1.,-1.),vec2f(-1.,1.),vec2f(-1.,1.),vec2f(1.,-1.),vec2f(1.,1.)); + return ShapeVertex(vec3f(p[vid],f32(id)),vec3f(0.,0.,1.)); +} +fn shade(id:u32,world:vec3f,normal:vec3f,ctx:ShapeContext)->Material { return Material(vec3f(.2,.5,.8),vec3f(.1),.4,0.); } +`; +const scanlines = "fn customPass(uv:vec2f,color:vec4f,ctx:PassContext)->vec4f { return color*vec4f(vec3f(.9+.1*sin(uv.y*200.)),1.); }"; +const scene = (options: Partial = {}) => prepareWorldScene({ instances: [primitive("sphere")], ...options }); + +for (const quality of ["low", "medium", "high"] as const) test(`all enabled post effects remain present at ${quality} quality`, () => { + const graph = buildWorldGraph(scene({ quality, floor: { mirror: true }, dof: { aperture: .7 }, shafts: { intensity: 1 }, lights: [{ position: [0, 8, 0] }] })); + const ids = graph.passes.map(pass => pass.id); + for (const id of ["reflection-sphere", "floor", "shafts", "dof-gather", "dof-composite", "bloom-composite", "tonemap"]) expect(ids).toContain(id); + const scale = quality === "low" ? .65 : quality === "medium" ? .85 : 1; + const target = (id: string) => graph.targets.find(target => target.id === id)!; + expect(target("reflect").scale).toBe(scale * .5); + expect(target("reflectDepth").scale).toBe(target("reflect").scale); + expect(target("dof").scale).toBe(scale * .5); + expect(target("shafts").scale).toBe(scale * .25); + expect(graph.uniformsSize).toBe(worldHeaderSize + worldInstanceSize); +}); + +test("animated floor, light and focus changes preserve topology while static quality controls change it", () => { + const a = scene({ dof: { focus: 12, aperture: .5 }, floor: { mirror: true, roughness: .1 }, shafts: { intensity: .5 }, lights: [{ intensity: 1 }] }); + const b = scene({ dof: { focus: { 0: 12, 2: 5 }, aperture: 1 }, floor: { mirror: true, roughness: .8 }, shafts: { intensity: 2 }, lights: [{ intensity: { 0: 1, 2: 4 } }] }); + expect(worldGraphTopology(a)).toEqual(worldGraphTopology(b)); + const graph = buildWorldGraph(scene({ dof: { scale: .25, samples: 7 }, floor: { mirror: true, scale: .3 }, shafts: { scale: .125, samples: 5 } })); + expect(graph.targets.find(target => target.id === "dof")?.scale).toBe(.25); + expect(graph.targets.find(target => target.id === "reflect")?.scale).toBe(.3); + expect(graph.targets.find(target => target.id === "shafts")?.scale).toBe(.125); +}); + +test("opaque floor clears the scene before translucent geometry loads its color and depth", () => { + const graph = buildWorldGraph(scene({ floor: { mirror: true }, instances: rowInstances({ id: "rows", count: 3 }) })); + const draws = graph.passes.filter(pass => pass.kind === "render"); + expect(draws.map(pass => pass.id)).toEqual(["reflection-box", "floor", "scene-box"]); + expect(draws[0].clear).not.toBe(false); + expect(draws[1].clear).toEqual(draws[0].clear); + expect(draws[2].clear).toBe(false); + const empty = buildWorldGraph(scene({ floor: {}, instances: [] })); + expect(empty.passes.filter(pass => pass.kind === "render").map(pass => pass.id)).toEqual(["floor"]); + const noFloor = buildWorldGraph(scene()); + expect(noFloor.passes.find(pass => pass.kind === "render")?.clear).not.toBe(false); +}); + +test("custom draw groups keep marker-local instance ids and SDF ranges outside primitive draws", () => { + const prepared = scene({ instances: [primitive("sphere"), ...customInstances({ wgsl: quad, vertexCount: 6, count: 3 }), + ...customInstances({ wgsl: quad, vertexCount: 6, count: 2 }), sdfInstance({ wgsl: "fn sdf(p:vec3f)->f32 { return length(p)-1.; }", bounds: [[-2, -2, -2], [2, 2, 2]] })] }); + const topology = worldGraphTopology(prepared); + expect(topology.counts).toEqual({ box: 0, ring: 0, sphere: 1, tube: 0 }); + expect(topology.customs?.map(draw => [draw.offset, draw.count])).toEqual([[1, 3], [4, 2]]); + expect(topology.sdfs?.map(draw => [draw.offset, draw.count])).toEqual([[6, 1]]); + const graph = buildWorldGraph(topology); + expect(graph.passes.filter(pass => pass.kind === "render").map(pass => pass.instanceCount)).toEqual([1, 3, 2, 1]); + expect(graph.uniformsSize).toBe(worldHeaderSize + 7 * worldInstanceSize); +}); + +test("postOrder rewires HDR stages and custom hooks retain the declared color-space boundary", () => { + const graph = buildWorldGraph(scene({ dof: {}, postOrder: ["bloom", "dof", "fog"], passes: [ + { id: "before-focus", wgsl: scanlines, before: "dof" }, { id: "ldr", wgsl: scanlines, after: "tonemap" }, + ] })); + const pass = (id: string) => graph.passes.find(pass => pass.id === id)!; + expect(pass("bloom-threshold").reads).toEqual(["hdr"]); + expect(pass("before-focus").reads?.[0]).toBe("bloom-composite"); + expect(pass("dof-gather").reads?.[0]).toBe("custom-post-0"); + expect(pass("fog").reads?.[0]).toBe("dof-composite"); + expect(pass("tonemap").kind).toBe("fullscreen"); + expect(pass("ldr").reads?.[0]).toBe("tonemapped"); + expect(pass("screen").reads).toEqual(["custom-post-1"]); + expect(graph.passes.filter(pass => pass.id === "tonemap")).toHaveLength(1); +}); + +test("invalid post stage permutations and placements fail rather than silently skipping hooks", () => { + for (const postOrder of [["fog", "fog", "bloom"], ["fog", "bloom"], ["fog", "dof", "unknown"]]) { + expect(() => buildWorldGraph({ quality: "high", counts: { box: 0, ring: 0, sphere: 0, tube: 0 }, backgroundVector: [0, 0, 0], postOrder })).toThrow("postOrder"); + } + for (const placement of [{ before: "scene" }, { after: "unknown" }, { before: "fog", after: "dof" }]) { + expect(() => buildWorldGraph(scene({ passes: [{ wgsl: scanlines, ...placement }] }))).toThrow("placement"); + } +}); + +test("custom ids cannot shadow built-in passes or another custom pass", () => { + for (const id of ["fog", "scene-sphere", "tonemap", "screen"]) { + expect(() => buildWorldGraph(scene({ passes: [{ id, wgsl: scanlines, after: "tonemap" }] }))).toThrow("pass id is duplicated"); + } + expect(() => buildWorldGraph(scene({ passes: [ + { id: "scanlines", wgsl: scanlines, after: "fog" }, { id: "scanlines", wgsl: scanlines, after: "bloom" }, + ] }))).toThrow("pass id is duplicated"); +}); From 5f1e5969bb5d32fa67acdd71f7dedf09510ce472 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 04:33:46 -0700 Subject: [PATCH 03/28] feat: add cinematic scenes and adaptive resolution --- apps/slides/examples/3d/cinematic-title.mdx | 82 ++++++++++++++++ apps/slides/src/TypeGPUShader.tsx | 22 +++-- .../src/__tests__/gpuRenderGraph.test.ts | 21 ++++ .../src/__tests__/typeGPUShader.test.ts | 35 ++++++- apps/slides/src/gpuRenderGraph.ts | 25 ++--- packages/presentation/src/GPUShader.tsx | 3 + packages/slides-effects/packs/world/README.md | 48 +++++++-- packages/slides-effects/packs/world/World.tsx | 27 +++++- packages/slides-effects/packs/world/post.ts | 12 ++- packages/slides-effects/packs/world/render.ts | 13 +++ .../slides-effects/packs/world/resolution.ts | 37 +++++++ packages/slides-effects/packs/world/scene.ts | 33 +++++-- .../scripts/renderGraphBenchmark.ts | 17 ++-- .../scripts/world-benchmark.mjs | 2 +- .../slides-effects/scripts/worldBenchmark.ts | 13 ++- packages/slides-effects/tests/world.test.ts | 33 ++++++- .../tests/worldPlayback.test.tsx | 68 ++++++++++++- .../slides-effects/tests/worldRender.test.ts | 27 +++++- .../tests/worldResolution.test.ts | 97 +++++++++++++++++++ 19 files changed, 547 insertions(+), 68 deletions(-) create mode 100644 apps/slides/examples/3d/cinematic-title.mdx create mode 100644 packages/slides-effects/packs/world/resolution.ts create mode 100644 packages/slides-effects/tests/worldResolution.test.ts diff --git a/apps/slides/examples/3d/cinematic-title.mdx b/apps/slides/examples/3d/cinematic-title.mdx new file mode 100644 index 000000000..970043bf0 --- /dev/null +++ b/apps/slides/examples/3d/cinematic-title.mdx @@ -0,0 +1,82 @@ +--- +title: Quiet Structures +width: 1920 +height: 1080 +transition: fade +theme: + backgroundColor: "#04060b" + color: "#f8fafc" +--- + +import { Background } from "@legend-apps/presentation" +import { Text } from "react-native" +import { World, Camera, Box, Ring, Orbs, Material, Floor, SpotLight, RimLight, Fog, Shafts, DOF, Bloom, Vignette, Grain } from "slides:world/World" + +export const structures = <> + + + + + + + + + + + + + + {structures} + + + + + + + + + + +# Quiet structures + +Light, air and a slow orbit + +--- + + + + + {structures} + + + + + + + + + + +# Through the air + +A quiet reveal + +--- + + + + + {structures} + + + + + + + + + + +# Room to breathe + +A wider view diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 78cb28a0c..943a4aa45 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -2,7 +2,7 @@ import { useContext, useEffect, useLayoutEffect, useRef, useState, type RefObjec import { AlphaType, ColorType, Skia } from "@shopify/react-native-skia"; import { PixelRatio, Platform, View } from "react-native"; import { Canvas, type CanvasRef, type RNCanvasContext } from "react-native-webgpu"; -import { makeMutable, runOnJS, runOnUI, useFrameCallback, useSharedValue, type SharedValue } from "react-native-reanimated"; +import { makeMutable, runOnJS, runOnUI, useAnimatedReaction, useFrameCallback, useSharedValue, type SharedValue } from "react-native-reanimated"; import { PresentationCanvasScaleContext, useBackgroundIntensity, useBackgroundSize, usePresentationValue, usePlayback, samplePlayback, usePreparationTask, type GPURenderGraphDefinition, type GPUShaderProps } from "@legend-apps/presentation"; import { prepareRenderGraph, encodeRenderGraph, type RenderGraphResources } from "./gpuRenderGraph"; import { reportSlideError } from "./slidesStore"; @@ -29,7 +29,7 @@ type Resources = { export function TypeGPUShader({ code, width: configuredWidth = 1696, height: configuredHeight = 650, background = false, particles = false, particleCount = 256, particleVertexCount = 6, particleBlend = "additive", previewTime = 4, clock: clockMode = "step", image, composite = false, frameData, frameDataSize = 0, borderRadius, - accessibilityLabel, active = true, children, maxTime, frameTimeIndex, renderGraph }: GPUShaderProps) { + accessibilityLabel, active = true, children, maxTime, frameTimeIndex, renderGraph, renderScale }: GPUShaderProps) { const size = useBackgroundSize(); const intensity = useBackgroundIntensity(); const displayScale = useContext(PresentationCanvasScaleContext); @@ -67,7 +67,7 @@ export function TypeGPUShader({ code, width: configuredWidth = 1696, height: con if (error) throw new Error(error); return - {resources && { setError(message); reportSlideError(message, errorContext.current.slideIndex, errorContext.current.isPreview); }} onReady={() => setPreparedResources(resources)} />} + {resources && { setError(message); reportSlideError(message, errorContext.current.slideIndex, errorContext.current.isPreview); }} onReady={() => setPreparedResources(resources)} />} {resources && children} ; } @@ -219,7 +219,7 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle return () => { cancelled = true; if (release) release(); else if (!transferred) device?.destroy(); }; } -function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, displayScale = 1, width, height, aspect, brightness, particleCount, particleVertexCount = 6, clockMode = "step", frameData, previewTime, onError, onReady }: { resources: Resources; effectActive?: boolean; maxTime?:number; frameTimeIndex?:number; displayScale?:number; width: number; height: number; aspect: number; brightness: number; particleCount: number; particleVertexCount?: number; clockMode?: "slide" | "step"; frameData?: SharedValue; previewTime: number; onError(message: string): void; onReady(): void }) { +function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, frameTimeIndex, displayScale = 1, width, height, aspect, brightness, particleCount, particleVertexCount = 6, clockMode = "step", frameData, previewTime, onError, onReady }: { resources: Resources; renderScale?: SharedValue; effectActive?: boolean; maxTime?:number; frameTimeIndex?:number; displayScale?:number; width: number; height: number; aspect: number; brightness: number; particleCount: number; particleVertexCount?: number; clockMode?: "slide" | "step"; frameData?: SharedValue; previewTime: number; onError(message: string): void; onReady(): void }) { const active = usePresentationValue("isActive"); const preview = usePresentationValue("isPreview"); const preparing = usePresentationValue("isPreparing"); @@ -243,10 +243,11 @@ function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, } simulationEpoch.value = { phase: state.phase, slideKey: state.slideKey, stepKey: state.stepKey }; const frameIndex = frozen ? Math.max(0, count.value - 1) : count.value; + const scale = renderScale?.value; try { const time = samplePlayback(state, previewTime, clockMode); device.queue.writeBuffer(clock, 0, new Float32Array([time, dt, frameIndex, state.stepIndex, aspect, brightness, samplePlayback(state, previewTime, "slide"), samplePlayback(state, previewTime, "step")])); - submitGPUFrame(r, frameIndex, particleCount, particleVertexCount, frameData); + submitGPUFrame(r, frameIndex, particleCount, particleVertexCount, frameData, scale); if (!frozen) count.value += 1; if (!rendered.value) { rendered.value = true; runOnJS(onReady)(); } } catch (caught) { @@ -254,6 +255,11 @@ function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, runOnJS(onError)(String(caught)); } }; + const scaleInputs = r.graph && renderScale ? { scale: renderScale, uniforms: frameData } : null; + // Order a frozen redraw after the uniform mapper has sampled the same scale. + useAnimatedReaction(() => scaleInputs ? { scale: scaleInputs.scale.value, uniforms: scaleInputs.uniforms?.value } : null, (snapshot, previous) => { + if (snapshot && previous && snapshot.scale !== previous.scale && rendered.value && playback.value.phase !== "playing") draw(0, true); + }); const frame = useFrameCallback(info => { if (!effectActive || playback.value.phase !== "playing") return; const state=playback.value, previous=simulationEpoch.value; @@ -275,7 +281,7 @@ function UIFrames({ resources: r, effectActive = true, maxTime, frameTimeIndex, else if (r.graph && rendered.value && (phase === "paused" || phase === "outgoing")) draw(0, true); })(); return () => frame.setActive(false); - }, [active, effectActive, maxTime, frameTimeIndex, preview, preparing, previewTime, step, frame, rendered, finished, brightness, playback, displayScale, width, height, clockMode, frameData, particleVertexCount]); + }, [active, effectActive, maxTime, frameTimeIndex, preview, preparing, previewTime, step, frame, rendered, finished, brightness, playback, displayScale, width, height, clockMode, frameData, renderScale, particleVertexCount]); useEffect(() => () => { // Queue destruction behind the final UI frame, never race a submitted callback. runOnUI(destroyGPU)(device, alive, disposed); @@ -321,14 +327,14 @@ async function loadImageTexture(device: GPUDevice, uri: string) { } finally { data.dispose(); } } -function submitGPUFrame(r: Resources, frame: number, particleCount: number, particleVertexCount: number, frameData?: SharedValue) { +function submitGPUFrame(r: Resources, frame: number, particleCount: number, particleVertexCount: number, frameData?: SharedValue, renderScale = 1) { "worklet"; const { device, context, clock, groups, compute, render, particles } = r; const data = frameData?.value; if (r.frameData && data) device.queue.writeBuffer(r.frameData, 0, new Float32Array(data!)); const encoder = device.createCommandEncoder(); if (r.graph) { - encodeRenderGraph(r.graph.value, device, encoder, context.getCurrentTexture()); + encodeRenderGraph(r.graph.value, device, encoder, context.getCurrentTexture(), renderScale); device.queue.submit([encoder.finish()]); context.present(); return; } diff --git a/apps/slides/src/__tests__/gpuRenderGraph.test.ts b/apps/slides/src/__tests__/gpuRenderGraph.test.ts index 21cb2af05..686a43706 100644 --- a/apps/slides/src/__tests__/gpuRenderGraph.test.ts +++ b/apps/slides/src/__tests__/gpuRenderGraph.test.ts @@ -79,6 +79,23 @@ test("graph rejects invalid references and shader diagnostics before a frame can expect(g.pipelines).toHaveLength(0); }); +test("global render scale resizes offscreen targets and dispatches without recompiling or scaling the screen", async () => { + const g = gpu(), r = await prepareRenderGraph(g.device, definition, "bgra8unorm", {}, {}); + const screen = { width: 1920, height: 1080, createView: () => "screen" }; + encodeRenderGraph(r, g.device, g.encoder, screen); + encodeRenderGraph(r, g.device, g.encoder, screen, .5); + expect(r.scale).toBe(.5); + expect(g.textures.slice(3).map(t => t.descriptor.size)).toEqual([[960, 540], [960, 540], [480, 270]]); + expect(g.textures.slice(0, 3).every(t => t.destroyed)).toBe(true); + expect(g.commands.filter(c => c[0] === "compute").at(-1)).toEqual(["compute", 120, 68]); + expect(g.commands.at(-2)[1].colorAttachments[0].view).toBe("screen"); + encodeRenderGraph(r, g.device, g.encoder, screen, .5); + expect(g.textures).toHaveLength(6); expect(g.groups).toHaveLength(8); expect(g.pipelines).toHaveLength(4); + encodeRenderGraph(r, g.device, g.encoder, { ...screen, width: 2, height: 2 }, .5); + expect(g.textures.at(-1).descriptor.size).toEqual([4, 4]); + for (const invalid of [0, -1, NaN, Infinity]) expect(() => encodeRenderGraph(r, g.device, g.encoder, screen, invalid)).toThrow("renderScale"); +}); + test("multipass encoding compiles for the host UI runtime", () => { const filename = fileURLToPath(new URL("../gpuRenderGraph.ts", import.meta.url)); @@ -132,5 +149,9 @@ test("native callback copies retain one mutable graph across draws and resizing" for (let frame = 0; frame < 6; frame++) submitGPUFrame(copy(resources), frame, 1, 6); expect(g.textures).toHaveLength(6); expect(g.groups).toHaveLength(8); expect(g.textures.slice(0, 3).every(texture => texture.destroyed)).toBe(true); + for (let frame = 0; frame < 6; frame++) submitGPUFrame(copy(resources), frame, 1, 6, undefined, .75); + expect(g.textures).toHaveLength(9); expect(g.groups).toHaveLength(12); + expect(resources.graph.value.scale).toBe(.75); + expect(g.textures.slice(3, 6).every(texture => texture.destroyed)).toBe(true); stop(); }); diff --git a/apps/slides/src/__tests__/typeGPUShader.test.ts b/apps/slides/src/__tests__/typeGPUShader.test.ts index ee01c331b..367007acb 100644 --- a/apps/slides/src/__tests__/typeGPUShader.test.ts +++ b/apps/slides/src/__tests__/typeGPUShader.test.ts @@ -14,10 +14,10 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, plugins: [require.resolve("react-native-worklets/plugin")], }); const effects = [], calls = [], uniforms = [], errors = []; - const frameData={value:[1,2,3,4]}; + const frameData={value:[1,2,3,4]}, renderScale={value:1}; const playback = { value: { phase: preparing ? "preparing" : preview ? "preview" : active ? "playing" : "paused", slideKey: "slide", stepKey: "step", stepIndex: 1, stepTime: 0, slideTime: 10 } }; - let callback, enabled = false, ready = 0; + let callback, reaction, reactionInputs, enabled = false, ready = 0; const frame = { setActive: value => { enabled = value; } }; const pass = kind => ({ setPipeline() {}, setBindGroup: (_, value) => calls.push([kind, value]), @@ -42,10 +42,11 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, "react-native-reanimated": { useSharedValue: value => ({ value }), useFrameCallback: fn => { callback = fn; return frame; }, + useAnimatedReaction: (prepare, fn) => { reactionInputs = prepare.__closure; let previous = prepare(); reaction = () => { const next = prepare(); fn(next, previous); previous = next; }; }, runOnJS: fn => fn, runOnUI: fn => fn, }, "./slidesStore": {}, - "./gpuRenderGraph": { encodeRenderGraph: () => calls.push(["graph"]) }, + "./gpuRenderGraph": { encodeRenderGraph: (_, __, ___, ____, scale) => calls.push(["graph", scale]) }, }; const UIFrames = new Function("require", "exports", code + "\nreturn UIFrames;")( name => mocks[name] ?? require(name), {}, @@ -53,10 +54,11 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, UIFrames({ resources: { alive: { value: true, set(value) { this.value = value; } }, disposed: { value: false }, device, context: { getCurrentTexture: () => ({ createView() {} }), present() {} }, clock: {}, graph: graph ? { value: {} } : undefined, states: [], groups: simulation ? ["A", "B"] : ["A"], compute: simulation ? {} : undefined, render: {}, particles: {}, frameData: data ? {} : undefined, composite: composite ? { pipeline: {}, group: "lens", texture: { createView() {} } } : undefined }, - effectActive, maxTime, frameTimeIndex, clockMode, frameData: data ? frameData : undefined, aspect: 2.61, brightness: 1, particleCount: 2048, previewTime: 7, onError: error => errors.push(error), onReady: () => ready++ }); + effectActive, maxTime, frameTimeIndex, clockMode, renderScale, frameData: data ? frameData : undefined, aspect: 2.61, brightness: 1, particleCount: 2048, previewTime: 7, onError: error => errors.push(error), onReady: () => ready++ }); const cleanup = effects.map(effect => effect()); return { calls, uniforms, errors, ready: () => ready, enabled: () => enabled, - playback, frameData, redraw: () => effects[0](), + playback, frameData, reactionInputs, redraw: () => effects[0](), + scale: value => { renderScale.value = value; reaction(); }, tick: () => { playback.value = { ...playback.value, stepTime: playback.value.stepTime + 0.016 }; callback({ timeSincePreviousFrame: 16 }); }, stop: () => cleanup.forEach(fn => fn?.()), device }; @@ -82,6 +84,29 @@ test("active GPU frames alternate storage buffers and dispatch the configured pa expect(f.calls.filter(call => call[0] === "destroy")).toHaveLength(1); }); +test("shared render scale redraws a paused graph at its frozen clock and frame", () => { + const f = frames({ graph: true, simulation: false }); + f.tick(); + f.playback.value = { ...f.playback.value, phase: "paused" }; + const previous = f.uniforms.at(-1); + f.scale(.75); + expect(f.calls.at(-1)).toEqual(["graph", .75]); + expect(f.uniforms.at(-1)).toEqual([previous[0], 0, previous[2], ...previous.slice(3)]); + const count = f.calls.length; + f.scale(.75); expect(f.calls).toHaveLength(count); + f.scale(1); expect(f.calls.at(-1)).toEqual(["graph", 1]); + f.stop(); +}); + +test("scale reaction subscribes to graph uniforms without subscribing legacy frame-data shaders", () => { + const contains = (value, target) => value === target || !!value && typeof value === "object" && Object.values(value).some(child => contains(child, target)); + for (const graph of [false, true]) { + const f = frames({ graph, data: true, simulation: false }); + expect(contains(f.reactionInputs, f.frameData)).toBe(graph); + f.stop(); + } +}); + test("preview renders one deterministic frame without starting a loop", () => { const f = frames({ preview: true }); expect(f.enabled()).toBe(false); diff --git a/apps/slides/src/gpuRenderGraph.ts b/apps/slides/src/gpuRenderGraph.ts index 8b2f6f051..40f29d49d 100644 --- a/apps/slides/src/gpuRenderGraph.ts +++ b/apps/slides/src/gpuRenderGraph.ts @@ -11,7 +11,7 @@ type PassResources = { pass: GPUPass; pipeline: GPUComputePipeline | GPURenderPi export type RenderGraphResources = { targets: GPUTarget[]; textures: Record; buffers: Record; passes: PassResources[]; clock: GPUBuffer; uniforms: GPUBuffer; sampler: GPUSampler; - width: number; height: number; + width: number; height: number; scale: number; }; function reference(value: string): [string, number] { @@ -36,13 +36,13 @@ function output(pass: GPUPass) { if (pass.kind === "fullscreen") return pass.mip === undefined ? pass.output : pass.output + "@" + pass.mip; return pass.kind === "render" ? pass.color[0] ?? pass.depth ?? "screen" : pass.writes?.[0] ?? "screen"; } -function dimensions(r: RenderGraphResources, value: string, width: number, height: number): [number, number] { +function dimensions(r: RenderGraphResources, value: string, width: number, height: number, renderScale = r.scale): [number, number] { "worklet"; const [id, mip] = reference(value); if (value === "screen" || r.buffers[id]) return [width, height]; const target = targetFor(r.targets, value); - if (r.textures[id] && r.width === width && r.height === height) return [Math.max(1, r.textures[id].width >> mip), Math.max(1, r.textures[id].height >> mip)]; - const scale = (target.scale ?? 1) / Math.pow(2, mip); + if (r.textures[id] && r.width === width && r.height === height && r.scale === renderScale) return [Math.max(1, r.textures[id].width >> mip), Math.max(1, r.textures[id].height >> mip)]; + const scale = (target.scale ?? 1) * renderScale / Math.pow(2, mip); return [Math.max(1, Math.floor(width * scale)), Math.max(1, Math.floor(height * scale))]; } @@ -59,7 +59,7 @@ export async function prepareRenderGraph(device: GPUDevice, definition: GPURende if (!Number.isFinite(target.scale ?? 1) || (target.scale ?? 1) <= 0 || !Number.isInteger(target.mips ?? 1) || (target.mips ?? 1) < 1) throw new Error("Invalid GPU target dimensions: " + target.id); } const r: RenderGraphResources = { targets, textures: {}, buffers: {}, passes: [], clock, uniforms, - sampler: device.createSampler({ minFilter: "linear", magFilter: "linear", mipmapFilter: "linear" }), width: 0, height: 0 }; + sampler: device.createSampler({ minFilter: "linear", magFilter: "linear", mipmapFilter: "linear" }), width: 0, height: 0, scale: 1 }; for (const buffer of buffers) { if (!Number.isInteger(buffer.size) || buffer.size <= 0 || buffer.size % 4 || (buffer.init && buffer.init.byteLength > buffer.size)) throw new Error("Invalid GPU buffer size: " + buffer.id); r.buffers[buffer.id] = device.createBuffer({ size: buffer.size, usage: 128 | 8 }); @@ -116,13 +116,14 @@ export async function prepareRenderGraph(device: GPUDevice, definition: GPURende return r; } -/** Recreate only texture views/bind groups when the drawing-buffer size changes. */ -export function resizeRenderGraph(r: RenderGraphResources, device: GPUDevice, width: number, height: number) { +/** Recreate only textures/views/bind groups when drawing-buffer size or render scale changes. */ +export function resizeRenderGraph(r: RenderGraphResources, device: GPUDevice, width: number, height: number, renderScale = 1) { "worklet"; - if (r.width === width && r.height === height) return; + if (!Number.isFinite(renderScale) || renderScale <= 0) throw new Error("GPU renderScale must be finite and positive"); + if (r.width === width && r.height === height && r.scale === renderScale) return; for (const target of r.targets) { const old = r.textures[target.id]; - const [w, h] = dimensions(r, target.id, width, height); + const [w, h] = dimensions(r, target.id, width, height, renderScale); const mips = target.mips ?? 1; // Allocate enough base pixels for all declared mip levels, even in tiny previews. const minSize = Math.pow(2, mips - 1); @@ -140,13 +141,13 @@ export function resizeRenderGraph(r: RenderGraphResources, device: GPUDevice, wi }), ] }); } - r.width = width; r.height = height; + r.width = width; r.height = height; r.scale = renderScale; } /** One encoder for every pass; the caller submits and presents the final screen. */ -export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, encoder: GPUCommandEncoder, screen: GPUTexture) { +export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, encoder: GPUCommandEncoder, screen: GPUTexture, renderScale = 1) { "worklet"; - resizeRenderGraph(r, device, screen.width, screen.height); + resizeRenderGraph(r, device, screen.width, screen.height, renderScale); for (const resource of r.passes) { const pass = resource.pass; if (pass.kind === "compute") { diff --git a/packages/presentation/src/GPUShader.tsx b/packages/presentation/src/GPUShader.tsx index 03fe741f7..ca7b0931a 100644 --- a/packages/presentation/src/GPUShader.tsx +++ b/packages/presentation/src/GPUShader.tsx @@ -10,6 +10,9 @@ export type GPUShaderProps = { active?: boolean; children?: ReactNode; maxTime?: number; frameTimeIndex?: number; renderGraph?: GPURenderGraphDefinition; + /** Positive shared multiplier for graph offscreen textures, including values above 1. + * Changes recreate/clear those textures; screen resolution stays fixed. Ignored without renderGraph. */ + renderScale?: SharedValue; borderRadius?: number; accessibilityLabel?: string; }; const GPUShaderContext = createContext | null>(null); diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index be7e8b33a..650820a41 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -26,12 +26,21 @@ const network = tree("root", [["a", ["a1", "a2"]], ["b", ["b1", "b2"]]]); ``` -`World` accepts `background`, `backgroundColor`, `quality`, `exposure`, `scrim`, +`World` accepts `background`, `backgroundColor`, `quality`, `renderScale`, `exposure`, `scrim`, `previewTime`, `active`, `borderRadius` and `fallback`. Normal React Native children are overlaid above the canvas; keep text in those children or in normal slide layers. A local GPU boundary displays `fallback` on unsupported adapters; the default is the dark background. GPU errors produce a diagnostic. A background -world fills the background host's dimensions. +world fills the background host's dimensions. Wrap it in presentation's +`` to place it behind the slide content: + +```tsx +import { Background } from "@legend-apps/presentation"; + + + + +``` `Camera.shots` are ordered keyframes with either a `step` or an `at` time in seconds. Times and step numbers should increase within their respective clock. @@ -82,8 +91,21 @@ height fog and volumetric lights, depth of field, a half-resolution bloom mip chain, chromatic aberration, ACES exposure/tonemapping, sRGB encoding, vignette and grain. `high` and `auto` use full-resolution scene targets and five bloom levels. `medium` uses 0.85 scene scale and five levels; `low` uses 0.65 scene -scale and three levels. Output resolution stays unchanged. `auto` currently -selects the high preset; it does not change quality based on frame timing. +scale and three levels. Output resolution stays unchanged. `renderScale` is a +static multiplier greater than zero and at most one for every offscreen target. +It combines with the preset and the per-effect scales; it does not rebuild pipelines. + +`auto` starts with the high preset and adjusts an additional multiplier from 1 +down to 0.5 in 0.125 steps. It samples live playback callback intervals, waits +through startup and resizing, lowers resolution after sustained intervals above +18.5ms, and requires several healthy windows before raising it. Paused and +outgoing frames keep the last live scale; preparation and previews use the +explicit static scale. A retained world may resize once when leaving preparation. +This policy changes +resolution only; camera motion and effect animation still sample the presentation +clock. Callback cadence is an indirect signal: CPU load or a low-refresh display +can also trigger reductions, and a GPU backlog may not immediately affect it. +Use a fixed quality and scale for repeatable image comparisons. For a raw WGSL escape hatch, use `GPURenderGraph` from `@legend-apps/presentation`. `buildWorldGraph` and the pure `scene.ts` helpers @@ -91,6 +113,10 @@ expose the same description and deterministic uniform samples for measurement. The scene hooks below share that pipeline. All animation samples the same clock; no frame history or per-frame React render is required. +The raw host accepts `renderScale` as a positive numeric SharedValue, defaulting +to 1. It multiplies offscreen target scales while retaining the drawing-buffer +size and compiled pipelines. Quality policy belongs to the pack. + Generate a standalone browser measurement page from the repository root: ```sh @@ -102,6 +128,9 @@ Open that file in Chrome with WebGPU and run Repeat at 3840×2160 and with each preset. Results report completed-submit frame times and per-pass GPU timestamps when available. Throughput measurements do not measure display refresh rate or prove a particular presentation frame rate. +Use fixed presets and `renderScale` for cost comparisons. Automatic adaptation +runs only with `paced: true` in the harness; its per-sample scale history is a +controller and callback-cadence check, rather than a steady-state cost at one resolution. ## Focus, floor and lighting @@ -140,6 +169,8 @@ Roughness blurs that reflection; reflectivity controls its strength. DOF uses a half-resolution gather and depth-aware composite. Shafts integrate point/spot scattering along the visible camera ray at quarter resolution. +Their fixed pixel jitter avoids synchronized integration bands; changing target +resolution changes that sampling pattern. Static `scale` overrides are available on DOF, Floor (reflection resolution) and Shafts; DOF and Shafts also accept `samples`. These change pipeline topology. Animated focus, aperture, blur radius, lighting and effect strengths update uniforms. @@ -162,8 +193,8 @@ import { PathPulse, Spray, Rows, Stage, flyTo } from "slides:world/World"; ``` Pulse paths accept positions or ids; bare graph ids resolve as `node:`. -The head follows the graph's curved edges. Its short straight trail joins two -sampled path positions and can cut across a bend. Ambient Spray accepts +The head and trail follow the graph's curved edges. The trail restarts at each +node so one tube never cuts across a junction. Ambient Spray accepts `position`, `spread`, `velocity` and `seed`; `under="leaves:lit"` emits beneath active leaves. Motion, birth and lifetime are analytic samples of absolute time. @@ -224,7 +255,8 @@ Use the function name `customPass`; `pass` is reserved in WGSL. Placement accept `scene`, `fog`, `dof`, `bloom` or `tonemap` through one `before` or `after` prop. Before tonemapping, colors are linear HDR. After tonemapping, colors are sRGB LDR; output then copies to the screen without a second tonemap. `before="scene"` is -invalid. Additional `reads` follow the fixed previous/depth/HDR texture bindings +invalid. A hook after tonemapping moves that stage into an offscreen target, so +its grain size follows the target resolution. Additional `reads` follow the fixed previous/depth/HDR texture bindings at group 0 binding 6 onward; declare their WGSL bindings explicitly. Raw `GPURenderGraph` exposes the full resource/pass contract. @@ -270,3 +302,5 @@ fn sdf(p:vec3f)->f32 { WGSL compilation/validation errors use the normal slide diagnostic path. The `world-tour.mdx` deck executes all three hook examples; `data-flow.mdx` shows pulse following, changed rows, stage flights and voxel reduction. +`cinematic-title.mdx` combines a mirror floor, lit monoliths, volumetric fog, +drifting particles and continuous camera moves behind normal text overlays. diff --git a/packages/slides-effects/packs/world/World.tsx b/packages/slides-effects/packs/world/World.tsx index 382dc5dca..722f1e45a 100644 --- a/packages/slides-effects/packs/world/World.tsx +++ b/packages/slides-effects/packs/world/World.tsx @@ -4,6 +4,7 @@ import { View } from "react-native"; import { makeMutable, useAnimatedReaction, useDerivedValue, useSharedValue, type SharedValue } from "react-native-reanimated"; import type { CameraOptions, CameraPose } from "./camera"; import { buildWorldGraph, worldGraphTopology } from "./render"; +import { advanceResolution, createResolutionState } from "./resolution"; import { compileTrack, customInstances, graphInstances, orbInstances, prepareWorldScene, primitive, pulseInstances, rowInstances, sampleWorldCamera, sampleWorldUniforms, sdfInstance, sprayInstances, voxelInstances, type Animated, type CustomOptions, type GraphOptions, type Instance, type LightOptions, type MaterialOptions, type OrbsOptions, @@ -62,6 +63,7 @@ class WorldGPU extends Component<{ fallback: ReactNode; children: ReactNode }, { export type WorldProps = { id?: string; width?: number; height?: number; background?: boolean; backgroundColor?: string; quality?: WorldScene["quality"]; exposure?: Animated; previewTime?: number; borderRadius?: number; + renderScale?: number; enter?: { from: "previous"; duration?: number }; scrim?: boolean | number; fallback?: ReactNode; active?: boolean; children?: ReactNode; accessibilityLabel?: string; postOrder?: WorldScene["postOrder"]; @@ -123,7 +125,8 @@ function worldContent(children: ReactNode, quality: WorldScene["quality"], expos return { prepared: prepareWorldScene(scene), overlay }; } function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHeight = 1080, background = false, backgroundColor = "#05060a", quality = "auto", - exposure = 1, previewTime = 4, borderRadius = 0, enter, scrim = false, fallback, active = true, children, accessibilityLabel, postOrder }: WorldProps) { + exposure = 1, previewTime = 4, borderRadius = 0, renderScale = 1, enter, scrim = false, fallback, active = true, children, accessibilityLabel, postOrder }: WorldProps) { + if (!Number.isFinite(renderScale) || renderScale <= 0 || renderScale > 1) throw new Error("World renderScale must be greater than zero and at most one"); const size = useBackgroundSize(); const width = background ? size.width : configuredWidth; const height = background ? size.height : configuredHeight; @@ -137,6 +140,22 @@ function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHei const sampleClock = samplePlayback; const cameraEvaluator = sampleWorldCamera; const sampleUniforms = sampleWorldUniforms; + const resolution = useSharedValue(createResolutionState()); + const resolutionEvaluator = advanceResolution; + const adaptive = quality === "auto"; + const adaptivePlayback = adaptive ? playback : null; + const scaleKey = `${width}:${height}:${renderScale}`; + useAnimatedReaction(() => adaptivePlayback?.value ?? null, state => { + "worklet"; + if (!state) return; + resolution.value = resolutionEvaluator(resolution.value, { timestamp: state.lastTimestamp, + playing: active && state.phase === "playing", key: `${state.slideKey}:${scaleKey}` }); + }, [adaptivePlayback, active, scaleKey]); + const effectiveScale = useDerivedValue(() => { + "worklet"; + const phase = adaptivePlayback?.value.phase; + return renderScale * (phase && phase !== "preview" && phase !== "preparing" ? resolution.value.scale : 1); + }, [renderScale, adaptivePlayback]); const enterDuration = enter?.duration ?? 1; const continuePrevious = enter?.from === "previous"; const phase = usePresentationValue("playbackPhase"); @@ -154,13 +173,13 @@ function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHei const clock = { ...state, slideTime: sampleClock(state, previewTime, "slide"), stepTime: sampleClock(state, previewTime, "step") }; const previous = state.phase === "preview" || state.phase === "preparing" ? null : origin.value.key === state.slideKey ? origin.value.pose : state.phase === "playing" ? entryPose?.value : null; - return sampleUniforms(prepared, clock, previous, enterDuration); - }, [prepared, previewTime, enterDuration, entryPose]); + return sampleUniforms(prepared, clock, previous, enterDuration, effectiveScale.value); + }, [prepared, previewTime, enterDuration, entryPose, effectiveScale]); return + borderRadius={borderRadius} previewTime={previewTime} clock="slide" active={active} renderScale={effectiveScale} /> {overlay.length > 0 && {overlay}} ; diff --git a/packages/slides-effects/packs/world/post.ts b/packages/slides-effects/packs/world/post.ts index 98e0de53c..6eb2949e7 100644 --- a/packages/slides-effects/packs/world/post.ts +++ b/packages/slides-effects/packs/world/post.ts @@ -22,9 +22,13 @@ ${depthCode()} let ray=worldRay(uv); let direction=normalize(ray); let distance=min(depthAt(uv),100.); let stride=distance/${steps}.; let density=max(uniforms[46],.0001); let anisotropy=clamp(uniforms[47],-.8,.8); + let pixel=vec2u(uv*vec2f(textureDimensions(depth))); + var seed=pixel.x*1973u+pixel.y*9277u+89173u; + seed=(seed^(seed>>16u))*2246822519u; seed=(seed^(seed>>13u))*3266489917u; + let jitter=f32((seed^(seed>>16u))&65535u)/65536.; var scattering=vec3f(0.); var opticalDepth=0.; for(var sampleIndex=0u;sampleIndex<${steps}u;sampleIndex++) { - let z=(f32(sampleIndex)+.5)*stride; let world=vector(0u)+ray*z; + let z=(f32(sampleIndex)+jitter)*stride; let world=vector(0u)+ray*z; let localDensity=density*exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); let transmission=exp(-opticalDepth); opticalDepth+=localDensity*stride*length(ray); for(var light=0u;light<${lights}u;light++) { @@ -65,7 +69,7 @@ fn coc(z:f32)->f32 { return clamp(abs(z-uniforms[32])/max(z,.05)*uniforms[33],0. return { gather: `${common} @fragment fn gather(@location(0) uv:vec2f)->@location(0) vec4f { - let centerDepth=depthAt(uv); let radius=coc(centerDepth)*uniforms[34]; + let centerDepth=depthAt(uv); let radius=coc(centerDepth)*uniforms[34]*uniforms[35]; let pixel=1./vec2f(textureDimensions(depth)); var color=vec3f(0.); var weight=0.; for(var i=0u;i<${samples}u;i++) { let angle=f32(i)*2.39996323; let disk=sqrt((f32(i)+.5)/${samples}.); @@ -118,12 +122,12 @@ fn digit(p:vec2f,value:u32)->f32 { if((mask&64u)!=0u) { distance=min(distance,rectangle(p,vec2f(.5,.92),vec2f(.32,.055))); } return 1.-smoothstep(0.,.025,distance); } -@fragment fn tonemap(@location(0) uv:vec2f)->@location(0) vec4f { +@fragment fn tonemap(@location(0) uv:vec2f,@builtin(position) pixel:vec4f)->@location(0) vec4f { let shift=(uv-.5)*uniforms[44]; var color=vec3f(textureSampleLevel(source,linearSampler,uv+shift,0.).r,textureSampleLevel(source,linearSampler,uv,0.).g,textureSampleLevel(source,linearSampler,uv-shift,0.).b); color=srgb(aces(color*uniforms[8]*frame.brightness)); color*=1.-uniforms[15]*smoothstep(.1,.8,length(uv-.5)); color*=1.-uniforms[17]*.65; - let noise=fract(sin(dot(uv*vec2f(textureDimensions(source)),vec2f(12.9898,78.233))+floor(frame.time*24.))*43758.5453)-.5; + let noise=fract(sin(dot(pixel.xy,vec2f(12.9898,78.233))+floor(frame.time*24.))*43758.5453)-.5; color+=noise*uniforms[16]; if(uniforms[53]>.5) { let value=floor(clamp(uniforms[56],0.,999999.)); var digits=1.; var remaining=u32(value); diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index 413ef03fd..41651dad3 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -265,8 +265,10 @@ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): passes.push({ kind: "fullscreen", id: "shafts", code: shaftCode(worldBindings, lights, topology.shaftSamples ?? samples), fragment: "shafts", reads: ["depth"], output: "shafts" }); } let color = "hdr", customIndex = 0; + const transientTargets = new Set(); const fullscreen = (id: string, code: string, fragment: string, reads: string[], output = id, targetScale = scale) => { if (output !== "screen") targets.push({ id: output, format: "rgba16float", scale: targetScale }); + if (output !== "screen" && output !== "dof" && targetScale === scale) transientTargets.add(output); passes.push({ kind: "fullscreen", id, code, fragment, reads, output }); color = output; }; const stages = ["scene", "fog", "dof", "bloom", "tonemap"]; @@ -311,5 +313,16 @@ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): if (passIds.has(pass.id)) throw new Error("World pass id is duplicated: " + pass.id); passIds.add(pass.id); } + if (!(topology.passes ?? []).some(pass => pass.reads?.length)) { + const aliases = new Map(), used = new Set(); + for (const pass of passes) { + if (pass.reads) pass.reads = pass.reads.map(id => aliases.get(id) ?? id); + if (pass.kind !== "fullscreen" || !transientTargets.has(pass.output)) continue; + const output = ["post-a", "post-b"].find(id => !pass.reads?.includes(id))!; + aliases.set(pass.output, output); used.add(output); pass.output = output; + } + for (let i = targets.length - 1; i >= 0; i--) if (transientTargets.has(targets[i].id)) targets.splice(i, 1); + for (const id of used) targets.push({ id, format: "rgba16float", scale }); + } return { targets, passes, uniformsSize: worldHeaderSize + instanceCount * worldInstanceSize }; } diff --git a/packages/slides-effects/packs/world/resolution.ts b/packages/slides-effects/packs/world/resolution.ts new file mode 100644 index 000000000..b59076303 --- /dev/null +++ b/packages/slides-effects/packs/world/resolution.ts @@ -0,0 +1,37 @@ +export type ResolutionState = { + scale: number; key: string; lastTimestamp: number | null; warmupUntil: number; + elapsed: number; frames: number; healthyWindows: number; cooldownUntil: number; +}; +export type ResolutionSample = { timestamp: number | null; playing: boolean; key: string }; + +export function createResolutionState(): ResolutionState { + "worklet"; + return { scale: 1, key: "", lastTimestamp: null, warmupUntil: 0, elapsed: 0, frames: 0, healthyWindows: 0, cooldownUntil: 0 }; +} + +export function advanceResolution(state: ResolutionState, sample: ResolutionSample): ResolutionState { + "worklet"; + const timestamp = sample.timestamp; + if (!sample.playing || timestamp === null || !Number.isFinite(timestamp)) { + return state.lastTimestamp === null ? state : { ...state, lastTimestamp: null, elapsed: 0, frames: 0, healthyWindows: 0 }; + } + if (state.key !== sample.key || state.lastTimestamp === null || timestamp < state.lastTimestamp) { + return { ...createResolutionState(), key: sample.key, scale: state.key === sample.key ? state.scale : 1, + lastTimestamp: timestamp, warmupUntil: timestamp + 500, cooldownUntil: state.key === sample.key ? state.cooldownUntil : 0 }; + } + if (timestamp === state.lastTimestamp) return state; + const interval = Math.min(100, timestamp - state.lastTimestamp); + if (timestamp < state.warmupUntil) return { ...state, lastTimestamp: timestamp }; + const elapsed = state.elapsed + interval, frames = state.frames + 1; + if (elapsed < 1000) return { ...state, lastTimestamp: timestamp, elapsed, frames }; + const mean = elapsed / frames; + const healthyWindows = mean <= 17.5 ? Math.min(5, state.healthyWindows + 1) : 0; + const lower = mean > 18.5 && state.scale > 0.5; + const raise = healthyWindows >= 5 && state.scale < 1; + const change = timestamp >= state.cooldownUntil && (lower || raise); + return { ...state, lastTimestamp: timestamp, elapsed: 0, frames: 0, + scale: change ? Math.max(0.5, Math.min(1, state.scale + (lower ? -0.125 : 0.125))) : state.scale, + healthyWindows: change ? 0 : healthyWindows, + warmupUntil: change ? timestamp + 500 : state.warmupUntil, + cooldownUntil: change ? timestamp + 3000 : state.cooldownUntil }; +} diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index aede7f47f..0827a1d43 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -135,7 +135,7 @@ export function orbInstances({ id = "orb", count = 48, colors = ["#22d3ee", "#b5 } export function pulseInstances(options: PulseOptions): Instance[] { - if (options.path.length < 2 || (options.duration ?? 2) <= 0) throw new Error("A world pulse needs at least two anchors and a positive duration"); + if (options.path.length < 2 || !Number.isFinite(options.duration ?? 2) || (options.duration ?? 2) <= 0 || !Number.isFinite(options.trail ?? 0.18) || (options.trail ?? 0.18) < 0) throw new Error("A world pulse needs at least two anchors, a positive duration and a nonnegative trail"); return [false, true].map(tail => ({ ...primitive(tail ? "tube" : "sphere", { id: tail ? `${options.id}:trail` : options.id, size: options.size ?? 0.08, thickness: (options.size ?? 0.08) * 0.45, color: options.color ?? "#22d3ee", emissive: options.emissive ?? 8 }), pulse: { options, anchors: [], curves: [], tail } })); @@ -236,15 +236,21 @@ function quadraticPoint(from: Vec3, to: Vec3, curve: number, t: number): Vec3 { const lift = Math.sqrt(dx * dx + dy * dy + dz * dz) * curve; return [from[0] + dx * t, from[1] + dy * t + 2 * (1 - t) * t * lift, from[2] + dz * t]; } +function pulseRange(pulse: NonNullable, clock: ClockSample) { + "worklet"; + const segments = pulse.anchors.length - 1; + const progress = Math.max(0, Math.min(1, triggerElapsed(pulse.options.step ?? 0, clock) / (pulse.options.duration ?? 2))); + const travel = progress * segments, segment = Math.min(segments - 1, Math.floor(travel)); + const head = travel - segment; + return { segment, head, tail: Math.max(0, head - (pulse.options.trail ?? 0.18) * segments) }; +} export function instancePosition(instance: Instance, clock: ClockSample): Vec3 { "worklet"; let position = sampleTrack(instance.position, clock); if (instance.pulse) { const pulse = instance.pulse; - const progress = Math.max(0, Math.min(1, triggerElapsed(pulse.options.step ?? 0, clock) / (pulse.options.duration ?? 2))); - const travel = Math.max(0, progress - (pulse.tail ? pulse.options.trail ?? 0.18 : 0)) * (pulse.anchors.length - 1); - const segment = Math.min(pulse.anchors.length - 2, Math.floor(travel)); - return quadraticPoint(instancePosition(pulse.anchors[segment], clock), instancePosition(pulse.anchors[segment + 1], clock), pulse.curves[segment], travel - segment); + const range = pulseRange(pulse, clock); + return quadraticPoint(instancePosition(pulse.anchors[range.segment], clock), instancePosition(pulse.anchors[range.segment + 1], clock), pulse.curves[range.segment], pulse.tail ? range.tail : range.head); } if (instance.spray) { const spray = instance.spray, life = spray.options.lifetime ?? 2; @@ -285,6 +291,10 @@ export function instanceOpacity(instance: Instance, clock: ClockSample): number } if (instance.pulse) { const elapsed = triggerElapsed(instance.pulse.options.step ?? 0, clock), duration = instance.pulse.options.duration ?? 2; + if (instance.pulse.tail) { + const range = pulseRange(instance.pulse, clock); + if (range.head === range.tail) return 0; + } return elapsed < 0 ? 0 : Math.max(0, Math.min(1, (duration + 0.25 - elapsed) / 0.25)); } if (instance.voxel) { @@ -334,7 +344,7 @@ export function sampleFocus(scene: PreparedWorldScene, clock: ClockSample, camer return sampleTrack({ initial: focusDepth(track.initial, scene, clock, camera), frames: track.frames.map(frame => ({ ...frame, value: focusDepth(frame.value, scene, clock, camera) })) }, clock); } -export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1): number[] { +export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1, renderScale = 1): number[] { "worklet"; const values = new Array(worldHeaderSize + scene.instances.length * worldInstanceSize).fill(0); const objects: Record = {}; @@ -348,6 +358,7 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl const fog = sampleTrack(scene.fogTrack, clock); values[20] = fog[0]; values[21] = fog[1]; values[22] = fog[2]; values[32] = sampleFocus(scene, clock, camera); + values[35] = renderScale; for (const field of scene.colorTracks) { const color = sampleTrack(field.track, clock); values[field.index] = color[0]; values[field.index + 1] = color[1]; values[field.index + 2] = color[2]; @@ -389,7 +400,15 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl values[index + 7] = emissive; values[index + 8] = scale[0]; values[index + 9] = scale[1]; values[index + 10] = scale[2]; values[index + 12] = endpoint[0]; values[index + 13] = endpoint[1]; values[index + 14] = endpoint[2]; - values[index + 15] = instance.curve; values[index + 19] = instance.thickness; + let curve = instance.curve; + if (instance.pulse?.tail) { + const pulse = instance.pulse, range = pulseRange(pulse, clock); + const from = instancePosition(pulse.anchors[range.segment], clock), to = instancePosition(pulse.anchors[range.segment + 1], clock); + const length = Math.hypot(to[0] - from[0], to[1] - from[1], to[2] - from[2]); + const chord = Math.hypot(endpoint[0] - position[0], endpoint[1] - position[1], endpoint[2] - position[2]); + curve = chord > 0 ? length * pulse.curves[range.segment] * Math.pow(range.head - range.tail, 2) / chord : 0; + } + values[index + 15] = curve; values[index + 19] = instance.thickness; values[index + 23] = instanceOpacity(instance, clock); const material = instance.materialTracks!, sampledAlbedo = sampleTrack(material.albedo, clock); const albedo = instance.material ? instance.graph ? mixVector(dim, sampledAlbedo, activation) : sampledAlbedo : [values[index + 4], values[index + 5], values[index + 6]]; diff --git a/packages/slides-effects/scripts/renderGraphBenchmark.ts b/packages/slides-effects/scripts/renderGraphBenchmark.ts index 0cb3b88ce..fe7d326b2 100644 --- a/packages/slides-effects/scripts/renderGraphBenchmark.ts +++ b/packages/slides-effects/scripts/renderGraphBenchmark.ts @@ -3,7 +3,8 @@ import { encodeRenderGraph, prepareRenderGraph } from "../../../apps/slides/src/ export type BenchmarkOptions = { definition: GPURenderGraphDefinition; - uniformsAt: (seconds: number) => number[]; + uniformsAt: (seconds: number, renderScale: number) => number[]; + renderScaleAt?: (timestamp: number) => number; width: number; height: number; warmup?: number; @@ -20,7 +21,7 @@ function summarize(values: number[]) { } /** Replay fixed clock samples, with optional browser cadence measurement. */ -export async function benchmarkRenderGraph({ definition, uniformsAt, width, height, warmup = 30, frames = 120, repetitions = 3, paced = false }: BenchmarkOptions) { +export async function benchmarkRenderGraph({ definition, uniformsAt, renderScaleAt, width, height, warmup = 30, frames = 120, repetitions = 3, paced = false }: BenchmarkOptions) { if (![width, height, warmup, frames, repetitions].every(Number.isInteger) || width < 1 || height < 1 || warmup < 0 || frames < 1 || repetitions < 1) { throw new Error("Benchmark dimensions and sample counts must be positive integers"); } @@ -40,7 +41,7 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig if (!context) { device.destroy(); throw new Error("No WebGPU canvas context"); } const format = navigator.gpu.getPreferredCanvasFormat(); context.configure({ device, format, alphaMode: "opaque" }); - const initial = uniformsAt(0); + const initial = uniformsAt(0, 1); const clock = device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }); const uniforms = device.createBuffer({ size: Math.max(16, initial.length * 4), usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST }); const passCount = definition.passes.length; @@ -48,7 +49,7 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig const queryBytes = passCount * 16; const resolve = query ? device.createBuffer({ size: queryBytes, usage: GPUBufferUsage.QUERY_RESOLVE | GPUBufferUsage.COPY_SRC }) : undefined; const readback = query ? device.createBuffer({ size: queryBytes, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ }) : undefined; - const samples: { repetition: number; frame: number; seconds: number; completedMs: number; intervalMs?: number; gpuMs?: number; passes?: Record }[] = []; + const samples: { repetition: number; frame: number; seconds: number; renderScale: number; completedMs: number; intervalMs?: number; gpuMs?: number; passes?: Record }[] = []; try { device.pushErrorScope("validation"); const graph = await prepareRenderGraph(device, definition, format, clock, uniforms); @@ -61,7 +62,8 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig previousFrame = now; const seconds = 2 + frame / 60; const start = performance.now(); - const values = uniformsAt(seconds); + const renderScale = renderScaleAt?.(now ?? start) ?? 1; + const values = uniformsAt(seconds, renderScale); if (values.length !== initial.length) throw new Error("Benchmark uniforms changed size"); device.queue.writeBuffer(clock, 0, new Float32Array([seconds, 1 / 60, frame, 1, width / height, 1, seconds, seconds])); device.queue.writeBuffer(uniforms, 0, new Float32Array(values)); @@ -78,7 +80,7 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig return encoder.beginComputePass({ ...descriptor, timestampWrites: { querySet: query, beginningOfPassWriteIndex: offset, endOfPassWriteIndex: offset + 1 } }); }, } as GPUCommandEncoder : encoder; - encodeRenderGraph(graph, device, timedEncoder, context!.getCurrentTexture()); + encodeRenderGraph(graph, device, timedEncoder, context!.getCurrentTexture(), renderScale); if (query && timed) { encoder.resolveQuerySet(query, 0, passCount * 2, resolve!, 0); encoder.copyBufferToBuffer(resolve!, 0, readback!, 0, queryBytes); @@ -95,7 +97,7 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig readback.unmap(); } if (failures.length) throw new Error(failures.join("\n")); - return { seconds, completedMs, ...(intervalMs === undefined ? {} : { intervalMs }), ...(gpuMs === undefined ? {} : { gpuMs, passes }) }; + return { seconds, renderScale, completedMs, ...(intervalMs === undefined ? {} : { intervalMs }), ...(gpuMs === undefined ? {} : { gpuMs, passes }) }; } for (let frame = 0; frame < warmup; frame++) await render(frame, false); for (let repetition = 0; repetition < repetitions; repetition++) { @@ -112,6 +114,7 @@ export async function benchmarkRenderGraph({ definition, uniformsAt, width, heig viewport: { width: innerWidth, height: innerHeight }, visibility: document.visibilityState }, methodology: `${paced ? "rAF-paced" : "Unpaced"} fixed-clock samples; completedMs includes uniform evaluation, writes, encode, submit and queue completion. GPU frame span is the last pass end minus the first pass beginning; pass scopes may overlap and are non-additive. GPU timestamps exclude CPU and presentation and are disabled in paced runs to avoid readback overhead. rAF cadence is browser callback cadence, not evidence of physical display presentation.`, pipeline: { targets: definition.targets, passes: definition.passes.map(pass => ({ id: pass.id, kind: pass.kind })) }, + renderScale: { min: Math.min(...samples.map(sample => sample.renderScale)), max: Math.max(...samples.map(sample => sample.renderScale)), final: samples.at(-1)!.renderScale }, completed: summarize(samples.map(sample => sample.completedMs)), gpu: timestamps ? summarize(samples.map(sample => sample.gpuMs!)) : null, passes: timestamps ? Object.fromEntries(definition.passes.map(pass => [pass.id, summarize(samples.map(sample => sample.passes![pass.id]))])) : null, diff --git a/packages/slides-effects/scripts/world-benchmark.mjs b/packages/slides-effects/scripts/world-benchmark.mjs index 86ebf8853..2b80d8ae8 100644 --- a/packages/slides-effects/scripts/world-benchmark.mjs +++ b/packages/slides-effects/scripts/world-benchmark.mjs @@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url"; const args = process.argv.slice(2); if (args.includes("--help") || args.length !== 2 || args[0] !== "--output") { - console.log("Usage: bun packages/slides-effects/scripts/world-benchmark.mjs --output /absolute/path/harness.html\nGenerates a standalone offline WebGPU measurement page. In Chrome, call await benchmarkWorld({width:1920,height:1080,quality:'high'}). Add allEffects:true for the full scene or paced:true for rAF interval measurements (without GPU timestamp readback).\nResult includes raw samples, per-pass GPU timestamps when available, completed-submit wall times and a PNG data URL.\nGenerate and launch the browser through the heavy gate; call the measurement through the fps gate. Keep generated pages/results outside the repository."); + console.log("Usage: bun packages/slides-effects/scripts/world-benchmark.mjs --output /absolute/path/harness.html\nGenerates a standalone offline WebGPU measurement page. In Chrome, call await benchmarkWorld({width:1920,height:1080,quality:'high'}). Add allEffects:true for the full scene, quality:'auto',paced:true for the shared adaptive controller (unpaced auto stays high/manual), renderScale:0.75 for a manual scale, or paced:true for rAF interval measurements (without GPU timestamp readback).\nResult includes raw samples, per-pass GPU timestamps when available, completed-submit wall times and a PNG data URL.\nGenerate and launch the browser through the heavy gate; call the measurement through the fps gate. Keep generated pages/results outside the repository."); process.exit(args.includes("--help") ? 0 : 2); } const output = resolve(args[1]); diff --git a/packages/slides-effects/scripts/worldBenchmark.ts b/packages/slides-effects/scripts/worldBenchmark.ts index 59cc92fb8..f489fff52 100644 --- a/packages/slides-effects/scripts/worldBenchmark.ts +++ b/packages/slides-effects/scripts/worldBenchmark.ts @@ -1,9 +1,10 @@ import { customInstances, graphInstances, orbInstances, prepareWorldScene, primitive, pulseInstances, rowInstances, sampleWorldUniforms, sdfInstance, sprayInstances, voxelInstances, type WorldScene } from "../packs/world/scene"; import { tree, wave } from "../packs/world/graph"; import { buildWorldGraph } from "../packs/world/render"; +import { advanceResolution, createResolutionState } from "../packs/world/resolution"; import { benchmarkRenderGraph, type BenchmarkOptions } from "./renderGraphBenchmark"; -type Options = Omit & { quality: NonNullable; allEffects?: boolean }; +type Options = Omit & { quality: NonNullable; allEffects?: boolean; renderScale?: number }; export async function benchmarkWorld(options: Options) { const data = tree("root", Array.from({ length: 3 }, (_, branch) => [ @@ -53,10 +54,16 @@ export async function benchmarkWorld(options: Options) { bloom: { intensity: 1.4, threshold: 1, radius: 0.8 }, fog: { density: 0.025, color: "#05060a" }, vignette: 0.3, grain: 0.015, exposure: 1.1, }); + const adaptiveActive = options.quality === "auto" && options.paced === true; + let resolution = createResolutionState(); const result = await benchmarkRenderGraph({ ...options, definition: buildWorldGraph(scene), - uniformsAt: seconds => sampleWorldUniforms(scene, { slideTime: seconds, stepTime: seconds, stepIndex: 1, stepOrigins: { 0: 0, 1: 0 } }) }); + renderScaleAt: timestamp => { + if (adaptiveActive) resolution = advanceResolution(resolution, { timestamp, playing: true, key: `${options.width}x${options.height}` }); + return (options.renderScale ?? 1) * (adaptiveActive ? resolution.scale : 1); + }, + uniformsAt: (seconds, renderScale) => sampleWorldUniforms(scene, { slideTime: seconds, stepTime: seconds, stepIndex: 1, stepOrigins: { 0: 0, 1: 0 } }, undefined, 1, renderScale) }); return { ...result, workload: { quality: options.quality, orbs: 64, graphNodes: 40, graphEdges: 39, rings: 1, - allEffects: !!options.allEffects, instanceCount: scene.instances.length, + allEffects: !!options.allEffects, instanceCount: scene.instances.length, adaptiveActive, manualRenderScale: options.renderScale ?? 1, effects: options.allEffects ? { reflection: { sceneScaleFactor: scene.floor?.scale }, dof: { sceneScaleFactor: scene.dof?.scale, samples: scene.dof?.samples }, shafts: { sceneScaleFactor: scene.shafts?.scale, samples: scene.shafts?.samples }, chromatic: 0.0025, lights: 2, pulseInstances: 2, sprayParticles: 48, rows: 8, voxels: 64, customQuads: 2, sdfObjects: 1, customPost: true } : null, diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index a4239675a..0396522ce 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { evaluateCamera, flyTo, type CameraOptions, type ClockSample } from "../packs/world/camera"; +import { evaluateCamera, flyTo, type CameraOptions, type ClockSample, type Vec3 } from "../packs/world/camera"; import { activationAt, layoutTree, subset, tree, wave } from "../packs/world/graph"; import { buildWorldGraph } from "../packs/world/render"; import { compileTrack, customInstances, graphInstances, instanceOpacity, instancePosition, orbInstances, prepareWorldScene, primitive, pulseInstances, @@ -183,6 +183,37 @@ test("pulse geometry, camera follow and focus share the exact curved path across expect(() => prepareWorldScene({ instances: pulseInstances({ id: "bad", path: ["bad", [0, 0, 0]] }) })).toThrow("cycle"); }); +test("pulse trails reconstruct exact curved subsegments and restart at a turn without extra instances", () => { + const scene = prepareWorldScene({ instances: [...graphInstances({ data: tree("root", [["a", ["leaf"]]]), edge: { curve: 0.65 } }), + ...pulseInstances({ id: "pulse", path: ["root", "a", "leaf"], step: 1, duration: 4, trail: 0.18 })] }); + const trail = scene.instances.find(instance => instance.id === "pulse:trail")!; + const index = worldHeaderSize + scene.instances.indexOf(trail) * worldInstanceSize; + const point = (from: Vec3, to: Vec3, curve: number, t: number): Vec3 => { + const length = Math.hypot(to[0] - from[0], to[1] - from[1], to[2] - from[2]); + return [from[0] + (to[0] - from[0]) * t, from[1] + (to[1] - from[1]) * t + 2 * (1 - t) * t * length * curve, from[2] + (to[2] - from[2]) * t]; + }; + const check = (time: number, sourceId: string, targetId: string, tailT: number, headT: number) => { + const sample = clock(time, 1, { 1: 0 }), values = sampleWorldUniforms(scene, sample); + const source = resolveAnchor(scene, sourceId, sample)!, target = resolveAnchor(scene, targetId, sample)!; + const start = values.slice(index, index + 3) as Vec3, end = values.slice(index + 12, index + 15) as Vec3; + for (const t of [0, 0.25, 0.5, 0.75, 1]) { + const actual = point(start, end, values[index + 15], t), expected = point(source, target, 0.65, tailT + (headT - tailT) * t); + actual.forEach((value, axis) => expect(value).toBeCloseTo(expected[axis], 10)); + } + return values; + }; + check(1, "root", "a", 0.14, 0.5); + const before = check(1.99, "root", "a", 0.635, 0.995); + const turn = sampleWorldUniforms(scene, clock(2, 1, { 1: 0 })); + expect(turn[index + 23]).toBe(0); expect(turn[index + 15]).toBe(0); + const after = check(2.01, "a", "leaf", 0, 0.005); + expect(after.slice(index, index + 3)).toEqual(resolveAnchor(scene, "a", clock(2.01, 1, { 1: 0 }))!); + expect(before[index + 23]).toBe(1); expect(after[index + 23]).toBe(1); + expect(scene.instances.filter(instance => instance.pulse)).toHaveLength(2); + expect(sampleWorldUniforms(scene, clock(1), undefined, 1, 0.75)[35]).toBe(0.75); + expect(sampleWorldUniforms(scene, clock(1))[35]).toBe(1); +}); + test("recycling transfers a flash with logical row identity and never to a reused slot", () => { const scene = prepareWorldScene({ instances: rowInstances({ id: "rows", count: 4, speed: 1, spacing: 1, changes: [{ row: 1, step: 1 }], flashDuration: 2 }), dof: { focus: "rows:row:1" }, camera: { shots: [{ position: [0, 0, 10], target: [0, 0, 0] }] } }); diff --git a/packages/slides-effects/tests/worldPlayback.test.tsx b/packages/slides-effects/tests/worldPlayback.test.tsx index 6c4ab3af5..5ddebea72 100644 --- a/packages/slides-effects/tests/worldPlayback.test.tsx +++ b/packages/slides-effects/tests/worldPlayback.test.tsx @@ -8,7 +8,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { createRequire } from "node:module"; -import { createPlaybackState, advancePlayback } from "../../presentation/src/playbackState"; +import { createPlaybackState, advancePlayback, transitionPlayback } from "../../presentation/src/playbackState"; import { samplePlayback } from "../../presentation/src/playback"; import { worldHeaderSize } from "../packs/world/scene"; @@ -92,6 +92,52 @@ test("equal-content parent renders and resize retain pipeline definitions while } finally { await act(() => tree?.unmount()); } }); +test("auto resolution adapts without React frames and preserves the frozen pause/outgoing scale", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + playback.value = createPlaybackState({ phase: "playing", slideKey: "adaptive", stepKey: "0", stepIndex: 0 }); + let renderer, commits = 0; + const content = (quality, active = true) => commits++}>; + try { + await act(() => { renderer = create(content("auto")); }); + const graph = () => renderer.root.findByType("render-graph").props; + const passes = graph().passes, initialCommits = commits; + for (let timestamp = 0; timestamp <= 6000; timestamp += 20) { + playback.value = advancePlayback(playback.value, timestamp); + reactUI(); + } + expect(graph().renderScale.value).toBeLessThan(0.8); + expect(graph().uniforms.value[35]).toBe(graph().renderScale.value); + expect(graph().passes).toBe(passes); expect(commits).toBe(initialCommits); + expect(playback.value.slideTime).toBeCloseTo(6); + const frozenScale = graph().renderScale.value; + playback.value = transitionPlayback(playback.value, { ...playback.value, phase: "paused" }); + reactUI(); + expect(graph().renderScale.value).toBe(frozenScale); expect(graph().uniforms.value[35]).toBe(frozenScale); + playback.value = transitionPlayback(playback.value, { ...playback.value, phase: "playing" }); + playback.value = advancePlayback(playback.value, 50000); + reactUI(); + expect(graph().renderScale.value).toBe(frozenScale); expect(playback.value.slideTime).toBeCloseTo(6); + playback.value = transitionPlayback(playback.value, { ...playback.value, phase: "outgoing" }); + reactUI(); + expect(graph().renderScale.value).toBe(frozenScale); + expect(graph().passes).toBe(passes); expect(commits).toBe(initialCommits); + await act(() => renderer.update(content("auto", false))); + reactUI(); + expect(graph().renderScale.value).toBe(frozenScale); expect(graph().passes).toBe(passes); + playback.value = createPlaybackState({ phase: "preview", slideKey: "preview", stepKey: "1", stepIndex: 1 }); + reactUI(); + expect(graph().renderScale.value).toBe(0.8); + playback.value = createPlaybackState({ phase: "playing", slideKey: "fixed", stepKey: "0", stepIndex: 0 }); + await act(() => renderer.update(content("high"))); + const fixedCommits = commits; + for (let timestamp = 0; timestamp <= 6000; timestamp += 20) { + playback.value = advancePlayback(playback.value, timestamp); + reactUI(); + } + expect(graph().renderScale.value).toBe(0.8); expect(commits).toBe(fixedCommits); + } finally { await act(() => renderer?.unmount()); } +}); + test("compiled recursive world evaluators survive an isolated UI-worklet closure", async () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "world-worklets-")); try { @@ -102,14 +148,14 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure if (!compiled.success) throw new Error(compiled.errors.join("\n")); const require = createRequire(import.meta.url); const module = { exports: {} }; - const callbacks = [], mutables = []; + const callbacks = [], mutables = [], preparations = [], reactions = []; const presentation = { GPURenderGraph: props => React.createElement("compiled-graph", props), usePlayback: () => playback, samplePlayback, useBackgroundSize: () => ({ width: 800, height: 500 }), usePresentationValue: () => playback.value.phase }; const reanimated = { makeMutable: value => { const shared = { value }; mutables.push(shared); return shared; }, useSharedValue: value => React.useRef({ value }).current, - useAnimatedReaction: () => {}, + useAnimatedReaction: (prepare, react) => { preparations.push(prepare); reactions.push(react); }, useDerivedValue: fn => { callbacks.push(fn); return { get value() { return fn(); } }; }, }; Function("require", "module", "exports", compiled.code)(name => { @@ -153,6 +199,8 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure for (const clock of [ { slideTime: 0, stepTime: 0, stepIndex: 0, stepOrigins: { 0: 0 } }, { slideTime: 3, stepTime: 1, stepIndex: 1, stepOrigins: { 0: 0, 1: 2 } }, + { slideTime: 3.99, stepTime: 1.99, stepIndex: 1, stepOrigins: { 0: 0, 1: 2 } }, + { slideTime: 4.01, stepTime: 2.01, stepIndex: 1, stepOrigins: { 0: 0, 1: 2 } }, { slideTime: 1, stepTime: 1, stepIndex: 2, stepOrigins: { 2: 0 } }, { slideTime: 0.5, stepTime: 0.5, stepIndex: 3, stepOrigins: { 3: 0 } }, ]) expect(sample(scene, clock)).toEqual(api.sampleWorldUniforms(scene, clock)); @@ -161,6 +209,15 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure try { playback.value = createPlaybackState({ phase: "playing", slideKey: "closure", stepKey: "0", stepIndex: 0 }); await act(() => { renderer = create(React.createElement(api.World, { id: "closure" })); }); + const autoClosure = reactions.at(-2).__closure; + const advanceScale = restore(autoClosure.resolutionEvaluator); + let isolated = autoClosure.resolution.value, expected = autoClosure.resolution.value; + for (let timestamp = 0; timestamp <= 6000; timestamp += 20) { + const input = { timestamp, playing: true, key: "compiled" }; + isolated = advanceScale(isolated, input); expected = autoClosure.resolutionEvaluator(expected, input); + expect(isolated).toEqual(expected); + } + expect(isolated.scale).toBeLessThan(1); const retained = mutables[0]; expect(Object.values(callbacks.at(-1).__closure)).not.toContain(retained); await act(() => renderer.unmount()); @@ -173,8 +230,11 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure expect(Object.values(callbacks.at(-1).__closure)).toContain(retained); await act(() => renderer.unmount()); const count = mutables.length; - await act(() => { renderer = create(React.createElement(api.World)); }); + await act(() => { renderer = create(React.createElement(api.World, { quality: "high" })); }); expect(mutables.length).toBe(count); + expect(Object.values(preparations.at(-2).__closure)).not.toContain(playback); + expect(restore(preparations.at(-2))()).toBeNull(); + expect(Object.values(callbacks.at(-2).__closure)).not.toContain(playback); } finally { await act(() => renderer?.unmount()); } } finally { fs.rmSync(directory, { recursive: true, force: true }); } }, 30000); diff --git a/packages/slides-effects/tests/worldRender.test.ts b/packages/slides-effects/tests/worldRender.test.ts index 3bebe1482..cc7c83409 100644 --- a/packages/slides-effects/tests/worldRender.test.ts +++ b/packages/slides-effects/tests/worldRender.test.ts @@ -65,16 +65,33 @@ test("postOrder rewires HDR stages and custom hooks retain the declared color-sp { id: "before-focus", wgsl: scanlines, before: "dof" }, { id: "ldr", wgsl: scanlines, after: "tonemap" }, ] })); const pass = (id: string) => graph.passes.find(pass => pass.id === id)!; + const output = (id: string) => { const item = pass(id); return item.kind === "fullscreen" ? item.output : undefined; }; expect(pass("bloom-threshold").reads).toEqual(["hdr"]); - expect(pass("before-focus").reads?.[0]).toBe("bloom-composite"); - expect(pass("dof-gather").reads?.[0]).toBe("custom-post-0"); - expect(pass("fog").reads?.[0]).toBe("dof-composite"); + expect(pass("before-focus").reads?.[0]).toBe(output("bloom-composite")); + expect(pass("dof-gather").reads?.[0]).toBe(output("before-focus")); + expect(pass("fog").reads?.[0]).toBe(output("dof-composite")); expect(pass("tonemap").kind).toBe("fullscreen"); - expect(pass("ldr").reads?.[0]).toBe("tonemapped"); - expect(pass("screen").reads).toEqual(["custom-post-1"]); + expect(pass("ldr").reads?.[0]).toBe(output("tonemap")); + expect(pass("screen").reads).toEqual([output("ldr")]); expect(graph.passes.filter(pass => pass.id === "tonemap")).toHaveLength(1); }); +test("HDR transients reuse two targets without aliasing DOF inputs, while explicit custom reads retain intermediates", () => { + for (const postOrder of [["fog", "dof", "bloom"], ["dof", "bloom", "fog"], ["bloom", "fog", "dof"]] as const) { + const graph = buildWorldGraph(scene({ dof: {}, postOrder: [...postOrder], passes: [{ wgsl: scanlines, after: "tonemap" }] })); + expect(graph.targets.filter(target => target.id.startsWith("post-"))).toHaveLength(2); + expect(graph.targets.some(target => target.id === "hdr")).toBe(true); + const gather = graph.passes.find(pass => pass.id === "dof-gather")!; + const composite = graph.passes.find(pass => pass.id === "dof-composite")!; + expect(composite.reads?.[0]).toBe(gather.reads?.[0]); + for (const pass of graph.passes) if (pass.kind === "fullscreen") expect(pass.reads).not.toContain(pass.output); + } + const retained = buildWorldGraph(scene({ dof: {}, passes: [{ wgsl: scanlines, after: "bloom", reads: ["fog"] }] })); + expect(retained.targets.some(target => target.id === "fog")).toBe(true); + expect(retained.targets.some(target => target.id.startsWith("post-"))).toBe(false); + expect(retained.passes.find(pass => pass.id === "custom-post-0")?.reads).toContain("fog"); +}); + test("invalid post stage permutations and placements fail rather than silently skipping hooks", () => { for (const postOrder of [["fog", "fog", "bloom"], ["fog", "bloom"], ["fog", "dof", "unknown"]]) { expect(() => buildWorldGraph({ quality: "high", counts: { box: 0, ring: 0, sphere: 0, tube: 0 }, backgroundVector: [0, 0, 0], postOrder })).toThrow("postOrder"); diff --git a/packages/slides-effects/tests/worldResolution.test.ts b/packages/slides-effects/tests/worldResolution.test.ts new file mode 100644 index 000000000..f31ad6338 --- /dev/null +++ b/packages/slides-effects/tests/worldResolution.test.ts @@ -0,0 +1,97 @@ +import { expect, test } from "bun:test"; +import { advanceResolution, createResolutionState, type ResolutionState } from "../packs/world/resolution"; +import { advancePlayback, createPlaybackState, transitionPlayback } from "../../presentation/src/playbackState"; + +function frames(state: ResolutionState, interval: number, count: number, key = "scene") { + let timestamp = state.lastTimestamp ?? 0; + const scales: number[] = []; + for (let index = 0; index < count; index++) { + timestamp += interval; + state = advanceResolution(state, { timestamp, playing: true, key }); + scales.push(state.scale); + } + return { state, scales }; +} + +test("ordinary 60 Hz and 120 Hz cadence keeps full resolution", () => { + for (const interval of [1000 / 60, 1000 / 120]) { + const { state, scales } = frames(createResolutionState(), interval, 2400); + expect(state.scale).toBe(1); + expect(scales.every(scale => scale === 1)).toBe(true); + } +}); + +test("sustained 50 fps steps down within bounds and healthy cadence recovers with hysteresis", () => { + const slow = frames(createResolutionState(), 20, 1000); + expect(slow.scales.slice(0, 60).every(scale => scale === 1)).toBe(true); + expect([...new Set(slow.scales)]).toEqual([1, 0.875, 0.75, 0.625, 0.5]); + expect(slow.state.scale).toBe(0.5); + expect(frames(createResolutionState(), 120, 200).state.scale).toBe(0.5); + const neutral = frames(slow.state, 18, 400); + expect(neutral.state.scale).toBe(0.5); + const shortRecovery = frames(neutral.state, 1000 / 60, 240); + expect(shortRecovery.state.scale).toBe(0.5); + const recovered = frames(shortRecovery.state, 1000 / 60, 1600); + expect(recovered.state.scale).toBe(1); + expect(recovered.scales.every(scale => scale >= 0.5 && scale <= 1)).toBe(true); +}); + +test("startup, isolated stalls, paused gaps and timestamp resets do not lower resolution", () => { + let state = frames(createResolutionState(), 1000 / 60, 120).state; + state = advanceResolution(state, { timestamp: state.lastTimestamp! + 50, playing: true, key: "scene" }); + state = frames(state, 1000 / 60, 120).state; + expect(state.scale).toBe(1); + const same = advanceResolution(state, { timestamp: state.lastTimestamp, playing: true, key: "scene" }); + expect(same).toBe(state); + const warmup = state.warmupUntil; + state = advanceResolution(state, { timestamp: state.lastTimestamp! + 5000, playing: true, key: "scene" }); + expect(state.scale).toBe(1); expect(state.warmupUntil).toBe(warmup); + state = advanceResolution(state, { timestamp: null, playing: false, key: "scene" }); + expect(state.lastTimestamp).toBeNull(); + state = advanceResolution(state, { timestamp: 20000, playing: true, key: "scene" }); + expect(state.warmupUntil).toBe(20500); expect(state.frames).toBe(0); + state = advanceResolution(state, { timestamp: 100, playing: true, key: "scene" }); + expect(state.warmupUntil).toBe(600); expect(state.scale).toBe(1); + state = advanceResolution({ ...state, scale: 0.5 }, { timestamp: 200, playing: true, key: "next" }); + expect(state.scale).toBe(1); expect(state.key).toBe("next"); +}); + +test("an isolated hitch preserves the current cooldown and recovery history", () => { + const state = frames(createResolutionState(), 20, 100).state; + expect(state.scale).toBe(0.875); + expect(state.cooldownUntil).toBeGreaterThan(state.lastTimestamp!); + const hitch = advanceResolution(state, { timestamp: state.lastTimestamp! + 500, playing: true, key: "scene" }); + expect(hitch.scale).toBe(state.scale); + expect(hitch.cooldownUntil).toBe(state.cooldownUntil); + expect(hitch.warmupUntil).toBe(state.warmupUntil); + const paused = advanceResolution(state, { timestamp: null, playing: false, key: "scene" }); + const resumed = advanceResolution(paused, { timestamp: state.lastTimestamp! + 20, playing: true, key: "scene" }); + expect(resumed.cooldownUntil).toBe(state.cooldownUntil); expect(resumed.scale).toBe(state.scale); + expect(advanceResolution(resumed, { timestamp: state.lastTimestamp! + 40, playing: true, key: "next" }).cooldownUntil).toBe(0); + const healthy = frames(createResolutionState(), 1000 / 60, 240).state; + expect(healthy.healthyWindows).toBeGreaterThan(0); + const interrupted = advanceResolution(healthy, { timestamp: healthy.lastTimestamp! + 500, playing: true, key: "scene" }); + expect(interrupted.healthyWindows).toBe(healthy.healthyWindows); +}); + +test("adaptation never changes playback clocks across a real pause and resume", () => { + let playback = createPlaybackState({ phase: "playing", slideKey: "clock", stepKey: "0", stepIndex: 0 }); + let resolution = createResolutionState(); + for (let timestamp = 0; timestamp <= 10000; timestamp += 20) { + playback = advancePlayback(playback, timestamp); + resolution = advanceResolution(resolution, { timestamp: playback.lastTimestamp, playing: playback.phase === "playing", key: playback.slideKey }); + } + expect(resolution.scale).toBeLessThan(1); + expect(playback.slideTime).toBeCloseTo(10); + playback = transitionPlayback(playback, { ...playback, phase: "paused" }); + resolution = advanceResolution(resolution, { timestamp: playback.lastTimestamp, playing: false, key: playback.slideKey }); + const paused = resolution.scale; + playback = advancePlayback(playback, 50000); + expect(playback.slideTime).toBeCloseTo(10); + playback = transitionPlayback(playback, { ...playback, phase: "playing" }); + playback = advancePlayback(playback, 50000); + resolution = advanceResolution(resolution, { timestamp: playback.lastTimestamp, playing: true, key: playback.slideKey }); + expect(resolution.scale).toBe(paused); + expect(resolution.frames).toBe(0); + expect(playback.slideTime).toBeCloseTo(10); +}); From 88a6da25a96c4ce8b1b81dfca3e29a6ac93f6b27 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 06:47:00 -0700 Subject: [PATCH 04/28] fix: refine world glow, reflections and lighting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use analytic sphere surfaces and radial emission for smooth round cores without tessellation cost. Refine local material lighting and rough mirror reflections while keeping the existing API. **Test plan** ✅ `bun test packages/slides-effects/tests/worldLook.test.ts packages/slides-effects/tests/world.test.ts packages/slides-effects/tests/worldRender.test.ts` ✅ Offline Chrome WebGPU compilation and smoke render with the full world graph. --- packages/slides-effects/packs/world/README.md | 24 ++++- packages/slides-effects/packs/world/render.ts | 87 ++++++++++++++----- .../slides-effects/tests/worldLook.test.ts | 28 ++++++ 3 files changed, 114 insertions(+), 25 deletions(-) create mode 100644 packages/slides-effects/tests/worldLook.test.ts diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index be7e8b33a..1a2aaf94d 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -133,10 +133,26 @@ both a camera target and a focus target. Materials can be declared by id or passed inline through a primitive's `material` prop. Albedo, emissive intensity, roughness, metal and rim strength support tracks. Point and spot lights support animated position, target, color, intensity, range -and spot angle (radians). At most four local lights are supported. `RimLight` -sets the shared rim color/intensity. The mirror redraws the scene about the floor -plane into a half-resolution reflection target and clips geometry below it. -Roughness blurs that reflection; reflectivity controls its strength. +and spot angle (radians). Intensity uses relative linear-light units, with +inverse-square attenuation beyond one world unit and a smooth cutoff at range. +Spot angle is the outer cone half-angle. Materials use GGX specular, Schlick +fresnel and energy-conserving diffuse; `metal` tints specular with albedo and +reduces diffuse. Surface radiance is capped at 4096 linear units to retain +half-float headroom for bloom. At most four local lights are supported. `RimLight` +adds a shared view-dependent rim color/intensity, including on dark materials. + +Spheres, orbs and spray use six-vertex analytic spheres with curved normals, +nonuniform scale and surface depth. Emissive spheres have a bright round center +and radial falloff; bloom supplies the surrounding halo. Emission contributes +HDR radiance but does not automatically create a local light. Place a point light +near an emissive subject when it should illuminate nearby surfaces. + +The mirror redraws the scene about the floor plane into a half-resolution +reflection target and clips geometry below it. Roughness broadens and fades the +reflection with distance; reflectivity controls its strength, with fresnel and +metal controlling its viewing-angle response. Reflection taps stay inside the +viewport and fade near the finite floor boundary. The regular fog stage also +fades the floor into the scene. DOF uses a half-resolution gather and depth-aware composite. Shafts integrate point/spot scattering along the visible camera ray at quarter resolution. diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index 413ef03fd..65f8904f6 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -29,12 +29,7 @@ fn corner(id:u32)->vec2f { } `; const mesh: Record = { - sphere: { count: 12 * 8 * 6, code: ` - let cell=vid/6u; let uv=(vec2f(f32(cell%12u),f32(cell/12u))+corner(vid))/vec2f(12.,8.); - let longitude=uv.x*6.2831853; let latitude=uv.y*3.14159265; - normal=vec3f(sin(latitude)*cos(longitude),cos(latitude),sin(latitude)*sin(longitude)); - point=normal*radius; -` }, + sphere: { count: 6, code: "" }, box: { count: 36, code: ` let face=vid/6u; let uv=corner(vid)*2.-1.; if(face==0u) { normal=vec3f(1.,0.,0.); point=vec3f(1.,uv.y,uv.x); } @@ -72,22 +67,28 @@ struct ShapeContext { time:f32, slideTime:f32, stepTime:f32, step:f32 } struct ShapeVertex { position:vec3f, normal:vec3f } fn shapeContext()->ShapeContext { return ShapeContext(frame.time,frame.slideTime,frame.stepTime,frame.step); } fn illuminate(world:vec3f,normal:vec3f,material:Material,rimColor:vec3f,rimIntensity:f32,activation:f32)->vec3f { - let view=normalize(eye()-world); var color=material.albedo*uniforms[52]+material.emissive; + let view=normalize(eye()-world); let nv=max(0.,dot(normal,view)); + var color=material.albedo*uniforms[52]+material.emissive; let roughness=clamp(material.roughness,.04,1.); let metal=clamp(material.metal,0.,1.); ${lightCount === 0 ? "color+=material.albedo*max(0.,dot(normal,normalize(vec3f(-.4,1.,.6))))*.55;" : ""} for(var light=0u;light<${lightCount}u;light++) { let base=64u+light*12u; let delta=vector(base)-world; let distance=max(length(delta),.001); let toward=delta/distance; - let range=max(.001,uniforms[base+7u]); let attenuation=pow(max(0.,1.-distance/range),2.)/(1.+distance*distance*.06); + let range=max(.001,uniforms[base+7u]); let cutoff=pow(max(0.,1.-pow(distance/range,4.)),2.); + let attenuation=cutoff/max(1.,distance*distance); var cone=1.; let cosine=uniforms[base+11u]; if(cosine>-.99) { cone=smoothstep(cosine,min(.9999,cosine+.08),dot(-toward,normalize(vector(base+8u)))); } let halfVector=normalize(toward+view+vec3f(.00001)); - let specular=pow(max(0.,dot(normal,halfVector)),mix(128.,3.,roughness))*mix(1.,.12,roughness); - let f0=mix(vec3f(.04),material.albedo,metal); - let diffuse=material.albedo*max(0.,dot(normal,toward))*(1.-metal); - color+=(diffuse+f0*specular)*vector(base+4u)*uniforms[base+3u]*attenuation*cone; + let nl=max(0.,dot(normal,toward)); let nh=max(0.,dot(normal,halfVector)); + let vh=max(0.,dot(view,halfVector)); let alpha=roughness*roughness; let a2=alpha*alpha; + let denominator=nh*nh*(a2-1.)+1.; + let distribution=a2/(3.14159265*denominator*denominator); + let visibility=.5/max(.0001,nl*sqrt(nv*nv*(1.-a2)+a2)+nv*sqrt(nl*nl*(1.-a2)+a2)); + let f0=mix(vec3f(.04),material.albedo,metal); let fresnel=f0+(1.-f0)*pow(1.-vh,5.); + let diffuse=material.albedo*(1.-fresnel)*(1.-metal)/3.14159265; + color+=(diffuse+fresnel*distribution*visibility)*nl*vector(base+4u)*max(0.,uniforms[base+3u])*attenuation*cone; } - let rim=pow(1.-abs(dot(normal,view)),3.); - return color+rim*(rimColor*rimIntensity+vector(48u)*uniforms[51]*activation); + let rim=pow(1.-nv,3.); + return clamp(color+rim*(rimColor*rimIntensity+vector(48u)*uniforms[51]*activation),vec3f(0.),vec3f(4096.)); } `; } @@ -98,6 +99,7 @@ struct WorldVertex { @builtin(position) position:vec4f, @location(0) world:vec3f `; function geometryCode(kind: PrimitiveKind, offset: number, lights: number, reflection: boolean, custom?: CustomGeometry, sdf?: SDFGeometry) { const shape = custom ? `let shapeVertex=shape(iid,vid,shapeContext()); point=shapeVertex.position*radius; normal=shapeVertex.normal;` : mesh[kind].code; + const sphere = kind === "sphere" && !custom && !sdf; const bounds = sdf ? sdf.bounds.map(v => `vec3f(${v.map(n => Number.isInteger(n) ? n + "." : n).join(",")})`) : []; return `${worldBindings}${triangleCorner}${lightingCode(lights, reflection)}${vertexFields} ${custom?.wgsl ?? sdf?.wgsl ?? ""} @@ -110,9 +112,47 @@ ${custom?.wgsl ?? sdf?.wgsl ?? ""} let scale=max(vector(base+8u),vec3f(.0001)); let world=origin+point*scale; var result:WorldVertex; result.position=projected(world); result.world=world; result.normal=normalize(normal/scale); result.color=vector(base+4u); result.emissive=uniforms[base+7u]; result.material=vec4f(uniforms[base+27u],uniforms[base+28u],uniforms[base+29u],uniforms[base+30u]); - result.albedo=vector(base+24u); result.opacity=uniforms[base+23u]; result.instance=iid; return result; + result.albedo=vector(base+24u); result.opacity=uniforms[base+23u]; result.instance=iid; + ${sphere ? ` + let center=projected(origin); let radii=max(vector(base+8u)*uniforms[base+3u],vec3f(.0001)); + let extent=max(radii.x,max(radii.y,radii.z)); + let z=max(.0001,center.w); let centerNDC=center.xy/z; + let margin=vec2f(extent)/(vec2f(uniforms[3]*frame.aspect,uniforms[3])*max(.0001,z-extent))*(1.+abs(centerNDC)*vec2f(uniforms[3]*frame.aspect,uniforms[3])); + let lower=select(max(vec2f(-1.),centerNDC-margin),vec2f(-1.),center.w<=extent); + let upper=select(min(vec2f(1.),centerNDC+margin),vec2f(1.),center.w<=extent); + let ndc=mix(lower,upper,corner(vid)); + var ray=cameraForward()+cameraRight()*ndc.x*frame.aspect*uniforms[3]+cameraUp()*ndc.y*uniforms[3]; + ${reflection ? "ray.y=-ray.y;" : ""} + result.position=vec4f(ndc,0.,1.); result.world=eye()+ray; + ` : ""} + return result; } -${sdf ? ` +${sphere ? ` +struct SphereResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } +@fragment fn worldFragment(in:WorldVertex)->SphereResult { + let base=${worldHeaderSize}u+(in.instance+${offset}u)*${worldInstanceSize}u; + let radii=max(vector(base+8u)*uniforms[base+3u],vec3f(.0001)); + let origin=(eye()-vector(base))/radii; let ray=normalize((in.world-eye())/radii); + let along=dot(origin,ray); let closest=origin-ray*along; let field=1.-dot(closest,closest); + let edge=max(fwidth(field),.00001); + if(field<=0. || in.opacity<=.001) { discard; } + let roots=vec2f(-along-sqrt(field),-along+sqrt(field)); + var local=vec3f(0.); var hit=vec3f(0.); var clip=vec4f(0.); var found=false; + for(var root=0u;root<2u;root++) { + if(roots[root]<0.) { continue; } + local=origin+ray*roots[root]; hit=vector(base)+local*radii; clip=projected(hit); + if(clip.w<=.05 || clip.z<0. || clip.z>clip.w) { continue; } + ${reflection ? "if(hit.ySDFResult { if(in.opacity<=.001) { discard; } @@ -137,7 +177,7 @@ struct SDFResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } }` : ` @fragment fn worldFragment(in:WorldVertex)->@location(0) vec4f { if(in.opacity<=.001) { discard; } - ${reflection ? "if(in.world.yFloorResult { let ray=cameraForward()+cameraRight()*(in.uv.x*2.-1.)*frame.aspect*uniforms[3]+cameraUp()*(1.-in.uv.y*2.)*uniforms[3]; - if(abs(ray.y)<.0001) { discard; } let distance=(uniforms[36]-vector(0u).y)/ray.y; + if(ray.y>=-.0001 || vector(0u).y<=uniforms[36]) { discard; } let distance=(uniforms[36]-vector(0u).y)/ray.y; let world=vector(0u)+ray*distance; if(distance<=.05 || distance>=500. || abs(world.x)>uniforms[39] || abs(world.z)>uniforms[39]) { discard; } let normal=vec3f(0.,1.,0.); let view=normalize(vector(0u)-world); let material=Material(vector(40u),vec3f(0.),uniforms[37],uniforms[43]); - var color=illuminate(world,normal,material,vec3f(0.),0.,1.); - ${mirror ? `let spread=uniforms[37]*.025; let pixel=vec2f(spread,spread*frame.aspect); + var color=illuminate(world,normal,material,vec3f(0.),0.,0.); + ${mirror ? `let roughness=clamp(uniforms[37],0.,1.); + let spread=roughness*roughness*min(distance,80.)*.0015; + let pixel=vec2f(spread,spread*frame.aspect); var reflected=textureSampleLevel(reflection,linearSampler,in.uv,0.).rgb*.4; reflected+=(textureSampleLevel(reflection,linearSampler,in.uv+pixel*vec2f(1.,0.),0.).rgb+textureSampleLevel(reflection,linearSampler,in.uv-pixel*vec2f(1.,0.),0.).rgb+textureSampleLevel(reflection,linearSampler,in.uv+pixel*vec2f(0.,1.),0.).rgb+textureSampleLevel(reflection,linearSampler,in.uv-pixel*vec2f(0.,1.),0.).rgb)*.15; let fresnel=.04+.96*pow(1.-clamp(dot(normal,view),0.,1.),5.); - color=mix(color,reflected,clamp(uniforms[38]*mix(fresnel,1.,uniforms[43]),0.,1.));` : ""} + let edgeFade=1.-smoothstep(uniforms[39]*.85,uniforms[39],max(abs(world.x),abs(world.z))); + let distanceFade=exp(-max(0.,uniforms[12])*distance*roughness); + let strength=clamp(uniforms[38],0.,1.)*mix(fresnel,1.,clamp(uniforms[43],0.,1.))*(1.-roughness*.7)*distanceFade*edgeFade; + color=mix(color,reflected,strength);` : ""} let clip=project(world); return FloorResult(vec4f(color,1.),clamp(clip.z/clip.w,0.,1.)); }`; } diff --git a/packages/slides-effects/tests/worldLook.test.ts b/packages/slides-effects/tests/worldLook.test.ts new file mode 100644 index 000000000..620d23d0e --- /dev/null +++ b/packages/slides-effects/tests/worldLook.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from "bun:test"; +import { buildWorldGraph } from "../packs/world/render"; +import { orbInstances, prepareWorldScene, primitive, sprayInstances } from "../packs/world/scene"; + +for (const quality of ["low", "medium", "high"] as const) { + test(`round particles use one instanced quad with surface depth at ${quality} quality`, () => { + const scene = prepareWorldScene({ quality, floor: { mirror: true }, instances: [ + primitive("box"), ...orbInstances({ count: 1000 }), ...sprayInstances({ rate: 1000, lifetime: 2 }), + ] }); + const graph = buildWorldGraph(scene); + for (const id of ["scene-sphere", "reflection-sphere"]) { + const draw = graph.passes.find(pass => pass.id === id)!; + expect(draw.kind).toBe("render"); + if (draw.kind !== "render") throw new Error("Expected sphere render pass"); + expect(draw.vertexCount).toBe(6); + expect(draw.instanceCount).toBe(3000); + expect(draw.code).toContain("@builtin(frag_depth)"); + expect(draw.code).toContain("if(field<=0."); + expect(draw.code).toContain("fwidth(field)"); + expect(draw.code).toContain("normal=normalize(local/radii)"); + expect(draw.code).toContain("(iid+1u)"); + } + const reflection = graph.targets.find(target => target.id === "reflect")!; + const hdr = graph.targets.find(target => target.id === "hdr")!; + expect(reflection.scale).toBe(hdr.scale! * .5); + expect(graph.passes.filter(pass => pass.kind === "render")).toHaveLength(5); + }); +} From 39fcda0777d28f2875fd9dbf9442dfacabf888ea Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 08:18:09 -0700 Subject: [PATCH 05/28] fix: cull hidden world spheres before rasterization Reject fully clipped and transparent sphere bounds in the vertex stage so hidden instances do not rasterize viewport quads. Preserve independent size and scale clamps for analytic intersections. Test plan: - 31 focused world tests passed. - Scoped renderer typecheck passed with zero diagnostics. - Nine Chrome WebGPU checks and offline 1080p/4K captures completed. - Fixed-workload ABBA renderer measurements completed at both resolutions. --- packages/slides-effects/packs/world/render.ts | 7 +++++-- packages/slides-effects/tests/worldLook.test.ts | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index 65f8904f6..ccaf396db 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -114,8 +114,11 @@ ${custom?.wgsl ?? sdf?.wgsl ?? ""} result.color=vector(base+4u); result.emissive=uniforms[base+7u]; result.material=vec4f(uniforms[base+27u],uniforms[base+28u],uniforms[base+29u],uniforms[base+30u]); result.albedo=vector(base+24u); result.opacity=uniforms[base+23u]; result.instance=iid; ${sphere ? ` - let center=projected(origin); let radii=max(vector(base+8u)*uniforms[base+3u],vec3f(.0001)); + let center=projected(origin); let radii=scale*radius; let extent=max(radii.x,max(radii.y,radii.z)); + if(result.opacity<=.001 || center.w+extent<=.05 || center.w-extent>=500.) { + result.position=vec4f(2.,2.,0.,1.); return result; + } let z=max(.0001,center.w); let centerNDC=center.xy/z; let margin=vec2f(extent)/(vec2f(uniforms[3]*frame.aspect,uniforms[3])*max(.0001,z-extent))*(1.+abs(centerNDC)*vec2f(uniforms[3]*frame.aspect,uniforms[3])); let lower=select(max(vec2f(-1.),centerNDC-margin),vec2f(-1.),center.w<=extent); @@ -131,7 +134,7 @@ ${sphere ? ` struct SphereResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } @fragment fn worldFragment(in:WorldVertex)->SphereResult { let base=${worldHeaderSize}u+(in.instance+${offset}u)*${worldInstanceSize}u; - let radii=max(vector(base+8u)*uniforms[base+3u],vec3f(.0001)); + let radii=max(vector(base+8u),vec3f(.0001))*max(uniforms[base+3u],.0001); let origin=(eye()-vector(base))/radii; let ray=normalize((in.world-eye())/radii); let along=dot(origin,ray); let closest=origin-ray*along; let field=1.-dot(closest,closest); let edge=max(fwidth(field),.00001); diff --git a/packages/slides-effects/tests/worldLook.test.ts b/packages/slides-effects/tests/worldLook.test.ts index 620d23d0e..eb8365218 100644 --- a/packages/slides-effects/tests/worldLook.test.ts +++ b/packages/slides-effects/tests/worldLook.test.ts @@ -13,6 +13,8 @@ for (const quality of ["low", "medium", "high"] as const) { expect(draw.kind).toBe("render"); if (draw.kind !== "render") throw new Error("Expected sphere render pass"); expect(draw.vertexCount).toBe(6); + const vertex = draw.code.slice(draw.code.indexOf("@vertex"), draw.code.indexOf("struct SphereResult")); + expect(vertex).toContain("result.opacity<=.001 || center.w+extent<=.05 || center.w-extent>=500."); expect(draw.instanceCount).toBe(3000); expect(draw.code).toContain("@builtin(frag_depth)"); expect(draw.code).toContain("if(field<=0."); From 5d6d70dfa2f5981e50e6b64a0d452666b2a79dd1 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 08:45:59 -0700 Subject: [PATCH 06/28] fix: preserve GPU backgrounds across slide navigation --- apps/slides/examples/3d/cinematic-title.mdx | 8 +- apps/slides/src/DeckRenderer.tsx | 2 +- apps/slides/src/TypeGPUShader.tsx | 18 ++-- .../slides/src/__tests__/deckRuntime.test.tsx | 48 +++++++++- apps/slides/src/__tests__/gpuSetup.test.ts | 93 +++++++++++++++++++ .../src/__tests__/typeGPUShader.test.ts | 7 +- packages/slides-effects/packs/world/README.md | 5 +- packages/slides-effects/packs/world/World.tsx | 51 ++++++---- .../tests/worldBackgroundPlayback.test.tsx | 87 +++++++++++++++++ .../tests/worldPlayback.test.tsx | 89 +++++++++++++++++- 10 files changed, 373 insertions(+), 35 deletions(-) create mode 100644 packages/slides-effects/tests/worldBackgroundPlayback.test.tsx diff --git a/apps/slides/examples/3d/cinematic-title.mdx b/apps/slides/examples/3d/cinematic-title.mdx index 970043bf0..01096a97c 100644 --- a/apps/slides/examples/3d/cinematic-title.mdx +++ b/apps/slides/examples/3d/cinematic-title.mdx @@ -13,7 +13,7 @@ import { Text } from "react-native" import { World, Camera, Box, Ring, Orbs, Material, Floor, SpotLight, RimLight, Fog, Shafts, DOF, Bloom, Vignette, Grain } from "slides:world/World" export const structures = <> - + @@ -39,7 +39,7 @@ export const structures = <> # Quiet structures -Light, air and a slow orbit +Light, air and a slow orbit --- @@ -59,7 +59,7 @@ export const structures = <> # Through the air -A quiet reveal +A quiet reveal --- @@ -79,4 +79,4 @@ export const structures = <> # Room to breathe -A wider view +A wider view diff --git a/apps/slides/src/DeckRenderer.tsx b/apps/slides/src/DeckRenderer.tsx index 70cd88fbb..3c5becb28 100644 --- a/apps/slides/src/DeckRenderer.tsx +++ b/apps/slides/src/DeckRenderer.tsx @@ -330,7 +330,7 @@ function BackgroundErrorBoundary({ children }: { children?: ReactNode }) { } export function SlideBackgroundHost(props: ComponentProps) { - return ; + return ; } export function SlideCanvas({ children, captureEnabled = true, targetIndex, isPreview = false, backgroundIsPreview = isPreview }: { children: ReactNode; captureEnabled?: boolean; targetIndex?: number; isPreview?: boolean; backgroundIsPreview?: boolean }) { diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 943a4aa45..65cf0893a 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -12,8 +12,10 @@ type Resources = { id: number; alive: SharedValue; disposed: SharedValue; + configured: SharedValue; device: GPUDevice; context: RNCanvasContext; + format: GPUTextureFormat; clock: GPUBuffer; states: GPUBuffer[]; groups: GPUBindGroup[]; @@ -126,12 +128,11 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle // Set the drawing buffer before transferring ownership to UIFrames. if (Platform.OS === "web") resizeBrowserSurface(context, width, height, displayScale); const format = navigator.gpu.getPreferredCanvasFormat(); - context.configure({ device, format, alphaMode: "premultiplied" }); if (renderGraph) { const clock = device.createBuffer({ size: 32, usage: 64 | 8 }); const frameData = device.createBuffer({ size: Math.max(16, frameDataSize * 4), usage: 128 | 8 }); const graph = await prepareRenderGraph(device, renderGraph, format, clock, frameData); - await transfer({ device, context, clock, frameData, graph, states: [], groups: [] }); + await transfer({ device, context, format, clock, frameData, graph, states: [], groups: [] }); return; } const fieldFormat = composite ? "rgba16float" : format; @@ -195,7 +196,7 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle }); compositeResources = { pipeline, group, texture }; } - await transfer({ device, context, clock, states, groups, compute, render, particles: particlePipeline, frameData, composite: compositeResources }); + await transfer({ device, context, format, clock, states, groups, compute, render, particles: particlePipeline, frameData, composite: compositeResources }); } catch (caught) { if (validationScope) { validationScope = false; @@ -205,15 +206,15 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle if (!cancelled) { setError(caught instanceof Error ? caught.message : String(caught)); reportError(caught); } } } - async function transfer(program: Omit & { graph?: RenderGraphResources }) { + async function transfer(program: Omit & { graph?: RenderGraphResources }) { validationScope = false; const validationError = await program.device.popErrorScope(); if (validationError) throw new Error(validationError.message); if (cancelled) { program.device.destroy(); return; } - const alive = makeMutable(true), disposed = makeMutable(false); + const alive = makeMutable(true), disposed = makeMutable(false), configured = makeMutable(false); release = () => runOnUI(destroyGPU)(program.device, alive, disposed); transferred = true; - setResources({ ...program, graph: program.graph ? makeMutable(program.graph) : undefined, alive, disposed, id: ++nextGPUResourceId }); + setResources({ ...program, graph: program.graph ? makeMutable(program.graph) : undefined, alive, disposed, configured, id: ++nextGPUResourceId }); } void setup(); return () => { cancelled = true; if (release) release(); else if (!transferred) device?.destroy(); }; @@ -330,6 +331,11 @@ async function loadImageTexture(device: GPUDevice, uri: string) { function submitGPUFrame(r: Resources, frame: number, particleCount: number, particleVertexCount: number, frameData?: SharedValue, renderScale = 1) { "worklet"; const { device, context, clock, groups, compute, render, particles } = r; + // Configure on UI after retirement of the previous owner, never during async setup. + if (!r.configured.value) { + context.configure({ device, format: r.format, alphaMode: "premultiplied" }); + r.configured.value = true; + } const data = frameData?.value; if (r.frameData && data) device.queue.writeBuffer(r.frameData, 0, new Float32Array(data!)); const encoder = device.createCommandEncoder(); diff --git a/apps/slides/src/__tests__/deckRuntime.test.tsx b/apps/slides/src/__tests__/deckRuntime.test.tsx index 45dc0625f..0f0987ea8 100644 --- a/apps/slides/src/__tests__/deckRuntime.test.tsx +++ b/apps/slides/src/__tests__/deckRuntime.test.tsx @@ -9,7 +9,7 @@ import React from "react"; import * as state from "@legendapp/state"; import * as stateReact from "@legendapp/state/react"; import { act, create } from "react-test-renderer"; -import { usePresentation } from "@legend-apps/presentation"; +import { GPURenderGraph, usePresentation } from "@legend-apps/presentation"; import { getSlidesState, setCurrentSlide, setSlidesState, slidesState$ } from "../slidesStore"; const titleSpeed$ = state.observable(1); @@ -470,3 +470,49 @@ test("brightness changes preserve the measured preview stage and mounted slide", setSlidesState(initial); } }); + +test("selected backgrounds retain the GPU renderer through registration, selection and removal", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + const initial = getSlidesState(); + const DeckRenderer = compiledRenderer(); + const SlideBackgroundHost = compiledRenderer("SlideBackgroundHost"); + const uniforms = { value: [1] }; + const passes = [{ kind: "fullscreen", id: "background", code: "@fragment fn shade()->@location(0) vec4f { return vec4f(0.,0.,0.,1.); }", fragment: "shade", output: "screen" }]; + let mounts = 0, unmounts = 0; + function BackgroundGraph() { + React.useEffect(() => { mounts++; return () => { unmounts++; }; }, []); + return ; + } + function Document({ components: { Deck, Slide, Background } }) { + return + + ; + } + const content = (slideIndex = 0, visible = true) => + {visible && } + ; + const log = spyOn(console, "error").mockImplementation(() => {}); + let renderer; + try { + setSlidesState({ component: Document, currentSlide: 0, slides: [], config: {}, runtimeErrors: [] }); + await act(() => { renderer = create(content()); }); + const shader = renderer.root.findByType("gpu-shader"); + expect(shader.props.renderGraph.passes).toBe(passes); + expect(shader.props.frameData).toBe(uniforms); + expect(mounts).toBe(1); expect(unmounts).toBe(0); + await act(() => renderer.update(content(1))); + expect(renderer.root.findAllByType("gpu-shader")).toHaveLength(0); + expect(unmounts).toBe(1); + await act(() => renderer.update(content())); + expect(renderer.root.findAllByType("gpu-shader")).toHaveLength(1); + expect(mounts).toBe(2); + await act(() => renderer.update(content(0, false))); + expect(renderer.root.findAllByType("gpu-shader")).toHaveLength(0); + expect(unmounts).toBe(2); + expect(getSlidesState().runtimeErrors).toEqual([]); + } finally { + if (renderer) await act(() => renderer.unmount()); + setSlidesState(initial); + log.mockRestore(); + } +}); diff --git a/apps/slides/src/__tests__/gpuSetup.test.ts b/apps/slides/src/__tests__/gpuSetup.test.ts index ccda35a00..18aec5a9c 100644 --- a/apps/slides/src/__tests__/gpuSetup.test.ts +++ b/apps/slides/src/__tests__/gpuSetup.test.ts @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; import { transformSync } from "@babel/core"; import { fileURLToPath } from "node:url"; +import { prepareRenderGraph, encodeRenderGraph } from "../gpuRenderGraph"; const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); const require = createRequire(filename); @@ -59,3 +60,95 @@ test("valid resources transfer only after validation and cancelled setup never t expect(cancelled.resources).toEqual([undefined]); expect(cancelled.errors).toEqual([]); }); + +test.each(["web", "macos"])("canvas device handoff waits for deferred frame retirement on %s", async platform => { + const deferred = () => { let resolve; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; }; + const ui = [], pendingDevices = [], events = [], errors = []; + let effects = [], callback, owner; + const context = { + canvas: { width: 100, height: 100 }, + configure: ({ device }) => { owner = device; events.push(["configure", device.id]); }, + getCurrentTexture: () => { + if (!owner) throw new Error("Texture requested before configuration"); + return { width: 100, height: 100, createView: () => ({ device: owner }) }; + }, + present() {}, + }; + const mutable = value => ({ value, set(next) { this.value = next; } }); + const playback = { value: { phase: "playing", slideKey: "slide", stepKey: "step", stepIndex: 0 } }; + const mocks = { + react: { useEffect: effect => effects.push(effect) }, + "react-native": { Platform: { OS: platform }, PixelRatio: { get: () => 1 } }, + "@shopify/react-native-skia": {}, "react-native-webgpu": {}, "./slidesStore": {}, + "@legend-apps/presentation": { usePresentationValue: field => field === "isActive", usePlayback: () => playback, samplePlayback: () => 0 }, + "./gpuRenderGraph": { prepareRenderGraph, encodeRenderGraph }, + "react-native-reanimated": { makeMutable: mutable, useSharedValue: mutable, useAnimatedReaction() {}, + useFrameCallback: fn => { callback = fn; return { setActive() {} }; }, runOnJS: fn => fn, + runOnUI: fn => (...args) => ui.push(() => fn(...args)) }, + }; + const { startShaderCanvas, UIFrames } = new Function("require", "exports", "navigator", code + "\nreturn { startShaderCanvas, UIFrames };")( + name => mocks[name] ?? require(name), {}, { gpu: { requestAdapter: async () => ({ requestDevice: async () => pendingDevices.shift() }), getPreferredCanvasFormat: () => "bgra8unorm" } }, + ); + const flushUI = () => { while (ui.length) ui.shift()(); }; + const begin = async id => { + const ready = deferred(), validation = deferred(), transferred = deferred(), resources = []; + const device = { id, lost: new Promise(() => {}), addEventListener() {}, pushErrorScope() {}, + popErrorScope: () => { ready.resolve(); return validation.promise; }, destroy: () => events.push(["destroy", id]), + createBuffer: () => ({}), createSampler: () => ({}), createBindGroupLayout: () => ({}), createPipelineLayout: () => ({}), createBindGroup: () => ({}), + createShaderModule: () => ({ getCompilationInfo: async () => ({ messages: [] }) }), createRenderPipeline: () => ({}), + queue: { writeBuffer() {}, submit() {} }, + createCommandEncoder: () => ({ finish() {}, beginRenderPass: descriptor => { + const view = descriptor.colorAttachments[0].view; + if (view.device !== device) throw new Error(`Device ${id} received a texture from device ${view.device.id}`); + events.push(["draw", id]); + return { setPipeline() {}, setBindGroup() {}, draw() {}, end() {} }; + } }), + }; + pendingDevices.push(device); + const stop = startShaderCanvas({ canvas: { current: { getContext: () => context } }, code: "", particles: false, particleCount: 1, + renderGraph: { targets: [], passes: [{ kind: "fullscreen", id: `screen-${id}`, code: "", fragment: "main", output: "screen" }] }, + setResources: value => { resources.push(value); if (value) transferred.resolve(value); }, setError: error => { if (error) errors.push(error); }, reportError: error => errors.push(String(error)) }); + await ready.promise; + return { stop, validation, transferred, resources }; + }; + const mount = resources => { + effects = []; + UIFrames({ resources, width: 100, height: 100, aspect: 1, brightness: 1, particleCount: 1, previewTime: 0, + onReady() {}, onError: error => errors.push(error) }); + const tick = callback, cleanup = effects.map(effect => effect()); + return { tick: () => tick({ timeSincePreviousFrame: 16 }), stop: () => cleanup.forEach(fn => fn?.()) }; + }; + const configurations = () => events.filter(event => event[0] === "configure").map(event => event[1]); + const old = await begin(1); + expect(configurations()).toEqual([]); + old.validation.resolve(null); + const oldFrames = mount(await old.transferred.promise); + expect(configurations()).toEqual([]); + flushUI(); oldFrames.tick(); + expect(configurations()).toEqual([1]); + old.stop(); oldFrames.stop(); + const replacement = await begin(2); + oldFrames.tick(); + expect(configurations()).toEqual([1]); + replacement.validation.resolve(null); + const newFrames = mount(await replacement.transferred.promise); + expect(configurations()).toEqual([1]); + flushUI(); + expect(configurations()).toEqual([1, 2]); + expect(events.findIndex(event => event[0] === "destroy" && event[1] === 1)).toBeLessThan(events.findIndex(event => event[0] === "configure" && event[1] === 2)); + const draws = events.filter(event => event[0] === "draw").length; + oldFrames.tick(); + expect(events.filter(event => event[0] === "draw")).toHaveLength(draws); + newFrames.tick(); + expect(events.at(-1)).toEqual(["draw", 2]); + const cancelled = await begin(3); + cancelled.stop(); cancelled.validation.resolve(null); + await new Promise(resolve => setImmediate(resolve)); + expect(cancelled.resources).toEqual([undefined]); + expect(configurations()).toEqual([1, 2]); + newFrames.tick(); + expect(errors).toEqual([]); + replacement.stop(); newFrames.stop(); flushUI(); + expect(events.filter(event => event[0] === "destroy" && event[1] === 1)).toHaveLength(1); + expect(events.filter(event => event[0] === "destroy" && event[1] === 2)).toHaveLength(1); +}); diff --git a/apps/slides/src/__tests__/typeGPUShader.test.ts b/apps/slides/src/__tests__/typeGPUShader.test.ts index 367007acb..ef14997fc 100644 --- a/apps/slides/src/__tests__/typeGPUShader.test.ts +++ b/apps/slides/src/__tests__/typeGPUShader.test.ts @@ -51,7 +51,7 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, const UIFrames = new Function("require", "exports", code + "\nreturn UIFrames;")( name => mocks[name] ?? require(name), {}, ); - UIFrames({ resources: { alive: { value: true, set(value) { this.value = value; } }, disposed: { value: false }, device, context: { getCurrentTexture: () => ({ createView() {} }), present() {} }, + UIFrames({ resources: { alive: { value: true, set(value) { this.value = value; } }, disposed: { value: false }, configured: { value: false }, format: "bgra8unorm", device, context: { configure() {}, getCurrentTexture: () => ({ createView() {} }), present() {} }, clock: {}, graph: graph ? { value: {} } : undefined, states: [], groups: simulation ? ["A", "B"] : ["A"], compute: simulation ? {} : undefined, render: {}, particles: {}, frameData: data ? {} : undefined, composite: composite ? { pipeline: {}, group: "lens", texture: { createView() {} } } : undefined }, effectActive, maxTime, frameTimeIndex, clockMode, renderScale, frameData: data ? frameData : undefined, aspect: 2.61, brightness: 1, particleCount: 2048, previewTime: 7, onError: error => errors.push(error), onReady: () => ready++ }); @@ -257,7 +257,8 @@ test("offscreen browser setup sizes first and allocates simulation resources onl ); const surface = {width:0,height:0}; const assertSize = () => expect([surface.width,surface.height]).toEqual([2016,1000]); - const context = {canvas:surface,configure:assertSize,getCurrentTexture:()=>{assertSize();return {createView(){}};}}; + let configurations = 0; + const context = {canvas:surface,configure:()=>{configurations++;assertSize();}}; const buffers=[],computes=[]; const device = { addEventListener(){},pushErrorScope(){},popErrorScope:async()=>null,lost:new Promise(()=>{}),destroy(){}, @@ -271,7 +272,7 @@ test("offscreen browser setup sizes first and allocates simulation resources onl for (const simulation of [false,true]) await new Promise((resolve,reject)=>{ start({canvas:{current:{getContext:()=>{surface.width=0;surface.height=0;return context;}}}, code:simulation ? "@compute @workgroup_size(64) fn simulate() {}" : "",particles:false,particleCount:1,width:2016,height:1000,displayScale:0.5, - setResources:r=>{if(r){r.context.getCurrentTexture();expect(r.states.length).toBe(simulation ? 2 : 0);expect(r.groups.length).toBe(simulation ? 2 : 1);resolve();}},setError:error=>{if(error)reject(new Error(error));},reportError:reject}); + setResources:r=>{if(r){assertSize();expect(r.configured.value).toBe(false);expect(configurations).toBe(0);expect(r.states.length).toBe(simulation ? 2 : 0);expect(r.groups.length).toBe(simulation ? 2 : 1);resolve();}},setError:error=>{if(error)reject(new Error(error));},reportError:reject}); }); expect(buffers.filter(b=>b.usage===(128|8))).toHaveLength(2); expect(buffers.filter(b=>b.usage===(64|8))).toHaveLength(2); diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index 650820a41..e1347a99d 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -42,6 +42,9 @@ import { Background } from "@legend-apps/presentation"; ``` +World animation time restarts on slide entry, including inside ``. +This includes drifting objects, scrolling rows, sprays and custom WGSL time uniforms. + `Camera.shots` are ordered keyframes with either a `step` or an `at` time in seconds. Times and step numbers should increase within their respective clock. Each shot inherits unspecified values from the pose sampled when it starts; @@ -95,7 +98,7 @@ scale and three levels. Output resolution stays unchanged. `renderScale` is a static multiplier greater than zero and at most one for every offscreen target. It combines with the preset and the per-effect scales; it does not rebuild pipelines. -`auto` starts with the high preset and adjusts an additional multiplier from 1 +`auto` starts each slide with the high preset and adjusts an additional multiplier from 1 down to 0.5 in 0.125 steps. It samples live playback callback intervals, waits through startup and resizing, lowers resolution after sustained intervals above 18.5ms, and requires several healthy windows before raising it. Paused and diff --git a/packages/slides-effects/packs/world/World.tsx b/packages/slides-effects/packs/world/World.tsx index 722f1e45a..4c68f93de 100644 --- a/packages/slides-effects/packs/world/World.tsx +++ b/packages/slides-effects/packs/world/World.tsx @@ -1,4 +1,4 @@ -import { GPURenderGraph, samplePlayback, useBackgroundSize, usePlayback, usePresentationValue } from "@legend-apps/presentation"; +import { GPURenderGraph, PresentationObservableProvider, samplePlayback, useBackgroundSize, usePlayback, usePresentation$, usePresentationValue, useRuntimeProjection } from "@legend-apps/presentation"; import { Children, Component, Fragment, isValidElement, useMemo, useState, type ReactNode } from "react"; import { View } from "react-native"; import { makeMutable, useAnimatedReaction, useDerivedValue, useSharedValue, type SharedValue } from "react-native-reanimated"; @@ -8,7 +8,7 @@ import { advanceResolution, createResolutionState } from "./resolution"; import { compileTrack, customInstances, graphInstances, orbInstances, prepareWorldScene, primitive, pulseInstances, rowInstances, sampleWorldCamera, sampleWorldUniforms, sdfInstance, sprayInstances, voxelInstances, type Animated, type CustomOptions, type GraphOptions, type Instance, type LightOptions, type MaterialOptions, type OrbsOptions, - type PostOptions, type PrimitiveOptions, type PulseOptions, type RowsOptions, type SDFOptions, type SprayOptions, type StageMotion, type StageOptions, type VoxelsOptions, type WorldScene } from "./scene"; + type PostOptions, type PreparedWorldScene, type PrimitiveOptions, type PulseOptions, type RowsOptions, type SDFOptions, type SprayOptions, type StageMotion, type StageOptions, type VoxelsOptions, type WorldScene } from "./scene"; export { tree, wave, subset, layoutTree } from "./graph"; export { evaluateCamera, flyTo } from "./camera"; @@ -69,8 +69,15 @@ export type WorldProps = { postOrder?: WorldScene["postOrder"]; }; export function World(props: WorldProps) { + const background = usePresentationValue("playbackScope") === "background"; // id defines a continuity boundary; changing it creates a fresh sampled origin. - return ; + const surface = ; + return background ? {surface} : surface; +} +function WorldSlideClock({ children }: { children: ReactNode }) { + const sourceRuntime$ = usePresentation$(); + const runtime$ = useRuntimeProjection(() => ({ ...sourceRuntime$.get(), playbackScope: "slide" }), [sourceRuntime$]); + return {children}; } function worldContent(children: ReactNode, quality: WorldScene["quality"], exposure: Animated, backgroundColor: string, scrim: boolean | number, postOrder: WorldScene["postOrder"]) { const scene: WorldScene = { instances: [], stages: [], materials: {}, lights: [], passes: [], quality, exposure, backgroundColor, postOrder, @@ -124,23 +131,35 @@ function worldContent(children: ReactNode, quality: WorldScene["quality"], expos collect(children); return { prepared: prepareWorldScene(scene), overlay }; } -function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHeight = 1080, background = false, backgroundColor = "#05060a", quality = "auto", - exposure = 1, previewTime = 4, borderRadius = 0, renderScale = 1, enter, scrim = false, fallback, active = true, children, accessibilityLabel, postOrder }: WorldProps) { - if (!Number.isFinite(renderScale) || renderScale <= 0 || renderScale > 1) throw new Error("World renderScale must be greater than zero and at most one"); +function WorldSurface(props: WorldProps) { + const { children, quality = "auto", exposure = 1, backgroundColor = "#05060a", scrim = false, postOrder, + width: configuredWidth = 1920, height: configuredHeight = 1080, background = false, borderRadius = 0, accessibilityLabel } = props; const size = useBackgroundSize(); const width = background ? size.width : configuredWidth; const height = background ? size.height : configuredHeight; - // GPU pipelines keep their identity across clock navigation and surface resizes. const { prepared, overlay } = useMemo(() => worldContent(children, quality, exposure, backgroundColor, scrim, postOrder), [children, quality, exposure, backgroundColor, scrim, postOrder]); const topology = JSON.stringify(worldGraphTopology(prepared)); + const [continuity] = useState(() => props.id ? retainedPose(props.id) : undefined); + const origin = useSharedValue<{ key: string; pose: CameraPose | null }>({ key: "", pose: null }); + const resolution = useSharedValue(createResolutionState()); + return + + {overlay.length > 0 && {overlay}} + ; +} +function WorldSceneRuntime({ width, height, background = false, quality = "auto", + previewTime = 4, borderRadius = 0, renderScale = 1, enter, fallback, active = true, prepared, topology, continuity, origin, resolution }: + WorldProps & { width: number; height: number; prepared: PreparedWorldScene; topology: string; continuity?: SharedValue; + origin: SharedValue<{ key: string; pose: CameraPose | null }>; resolution: SharedValue> }) { + if (!Number.isFinite(renderScale) || renderScale <= 0 || renderScale > 1) throw new Error("World renderScale must be greater than zero and at most one"); + // Each pipeline generation retains its own uniform SharedValue until UI retirement. const graph = useMemo(() => buildWorldGraph(JSON.parse(topology)), [topology]); const playback = usePlayback(); - const [continuity] = useState(() => id ? retainedPose(id) : undefined); - const origin = useSharedValue<{ key: string; pose: CameraPose | null }>({ key: "", pose: null }); const sampleClock = samplePlayback; const cameraEvaluator = sampleWorldCamera; const sampleUniforms = sampleWorldUniforms; - const resolution = useSharedValue(createResolutionState()); const resolutionEvaluator = advanceResolution; const adaptive = quality === "auto"; const adaptivePlayback = adaptive ? playback : null; @@ -175,12 +194,8 @@ function WorldSurface({ id, width: configuredWidth = 1920, height: configuredHei : origin.value.key === state.slideKey ? origin.value.pose : state.phase === "playing" ? entryPose?.value : null; return sampleUniforms(prepared, clock, previous, enterDuration, effectiveScale.value); }, [prepared, previewTime, enterDuration, entryPose, effectiveScale]); - return - - - - {overlay.length > 0 && {overlay}} - ; + return + + ; } diff --git a/packages/slides-effects/tests/worldBackgroundPlayback.test.tsx b/packages/slides-effects/tests/worldBackgroundPlayback.test.tsx new file mode 100644 index 000000000..fe2d8553c --- /dev/null +++ b/packages/slides-effects/tests/worldBackgroundPlayback.test.tsx @@ -0,0 +1,87 @@ +// @ts-nocheck Exercise actual background registration and presentation clocks without a GPU. +import "../../../apps/slides/src/__tests__/nativeMock"; +import { frames, flushUI } from "../../../apps/slides/src/__tests__/uiClockMock"; +import { expect, test } from "bun:test"; +import { observable } from "@legendapp/state"; +import { Background, BackgroundHost, GPUShaderProvider, PresentationObservableProvider, usePlayback } from "@legend-apps/presentation"; +import React from "react"; +import { act, create } from "react-test-renderer"; +import { Camera, Sphere, World } from "../packs/world/World"; +import { worldHeaderSize } from "../packs/world/scene"; + +test("selected background worlds restart slide time while preserving camera continuity and step sampling", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + const runtime = slideIndex => observable({ currentSlide: 0, currentStep: 0, slideIndex, stepIndex: 0, + slideCount: 2, stepCount: 2, startedAt: 0, stepStartedAt: 0, isActive: slideIndex === 0, + isPreview: false, isPreparing: slideIndex !== 0, playbackPhase: slideIndex === 0 ? "playing" : "preparing", + playbackScope: "slide", goTo() {}, next() {}, previous() {} }); + const first$ = runtime(0), second$ = runtime(1); + function Renderer(props) { return ; } + function BackgroundClock() { return ; } + const declarations = <> + + + + + + + + ; + const content = (slideIndex, isPreview = false) => + {declarations} + ; + let renderer; + const graph = () => renderer.root.findByType("world-gpu").props; + const values = () => graph().frameData.value; + const tick = timestamp => frames.forEach(frame => frame.tick(timestamp)); + try { + await act(() => { renderer = create(content(0)); }); + flushUI(); tick(0); tick(10000); + const previous = values().slice(0, 3); + expect(graph().clockState.value.slideTime).toBe(10); + expect(previous[0]).toBeCloseTo(Math.sin(0.45) * 20); + + await act(() => { + first$.assign({ currentSlide: 1, playbackPhase: "outgoing", isActive: false }); + second$.assign({ currentSlide: 1, currentStep: 1, stepIndex: 1, startedAt: 10000, stepStartedAt: 10000, + isActive: true, isPreparing: false, playbackPhase: "playing" }); + renderer.update(content(1)); + }); + flushUI(); tick(10000); + expect(graph().clockState.value.slideTime).toBe(0); + expect(graph().clockState.value.slideKey).toBe("1:10000"); + const backgroundClock = renderer.root.findByType("background-clock").props.value.value; + expect(backgroundClock.slideTime).toBe(10); + expect(backgroundClock.slideKey).toBe("background"); + values().slice(0, 3).forEach((value, index) => expect(value).toBeCloseTo(previous[index])); + expect(values()[worldHeaderSize]).toBe(0); + tick(11000); + expect(graph().clockState.value.slideTime).toBe(1); + expect(values()[worldHeaderSize]).toBeCloseTo(2); + tick(11500); + expect(values().slice(0, 3)).toEqual([8, 5, 20]); + tick(13350); + expect(values()[2]).toBeGreaterThan(20); + expect(values()[2]).toBeLessThan(34); + tick(17800); + expect(values().slice(0, 3)).toEqual([0, 7, 34]); + + await act(() => second$.assign({ currentStep: 0, stepIndex: 0, stepStartedAt: 17800 })); + flushUI(); tick(17800); + expect(values()[worldHeaderSize]).toBe(0); + expect(graph().clockState.value.slideTime).toBeCloseTo(7.8); + await act(() => renderer.unmount()); + await act(() => { + second$.assign({ currentStep: 1, stepIndex: 1, stepStartedAt: 18000 }); + renderer = create(content(1, true)); + }); + flushUI(); + const preview = values(); + expect(graph().clockState.value.phase).toBe("preview"); + expect(preview[worldHeaderSize]).toBe(4); + expect(preview[2]).toBeGreaterThan(20); + expect(preview[2]).toBeLessThan(34); + tick(40000); + expect(values()).toEqual(preview); + } finally { await act(() => renderer?.unmount()); flushUI(); } +}); diff --git a/packages/slides-effects/tests/worldPlayback.test.tsx b/packages/slides-effects/tests/worldPlayback.test.tsx index 5ddebea72..4dc6c5eb1 100644 --- a/packages/slides-effects/tests/worldPlayback.test.tsx +++ b/packages/slides-effects/tests/worldPlayback.test.tsx @@ -10,10 +10,19 @@ import path from "node:path"; import { createRequire } from "node:module"; import { createPlaybackState, advancePlayback, transitionPlayback } from "../../presentation/src/playbackState"; import { samplePlayback } from "../../presentation/src/playback"; +import { PresentationObservableProvider, usePresentation$ } from "../../presentation/src/runtime"; +import { useRuntimeProjection } from "../../presentation/src/useRuntimeProjection"; import { worldHeaderSize } from "../packs/world/scene"; const playback = { value: createPlaybackState({ phase: "playing", slideKey: "a", stepKey: "0", stepIndex: 0 }) }; +const animated = { ...await import("react-native-reanimated") }; +mock.module("react-native-reanimated", () => ({ ...animated, useDerivedValue(read) { + const latest = React.useRef(read); + latest.current = read; + return React.useRef({ _isReanimatedSharedValue: true, get value() { return latest.current(); } }).current; +} })); mock.module("@legend-apps/presentation", () => ({ + PresentationObservableProvider, usePresentation$, useRuntimeProjection, GPURenderGraph: props => React.createElement("render-graph", props), usePlayback: () => playback, samplePlayback, useBackgroundSize: () => ({ width: 1280, height: 720 }), usePresentationValue: () => playback.value.phase, @@ -92,6 +101,83 @@ test("equal-content parent renders and resize retain pipeline definitions while } finally { await act(() => tree?.unmount()); } }); +test("topology transitions retire a uniform generation with its original capacity", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + playback.value = createPlaybackState({ phase: "playing", slideKey: "generation-a", stepKey: "0", stepIndex: 0 }); + let renderer, overlayMounts = 0, overlayUnmounts = 0; + function Overlay() { + const [identity] = React.useState(() => ++overlayMounts); + React.useEffect(() => () => { overlayUnmounts++; }, []); + return ; + } + const content = (count, width = 800, exposure = 1) => + + ; + const graph = () => renderer.root.findByType("render-graph").props; + try { + await act(() => { renderer = create(content(24)); }); + const first = graph(); + expect(first.uniforms.value).toHaveLength(first.uniformsSize); + await act(() => renderer.update(content(24, 1000, 2))); + expect(graph().uniforms).toBe(first.uniforms); + expect(graph().passes).toBe(first.passes); + expect(graph().uniforms.value[8]).toBe(2); + playback.value = createPlaybackState({ phase: "playing", slideKey: "generation-b", stepKey: "0", stepIndex: 0 }); + await act(() => renderer.update(content(64))); + const second = graph(); + expect(second.uniforms).not.toBe(first.uniforms); + expect(second.uniforms.value).toHaveLength(second.uniformsSize); + expect(first.uniforms.value).toHaveLength(first.uniformsSize); + playback.value = createPlaybackState({ phase: "playing", slideKey: "generation-c", stepKey: "0", stepIndex: 0 }); + await act(() => renderer.update(content(24))); + expect(graph().uniforms).not.toBe(first.uniforms); + expect(graph().uniforms).not.toBe(second.uniforms); + expect(graph().uniforms.value).toHaveLength(first.uniformsSize); + expect(second.uniforms.value).toHaveLength(second.uniformsSize); + expect(first.uniforms.value).toHaveLength(first.uniformsSize); + expect(renderer.root.findByType("world-overlay").props.identity).toBe(1); + expect(overlayMounts).toBe(1); expect(overlayUnmounts).toBe(0); + for (let timestamp = 0; timestamp <= 6000; timestamp += 20) { + playback.value = advancePlayback(playback.value, timestamp); + reactUI(); + } + const scale = graph().renderScale.value; + expect(scale).toBeLessThan(1); + await act(() => renderer.update(content(64))); + reactUI(); + expect(graph().renderScale.value).toBe(scale); + } finally { await act(() => renderer?.unmount()); } +}); + +test("a topology change during camera entry preserves the original slide anchor", async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + playback.value = createPlaybackState({ phase: "playing", slideKey: "entry-origin", stepKey: "0", stepIndex: 0 }); + let renderer; + const content = (count, entering = false) => + + ; + const values = () => renderer.root.findByType("render-graph").props.uniforms.value; + try { + await act(() => { renderer = create(content(24)); }); + reactUI(); + playback.value = createPlaybackState({ phase: "playing", slideKey: "entry-target", stepKey: "0", stepIndex: 0 }); + await act(() => renderer.update(content(24, true))); + reactUI(); + playback.value = advancePlayback(advancePlayback(playback.value, 0), 1000); + reactUI(); + expect(values()[2]).toBe(20); + await act(() => renderer.update(content(64, true))); + reactUI(); + expect(values()[2]).toBe(20); + playback.value = advancePlayback(playback.value, 1500); + reactUI(); + expect(values()[2]).toBeCloseTo(28.75); + playback.value = advancePlayback(playback.value, 2000); + reactUI(); + expect(values()[2]).toBe(30); + } finally { await act(() => renderer?.unmount()); } +}); + test("auto resolution adapts without React frames and preserves the frozen pause/outgoing scale", async () => { globalThis.IS_REACT_ACT_ENVIRONMENT = true; playback.value = createPlaybackState({ phase: "playing", slideKey: "adaptive", stepKey: "0", stepIndex: 0 }); @@ -149,7 +235,8 @@ test("compiled recursive world evaluators survive an isolated UI-worklet closure const require = createRequire(import.meta.url); const module = { exports: {} }; const callbacks = [], mutables = [], preparations = [], reactions = []; - const presentation = { GPURenderGraph: props => React.createElement("compiled-graph", props), + const presentation = { PresentationObservableProvider, usePresentation$, useRuntimeProjection, + GPURenderGraph: props => React.createElement("compiled-graph", props), usePlayback: () => playback, samplePlayback, useBackgroundSize: () => ({ width: 800, height: 500 }), usePresentationValue: () => playback.value.phase }; const reanimated = { From db5caf4fff6001064ac748750b296175a32a5d78 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 10:21:27 -0700 Subject: [PATCH 07/28] fix: round bloom halos for small bright objects The bloom chain filtered with an X-shaped five-tap kernel and a hard threshold, so tiny HDR points spread into box-shaped halos down the mip chain. Downsample with a 13-tap filter (Karis-averaged on the first level), upsample with a 3x3 tent, and threshold with a soft knee. Bloom props are unchanged. --- packages/slides-effects/packs/world/README.md | 2 +- packages/slides-effects/packs/world/render.ts | 30 ++++++++++++------- .../slides-effects/tests/worldLook.test.ts | 9 ++++++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index d31cecd9f..db513617d 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -91,7 +91,7 @@ upload still scale with the instance count; the graph API targets 20–60 nodes. The pipeline is an instanced HDR scene with depth, optional planar reflections, height fog and volumetric lights, depth of field, a half-resolution bloom mip -chain, chromatic aberration, ACES exposure/tonemapping, sRGB encoding, vignette +chain with a soft threshold knee and round 13-tap/tent filters, chromatic aberration, ACES exposure/tonemapping, sRGB encoding, vignette and grain. `high` and `auto` use full-resolution scene targets and five bloom levels. `medium` uses 0.85 scene scale and five levels; `low` uses 0.65 scene scale and three levels. Output resolution stays unchanged. `renderScale` is a diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index 18a6949bf..ba98b00aa 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -217,25 +217,33 @@ struct FloorResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } } const filterCode = `${worldBindings} @group(0) @binding(3) var source:texture_2d; -fn tentFilter(uv:vec2f)->vec3f { - let pixel=1./vec2f(textureDimensions(source)); - return textureSampleLevel(source,linearSampler,uv,0.).rgb*.25 - +(textureSampleLevel(source,linearSampler,uv+pixel*vec2f(-1.,-1.),0.).rgb - +textureSampleLevel(source,linearSampler,uv+pixel*vec2f(1.,-1.),0.).rgb - +textureSampleLevel(source,linearSampler,uv+pixel*vec2f(-1.,1.),0.).rgb - +textureSampleLevel(source,linearSampler,uv+pixel*vec2f(1.,1.),0.).rgb)*.1875; +fn tap(uv:vec2f,offset:vec2f)->vec3f { return textureSampleLevel(source,linearSampler,uv+offset/vec2f(textureDimensions(source)),0.).rgb; } +fn box(color:vec3f,weight:f32,karis:bool)->vec4f { let w=weight*select(1.,1./(1.+max(color.r,max(color.g,color.b))),karis); return vec4f(color*w,w); } +// 13 taps as five overlapping 2x2 boxes; the Karis average keeps single HDR pixels from flaring. +fn downsample13(uv:vec2f,karis:bool)->vec3f { + let a=tap(uv,vec2f(-2.,-2.)); let b=tap(uv,vec2f(0.,-2.)); let c=tap(uv,vec2f(2.,-2.)); + let d=tap(uv,vec2f(-2.,0.)); let e=tap(uv,vec2f(0.,0.)); let f=tap(uv,vec2f(2.,0.)); + let g=tap(uv,vec2f(-2.,2.)); let h=tap(uv,vec2f(0.,2.)); let i=tap(uv,vec2f(2.,2.)); + let j=tap(uv,vec2f(-1.,-1.)); let k=tap(uv,vec2f(1.,-1.)); let l=tap(uv,vec2f(-1.,1.)); let m=tap(uv,vec2f(1.,1.)); + let sum=box((j+k+l+m)*.25,.5,karis)+box((a+b+d+e)*.25,.125,karis)+box((b+c+e+f)*.25,.125,karis)+box((d+e+g+h)*.25,.125,karis)+box((e+f+h+i)*.25,.125,karis); + return sum.rgb/sum.a; +} +fn tent9(uv:vec2f)->vec3f { + return (tap(uv,vec2f(0.,0.))*4.+(tap(uv,vec2f(-1.,0.))+tap(uv,vec2f(1.,0.))+tap(uv,vec2f(0.,-1.))+tap(uv,vec2f(0.,1.)))*2. + +tap(uv,vec2f(-1.,-1.))+tap(uv,vec2f(1.,-1.))+tap(uv,vec2f(-1.,1.))+tap(uv,vec2f(1.,1.)))/16.; } `; const downsample = `${filterCode} -@fragment fn downsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(tentFilter(uv),1.); }`; +@fragment fn downsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(downsample13(uv,false),1.); }`; const threshold = `${filterCode} @fragment fn threshold(@location(0) uv:vec2f)->@location(0) vec4f { - let color=tentFilter(uv); let brightness=max(color.r,max(color.g,color.b)); - return vec4f(color*max(0.,brightness-uniforms[10])/max(.0001,brightness),1.); + let color=downsample13(uv,true); let brightness=max(color.r,max(color.g,color.b)); + let knee=uniforms[10]*.5; let soft=clamp(brightness-uniforms[10]+knee,0.,2.*knee); + return vec4f(color*max(soft*soft/(4.*knee+.0001),brightness-uniforms[10])/max(.0001,brightness),1.); }`; const upsample = `${filterCode} @group(0) @binding(4) var high:texture_2d; -@fragment fn upsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(textureSampleLevel(high,linearSampler,uv,0.).rgb+tentFilter(uv)*uniforms[11],1.); }`; +@fragment fn upsample(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(textureSampleLevel(high,linearSampler,uv,0.).rgb+tent9(uv)*uniforms[11],1.); }`; const bloomComposite = `${worldBindings} @group(0) @binding(3) var source:texture_2d; @group(0) @binding(4) var bloom:texture_2d; diff --git a/packages/slides-effects/tests/worldLook.test.ts b/packages/slides-effects/tests/worldLook.test.ts index eb8365218..16aa22223 100644 --- a/packages/slides-effects/tests/worldLook.test.ts +++ b/packages/slides-effects/tests/worldLook.test.ts @@ -28,3 +28,12 @@ for (const quality of ["low", "medium", "high"] as const) { expect(graph.passes.filter(pass => pass.kind === "render")).toHaveLength(5); }); } + +test("bloom uses round kernels with a soft threshold knee", () => { + const graph = buildWorldGraph(prepareWorldScene({ instances: [], bloom: {} })); + const code = (id: string) => graph.passes.find(pass => pass.id === id)!.code; + expect(code("bloom-threshold")).toContain("downsample13(uv,true)"); + expect(code("bloom-threshold")).toContain("soft*soft"); + expect(code("bloom-down-1")).toContain("downsample13(uv,false)"); + expect(code("bloom-up-0")).toContain("tent9(uv)"); +}); From 3f2eca8e637488022d2f2c490388e7bebb240e75 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 11:55:02 -0700 Subject: [PATCH 08/28] feat: make the data flow example full-bleed and denser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the data-flow scenes behind short slide overlays and expand the tree to 53 nodes. Contrast the magenta cascade with a cyan pulse whose thinner, fainter curved trail leaves its spherical head distinct. Keep recycled-row focus, stage flights and voxel reduction on the shared playback clock, with stronger bloom, fog and lighting throughout. **Test plan** - ✅ Run the scoped World suites in isolated Bun processes, including the compiled deck's scene/uniform assertions. - ✅ Validate `data-flow.mdx` with the real deck compiler. - Native visual and performance verification follows separately. --- apps/slides/examples/3d/data-flow.mdx | 210 +++++++++++------- packages/slides-effects/packs/world/README.md | 6 +- packages/slides-effects/packs/world/scene.ts | 3 +- packages/slides-effects/tests/world.test.ts | 16 ++ .../tests/worldDataFlow.test.tsx | 124 +++++++++++ 5 files changed, 281 insertions(+), 78 deletions(-) create mode 100644 packages/slides-effects/tests/worldDataFlow.test.tsx diff --git a/apps/slides/examples/3d/data-flow.mdx b/apps/slides/examples/3d/data-flow.mdx index 71ff2b784..19454df68 100644 --- a/apps/slides/examples/3d/data-flow.mdx +++ b/apps/slides/examples/3d/data-flow.mdx @@ -8,95 +8,155 @@ theme: color: "#f8fafc" --- +import { Background } from "@legend-apps/presentation" import { Text } from "react-native" -import { World, Camera, Graph, tree, wave, subset, PathPulse, Rows, Voxels, WorldCounter, Stage, flyTo, Box, Ring, Floor, PointLight, RimLight, Bloom, Fog, DOF, Vignette } from "slides:world/World" +import { World, Camera, Graph, tree, wave, subset, PathPulse, Spray, Orbs, Rows, Voxels, WorldCounter, Stage, flyTo, Box, Ring, Floor, PointLight, RimLight, Bloom, Fog, DOF, Vignette, Grain } from "slides:world/World" export const network = tree("root", [ - ["north", [["n1", ["n11", "n12", "n13"]], ["n2", ["n21", "n22", "n23"]]]], - ["east", [["e1", ["e11", "e12", "e13"]], ["e2", ["e21", "e22", "e23"]]]], - ["west", [["w1", ["w11", "w12", "w13"]], ["w2", ["w21", "w22", "w23"]]]], + ["north", [["n1", ["n11", "n12", "n13"]], ["n2", ["n21", "n22", "n23"]], ["n3", ["n31", "n32", "n33"]]]], + ["east", [["e1", ["e11", "e12", "e13"]], ["e2", ["e21", "e22", "e23"]], ["e3", ["e31", "e32", "e33"]]]], + ["south", [["s1", ["s11", "s12", "s13"]], ["s2", ["s21", "s22", "s23"]], ["s3", ["s31", "s32", "s33"]]]], + ["west", [["w1", ["w11", "w12", "w13"]], ["w2", ["w21", "w22", "w23"]], ["w3", ["w31", "w32", "w33"]]]], ]) -# Follow one branch - - - - - - - - -Across the tree -Only this branch +export const branching = { data: network, radius: 6.8, levelHeight: 1.7, position: [0, 3.2, 0], edge: { curve: 0.45, thickness: 0.025, glow: 3 } } + +export const atmosphere = <> + + + + + + + + + + + + + + + {atmosphere} + + + + + + {step => {step === 2 ? "Pruned subset" : "Cascade"}} + + --- -# A single path - - - - - - - - - -Follow the pulse + + + + + + + + + + {atmosphere} + + + + + + {() => One path} + + --- -# Change in a moving column - - - - - - - - - - -One changed row -Focus follows the next change + + + + + + + + + {atmosphere} + + + + + + {() => Changed rows} + + --- -# Three places to land - - - - - - - - - - - - - - - - - - - - -A softer glow -A quiet landing + + + + + + + + + + + + + + + + + + + + + + + {atmosphere} + + + + + + {() => Three stages} + + --- -# Less to carry - - - - - - Active cells - - - - - -About one fifth remains + + + + + + + + + + {atmosphere} + + + + + + {() => Less to carry} + + + + Active cells + diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index db513617d..3bcebc36a 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -212,8 +212,10 @@ import { PathPulse, Spray, Rows, Stage, flyTo } from "slides:world/World"; ``` Pulse paths accept positions or ids; bare graph ids resolve as `node:`. -The head and trail follow the graph's curved edges. The trail restarts at each -node so one tube never cuts across a junction. Ambient Spray accepts +The head is a round emissive orb of radius `size`. Its thinner trail uses 18% +of that radius and 30% of the head's emissive intensity so the orb stays distinct. +Both follow the graph's curved edges. The trail restarts at each node so one tube +never cuts across a junction. Ambient Spray accepts `position`, `spread`, `velocity` and `seed`; `under="leaves:lit"` emits beneath active leaves. Motion, birth and lifetime are analytic samples of absolute time. diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index 0827a1d43..8cb1c0db4 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -137,7 +137,7 @@ export function orbInstances({ id = "orb", count = 48, colors = ["#22d3ee", "#b5 export function pulseInstances(options: PulseOptions): Instance[] { if (options.path.length < 2 || !Number.isFinite(options.duration ?? 2) || (options.duration ?? 2) <= 0 || !Number.isFinite(options.trail ?? 0.18) || (options.trail ?? 0.18) < 0) throw new Error("A world pulse needs at least two anchors, a positive duration and a nonnegative trail"); return [false, true].map(tail => ({ ...primitive(tail ? "tube" : "sphere", { id: tail ? `${options.id}:trail` : options.id, - size: options.size ?? 0.08, thickness: (options.size ?? 0.08) * 0.45, color: options.color ?? "#22d3ee", emissive: options.emissive ?? 8 }), + size: options.size ?? 0.08, thickness: (options.size ?? 0.08) * 0.18, color: options.color ?? "#22d3ee", emissive: options.emissive ?? 8 }), pulse: { options, anchors: [], curves: [], tail } })); } export function sprayInstances(options: SprayOptions = {}): Instance[] { @@ -392,6 +392,7 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl values[index] = position[0]; values[index + 1] = position[1]; values[index + 2] = position[2]; values[index + 3] = sampleTrack(instance.size, clock) * (instance.voxel ? instanceOpacity(instance, clock) : 1); let emissive = sampleTrack(instance.emissive, clock) * activation; + if (instance.pulse?.tail) emissive *= 0.3; if (instance.rows) { const flash = rowFlash(instance, clock); color = mixVector(color, instance.rows.flashColor, flash); emissive += flash * 8; } for (const stage of instance.stages ?? []) emissive *= sampleTrack(stage.glow, clock); values[index + 4] = dim[0] + (color[0] - dim[0]) * activation; diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index 0396522ce..7f0c45c25 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -183,6 +183,22 @@ test("pulse geometry, camera follow and focus share the exact curved path across expect(() => prepareWorldScene({ instances: pulseInstances({ id: "bad", path: ["bad", [0, 0, 0]] }) })).toThrow("cycle"); }); +test("pulse keeps a spherical head distinct from its thin, fainter curved trail", () => { + const scene = prepareWorldScene({ instances: pulseInstances({ id: "pulse", path: [[0, 3, 0], [4, 0, 0]], size: 0.2, emissive: 12 }) }); + const head = scene.instances.find(instance => instance.id === "pulse")!; + const trail = scene.instances.find(instance => instance.id === "pulse:trail")!; + const values = sampleWorldUniforms(scene, clock(1)); + const at = (instance: typeof head) => worldHeaderSize + scene.instances.indexOf(instance) * worldInstanceSize; + expect(head.kind).toBe("sphere"); expect(trail.kind).toBe("tube"); + expect(values.slice(at(head) + 8, at(head) + 11)).toEqual([1, 1, 1]); + expect(values[at(head) + 3]).toBe(0.2); + expect(values[at(trail) + 19]).toBeCloseTo(0.036); + expect(values[at(trail) + 7]).toBeCloseTo(3.6); + expect(values[at(head) + 7]).toBe(12); + const sphere = buildWorldGraph(scene).passes.find(pass => pass.id === "scene-sphere"); + expect(sphere).toMatchObject({ kind: "render", vertexCount: 6, instanceCount: 1 }); +}); + test("pulse trails reconstruct exact curved subsegments and restart at a turn without extra instances", () => { const scene = prepareWorldScene({ instances: [...graphInstances({ data: tree("root", [["a", ["leaf"]]]), edge: { curve: 0.65 } }), ...pulseInstances({ id: "pulse", path: ["root", "a", "leaf"], step: 1, duration: 4, trail: 0.18 })] }); diff --git a/packages/slides-effects/tests/worldDataFlow.test.tsx b/packages/slides-effects/tests/worldDataFlow.test.tsx new file mode 100644 index 000000000..384b93ffd --- /dev/null +++ b/packages/slides-effects/tests/worldDataFlow.test.tsx @@ -0,0 +1,124 @@ +// @ts-nocheck Inspect the real compiled deck and clock uniforms without a GPU. +import "../../../apps/slides/src/__tests__/nativeMock"; +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { createRequire } from "node:module"; +import path from "node:path"; +import React from "react"; +import { act, create } from "react-test-renderer"; +import { compileDeck } from "../../presentation/src/compiler"; +import { layoutTree } from "../packs/world/graph"; +import { colorVector, worldHeaderSize, worldInstanceSize } from "../packs/world/scene"; + +const playback = { value: { phase: "playing", slideTime: 0, stepTime: 0, stepIndex: 0, stepOrigins: { 0: 0 } } }; +let renderer, network; +const closeVector = (actual, expected) => expected.forEach((value, axis) => expect(actual[axis]).toBeCloseTo(value, 10)); +const values = index => renderer.root.findAllByType("world-gpu")[index].props.uniforms.value; +const instances = index => { + const data = values(index), result = []; + for (let offset = worldHeaderSize; offset < data.length; offset += worldInstanceSize) result.push(data.slice(offset, offset + worldInstanceSize)); + return result; +}; +function sample(time, step = 0) { + playback.value = { ...playback.value, slideTime: time, stepTime: time, stepIndex: step, stepOrigins: { 0: 0, [step]: 0 } }; +} + +beforeAll(async () => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + const root = path.resolve(import.meta.dirname, "../../.."); + const compiled = await compileDeck(path.join(root, "apps/slides/examples/3d/data-flow.mdx"), { + effectPacksDirectory: path.join(root, "packages/slides-effects/packs"), + }); + if (!compiled.success) throw new Error(compiled.errors.join("\n")); + const require = createRequire(import.meta.url), module = { exports: {} }; + Function("require", "module", "exports", compiled.code)(name => { + if (name === "@legend-apps/presentation") return { + defineTransition: definition => definition, + Background: "background", GPURenderGraph: props => React.createElement("world-gpu", props), + usePresentationValue: key => key === "playbackScope" ? "slide" : "playing", + useBackgroundSize: () => ({ width: 1920, height: 1080 }), + usePlayback: () => playback, samplePlayback: (state, _preview, clock) => clock === "step" ? state.stepTime : state.slideTime, + }; + if (name === "react-native-reanimated") return { + useSharedValue: value => React.useRef({ value }).current, + useDerivedValue: read => ({ get value() { return read(); } }), useAnimatedReaction() {}, + }; + return require(name); + }, module, module.exports); + network = module.exports.network; + await act(() => { + renderer = create(React.createElement(module.exports.default, { components: { + Deck: "deck", Slide: "slide", View: "view", + Steps: ({ children }) => children(playback.value.stepIndex), + } })); + }); +}, 30000); +afterAll(async () => { await act(() => renderer?.unmount()); }); + +test("all five scenes fill the background and retain short overlay titles and navigation states", () => { + const slides = renderer.root.findAllByType("slide"); + expect(slides).toHaveLength(5); + for (const slide of slides) { + const backgrounds = slide.findAllByType("background"); + expect(backgrounds).toHaveLength(1); + const graph = backgrounds[0].findByType("world-gpu").props; + expect(graph.background).toBe(true); + expect([graph.width, graph.height]).toEqual([1920, 1080]); + expect(slide.findAllByType("text").some(text => text.props.className?.includes("text-6xl"))).toBe(true); + } + const navigation = renderer.root.findAll(node => typeof node.type === "function" && node.type.name === "Steps"); + expect(navigation.map(node => node.props.count)).toEqual([3, 2, 3, 3, 2]); +}); + +test("the 53-node cascade heats descendants then prunes to one branch while keeping dim geometry", () => { + expect(layoutTree(network).nodes).toHaveLength(53); + sample(0); + const nodes = () => instances(0).filter(item => item[3] === 0.24); + const edges = instances(0).filter(item => item[15] === 0.45); + expect(nodes()).toHaveLength(53); expect(edges).toHaveLength(52); + for (const item of [...nodes(), ...edges]) { + expect(item[23]).toBe(1); + expect(item.slice(4, 7)).toEqual(colorVector("#647b9a")); + } + expect(values(0)[1]).toBeCloseTo(1.8); + expect(values(0)[9]).toBe(1.8); expect(values(0)[12]).toBe(0.035); + sample(0.5, 1); + const partial = nodes().filter(item => item[30] > 0); + expect(partial.length).toBeGreaterThan(1); expect(partial.length).toBeLessThan(53); + sample(2, 1); + expect(nodes().every(item => item[30] === 1)).toBe(true); + closeVector(nodes()[0].slice(4, 7), colorVector("#ff36c8")); + sample(2, 2); + expect(nodes().filter(item => item[30] === 1)).toHaveLength(13); + const target = layoutTree(network, { radius: 6.8, levelHeight: 1.7, position: [0, 3.2, 0] }).nodes.find(node => node.id === "e2").position; + closeVector(values(0).slice(4, 7), target); + expect(values(0)[32]).toBeCloseTo(10); +}); + +test("the cyan orb follows one curved edge with a fainter trail and matching camera focus", () => { + sample(2, 1); + const head = instances(1).find(item => item[7] === 14); + const trail = instances(1).find(item => item[7] === 14 * 0.3); + expect(head.slice(4, 7)).toEqual(colorVector("#22d3ee")); + expect(head.slice(8, 11)).toEqual([1, 1, 1]); + expect(trail[19]).toBeCloseTo(head[3] * 0.18); + expect(trail.slice(12, 15)).toEqual(head.slice(0, 3)); + expect(trail[15]).toBeGreaterThan(0); + expect(values(1).slice(4, 7)).toEqual(head.slice(0, 3)); + expect(values(1)[32]).toBeCloseTo(7); + expect(instances(1).filter(item => item[7] > 0 && item[3] === 0.2)).toHaveLength(2); +}); + +test("rows, stage flights and voxel reduction retain their clock-driven behavior", () => { + sample(1, 1); + expect(instances(2).filter(item => item[3] === 0.5 && item[7] > 1)).toHaveLength(1); + expect(renderer.root.findAllByType("world-gpu")[2].props.passes.find(pass => pass.id === "scene-box").instanceCount).toBe(15); + sample(3, 2); + expect(values(3).slice(4, 7)).toEqual([10, 0, 0]); + expect(values(3).slice(0, 3)).toEqual([12, 2.5, 10]); + expect(values(3)[32]).toBeCloseTo(Math.hypot(2, 2.5, 10)); + sample(0); + expect(values(4)[56]).toBe(1000); + sample(2.4, 1); + expect(values(4)[56]).toBe(200); + expect(instances(4).filter(item => item[3] > 0 && item[3] <= 0.17)).toHaveLength(200); +}); From 5460f61a8ed40c3fb82102eadaef6b05fa3df9da Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 12:00:07 -0700 Subject: [PATCH 09/28] fix: lower compiled deck classes for Hermes Deck-local error boundaries evaluate to undefined when unlowered classes reach Hermes through Function evaluation, preventing World from rendering. Apply the native Babel preset with inline helpers while preserving host imports and captured loop bindings. Test plan: - Pass the focused compiler, effect-pack, World playback/render/resolution and GPU host suites (8 files). - Pass the scoped compiler typecheck. - Confirm World renders in the native macOS Debug audience window; full native presentation validation remains in progress. --- bun.lock | 2 +- packages/presentation/package.json | 2 +- .../src/__tests__/compileDeck.test.ts | 18 +++++++++++++++--- .../presentation/src/compiler/compileDeck.ts | 9 +++++---- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/bun.lock b/bun.lock index 784adca03..7585d4672 100644 --- a/bun.lock +++ b/bun.lock @@ -572,9 +572,9 @@ "version": "0.0.0", "dependencies": { "@babel/core": "^7.29.0", - "@babel/plugin-transform-block-scoping": "^7.28.6", "@legendapp/state": "catalog:", "@mdx-js/mdx": "^3.1.1", + "@react-native/babel-preset": "0.81.6", "@shopify/react-native-skia": "^2.11.1", "babel-plugin-react-compiler": "1.0.0", "esbuild": "^0.25.9", diff --git a/packages/presentation/package.json b/packages/presentation/package.json index 3284b3dd4..d1158af21 100644 --- a/packages/presentation/package.json +++ b/packages/presentation/package.json @@ -8,10 +8,10 @@ }, "dependencies": { "@babel/core": "^7.29.0", - "@babel/plugin-transform-block-scoping": "^7.28.6", "@legendapp/state": "catalog:", "@shopify/react-native-skia": "^2.11.1", "@mdx-js/mdx": "^3.1.1", + "@react-native/babel-preset": "0.81.6", "babel-plugin-react-compiler": "1.0.0", "esbuild": "^0.25.9", "react": "catalog:", diff --git a/packages/presentation/src/__tests__/compileDeck.test.ts b/packages/presentation/src/__tests__/compileDeck.test.ts index 52c6d84f1..577808e86 100644 --- a/packages/presentation/src/__tests__/compileDeck.test.ts +++ b/packages/presentation/src/__tests__/compileDeck.test.ts @@ -386,17 +386,24 @@ describe("compileDeck", () => { expect(result.code).toContain('template":false'); }); - test.skipIf(process.platform !== "darwin")("preserves template exports and captured loop bindings in Hermes", async () => { + test.skipIf(process.platform !== "darwin")("preserves classes, template exports and captured loop bindings in Hermes", async () => { const deckPath = createDeck({ "deck.mdx": [ "---", "template: ./Frame.tsx", "---", + 'export { Boundary } from "./Boundary"', "export function loopValues() { const callbacks = []; for (let i = 0; i < 3; i++) callbacks.push(() => i); return callbacks.map(fn => fn()); }", "", "# Framed slide", ].join("\n"), "Frame.tsx": "export default function Frame({ children }) { return children; }", + "Boundary.tsx": `import { Component } from "react"; + export class Boundary extends Component { + state = { failed: false }; + static getDerivedStateFromError() { return { failed: true }; } + render() { return this.state.failed ? this.props.fallback : this.props.children; } + }`, }); const result = await compileDeck(deckPath); expect(result).toMatchObject({ success: true }); @@ -404,11 +411,16 @@ describe("compileDeck", () => { const probePath = path.join(path.dirname(deckPath), "probe.js"); fs.writeFileSync(probePath, [ "var module = { exports: {} };", - `Function("require", "module", "exports", ${JSON.stringify(result.code)})(function() { return {}; }, module, module.exports);`, + "function Component(props) { this.props = props; }", + `Function("require", "module", "exports", ${JSON.stringify(result.code)})(function(name) { return name === "react" ? { Component: Component } : {}; }, module, module.exports);`, + 'var boundary = new module.exports.Boundary({ children: "scene", fallback: "fallback" });', + "var content = boundary.render();", + "boundary.state = module.exports.Boundary.getDerivedStateFromError(new Error());", "print(JSON.stringify({", ' componentType: typeof module.exports.default,', ' framed: module.exports.__legendSlidesTemplates["./Frame.tsx"]({ children: "content" }),', " loop: module.exports.loopValues(),", + ' content: content, fallback: boundary.render(), inherited: boundary instanceof Component,', "}));", ].join("\n")); const reactNativeRoot = path.dirname(createRequire(import.meta.url).resolve("react-native/package.json")); @@ -417,7 +429,7 @@ describe("compileDeck", () => { expect(probe.error).toBeUndefined(); expect(probe.stderr).toBe(""); expect(probe.status).toBe(0); - expect(JSON.parse(probe.stdout)).toEqual({ componentType: "function", framed: "content", loop: [0, 1, 2] }); + expect(JSON.parse(probe.stdout)).toEqual({ componentType: "function", framed: "content", loop: [0, 1, 2], content: "scene", fallback: "fallback", inherited: true }); } }); diff --git a/packages/presentation/src/compiler/compileDeck.ts b/packages/presentation/src/compiler/compileDeck.ts index 5bfb8655d..a1b8a89e4 100644 --- a/packages/presentation/src/compiler/compileDeck.ts +++ b/packages/presentation/src/compiler/compileDeck.ts @@ -24,7 +24,7 @@ import { effectLockPath, installedPacksDirectory, readEffectLock, writeEffectLoc // Pass plugin modules rather than paths so the shipped compiler can be a single bundle. const workletsPlugin = require("react-native-worklets/plugin"); const reactCompilerPlugin = require("babel-plugin-react-compiler"); -const blockScopingPlugin = require("@babel/plugin-transform-block-scoping"); +const reactNativePreset = require("@react-native/babel-preset"); export const hostModules = new Set([ "react", "react/compiler-runtime", @@ -345,14 +345,15 @@ export async function compileDeck(deckPath: string, options: { source?: string; } // Built-in transition sources enter through a separate loader, so serialize // their worklets here too. Imported deck source was serialized before bundling. - // Hermes shares captured loop bindings without Babel's block-scope transform. - // Run it after bundling so esbuild's generated export getters are lowered too. + // Lower classes and scoped bindings for Hermes after bundling so generated + // export getters are covered too. Deck helpers must stay inline. const code = transformSync(output.text, { babelrc: false, compact: false, configFile: false, filename: "deck.js", - plugins: [[workletsPlugin, { disableSourceMaps: true }], blockScopingPlugin], + presets: [[reactNativePreset, { disableImportExportTransform: true, disableStaticViewConfigsCodegen: true, enableBabelRuntime: false, unstable_transformProfile: "hermes-stable" }]], + plugins: [[workletsPlugin, { disableSourceMaps: true }]], sourceMaps: "inline", sourceType: "script", })?.code; From 65bde91e72e4fbb3df15ed5091b7872272d162ac Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 13:51:42 -0700 Subject: [PATCH 10/28] fix: skip zero-count GPU draws in render graphs A World with no instances still encodes an empty render pass to clear its target, and that pass drew zero instances. Dawn reports such draws through a device logging callback that react-native-wgpu forwards to console.warn on the JS runtime from the UI thread, which raced the JS thread and crashed Hermes when the next slide was prepared. Keep the pass (it still clears) but skip draws and per-instance dispatches whose count is zero. Test plan: - New render-graph test fails before and passes after; the focused compiler, effect-pack, World and GPU host suites pass (8 files, 94 tests); scoped typecheck is clean. - Native Debug macOS app: unfixed, traversing the World tour crashed entering slide 9 with the same native stack; fixed, it traversed every slide and step of all three 3D example decks without Dawn zero-count warnings or crashes. --- apps/slides/src/__tests__/gpuRenderGraph.test.ts | 11 +++++++++++ apps/slides/src/gpuRenderGraph.ts | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/slides/src/__tests__/gpuRenderGraph.test.ts b/apps/slides/src/__tests__/gpuRenderGraph.test.ts index 686a43706..0814269db 100644 --- a/apps/slides/src/__tests__/gpuRenderGraph.test.ts +++ b/apps/slides/src/__tests__/gpuRenderGraph.test.ts @@ -69,6 +69,17 @@ test("multipass setup binds declared resources, initializes buffers, and submits expect(g.textures.slice(3).map(t => t.descriptor.size)).toEqual([[3840, 2160], [3840, 2160], [1920, 1080]]); }); +test("empty draws and dispatches still run their passes without a zero-count GPU call", async () => { + const g = gpu(); + const r = await prepareRenderGraph(g.device, { targets: definition.targets, passes: [ + { kind: "compute", id: "instances", code: "", entry: "main", writes: ["hdr"], workgroups: { perInstance: 0 } }, + { kind: "render", id: "empty", code: "", vertex: "vertex", fragment: "fragment", color: ["hdr"], depth: "depth", vertexCount: 36, instanceCount: 0, clear: [0, 0, 0, 1] }, + ] }, "bgra8unorm", {}, {}); + encodeRenderGraph(r, g.device, g.encoder, { width: 64, height: 64, createView: () => "screen" }); + expect(g.commands.map(c => c[0])).toEqual(["render"]); + expect(g.commands[0][1].colorAttachments[0].loadOp).toBe("clear"); +}); + test("graph rejects invalid references and shader diagnostics before a frame can receive pipelines", async () => { const g = gpu(); await expect(prepareRenderGraph(g.device, { ...definition, passes: [{ ...definition.passes[3], reads: ["bloom@3"] }] }, "bgra8unorm", {}, {})).rejects.toThrow("Invalid GPU mip"); diff --git a/apps/slides/src/gpuRenderGraph.ts b/apps/slides/src/gpuRenderGraph.ts index 40f29d49d..1205db097 100644 --- a/apps/slides/src/gpuRenderGraph.ts +++ b/apps/slides/src/gpuRenderGraph.ts @@ -157,7 +157,7 @@ export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, en const groups = pass.workgroups; if (groups === "perPixel") command.dispatchWorkgroups(Math.ceil(width / 8), Math.ceil(height / 8)); else if (Array.isArray(groups)) command.dispatchWorkgroups(groups[0], groups[1] ?? 1, groups[2] ?? 1); - else command.dispatchWorkgroups(Math.ceil(groups.perInstance / 64)); + else if (groups.perInstance > 0) command.dispatchWorkgroups(Math.ceil(groups.perInstance / 64)); command.end(); } else { const colors = pass.kind === "render" ? pass.color : [output(pass)]; @@ -166,7 +166,11 @@ export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, en clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" : "clear", storeOp: "store" })), ...(pass.kind === "render" && pass.depth ? { depthStencilAttachment: { view: view(r, pass.depth), depthClearValue: 1, depthLoadOp: clear === false ? "load" as const : "clear" as const, depthStoreOp: "store" as const } } : {}) }); command.setPipeline(resource.pipeline as GPURenderPipeline); command.setBindGroup(0, resource.group!); - command.draw(pass.kind === "render" ? pass.vertexCount : 3, pass.kind === "render" ? pass.instanceCount ?? 1 : 1); command.end(); + // Dawn reports zero-count draws through a logging callback that calls the JS runtime from this UI thread, which can crash Hermes. + // An empty pass still clears its attachments. + const vertices = pass.kind === "render" ? pass.vertexCount : 3, instances = pass.kind === "render" ? pass.instanceCount ?? 1 : 1; + if (vertices > 0 && instances > 0) command.draw(vertices, instances); + command.end(); } } } From 061829edf092bc385dd5a213d04f9e810c1f037f Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Thu, 29 Oct 2026 15:52:33 -0700 Subject: [PATCH 11/28] fix: keep changed rows highlighted after their flash Changed rows now hold their flash color with a steady glow for the rest of the slide while unchanged rows dim, so the settled state still shows what changed. The data flow example uses magenta heat and a low camera so depth of field separates the changed rows. --- apps/slides/examples/3d/data-flow.mdx | 4 +-- packages/slides-effects/packs/world/README.md | 3 ++- packages/slides-effects/packs/world/scene.ts | 25 +++++++++++++------ packages/slides-effects/tests/world.test.ts | 21 ++++++++++++++++ 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/apps/slides/examples/3d/data-flow.mdx b/apps/slides/examples/3d/data-flow.mdx index 19454df68..5f3237045 100644 --- a/apps/slides/examples/3d/data-flow.mdx +++ b/apps/slides/examples/3d/data-flow.mdx @@ -80,8 +80,8 @@ export const atmosphere = <> - - + + diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index 3bcebc36a..35ddd0f96 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -220,7 +220,8 @@ never cuts across a junction. Ambient Spray accepts active leaves. Motion, birth and lifetime are analytic samples of absolute time. Rows keep a fixed instance pool while logical row ids advance with the clock. -Only listed changes flash; recycled slots do not inherit a previous row's flash. +Listed changes flash on entry, then stay in `flashColor` with a steady glow while +the other rows dim; recycled slots do not inherit a previous row's highlight. Use `rows:row:4` to follow or focus logical row 4 while it is visible. `width`, `depth`, `start`, `flashDuration` and `position` configure the column. Stages offset nested primitives and multiply their glow. Stage positions and glow diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index 8cb1c0db4..2ee87c970 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -219,16 +219,21 @@ export function rowLogicalIndex(instance: Instance, clock: ClockSample) { const rows = instance.rows!; return rows.logical ?? Math.floor((rows.options.start ?? 0) + clock.slideTime * (rows.options.speed ?? 0.3)) + rows.slot; } -function rowFlash(instance: Instance, clock: ClockSample) { +function rowHighlight(instance: Instance, clock: ClockSample) { "worklet"; const rows = instance.rows!; - const logical = rowLogicalIndex(instance, clock); - let flash = 0; - for (const change of rows.options.changes ?? []) if (change.row === logical) { - const elapsed = triggerElapsed(change.step, clock), duration = rows.options.flashDuration ?? 0.8; - if (elapsed >= 0 && elapsed < duration) flash = Math.max(flash, Math.pow(1 - elapsed / duration, 2)); + const logical = rowLogicalIndex(instance, clock), duration = rows.options.flashDuration ?? 0.8; + let changed = false, active = false, flash = 0; + for (const change of rows.options.changes ?? []) { + const elapsed = triggerElapsed(change.step, clock); + if (elapsed < 0) continue; + active = true; + if (change.row !== logical) continue; + changed = true; + if (elapsed < duration) flash = Math.max(flash, Math.pow(1 - elapsed / duration, 2)); } - return flash; + // Changed rows keep their color and glow once their step has started; the others dim behind them. + return { changed, dimmed: active && !changed, flash }; } function quadraticPoint(from: Vec3, to: Vec3, curve: number, t: number): Vec3 { "worklet"; @@ -393,7 +398,11 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl values[index + 3] = sampleTrack(instance.size, clock) * (instance.voxel ? instanceOpacity(instance, clock) : 1); let emissive = sampleTrack(instance.emissive, clock) * activation; if (instance.pulse?.tail) emissive *= 0.3; - if (instance.rows) { const flash = rowFlash(instance, clock); color = mixVector(color, instance.rows.flashColor, flash); emissive += flash * 8; } + if (instance.rows) { + const highlight = rowHighlight(instance, clock); + if (highlight.changed) { color = instance.rows.flashColor; emissive += 1.5 + highlight.flash * 8; } + else if (highlight.dimmed) color = mixVector(color, [0, 0, 0], 0.8); + } for (const stage of instance.stages ?? []) emissive *= sampleTrack(stage.glow, clock); values[index + 4] = dim[0] + (color[0] - dim[0]) * activation; values[index + 5] = dim[1] + (color[1] - dim[1]) * activation; diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index 7f0c45c25..b48a5b197 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -230,6 +230,27 @@ test("pulse trails reconstruct exact curved subsegments and restart at a turn wi expect(sampleWorldUniforms(scene, clock(1))[35]).toBe(1); }); +test("changed rows stay highlighted for the whole step while the others dim", () => { + const scene = prepareWorldScene({ instances: rowInstances({ id: "rows", count: 4, speed: 0, color: "#808080", flashColor: "#ff0000", changes: [{ row: 1, step: 1 }, { row: 2, step: 2 }], flashDuration: 0.5 }), + camera: { shots: [{ position: [0, 0, 10], target: [0, 0, 0] }] } }); + const row = (slot: number, sample: ClockSample) => { + const values = sampleWorldUniforms(scene, sample), index = worldHeaderSize + slot * worldInstanceSize; + return { color: [...values.slice(index + 4, index + 7)], emissive: values[index + 7] }; + }; + const grey = row(0, clock(5)).color; + expect(row(1, clock(5)).emissive).toBe(0.1); + const entry = row(1, clock(1.1, 1, { 1: 1 })), settled = row(1, clock(9, 1, { 1: 1 })); + expect(entry.emissive).toBeGreaterThan(settled.emissive); + expect(settled.color).toEqual([1, 0, 0]); + expect(settled.emissive).toBeCloseTo(1.6); + expect(row(0, clock(9, 1, { 1: 1 })).color[0]).toBeCloseTo(grey[0] * 0.2); + expect(row(2, clock(9, 1, { 1: 1 })).emissive).toBe(0.1); + const later = clock(20, 2, { 1: 1, 2: 10 }); + expect(row(1, later).color).toEqual([1, 0, 0]); + expect(row(2, later).color).toEqual([1, 0, 0]); + expect(row(3, later).color[0]).toBeCloseTo(grey[0] * 0.2); +}); + test("recycling transfers a flash with logical row identity and never to a reused slot", () => { const scene = prepareWorldScene({ instances: rowInstances({ id: "rows", count: 4, speed: 1, spacing: 1, changes: [{ row: 1, step: 1 }], flashDuration: 2 }), dof: { focus: "rows:row:1" }, camera: { shots: [{ position: [0, 0, 10], target: [0, 0, 0] }] } }); From bbb47af155664a6104cebc2e64abb92be6433bfa Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 05:36:16 -0700 Subject: [PATCH 12/28] fix: tighten the changed rows glow Changed rows went white-hot and bloomed magenta over their neighbours. The steady glow drops from +1.5 to +0.5 emissive and the entry flash peak from +8 to +4, so changed rows stay saturated magenta with a tight halo while the others dim. Test plan: - world.test.ts row highlight test fails before (entry 6.72, expected < 4) and passes after; world, worldRender, worldLook, worldDataFlow, worldPlayback and worldBackgroundPlayback suites: 47 pass. - Scoped TypeScript on scene.ts: 0 diagnostics. - Native Debug app, Data Flow deck: all 13 states rendered with no runtime errors; checked settled and 550 ms entry stills of the rows slide. --- packages/slides-effects/packs/world/scene.ts | 2 +- packages/slides-effects/tests/world.test.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index 2ee87c970..93a4f63b7 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -400,7 +400,7 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl if (instance.pulse?.tail) emissive *= 0.3; if (instance.rows) { const highlight = rowHighlight(instance, clock); - if (highlight.changed) { color = instance.rows.flashColor; emissive += 1.5 + highlight.flash * 8; } + if (highlight.changed) { color = instance.rows.flashColor; emissive += 0.5 + highlight.flash * 4; } else if (highlight.dimmed) color = mixVector(color, [0, 0, 0], 0.8); } for (const stage of instance.stages ?? []) emissive *= sampleTrack(stage.glow, clock); diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index b48a5b197..a07d91b08 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -241,8 +241,9 @@ test("changed rows stay highlighted for the whole step while the others dim", () expect(row(1, clock(5)).emissive).toBe(0.1); const entry = row(1, clock(1.1, 1, { 1: 1 })), settled = row(1, clock(9, 1, { 1: 1 })); expect(entry.emissive).toBeGreaterThan(settled.emissive); + expect(entry.emissive).toBeLessThan(4); expect(settled.color).toEqual([1, 0, 0]); - expect(settled.emissive).toBeCloseTo(1.6); + expect(settled.emissive).toBeCloseTo(0.6); expect(row(0, clock(9, 1, { 1: 1 })).color[0]).toBeCloseTo(grey[0] * 0.2); expect(row(2, clock(9, 1, { 1: 1 })).emissive).toBe(0.1); const later = clock(20, 2, { 1: 1, 2: 10 }); From 7c602feaa834c6ffd0876c9ef1eea765bfc59419 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 05:51:35 -0700 Subject: [PATCH 13/28] fix: keep one source per spray particle Each spray particle embedded every candidate source, and each source embedded the whole graph layout. The Data Flow title scene grew to 1.5M copied values from 4k unique objects. Every step re-prepares the scene and copies it into two UI mappers, so the UI heap climbed to 1.6 GB and the UI thread stalled for seconds per step. A particle only samples one source, so preparation now keeps just that one; rendering is unchanged. Test plan: - New world.test.ts spray test fails before (6 sources per particle, expected 1) and passes after. world, worldRender, worldLook, worldDataFlow, worldPlayback and worldBackgroundPlayback: 48 pass. - Scoped TypeScript on scene.ts: 0 diagnostics. - Native Debug app, fresh launch, Data Flow slide 0 steps 0-1-2-1-0 three times: UI heap peak 1640 MB before, 172 MB after; copied values per prepare 1.53M before, 118k after; max RSS 2.07 GB before, 0.68 GB after; no stalled UI samples and no runtime errors after. --- packages/slides-effects/packs/world/scene.ts | 13 +++++++------ packages/slides-effects/tests/world.test.ts | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index 93a4f63b7..d1fe6cabf 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -30,7 +30,7 @@ export type Instance = { stages?: StageMotion[]; material?: string | MaterialOptions; materialTracks?: { albedo: Track; roughness: Track; metal: Track; rim: Track }; pulse?: { options: PulseOptions; anchors: Instance[]; curves: number[]; tail: boolean }; - spray?: { options: SprayOptions; sources: Instance[]; phase: number }; + spray?: { options: SprayOptions; source?: Instance; phase: number }; rows?: { options: RowsOptions; slot: number; logical?: number; flashColor: Vec3 }; voxel?: { options: VoxelsOptions; index: number; count: number; target: Vec3 }; custom?: { wgsl: string; vertexCount: number }; sdf?: { wgsl: string; bounds: [Vec3, Vec3]; steps?: number }; @@ -145,7 +145,7 @@ export function sprayInstances(options: SprayOptions = {}): Instance[] { if (!Number.isFinite(count) || count < 0 || count > 10000 || (options.lifetime ?? 2) <= 0) throw new Error("World spray requires a positive lifetime and at most 10000 particles"); return orbInstances({ id: options.id ?? "spray", count, colors: [options.color ?? "#22d3ee"], radius: options.size ?? 0.025, spread: [0, 0, 0], position: options.position, seed: options.seed, drift: 0, emissive: 5 }).map((instance, index) => ({ - ...instance, spray: { options, sources: [], phase: ((index + 1) * 0.61803398875) % 1 }, + ...instance, spray: { options, phase: ((index + 1) * 0.61803398875) % 1 }, })); } export function rowInstances(options: RowsOptions): Instance[] { @@ -261,7 +261,7 @@ export function instancePosition(instance: Instance, clock: ClockSample): Vec3 { const spray = instance.spray, life = spray.options.lifetime ?? 2; const elapsed = spray.options.step === undefined ? clock.slideTime : triggerElapsed(spray.options.step, clock); const age = ((Math.max(0, elapsed) / life + spray.phase) % 1) * life; - if (spray.sources.length) position = instancePosition(spray.sources[Math.floor(spray.phase * spray.sources.length) % spray.sources.length], clock); + if (spray.source) position = instancePosition(spray.source, clock); else { const offset = stageOffset(instance.stages, clock); position = [position[0] + offset[0], position[1] + offset[1], position[2] + offset[2]]; @@ -309,7 +309,7 @@ export function instanceOpacity(instance: Instance, clock: ClockSample): number const spray = instance.spray, elapsed = spray.options.step === undefined ? clock.slideTime : triggerElapsed(spray.options.step, clock); if (elapsed < 0) return 0; const phase = (elapsed / (spray.options.lifetime ?? 2) + spray.phase) % 1; - const source = spray.sources[Math.floor(spray.phase * spray.sources.length) % spray.sources.length]; + const source = spray.source; const lit = source?.graph ? activationAt(source.graph.node, source.graph.nodes, source.graph.activate, clock) : 1; return Math.sin(phase * Math.PI) * lit; } @@ -471,7 +471,8 @@ export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { const sources = under === "leaves:lit" ? instances.filter(item => item.kind === "sphere" && item.graph && !item.graph.nodes.some(node => node.parent === item.graph!.node.id)) : under ? [byId.get(under) ?? byId.get(`node:${under}`)].filter((item): item is typeof instances[number] => !!item) : []; if (under && sources.length === 0) throw new Error(`Unknown world spray source: ${under}`); - instance.spray = { ...instance.spray, sources }; + // Keep only the source this particle samples; serializing every candidate per particle multiplies the scene copied to the UI. + instance.spray = { ...instance.spray, source: sources[Math.floor(instance.spray.phase * sources.length) % sources.length] }; } } const visiting = new Set(), visited = new Set(); @@ -479,7 +480,7 @@ export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { if (visiting.has(instance)) throw new Error("World motion anchors cannot form a cycle"); if (visited.has(instance)) return; visiting.add(instance); - for (const anchor of instance.pulse?.anchors ?? instance.spray?.sources ?? []) verifyAnchors(anchor); + for (const anchor of instance.pulse?.anchors ?? (instance.spray?.source ? [instance.spray.source] : [])) verifyAnchors(anchor); visiting.delete(instance); visited.add(instance); } instances.forEach(verifyAnchors); diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index a07d91b08..d630aa0e6 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -347,6 +347,20 @@ test("spray sampling is stateless and gates against lit leaves", () => { expect(particles.map(instance => instancePosition(instance, clock(2, 1, { 1: 0 })))).toEqual(positions); }); +test("each spray particle carries only the leaf it samples", () => { + const scene = prepareWorldScene({ instances: [...graphInstances({ data: tree("root", [["a", ["a1", "a2", "a3"]], ["b", ["b1", "b2", "b3"]]]) }), + ...sprayInstances({ id: "spray", under: "leaves:lit", rate: 10, lifetime: 2 })] }); + const leaves = scene.instances.filter(instance => instance.kind === "sphere" && /^node:[ab]\d$/.test(instance.id ?? "")); + const carried = scene.instances.filter(instance => instance.spray).map(particle => { + const found = new Set(); + const walk = (value: unknown) => { if (!value || typeof value !== "object") return; if (leaves.includes(value as never)) { found.add(value); return; } Object.values(value).forEach(walk); }; + walk(particle.spray); + expect(found.size).toBe(1); + return [...found][0]; + }); + expect(new Set(carried).size).toBe(leaves.length); +}); + test("explicit materials and rim lights preserve dim graph nodes until activation", () => { const scene = prepareWorldScene({ instances: graphInstances({ data: tree("root"), nodes: { root: { material: "bright" } }, activate: [wave({ from: "root", step: 1 })] }), materials: { bright: { albedo: "#ffffff", rim: 2 } }, rim: { intensity: 3 } }); From e56b16b9c456c6ad4730fa0a714a0b2c131b0d56 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 06:33:58 -0700 Subject: [PATCH 14/28] fix: keep Worklets frames on every macOS display pulse On macOS, Worklets stopped its CVDisplayLink on every frame and restarted it only after the UI runtime's callbacks finished. When a frame took more than about 6 ms, the restart missed the next pulse, so UI animations ran at 30 fps. Keeping the link running fixes that, but React Native macOS's display link queues a main-thread tick for every pulse. After a main thread stall, those queued ticks would run back to back. Worklets now uses its own macOS display link in the existing patched file. It keeps at most one tick queued or running, runs while frames are requested, and stops on an idle tick (rechecked under the queue lock). After invalidation it ignores Start requests and drops queued ticks. The iOS path matches upstream again, and frame timestamps are read the same way as before. Test plan: - `patches/tests/run-react-native-worklets-frame-queue.sh` compiles the patched AnimationFrameQueue.mm with RCTPlatformDisplayLink and drives a real CVDisplayLink (60 Hz). Old source: 4 failures (8 and 12 ms of work per frame run at 30 fps; a queued tick runs after invalidate; a request after invalidate restarts the link). New source: passes all 5 runs. The one-hunk variant without coalescing runs 9 frames 0 ms apart after a 133 ms stall; the new link runs one per pulse. - Slides Debug rebuilt with the patched source (cold build into its existing DerivedData path). The new debug dylib contains WorkletsFrameLink; the old product's does not. UI runtime rAF probe on data-flow, fullscreen 3420x2040 on a 60 Hz virtual display. These runs were not under the fps lock (it was held by another job) and are diagnostic only: - old app: 0/4/8/12 ms of work per frame = 60/33/30/30 fps; voxel slide 32 fps; rows slide 59 fps - new app: 60/60/59.5/60 fps; voxel slide 60 fps; rows slide 60 fps - Native smoke on the new app: world-tour (22 states), data-flow (13) and cinematic-title (4) all ok, with no runtime errors and max RSS 1.15 GB. - Not run: Release build, fps-locked present-commit series, iOS build. --- patches/react-native-worklets@0.8.3.patch | 175 +++++++++++++++-- .../react-native-worklets-frame-queue.test.mm | 183 ++++++++++++++++++ .../run-react-native-worklets-frame-queue.sh | 21 ++ 3 files changed, 366 insertions(+), 13 deletions(-) create mode 100644 patches/tests/react-native-worklets-frame-queue.test.mm create mode 100755 patches/tests/run-react-native-worklets-frame-queue.sh diff --git a/patches/react-native-worklets@0.8.3.patch b/patches/react-native-worklets@0.8.3.patch index e27ec6d9e..742c82f19 100644 --- a/patches/react-native-worklets@0.8.3.patch +++ b/patches/react-native-worklets@0.8.3.patch @@ -1,22 +1,171 @@ diff --git a/apple/worklets/apple/AnimationFrameQueue.mm b/apple/worklets/apple/AnimationFrameQueue.mm -index b93cd2c625ef8a03415ce23c6fc584e51d3696a9..08e340e6e7cf9025192ffc3c835395e676862b49 100644 +index b93cd2c625ef8a03415ce23c6fc584e51d3696a9..e955364572c134b57ae54bf0a2bce1c9c35dd05a 100644 --- a/apple/worklets/apple/AnimationFrameQueue.mm +++ b/apple/worklets/apple/AnimationFrameQueue.mm -@@ -87,7 +87,6 @@ - (void)executeQueue:(WorkletsDisplayLink *)displayLink +@@ -6,7 +6,9 @@ + #import + + #if TARGET_OS_OSX ++#import + #import ++#import + #endif // TARGET_OS_OSX + + #import +@@ -20,6 +22,124 @@ + const auto POOR = CAFrameRateRangeMake(10, 24, 24); + } // namespace FrameRateRange + ++#if TARGET_OS_OSX ++// RCTPlatformDisplayLink queues a main run loop tick for every display pulse, so ticks queued while the ++// main thread is busy run back to back. This link keeps at most one tick queued or running, and an ++// invalidated link never restarts or delivers a queued tick. ++@interface WorkletsFrameLink : RCTPlatformDisplayLink ++@end ++ ++@implementation WorkletsFrameLink { ++ CVDisplayLinkRef link_; ++ __weak id target_; ++ SEL selector_; ++ NSRunLoop *runLoop_; ++ NSRunLoopMode mode_; ++ // Orders Start/Stop with invalidation. The display link callback never takes it. ++ std::mutex controlMutex_; ++ std::atomic tickPending_; ++ std::atomic invalidated_; ++} ++ +++ (instancetype)displayLinkWithTarget:(id)target selector:(SEL)selector ++{ ++ WorkletsFrameLink *link = [self new]; ++ link->target_ = target; ++ link->selector_ = selector; ++ return link; ++} ++ ++static CVReturn WorkletsFrameLinkCallback( ++ CVDisplayLinkRef, ++ const CVTimeStamp *, ++ const CVTimeStamp *, ++ CVOptionFlags, ++ CVOptionFlags *, ++ void *context) ++{ ++ WorkletsFrameLink *link = (__bridge WorkletsFrameLink *)context; ++ if (link->invalidated_ || link->tickPending_.exchange(true)) { ++ return kCVReturnSuccess; ++ } ++ CFRunLoopRef runLoop = link->runLoop_.getCFRunLoop; ++ CFRunLoopPerformBlock(runLoop, (__bridge CFTypeRef)link->mode_, ^{ ++ @autoreleasepool { ++ id target = link->target_; ++ if (!link->invalidated_ && target) { ++ ((void (*)(id, SEL, WorkletsFrameLink *))[target methodForSelector:link->selector_])( ++ target, link->selector_, link); ++ } ++ link->tickPending_ = false; ++ } ++ }); ++ CFRunLoopWakeUp(runLoop); ++ return kCVReturnSuccess; ++} ++ ++- (void)dealloc ++{ ++ if (link_ != NULL) { ++ CVDisplayLinkStop(link_); ++ CVDisplayLinkRelease(link_); ++ } ++} ++ ++- (void)addToRunLoop:(NSRunLoop *)runLoop forMode:(NSRunLoopMode)mode ++{ ++ runLoop_ = runLoop; ++ mode_ = mode; ++ if (CVDisplayLinkCreateWithActiveCGDisplays(&link_) != kCVReturnSuccess) { ++ CVDisplayLinkCreateWithCGDisplay(CGMainDisplayID(), &link_); ++ } ++ RCTAssert(link_ != NULL, @"Cannot create display link"); ++ CVDisplayLinkSetOutputCallback(link_, &WorkletsFrameLinkCallback, (__bridge void *)self); ++} ++ ++- (void)setPaused:(BOOL)paused ++{ ++ std::lock_guard lock(controlMutex_); ++ if (link_ == NULL || invalidated_) { ++ return; ++ } ++ if (paused) { ++ CVDisplayLinkStop(link_); ++ } else { ++ CVDisplayLinkStart(link_); ++ } ++} ++ ++- (void)invalidate ++{ ++ std::lock_guard lock(controlMutex_); ++ invalidated_ = true; ++ if (link_ != NULL) { ++ CVDisplayLinkStop(link_); ++ } ++} ++ ++- (BOOL)isPaused ++{ ++ return link_ == NULL || !CVDisplayLinkIsRunning(link_); ++} ++ ++- (NSTimeInterval)timestamp ++{ ++ CVTimeStamp now = {}; ++ CVDisplayLinkGetCurrentTime(link_, &now); ++ return (NSTimeInterval)now.hostTime / (NSTimeInterval)CVGetHostClockFrequency(); ++} ++ ++- (NSTimeInterval)duration ++{ ++ const CVTime period = CVDisplayLinkGetNominalOutputVideoRefreshPeriod(link_); ++ return (period.flags & kCVTimeIsIndefinite) ? 0 : (NSTimeInterval)period.timeValue / period.timeScale; ++} ++ ++@end ++#else // TARGET_OS_OSX ++typedef WorkletsDisplayLink WorkletsFrameLink; ++#endif // TARGET_OS_OSX ++ + @implementation AnimationFrameQueue { + /* DisplayLink is thread safe. */ + WorkletsDisplayLink *displayLink_; +@@ -45,9 +165,9 @@ - (instancetype)init + + SEL frameCallback = supportsProMotion ? @selector(executeQueueForProMotion:) : @selector(executeQueue:); + currentFrameRate_ = supportsProMotion ? FrameRateRange::BEST : FrameRateRange::STANDARD; +- displayLink_ = [WorkletsDisplayLink displayLinkWithTarget:self selector:frameCallback]; ++ displayLink_ = [WorkletsFrameLink displayLinkWithTarget:self selector:frameCallback]; + } else { +- displayLink_ = [WorkletsDisplayLink displayLinkWithTarget:self selector:@selector(executeQueue:)]; ++ displayLink_ = [WorkletsFrameLink displayLinkWithTarget:self selector:@selector(executeQueue:)]; + } + + #if TARGET_OS_OSX +@@ -87,7 +207,19 @@ - (void)executeQueue:(WorkletsDisplayLink *)displayLink RCTAssertMainQueue(); auto frameCallbacks = [self pullCallbacks]; -- [displayLink_ setPaused:TRUE]; ++#if TARGET_OS_OSX ++ // The link keeps running while frames are requested, so a long frame still makes the next pulse. ++ // Stop it on an idle tick; the recheck under the lock keeps a concurrent request from being lost. ++ if (frameCallbacks.empty()) { ++ std::lock_guard lock(callbacksMutex_); ++ if (frameCallbacks_.empty()) { ++ [displayLink_ setPaused:TRUE]; ++ } ++ return; ++ } ++#else // TARGET_OS_OSX + [displayLink_ setPaused:TRUE]; ++#endif // TARGET_OS_OSX #if TARGET_OS_OSX auto targetTimestamp = displayLink.timestamp + displayLink.duration; -@@ -95,6 +94,9 @@ - (void)executeQueue:(WorkletsDisplayLink *)displayLink - auto targetTimestamp = displayLink.targetTimestamp; - #endif // TARGET_OS_OSX - targetTimestamp = worklets::calculateTimestampWithSlowAnimations(targetTimestamp); -+ // macOS cannot read the CVDisplayLink clock after it is stopped. Capture -+ // the timestamp first, then pause before callbacks can request another frame. -+ [displayLink_ setPaused:TRUE]; - for (const auto &callback : frameCallbacks) { - callback(targetTimestamp); - } diff --git a/patches/tests/react-native-worklets-frame-queue.test.mm b/patches/tests/react-native-worklets-frame-queue.test.mm new file mode 100644 index 000000000..2f8deed55 --- /dev/null +++ b/patches/tests/react-native-worklets-frame-queue.test.mm @@ -0,0 +1,183 @@ +#import +#import +#import +#include +#include +#include +#include +#include + +// Compiles the production AnimationFrameQueue.mm and RCTPlatformDisplayLink.m against a real CVDisplayLink. +extern "C" BOOL RCTIsMainQueue(void) { return NSThread.isMainThread; } +extern "C" void _RCTAssertFormat(const char *condition, const char *file, int line, const char *, NSString *, ...) +{ + fprintf(stderr, "RCTAssert %s %s:%d\n", condition, file, line); + abort(); +} + +static int failures = 0; +#define CHECK(cond, ...) \ + do { \ + bool ok_ = (cond); \ + if (!ok_) failures++; \ + printf("%s ", ok_ ? "PASS" : "FAIL"); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } while (0) + +static double nowMs() { return CACurrentMediaTime() * 1000; } +static void runFor(double ms) { [NSRunLoop.mainRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:ms / 1000]]; } +static void busy(double ms) { for (double end = nowMs() + ms; nowMs() < end;) {} } +static dispatch_queue_t turboModuleQueue() +{ + static dispatch_queue_t queue = dispatch_queue_create("com.meta.react.turbomodulemanager.queue", DISPATCH_QUEUE_SERIAL); + return queue; +} +static BOOL linkPaused(AnimationFrameQueue *queue) { return [[queue valueForKey:@"displayLink_"] isPaused]; } + +// A self-requeueing frame loop, as the UI runtime does with nativeFlushQueue. +struct Loop { + AnimationFrameQueue *queue; + double busyMs = 0; + bool running = true; + int stallAtFrame = -1; + double stallMs = 0; + double stallEnd = 0; + std::vector calls, timestamps; + std::function frame; + Loop(AnimationFrameQueue *q) : queue(q) + { + frame = [this](double timestamp) { + calls.push_back(nowMs()); + timestamps.push_back(timestamp); + busy(busyMs); + if ((int)calls.size() == stallAtFrame) { + usleep(stallMs * 1000); + stallEnd = nowMs(); + } + if (running) [queue requestAnimationFrame:frame]; + }; + } + void start() { [queue requestAnimationFrame:frame]; } +}; + +static double median(std::vector values) +{ + std::sort(values.begin(), values.end()); + return values.empty() ? 0 : values[values.size() / 2]; +} + +static std::vector deltas(const std::vector &values, size_t from = 1) +{ + std::vector result; + for (size_t i = std::max(from, 1); i < values.size(); i++) result.push_back(values[i] - values[i - 1]); + return result; +} + +int main() +{ + setvbuf(stdout, NULL, _IOLBF, 0); + @autoreleasepool { + NSThread.currentThread.name = @"com.facebook.react.runtime.JavaScript"; + CVDisplayLinkRef probe; + CVDisplayLinkCreateWithActiveCGDisplays(&probe); + CVTime nominal = CVDisplayLinkGetNominalOutputVideoRefreshPeriod(probe); + const double period = 1000.0 * nominal.timeValue / nominal.timeScale; + CVDisplayLinkRelease(probe); + printf("INFO display period %.2f ms\n", period); + + for (double busyMs : {0.0, 4.0, 8.0, 12.0}) { + AnimationFrameQueue *queue = [AnimationFrameQueue new]; + Loop loop(queue); + loop.busyMs = busyMs; + loop.start(); + runFor(1000); + loop.running = false; + runFor(4 * period); + auto frameDeltas = deltas(loop.calls, 5); + double fps = 1000.0 * (loop.calls.size() - 6) / (loop.calls.back() - loop.calls[5]); + printf("INFO busy %.0f ms: %.1f fps, median %.2f ms\n", busyMs, fps, median(frameDeltas)); + CHECK(median(frameDeltas) < 1.5 * period, "busy %.0f ms keeps one frame per pulse (median %.2f ms)", busyMs, median(frameDeltas)); + auto timestampDeltas = deltas(loop.timestamps); + CHECK(*std::min_element(timestampDeltas.begin(), timestampDeltas.end()) > 0, "busy %.0f ms timestamps increase", busyMs); + CHECK(fabs(loop.timestamps.back() - loop.calls.back()) < 3 * period, "timestamps are milliseconds near the media clock"); + dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); + } + + { + AnimationFrameQueue *queue = [AnimationFrameQueue new]; + Loop loop(queue); + loop.stallAtFrame = 20; + loop.stallMs = 8 * period; + loop.start(); + runFor(600); + loop.running = false; + runFor(4 * period); + std::vector after; + for (size_t i = 20; i < loop.calls.size() && after.size() < 6; i++) after.push_back(loop.calls[i]); + auto afterDeltas = deltas(after); + double minDelta = *std::min_element(afterDeltas.begin(), afterDeltas.end()); + auto timestampDeltas = deltas(std::vector(loop.timestamps.begin() + 20, loop.timestamps.begin() + 26)); + double minTimestampDelta = *std::min_element(timestampDeltas.begin(), timestampDeltas.end()); + int burst = 0; + for (double call : loop.calls) burst += call > loop.stallEnd && call < loop.stallEnd + period / 2; + printf("INFO stall %.0f ms: %d frames within half a period after it, min gap %.2f ms, min timestamp delta %.2f ms\n", + loop.stallMs, burst, minDelta, minTimestampDelta); + CHECK(burst <= 1, "main thread stall does not replay queued pulses (%d frames)", burst); + CHECK(minDelta > period / 2, "frames after the stall stay one pulse apart (min %.2f ms)", minDelta); + CHECK(minTimestampDelta > period / 2, "timestamps after the stall stay one pulse apart (min %.2f ms)", minTimestampDelta); + dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); + } + + { + AnimationFrameQueue *queue = [AnimationFrameQueue new]; + std::vector parent, children; + bool requeued = false; + std::function child = [&](double timestamp) { children.push_back(timestamp); }; + [queue requestAnimationFrame:[&](double timestamp) { + parent.push_back(timestamp); + for (int i = 0; i < 3; i++) [queue requestAnimationFrame:child]; + requeued = true; + }]; + runFor(6 * period); + CHECK(parent.size() == 1 && children.size() == 3, "each registration runs once (%zu parent, %zu children)", parent.size(), children.size()); + CHECK(requeued && children.size() == 3 && children[0] == children[2] && children[0] > parent[0], + "callbacks requested during a frame share the next frame"); + runFor(4 * period); + CHECK(linkPaused(queue), "link stops after an idle tick"); + double requested = nowMs(); + double resumed = 0; + [queue requestAnimationFrame:[&](double) { resumed = nowMs(); }]; + runFor(4 * period); + CHECK(resumed > 0 && resumed - requested < 3 * period, "request after idle resumes (%.2f ms)", resumed - requested); + + static std::atomic requests = 0, called = 0; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + for (int i = 0; i < 300; i++) { + usleep(arc4random_uniform((useconds_t)(2 * period * 1000))); + [queue requestAnimationFrame:[](double) { called++; }]; + requests++; + } + }); + for (double end = nowMs() + 300 * 2 * period; nowMs() < end && requests < 300;) runFor(50); + runFor(6 * period); + CHECK(requests == 300 && called == 300, "requests racing idle stops all run (%d/%d)", called.load(), requests.load()); + dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); + } + + { + AnimationFrameQueue *queue = [AnimationFrameQueue new]; + int called = 0; + [queue requestAnimationFrame:[&](double) { called++; }]; + usleep(3 * period * 1000); + dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); + runFor(4 * period); + CHECK(called == 0, "tick queued before invalidate does not deliver (%d)", called); + [queue requestAnimationFrame:[&](double) { called++; }]; + runFor(4 * period); + CHECK(called == 0 && linkPaused(queue), "request after invalidate does not restart the link"); + } + } + printf("%s %d failure(s)\n", failures ? "FAILED" : "OK", failures); + return failures ? 1 : 0; +} diff --git a/patches/tests/run-react-native-worklets-frame-queue.sh b/patches/tests/run-react-native-worklets-frame-queue.sh new file mode 100755 index 000000000..165c97071 --- /dev/null +++ b/patches/tests/run-react-native-worklets-frame-queue.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Runs the patched Worklets AnimationFrameQueue against a real display link. Needs a display. +# Usage: run-react-native-worklets-frame-queue.sh [AnimationFrameQueue.mm] +set -euo pipefail +repo="$(cd "$(dirname "$0")/../.." && pwd)" +modules="${SPARK_NODE_MODULES:-$repo/shell/.legend/spark/node_modules}" +worklets="$modules/react-native-worklets" +rn="$modules/react-native-macos" +queue="${1:-$worklets/apple/worklets/apple/AnimationFrameQueue.mm}" +dir="$(mktemp -d "${TMPDIR:-/tmp}/worklets-frame-queue.XXXXXX")" +trap 'rm -rf "$dir"' EXIT +mkdir "$dir/include" +ln -s "$rn/React/Base" "$dir/include/React" +flags=(-fobjc-arc -O1 -DNDEBUG -Wno-deprecated-declarations -I"$dir/include" -I"$rn/React/Base" -I"$rn/ReactCommon" + -I"$worklets/apple" -I"$worklets/Common/cpp") +clang -x objective-c "${flags[@]}" -c "$rn/React/Base/macOS/RCTPlatformDisplayLink.m" -o "$dir/link.o" +for source in "$queue" "$worklets/apple/worklets/apple/SlowAnimations.mm" "$(dirname "$0")/react-native-worklets-frame-queue.test.mm"; do + clang++ -x objective-c++ -std=c++20 "${flags[@]}" -c "$source" -o "$dir/$(basename "$source").o" +done +clang++ -fobjc-arc "$dir"/*.o -framework Foundation -framework QuartzCore -framework CoreVideo -framework CoreGraphics -o "$dir/test" +"$dir/test" From 20a25a4188375bf974c1e510c7d1b484dc69b1e0 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 06:53:43 -0700 Subject: [PATCH 15/28] fix: stop empty particle work and render-time scale reads in GPU frames React Compiler memoized the frame worklet on `renderScale?.value`, so every UIFrames render read the shared value on JS. A native World smoke logged 280 Reanimated "Reading from `value` during component render" warnings from this read. The worklet now reads it with `get()`, which the compiler does not hoist into a render-time dependency. Frames with zero particles or zero particle vertices no longer dispatch compute or draw particles. Fullscreen and composite draws, attachment clears, submit and present are unchanged. Frame data longer than its GPU buffer now throws before the upload, which stops the surface and reports through the existing frame error path. Test plan: - typeGPUShader.test.ts: new tests fail on the base and pass here: render under React Compiler + worklets reads `renderScale` 3 times in 3 renders (now 0); count 0 / vertices 0 matrix with compute and composite records the empty dispatch/draw (now skipped, with clears, ends, submit and present unchanged); oversize frame data uploads (now stops and reports once). 20/20 pass. - Debug app (Oct 28 product, task Metro): data-flow, world-tour and cinematic-title, 39 slide/step states, 0 render-read warnings, 0 errors. Two UIFrames mounted under World runtimes; served module uses `get()`. --- apps/slides/src/TypeGPUShader.tsx | 11 ++- .../src/__tests__/typeGPUShader.test.ts | 88 ++++++++++++++++--- 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 65cf0893a..695248852 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -244,7 +244,7 @@ function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, fra } simulationEpoch.value = { phase: state.phase, slideKey: state.slideKey, stepKey: state.stepKey }; const frameIndex = frozen ? Math.max(0, count.value - 1) : count.value; - const scale = renderScale?.value; + const scale = renderScale?.get(); try { const time = samplePlayback(state, previewTime, clockMode); device.queue.writeBuffer(clock, 0, new Float32Array([time, dt, frameIndex, state.stepIndex, aspect, brightness, samplePlayback(state, previewTime, "slide"), samplePlayback(state, previewTime, "step")])); @@ -337,7 +337,10 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part r.configured.value = true; } const data = frameData?.value; - if (r.frameData && data) device.queue.writeBuffer(r.frameData, 0, new Float32Array(data!)); + if (r.frameData && data) { + if (data.length * 4 > r.frameData.size) throw new Error(`Frame data has ${data.length} values; its buffer holds ${r.frameData.size / 4}`); + device.queue.writeBuffer(r.frameData, 0, new Float32Array(data)); + } const encoder = device.createCommandEncoder(); if (r.graph) { encodeRenderGraph(r.graph.value, device, encoder, context.getCurrentTexture(), renderScale); @@ -346,7 +349,7 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part } const source = compute ? frame % 2 : 0; const target = compute ? 1-source : 0; - if (compute) { + if (compute && particleCount > 0) { const simulation = encoder.beginComputePass(); simulation.setPipeline(compute); simulation.setBindGroup(0, groups[source]); simulation.dispatchWorkgroups(Math.ceil(particleCount / 64)); simulation.end(); @@ -357,7 +360,7 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part }] }); pass.setBindGroup(0, groups[target]); pass.setPipeline(render!); pass.draw(3); - if (particles) { pass.setPipeline(particles); pass.draw(particleVertexCount, particleCount); } + if (particles && particleCount > 0 && particleVertexCount > 0) { pass.setPipeline(particles); pass.draw(particleVertexCount, particleCount); } pass.end(); if (r.composite) { const compositePass = encoder.beginRenderPass({ colorAttachments: [{ diff --git a/apps/slides/src/__tests__/typeGPUShader.test.ts b/apps/slides/src/__tests__/typeGPUShader.test.ts index ef14997fc..dc6f82108 100644 --- a/apps/slides/src/__tests__/typeGPUShader.test.ts +++ b/apps/slides/src/__tests__/typeGPUShader.test.ts @@ -5,7 +5,7 @@ import { createRequire } from "node:module"; import { transformSync } from "@babel/core"; import { fileURLToPath } from "node:url"; -function frames({ preview = false, active = true, effectActive = true, maxTime, frameTimeIndex, preparing = false, composite = false, data = false, clockMode = "step", simulation = true, graph = false } = {}) { +function frames({ preview = false, active = true, effectActive = true, maxTime, frameTimeIndex, preparing = false, composite = false, data = false, clockMode = "step", simulation = true, graph = false, particleCount = 2048, particleVertexCount } = {}) { const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); const require = createRequire(filename); const { code } = transformSync(readFileSync(filename, "utf8"), { @@ -13,8 +13,8 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, presets: [require.resolve("@react-native/babel-preset")], plugins: [require.resolve("react-native-worklets/plugin")], }); - const effects = [], calls = [], uniforms = [], errors = []; - const frameData={value:[1,2,3,4]}, renderScale={value:1}; + const effects = [], calls = [], events = [], uniforms = [], errors = []; + const frameData={value:[1,2,3,4]}, renderScale={value:1,get(){return this.value;}}; const playback = { value: { phase: preparing ? "preparing" : preview ? "preview" : active ? "playing" : "paused", slideKey: "slide", stepKey: "step", stepIndex: 1, stepTime: 0, slideTime: 10 } }; let callback, reaction, reactionInputs, enabled = false, ready = 0; @@ -22,12 +22,13 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, const pass = kind => ({ setPipeline() {}, setBindGroup: (_, value) => calls.push([kind, value]), dispatchWorkgroups: count => calls.push(["dispatch", count]), - draw: (...args) => calls.push(["draw", ...args]), end() {}, + draw: (...args) => calls.push(["draw", ...args]), end: () => events.push(["end", kind]), }); const device = { - queue: { writeBuffer: (_, __, values) => uniforms.push([...values]), submit() {} }, + queue: { writeBuffer: (_, __, values) => uniforms.push([...values]), submit: () => events.push(["submit"]) }, createCommandEncoder: () => ({ - beginComputePass: () => pass("compute"), beginRenderPass: () => pass("render"), finish() {}, + beginComputePass: () => { events.push(["begin", "compute"]); return pass("compute"); }, + beginRenderPass: ({ colorAttachments }) => { events.push(["begin", "render", colorAttachments[0].loadOp]); return pass("render"); }, finish() {}, }), destroy: () => calls.push(["destroy"]), }; @@ -51,12 +52,12 @@ function frames({ preview = false, active = true, effectActive = true, maxTime, const UIFrames = new Function("require", "exports", code + "\nreturn UIFrames;")( name => mocks[name] ?? require(name), {}, ); - UIFrames({ resources: { alive: { value: true, set(value) { this.value = value; } }, disposed: { value: false }, configured: { value: false }, format: "bgra8unorm", device, context: { configure() {}, getCurrentTexture: () => ({ createView() {} }), present() {} }, - clock: {}, graph: graph ? { value: {} } : undefined, states: [], groups: simulation ? ["A", "B"] : ["A"], compute: simulation ? {} : undefined, render: {}, particles: {}, frameData: data ? {} : undefined, + UIFrames({ resources: { alive: { value: true, set(value) { this.value = value; } }, disposed: { value: false }, configured: { value: false }, format: "bgra8unorm", device, context: { configure() {}, getCurrentTexture: () => ({ createView() {} }), present: () => events.push(["present"]) }, + clock: {}, graph: graph ? { value: {} } : undefined, states: [], groups: simulation ? ["A", "B"] : ["A"], compute: simulation ? {} : undefined, render: {}, particles: {}, frameData: data ? { size: 16 } : undefined, composite: composite ? { pipeline: {}, group: "lens", texture: { createView() {} } } : undefined }, - effectActive, maxTime, frameTimeIndex, clockMode, renderScale, frameData: data ? frameData : undefined, aspect: 2.61, brightness: 1, particleCount: 2048, previewTime: 7, onError: error => errors.push(error), onReady: () => ready++ }); + effectActive, maxTime, frameTimeIndex, clockMode, renderScale, frameData: data ? frameData : undefined, aspect: 2.61, brightness: 1, particleCount, particleVertexCount, previewTime: 7, onError: error => errors.push(error), onReady: () => ready++ }); const cleanup = effects.map(effect => effect()); - return { calls, uniforms, errors, ready: () => ready, enabled: () => enabled, + return { calls, events, uniforms, errors, ready: () => ready, enabled: () => enabled, playback, frameData, reactionInputs, redraw: () => effects[0](), scale: value => { renderScale.value = value; reaction(); }, tick: () => { playback.value = { ...playback.value, stepTime: playback.value.stepTime + 0.016 }; @@ -84,6 +85,23 @@ test("active GPU frames alternate storage buffers and dispatch the configured pa expect(f.calls.filter(call => call[0] === "destroy")).toHaveLength(1); }); +test("particle-free frames skip empty particle work but still clear, draw fullscreen passes and present", () => { + for (const [particleCount, particleVertexCount] of [[0, 6], [2, 0], [2, 6]]) for (const composite of [false, true]) for (const simulation of [false, true]) { + const f = frames({ particleCount, particleVertexCount, composite, simulation }); + f.tick(); + const dispatches = simulation && particleCount > 0; + const frame = { + draws: [["draw", 3], ...(particleCount && particleVertexCount ? [["draw", particleVertexCount, particleCount]] : []), ...(composite ? [["draw", 3]] : [])], + events: [...(dispatches ? [["begin", "compute"], ["end", "compute"]] : []), ["begin", "render", "clear"], ["end", "render"], + ...(composite ? [["begin", "render", "clear"], ["end", "render"]] : []), ["submit"], ["present"]], + }; + expect(f.calls.filter(call => call[0] === "dispatch")).toEqual(dispatches ? [["dispatch", 1], ["dispatch", 1]] : []); + expect(f.calls.filter(call => call[0] === "draw")).toEqual([...frame.draws, ...frame.draws]); + expect(f.events).toEqual([...frame.events, ...frame.events]); + f.stop(); + } +}); + test("shared render scale redraws a paused graph at its frozen clock and frame", () => { const f = frames({ graph: true, simulation: false }); f.tick(); @@ -169,6 +187,21 @@ test("GPU submission errors stop subsequent rendering", () => { f.stop(); }); +test("frame data larger than its GPU buffer stops before upload and reports once", () => { + const f = frames({ data: true }); + expect(f.uniforms.at(-1)).toEqual([1, 2, 3, 4]); + f.frameData.value = [1, 2, 3, 4, 5]; + const count = f.uniforms.length; + f.tick(); + expect(f.uniforms.slice(count).some(values => values.length === 5)).toBe(false); + expect(f.errors).toHaveLength(1); + const submitted = f.events.length; + f.tick(); + expect(f.events).toHaveLength(submitted); + expect(f.errors).toHaveLength(1); + f.stop(); +}); + test("the native GPU bridge compiles with React Compiler and the worklet transform", () => { const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); const require = createRequire(filename); @@ -358,3 +391,38 @@ test("frame failures report the latest slide context as well as stopping the sur expect(errors).toEqual(["draw failed"]); expect(reports).toEqual([["draw failed", 4, true]]); }); + +test("React Compiler renders frame surfaces without reading shared values", () => { + const filename = fileURLToPath(new URL("../TypeGPUShader.tsx", import.meta.url)); + const require = createRequire(filename); + const { code } = transformSync(readFileSync(filename, "utf8"), { + filename, babelrc: false, configFile: false, + presets: [require.resolve("@react-native/babel-preset")], + plugins: [ + [require.resolve("babel-plugin-react-compiler"), { target: "19" }], + require.resolve("react-native-worklets/plugin"), + ], + }); + let rendering = false, cache; + const reads = []; + const shared = (name, value) => ({ get value() { if (rendering) reads.push(name); return value; }, set value(next) { value = next; }, + get() { if (rendering) reads.push(name); return value; }, set(next) { value = next; } }); + const mocks = { + react: { useEffect() {} }, + "react/compiler-runtime": { c: size => cache ??= new Array(size).fill(Symbol.for("react.memo_cache_sentinel")) }, + "react-native": {}, "@shopify/react-native-skia": {}, "react-native-webgpu": {}, "./slidesStore": {}, "./gpuRenderGraph": {}, + "@legend-apps/presentation": { usePresentationValue: () => false, usePlayback: () => playback, samplePlayback() {} }, + "react-native-reanimated": { useSharedValue: value => shared("local", value), useFrameCallback: () => ({ setActive() {} }), + useAnimatedReaction() {}, runOnJS: fn => fn, runOnUI: fn => fn }, + }; + const playback = shared("playback", { phase: "playing" }); + const UIFrames = new Function("require", "exports", code + "\nreturn UIFrames;")(name => mocks[name] ?? require(name), {}); + const props = { resources: { alive: shared("alive", true), disposed: shared("disposed", false), configured: shared("configured", false), + graph: shared("graph", {}), device: {}, context: {}, clock: {}, groups: [] }, renderScale: shared("renderScale", 1), + frameData: shared("frameData", [0]), frameTimeIndex: 0, width: 1, height: 1, aspect: 1, brightness: 1, particleCount: 0, previewTime: 0, + onError() {}, onReady() {} }; + rendering = true; + UIFrames(props); UIFrames(props); UIFrames({ ...props, particleCount: 1, frameTimeIndex: undefined, maxTime: 2 }); + rendering = false; + expect(reads).toEqual([]); +}); From 028f0a615aaf184cf188d52512a5f98c5e4cd20f Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 06:53:50 -0700 Subject: [PATCH 16/28] fix: keep pure annotations in compiled decks The final React Native preset pass strips comments, which removed esbuild's PURE annotations from the executable deck output. The final transform now keeps comments; Hermes lowering, worklets and inline helpers are unchanged, and there is still one source map comment. Test plan: - compileDeck.test.ts: new pure-annotation test fails on the base and passes here; whole file 40/40, including the Hermes class/loop test. - Packaged compiler helper test (signs a helper bundle, compiles local worklets and Webview components) passes. --- .../presentation/src/__tests__/compileDeck.test.ts | 11 +++++++++++ packages/presentation/src/compiler/compileDeck.ts | 1 + 2 files changed, 12 insertions(+) diff --git a/packages/presentation/src/__tests__/compileDeck.test.ts b/packages/presentation/src/__tests__/compileDeck.test.ts index 577808e86..711eeeacc 100644 --- a/packages/presentation/src/__tests__/compileDeck.test.ts +++ b/packages/presentation/src/__tests__/compileDeck.test.ts @@ -331,6 +331,17 @@ describe("compileDeck", () => { expect(result.uniwindCode).toContain("bg-fuchsia-500"); }); + test("keeps pure annotations in executable output", async () => { + const deckPath = createDeck({ + "deck.mdx": `import { Card } from "./Card"\n\n`, + "Card.tsx": `import { View } from "react-native"; export function Card() { return ; }`, + }); + const result = await compileDeck(deckPath); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.code.split("//# sourceMappingURL=")[0]).toMatch(/\/\*\s*[@#]__PURE__\s*\*\/\s*\(0, import_jsx_runtime\d*\.jsx\)\(import_react_native\d*\.View/); + }); + test("bundles a template selected by file name in deck frontmatter", async () => { const deckPath = createDeck({ "deck.mdx": [ diff --git a/packages/presentation/src/compiler/compileDeck.ts b/packages/presentation/src/compiler/compileDeck.ts index a1b8a89e4..7d619c28a 100644 --- a/packages/presentation/src/compiler/compileDeck.ts +++ b/packages/presentation/src/compiler/compileDeck.ts @@ -349,6 +349,7 @@ export async function compileDeck(deckPath: string, options: { source?: string; // export getters are covered too. Deck helpers must stay inline. const code = transformSync(output.text, { babelrc: false, + comments: true, compact: false, configFile: false, filename: "deck.js", From 9a0e3d233055c91f31426f0dc41104c727cf46d1 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 06:53:50 -0700 Subject: [PATCH 17/28] fix: order World draws by code unit World sorted draw keys, which include custom WGSL, with localeCompare, so draw order depended on the engine's collation. It now compares UTF-16 code units. Test plan: - worldRender.test.ts: custom shaders differing only in letter case keep locale order on the base and code-unit order here, with matching draw offsets, instance counts and uniform size. 11/11 pass. --- packages/slides-effects/packs/world/scene.ts | 2 +- packages/slides-effects/tests/worldRender.test.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/slides-effects/packs/world/scene.ts b/packages/slides-effects/packs/world/scene.ts index d1fe6cabf..97e34cf18 100644 --- a/packages/slides-effects/packs/world/scene.ts +++ b/packages/slides-effects/packs/world/scene.ts @@ -487,7 +487,7 @@ export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { if (scene.counter && !instances.some(instance => instance.voxel?.options.id === scene.counter!.source)) throw new Error(`Unknown world counter source: ${scene.counter.source}`); const order = (instance: Instance) => instance.custom ? `1:${instance.custom.wgsl}:${instance.custom.vertexCount}` : instance.sdf ? `2:${instance.sdf.wgsl}` : `0:${instance.kind}`; - instances.sort((a, b) => order(a).localeCompare(order(b))); + instances.sort((a, b) => { const x = order(a), y = order(b); return x < y ? -1 : x > y ? 1 : 0; }); const fields: [number, Animated][] = [[9, scene.bloom?.intensity ?? 0], [10, scene.bloom?.threshold ?? 1], [11, scene.bloom?.radius ?? 0.8], [12, scene.fog?.density ?? 0], [13, scene.fog?.height ?? 0], [14, scene.fog?.falloff ?? 0.15], [15, scene.vignette ?? 0.25], [16, scene.grain ?? 0.015], [33, scene.dof ? scene.dof.aperture ?? 0.6 : 0], [34, scene.dof?.maxBlur ?? 12], [36, scene.floor?.y ?? -5], [37, scene.floor?.roughness ?? 0.06], diff --git a/packages/slides-effects/tests/worldRender.test.ts b/packages/slides-effects/tests/worldRender.test.ts index cc7c83409..4cdf52213 100644 --- a/packages/slides-effects/tests/worldRender.test.ts +++ b/packages/slides-effects/tests/worldRender.test.ts @@ -60,6 +60,16 @@ test("custom draw groups keep marker-local instance ids and SDF ranges outside p expect(graph.uniformsSize).toBe(worldHeaderSize + 7 * worldInstanceSize); }); +test("custom draw order compares WGSL by code unit rather than locale", () => { + const lower = quad.replace("vec3f(.2,.5,.8)", "vec3f(.2,.5,.8)*alpha"), upper = quad.replace("vec3f(.2,.5,.8)", "vec3f(.2,.5,.8)*Beta"); + const topology = worldGraphTopology(scene({ instances: [primitive("sphere"), ...customInstances({ wgsl: lower, vertexCount: 6, count: 2 }), + ...customInstances({ wgsl: upper, vertexCount: 3, count: 3 })] })); + expect(topology.customs?.map(draw => [draw.wgsl, draw.offset, draw.count])).toEqual([[upper, 1, 3], [lower, 4, 2]]); + const graph = buildWorldGraph(topology); + expect(graph.passes.filter(pass => pass.kind === "render").map(pass => [pass.vertexCount, pass.instanceCount])).toEqual([[expect.any(Number), 1], [3, 3], [6, 2]]); + expect(graph.uniformsSize).toBe(worldHeaderSize + 6 * worldInstanceSize); +}); + test("postOrder rewires HDR stages and custom hooks retain the declared color-space boundary", () => { const graph = buildWorldGraph(scene({ dof: {}, postOrder: ["bloom", "dof", "fog"], passes: [ { id: "before-focus", wgsl: scanlines, before: "dof" }, { id: "ldr", wgsl: scanlines, after: "tonemap" }, From 70961b8551af6434f835e31833fe0b7b02edd359 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 06:53:58 -0700 Subject: [PATCH 18/28] test: run the real World decks in Hermes and Dawn The World example decks were not compiled or executed by any check. New exampleShaders tests compile world-tour, data-flow and cinematic-title from their source files and: - check the compiled module has no class syntax, requires only host modules and has no Babel runtime or regenerator references; - on macOS, evaluate each module in React Native's Hermes binary with a strict host-module allowlist and expect a default export function; - render every slide, including Background content, and collect each World graph pass, covering custom geometry, SDF, custom post passes and shaft sample-count variants; - on macOS, validate every unique pass, plus all-effects graphs at low/medium/high/auto quality, with the existing Dawn shader module validator. A broken real World shader must fail it. This checks WGSL module compilation only, not pipeline layout or rendering. worldDataFlow now passes the deck source (so no effect lock is written) and asserts the named node and orb trail exist before reading them. A new workflow runs the portable tests on Linux and the Hermes tests on a hosted macOS runner. Dawn needs a Metal adapter; no runner for that is provisioned, so the Dawn test runs locally only. Test plan: - exampleShaders.test.ts under the heavy gate: 5/5 pass, including Dawn validation and the broken-shader failure. - Hermes rejects class syntax and unknown host modules in the same probe form (manual check). - worldDataFlow.test.tsx 4/4, compileDeck.test.ts -t "in Hermes" 1/1. - The workflow file parses; it has not run on GitHub. --- .github/workflows/slides-world.yml | 48 +++++++ .../src/__tests__/exampleShaders.test.ts | 130 +++++++++++++++++- .../tests/worldDataFlow.test.tsx | 13 +- 3 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/slides-world.yml diff --git a/.github/workflows/slides-world.yml b/.github/workflows/slides-world.yml new file mode 100644 index 000000000..f912ea32f --- /dev/null +++ b/.github/workflows/slides-world.yml @@ -0,0 +1,48 @@ +name: Slides World + +on: + pull_request: + paths: + - apps/slides/** + - packages/presentation/** + - packages/slides-effects/** + - patches/** + - package.json + - bun.lock + - .github/workflows/slides-world.yml + workflow_dispatch: + +jobs: + # Compiles the real World decks and runs the mocked host tests. Darwin-only + # Hermes and Dawn tests are skipped here, not validated. + portable: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + - run: bun install --frozen-lockfile + # Bun shares module mocks across files, so each file gets its own process. + - run: | + for file in \ + apps/slides/src/__tests__/typeGPUShader.test.ts \ + apps/slides/src/__tests__/gpuRenderGraph.test.ts \ + apps/slides/src/__tests__/exampleShaders.test.ts \ + packages/presentation/src/__tests__/compileDeck.test.ts \ + packages/slides-effects/tests/worldDataFlow.test.tsx \ + packages/slides-effects/tests/worldRender.test.ts; do + bun test --timeout 60000 --preload ./apps/slides/src/__tests__/nativeMock.ts "$file" + done + + # Executes compiled decks in React Native's macOS Hermes runtime; needs no GPU. + hermes: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + - run: bun install --frozen-lockfile + - run: bun test --timeout 60000 --preload ./apps/slides/src/__tests__/nativeMock.ts packages/presentation/src/__tests__/compileDeck.test.ts -t "in Hermes" + - run: bun test --timeout 60000 --preload ./apps/slides/src/__tests__/nativeMock.ts apps/slides/src/__tests__/exampleShaders.test.ts -t "evaluate in Hermes" diff --git a/apps/slides/src/__tests__/exampleShaders.test.ts b/apps/slides/src/__tests__/exampleShaders.test.ts index 22be44cf4..2e82f994a 100644 --- a/apps/slides/src/__tests__/exampleShaders.test.ts +++ b/apps/slides/src/__tests__/exampleShaders.test.ts @@ -5,16 +5,38 @@ import os from "node:os"; import path from "node:path"; import { createRequire } from "node:module"; import { spawnSync } from "node:child_process"; +import React from "react"; +import { act, create } from "react-test-renderer"; +import { transformSync } from "@babel/core"; import { cloudProgram } from "../../examples/components/CloudProgram"; import { compileDeck } from "../../../../packages/presentation/src/compiler"; +import { hostModules } from "../../../../packages/presentation/src/compiler/compileDeck"; +import { buildWorldGraph } from "../../../../packages/slides-effects/packs/world/render"; +import { customInstances, prepareWorldScene, primitive, sdfInstance } from "../../../../packages/slides-effects/packs/world/scene"; import { gameOfLifeShader } from "../../examples/components/GameOfLifeProgram"; import { boidsProgram } from "../../examples/components/TypeGPUBoids"; +const require = createRequire(import.meta.url); const programs = { cloudProgram, gameOfLifeShader, boidsProgram }; -test.skipIf(process.platform !== "darwin")("example programs compile with the app's native WebGPU shader compiler", () => { - const require = createRequire(import.meta.url); +const root = path.resolve(import.meta.dir, "../../../.."); +const worldDecks = ["world-tour", "data-flow", "cinematic-title"]; +const compiledDecks = new Map(); + +function compileWorldDeck(name) { + const file = path.join(root, "apps/slides/examples/3d", `${name}.mdx`); + if (!compiledDecks.has(name)) compiledDecks.set(name, compileDeck(file, { + source: fs.readFileSync(file, "utf8"), effectPacksDirectory: path.join(root, "packages/slides-effects/packs"), + }).then(result => { + if (!result.success) throw new Error(`${name}: ${result.errors.join("\n")}`); + return result; + })); + return compiledDecks.get(name); +} + +// Dawn's createShaderModule + getCompilationInfo: WGSL module validation only, not pipeline layout or rendering. +function validateShaders(shaders) { const webgpu = path.dirname(require.resolve("react-native-webgpu/package.json")); const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "slides-shader-test-")); try { @@ -24,15 +46,111 @@ test.skipIf(process.platform !== "darwin")("example programs compile with the ap "-framework", "Metal", "-framework", "Foundation", "-framework", "QuartzCore", "-framework", "IOSurface", "-framework", "IOKit", "-o", executable], { encoding: "utf8", timeout: 30_000 }); expect(build.stderr).toBe(""); expect(build.status).toBe(0); - const files = Object.entries(programs).map(([name, code]) => { + const files = Object.entries(shaders).map(([name, code]) => { const file = path.join(temporary, `${name}.wgsl`); fs.writeFileSync(file, code); return file; }); - const result = spawnSync(executable, files, { encoding: "utf8", timeout: 30_000 }); - expect(result.stderr).toBe(""); - expect(result.status).toBe(0); + return spawnSync(executable, files, { encoding: "utf8", timeout: 60_000 }); } finally { fs.rmSync(temporary, { recursive: true, force: true }); } +} + +// Renders every slide of the real compiled deck, including Background content, and keeps each World graph pass. +async function worldPasses(name) { + const compiled = await compileWorldDeck(name); + const playback = { value: { phase: "preparing", slideKey: "", stepKey: "", slideTime: 0, stepTime: 0, stepIndex: 0, stepOrigins: { 0: 0 } } }; + const presentation = require("@legend-apps/presentation"); + const module = { exports: {} }, passes = []; + Function("require", "module", "exports", "global", compiled.code)(id => { + if (id === "@legend-apps/presentation") return { ...presentation, + Background: ({ children }) => React.createElement("background", null, children), + GPURenderGraph: props => { passes.push(...props.passes.map(pass => ({ ...pass, label: `${name}-${passes.length}-${pass.id}` }))); return null; }, + usePresentationValue: () => undefined, useBackgroundSize: () => ({ width: 1920, height: 1080 }), usePlayback: () => playback, + samplePlayback: (state, _preview, clock) => clock === "step" ? state.stepTime : state.slideTime }; + if (id === "react-native-reanimated") return { ...require(id), + makeMutable: value => ({ value }), useSharedValue: value => React.useRef({ value }).current, useDerivedValue: read => ({ get value() { return read(); } }), useAnimatedReaction() {} }; + return require(id); + }, module, module.exports, globalThis); + let renderer; + globalThis.IS_REACT_ACT_ENVIRONMENT = true; + await act(() => { + renderer = create(React.createElement(module.exports.default, { components: { + Deck: "deck", Slide: "slide", View: "view", Step: ({ children }) => children, Steps: ({ children }) => children(0), + } })); + }); + await act(() => renderer.unmount()); + return passes; +} + +test("World decks compile to Hermes-ready modules that require only host packages", async () => { + for (const name of worldDecks) { + const { code } = await compileWorldDeck(name); + const requires = new Set(); + let classes = 0; + transformSync(code, { babelrc: false, configFile: false, code: false, sourceType: "script", plugins: [() => ({ visitor: { + Class() { classes++; }, + CallExpression({ node }) { if (node.callee.name === "require") requires.add(node.arguments[0].value); }, + } })] }); + expect(classes, name).toBe(0); + expect([...requires].filter(id => !hostModules.has(id)), name).toEqual([]); + expect(code, name).not.toMatch(/regeneratorRuntime|@babel\/runtime/); + } }, 60_000); +test.skipIf(process.platform !== "darwin")("World decks evaluate in Hermes with strict host modules", async () => { + const hermesPath = path.join(path.dirname(require.resolve("react-native/package.json")), "sdks/hermesc/osx-bin/hermes"); + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "slides-world-hermes-")); + try { + for (const name of worldDecks) { + const probePath = path.join(temporary, `${name}.js`); + fs.writeFileSync(probePath, [ + `var hosts = ${JSON.stringify([...hostModules])};`, + "function host() { return new Proxy(function() {}, { get: function(_, key) { return key === Symbol.iterator ? [][Symbol.iterator] : key === Symbol.toPrimitive ? function() { return 0; } : host(); }, apply: host, construct: host }); }", + "var module = { exports: {} };", + `Function("require", "module", "exports", "global", ${JSON.stringify((await compileWorldDeck(name)).code)})(function(id) {`, + ' if (hosts.indexOf(id) < 0) throw new Error("Deck requested unavailable host module " + id);', + " return host();", + "}, module, module.exports, globalThis);", + "print(typeof module.exports.default);", + ].join("\n")); + const probe = spawnSync(hermesPath, [probePath], { encoding: "utf8", timeout: 30_000 }); + expect(probe.error, name).toBeUndefined(); + expect(probe.stderr, name).toBe(""); + expect(probe.status, name).toBe(0); + expect(probe.stdout.trim(), name).toBe("function"); + } + } finally { fs.rmSync(temporary, { recursive: true, force: true }); } +}, 60_000); + +test("World decks render their real graphs on every slide", async () => { + const passes = Object.fromEntries(await Promise.all(worldDecks.map(async name => [name, await worldPasses(name)]))); + for (const name of worldDecks) expect(passes[name].length, name).toBeGreaterThan(0); + const ids = name => new Set(passes[name].map(pass => pass.id)); + for (const id of ["scene-custom-0", "scene-sdf-0", "custom-post-0", "dof-gather", "shafts", "floor"]) expect(ids("world-tour")).toContain(id); + const shafts = name => new Set(passes[name].filter(pass => pass.id === "shafts").map(pass => pass.code)); + expect(shafts("cinematic-title").size).toBeGreaterThan(0); + expect([...shafts("cinematic-title")].some(code => !shafts("world-tour").has(code))).toBe(true); +}, 60_000); + +const qualityGraphs = () => ["low", "medium", "high", "auto"].flatMap(quality => buildWorldGraph(prepareWorldScene({ quality, + instances: [primitive("sphere"), primitive("box"), primitive("ring"), primitive("tube"), + ...customInstances({ wgsl: "fn shape(id:u32,vid:u32,ctx:ShapeContext)->ShapeVertex { return ShapeVertex(vec3f(f32(vid)),vec3f(0.,0.,1.)); }\nfn shade(id:u32,world:vec3f,normal:vec3f,ctx:ShapeContext)->Material { return Material(vec3f(.5),vec3f(0.),.5,0.); }", vertexCount: 3 }), + sdfInstance({ wgsl: "fn sdf(p:vec3f)->f32 { return length(p)-1.; }", bounds: [[-2, -2, -2], [2, 2, 2]] })], + floor: { mirror: true }, dof: { aperture: .7 }, shafts: { intensity: 1 }, lights: [{ position: [0, 8, 0] }, { kind: "spot" }], fog: { density: .02 }, + bloom: { intensity: 1 }, chromatic: .2, rim: { intensity: 1 }, passes: [{ id: "lines", wgsl: "fn customPass(uv:vec2f,color:vec4f,ctx:PassContext)->vec4f { return color; }" }], +})).passes.map(pass => ({ ...pass, label: `quality-${quality}-${pass.id}` }))); + +test.skipIf(process.platform !== "darwin")("example and World programs compile with the app's native WebGPU shader compiler", async () => { + const world = [...(await Promise.all(worldDecks.map(worldPasses))).flat(), ...qualityGraphs()]; + const unique = new Map(world.map(pass => [pass.code, pass.label])); + const result = validateShaders({ ...programs, ...Object.fromEntries([...unique].map(([code, label]) => [label, code])) }); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toHaveLength(Object.keys(programs).length + unique.size); + const [code, label] = [...unique].find(([, label]) => label.startsWith("world-tour-") && label.endsWith("scene-custom-0")); + const invalid = validateShaders({ [label]: code.replace("fn shape(", "fn shape(undeclared_token ") }); + expect(invalid.status).toBe(1); + expect(invalid.stderr).toMatch(/error|expected/i); +}, 120_000); + test("documented GPU decks compile using the shared-clock shader host", async () => { for (const name of ["examples/showcase.mdx"]) { const file = path.resolve(import.meta.dir, "../..", name); diff --git a/packages/slides-effects/tests/worldDataFlow.test.tsx b/packages/slides-effects/tests/worldDataFlow.test.tsx index 384b93ffd..ae54cc2ad 100644 --- a/packages/slides-effects/tests/worldDataFlow.test.tsx +++ b/packages/slides-effects/tests/worldDataFlow.test.tsx @@ -1,6 +1,7 @@ // @ts-nocheck Inspect the real compiled deck and clock uniforms without a GPU. import "../../../apps/slides/src/__tests__/nativeMock"; import { afterAll, beforeAll, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import React from "react"; @@ -25,8 +26,9 @@ function sample(time, step = 0) { beforeAll(async () => { globalThis.IS_REACT_ACT_ENVIRONMENT = true; const root = path.resolve(import.meta.dirname, "../../.."); - const compiled = await compileDeck(path.join(root, "apps/slides/examples/3d/data-flow.mdx"), { - effectPacksDirectory: path.join(root, "packages/slides-effects/packs"), + const deckPath = path.join(root, "apps/slides/examples/3d/data-flow.mdx"); + const compiled = await compileDeck(deckPath, { + source: readFileSync(deckPath, "utf8"), effectPacksDirectory: path.join(root, "packages/slides-effects/packs"), }); if (!compiled.success) throw new Error(compiled.errors.join("\n")); const require = createRequire(import.meta.url), module = { exports: {} }; @@ -89,8 +91,9 @@ test("the 53-node cascade heats descendants then prunes to one branch while keep closeVector(nodes()[0].slice(4, 7), colorVector("#ff36c8")); sample(2, 2); expect(nodes().filter(item => item[30] === 1)).toHaveLength(13); - const target = layoutTree(network, { radius: 6.8, levelHeight: 1.7, position: [0, 3.2, 0] }).nodes.find(node => node.id === "e2").position; - closeVector(values(0).slice(4, 7), target); + const target = layoutTree(network, { radius: 6.8, levelHeight: 1.7, position: [0, 3.2, 0] }).nodes.find(node => node.id === "e2"); + expect(target, "node e2").toBeDefined(); + closeVector(values(0).slice(4, 7), target.position); expect(values(0)[32]).toBeCloseTo(10); }); @@ -98,6 +101,8 @@ test("the cyan orb follows one curved edge with a fainter trail and matching cam sample(2, 1); const head = instances(1).find(item => item[7] === 14); const trail = instances(1).find(item => item[7] === 14 * 0.3); + expect(head, "orb head").toBeDefined(); + expect(trail, "orb trail").toBeDefined(); expect(head.slice(4, 7)).toEqual(colorVector("#22d3ee")); expect(head.slice(8, 11)).toEqual([1, 1, 1]); expect(trail[19]).toBeCloseTo(head[3] * 0.18); From 233fcd38778174c9036ef6e5421417c86781d5eb Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 06:59:39 -0700 Subject: [PATCH 19/28] test: guard Worklets frame samples before measuring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Report insufficient frame delivery before indexing or slicing samples. Require a 60 Hz display for the fixed workloads and capture windows. **Test plan** ```sh patches/tests/run-react-native-worklets-frame-queue.sh "$patched_queue" ``` ✅ The real patched native queue completes the corrected harness without failures. A task-local zero-duration capture exits 1 with an insufficient-samples diagnostic instead of evaluating invalid ranges. --- .../react-native-worklets-frame-queue.test.mm | 21 ++++++++++++++++--- .../run-react-native-worklets-frame-queue.sh | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/patches/tests/react-native-worklets-frame-queue.test.mm b/patches/tests/react-native-worklets-frame-queue.test.mm index 2f8deed55..52017cf40 100644 --- a/patches/tests/react-native-worklets-frame-queue.test.mm +++ b/patches/tests/react-native-worklets-frame-queue.test.mm @@ -80,11 +80,18 @@ int main() @autoreleasepool { NSThread.currentThread.name = @"com.facebook.react.runtime.JavaScript"; CVDisplayLinkRef probe; - CVDisplayLinkCreateWithActiveCGDisplays(&probe); + if (CVDisplayLinkCreateWithActiveCGDisplays(&probe) != kCVReturnSuccess) { + CHECK(false, "a display link is available"); + return 1; + } CVTime nominal = CVDisplayLinkGetNominalOutputVideoRefreshPeriod(probe); const double period = 1000.0 * nominal.timeValue / nominal.timeScale; CVDisplayLinkRelease(probe); printf("INFO display period %.2f ms\n", period); + if (fabs(period - 1000.0 / 60) > 0.5 || !std::isfinite(period)) { + CHECK(false, "frame queue tests require a 60 Hz display"); + return 1; + } for (double busyMs : {0.0, 4.0, 8.0, 12.0}) { AnimationFrameQueue *queue = [AnimationFrameQueue new]; @@ -94,6 +101,11 @@ int main() runFor(1000); loop.running = false; runFor(4 * period); + dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); + if (loop.calls.size() < 7 || loop.timestamps.size() < 2) { + CHECK(false, "busy %.0f ms delivered too few frame samples (%zu calls, %zu timestamps)", busyMs, loop.calls.size(), loop.timestamps.size()); + return 1; + } auto frameDeltas = deltas(loop.calls, 5); double fps = 1000.0 * (loop.calls.size() - 6) / (loop.calls.back() - loop.calls[5]); printf("INFO busy %.0f ms: %.1f fps, median %.2f ms\n", busyMs, fps, median(frameDeltas)); @@ -101,7 +113,6 @@ int main() auto timestampDeltas = deltas(loop.timestamps); CHECK(*std::min_element(timestampDeltas.begin(), timestampDeltas.end()) > 0, "busy %.0f ms timestamps increase", busyMs); CHECK(fabs(loop.timestamps.back() - loop.calls.back()) < 3 * period, "timestamps are milliseconds near the media clock"); - dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); } { @@ -113,6 +124,11 @@ int main() runFor(600); loop.running = false; runFor(4 * period); + dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); + if (loop.calls.size() < 26 || loop.timestamps.size() < 26) { + CHECK(false, "stall test delivered too few frame samples (%zu calls, %zu timestamps)", loop.calls.size(), loop.timestamps.size()); + return 1; + } std::vector after; for (size_t i = 20; i < loop.calls.size() && after.size() < 6; i++) after.push_back(loop.calls[i]); auto afterDeltas = deltas(after); @@ -126,7 +142,6 @@ int main() CHECK(burst <= 1, "main thread stall does not replay queued pulses (%d frames)", burst); CHECK(minDelta > period / 2, "frames after the stall stay one pulse apart (min %.2f ms)", minDelta); CHECK(minTimestampDelta > period / 2, "timestamps after the stall stay one pulse apart (min %.2f ms)", minTimestampDelta); - dispatch_sync(turboModuleQueue(), ^{ [queue invalidate]; }); } { diff --git a/patches/tests/run-react-native-worklets-frame-queue.sh b/patches/tests/run-react-native-worklets-frame-queue.sh index 165c97071..4b47ad4e1 100755 --- a/patches/tests/run-react-native-worklets-frame-queue.sh +++ b/patches/tests/run-react-native-worklets-frame-queue.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runs the patched Worklets AnimationFrameQueue against a real display link. Needs a display. +# Runs the patched Worklets AnimationFrameQueue against a real 60 Hz display link. # Usage: run-react-native-worklets-frame-queue.sh [AnimationFrameQueue.mm] set -euo pipefail repo="$(cd "$(dirname "$0")/../.." && pwd)" From e9c5dce4232abaf5c85cd493fdbb962535f79ffc Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 08:36:14 -0700 Subject: [PATCH 20/28] ci: complete Slides verification entry points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run World checks when shared compiler inputs change and document the native frame harness's local display and source prerequisites. **Test plan** ✅ Parsed the workflow with Ruby YAML and verified all three compiler input paths select the portable and Hermes jobs. Reviewed the documented command against the existing runner; no runtime source changed. --- .github/workflows/slides-world.yml | 3 +++ apps/slides/README.md | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/slides-world.yml b/.github/workflows/slides-world.yml index f912ea32f..b469a9b57 100644 --- a/.github/workflows/slides-world.yml +++ b/.github/workflows/slides-world.yml @@ -7,6 +7,9 @@ on: - packages/presentation/** - packages/slides-effects/** - patches/** + - shell/src/global.css + - shell/package.json + - scripts/compile-slides.ts - package.json - bun.lock - .github/workflows/slides-world.yml diff --git a/apps/slides/README.md b/apps/slides/README.md index 2f366bbfc..9a9c4d1a0 100644 --- a/apps/slides/README.md +++ b/apps/slides/README.md @@ -869,3 +869,17 @@ boundaries in the compiler, source editor, and speaker-notes editor. A top-level dash separator starts a slide even directly after text; use `## Heading` for a heading instead of setext underline syntax. Separators inside code and JSX stay inside their containing slide. + +### Native frame scheduler check + +After normal macOS native preparation applies the tracked Worklets patch, run +this local check from the repository root with a 60 Hz display: + +```sh +patches/tests/run-react-native-worklets-frame-queue.sh +``` + +The check compiles the installed source under `shell/.legend/spark/node_modules`; +it does not apply the patch. To check another candidate, pass its exact +`AnimationFrameQueue.mm` path and set `SPARK_NODE_MODULES` to the matching native +dependencies. This display-dependent check is separate from hosted CI. From 5ac67d6a9988634ee5b6747cedc27fa11d6e5cdb Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 10:39:53 -0700 Subject: [PATCH 21/28] feat: resolve effect packs next to the deck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3D example decks import slides:world, which is not published, so a fresh checkout failed with 'Effect pack "world" is not in the catalog' unless LEGEND_SLIDES_EFFECT_PACKS was set. A pack at /effects//pack.json now resolves from source before installed snapshots or the catalog, so no install prompt appears. The lockfile records it as { "path": "effects/" } instead of a version and checksum, so edits neither fail verification nor rewrite the lock. A lock path can point elsewhere in the repository; the 3D example decks and the world example use one to reach packages/slides-effects/packs/world. The pack's manifest is already a compile dependency, so the app watches its folder like the deck's. **Test plan** ```sh bun test packages/presentation/src/__tests__/effectPacks.test.ts ``` ✅ New tests cover local-before-installed resolution, the relative lock entry staying byte-identical across pack edits, lock paths outside the deck folder, a missing local path failing without an install request, and installation preserving local lock entries without the catalog. ✅ All four World decks compile with LEGEND_SLIDES_EFFECT_PACKS unset and no installed packs, tracking the pack's source files. ✅ Scoped typecheck of the changed compiler and lock consumers: 0 diagnostics. --- .../3d/cinematic-title.mdx.effects.lock.json | 8 ++++ .../3d/data-flow.mdx.effects.lock.json | 8 ++++ .../3d/world-tour.mdx.effects.lock.json | 8 ++++ .../src/__tests__/effectPacks.test.ts | 44 ++++++++++++++++++- .../presentation/src/compiler/compileDeck.ts | 9 ++-- .../presentation/src/compiler/effectPacks.ts | 24 ++++++---- .../presentation/src/effectPacks/schema.ts | 15 +++++-- packages/slides-effects/README.md | 3 ++ .../world/example.mdx.effects.lock.json | 8 ++++ 9 files changed, 111 insertions(+), 16 deletions(-) create mode 100644 apps/slides/examples/3d/cinematic-title.mdx.effects.lock.json create mode 100644 apps/slides/examples/3d/data-flow.mdx.effects.lock.json create mode 100644 apps/slides/examples/3d/world-tour.mdx.effects.lock.json create mode 100644 packages/slides-effects/examples/world/example.mdx.effects.lock.json diff --git a/apps/slides/examples/3d/cinematic-title.mdx.effects.lock.json b/apps/slides/examples/3d/cinematic-title.mdx.effects.lock.json new file mode 100644 index 000000000..35fed5574 --- /dev/null +++ b/apps/slides/examples/3d/cinematic-title.mdx.effects.lock.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "packs": { + "world": { + "path": "../../../../packages/slides-effects/packs/world" + } + } +} diff --git a/apps/slides/examples/3d/data-flow.mdx.effects.lock.json b/apps/slides/examples/3d/data-flow.mdx.effects.lock.json new file mode 100644 index 000000000..35fed5574 --- /dev/null +++ b/apps/slides/examples/3d/data-flow.mdx.effects.lock.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "packs": { + "world": { + "path": "../../../../packages/slides-effects/packs/world" + } + } +} diff --git a/apps/slides/examples/3d/world-tour.mdx.effects.lock.json b/apps/slides/examples/3d/world-tour.mdx.effects.lock.json new file mode 100644 index 000000000..35fed5574 --- /dev/null +++ b/apps/slides/examples/3d/world-tour.mdx.effects.lock.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "packs": { + "world": { + "path": "../../../../packages/slides-effects/packs/world" + } + } +} diff --git a/packages/presentation/src/__tests__/effectPacks.test.ts b/packages/presentation/src/__tests__/effectPacks.test.ts index 9dcd863a8..77edf48d1 100644 --- a/packages/presentation/src/__tests__/effectPacks.test.ts +++ b/packages/presentation/src/__tests__/effectPacks.test.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import { compileDeck } from "../compiler"; import { resolveEffectPack } from "../compiler/effectPacks"; -import { effectPackIncompatibility } from "../effectPacks/schema"; +import { effectPackIncompatibility, resolveInstallation } from "../effectPacks/schema"; const temporary = []; function fixture(extra = {}) { @@ -56,6 +56,48 @@ test("pack imports do not relax deck or pack containment", async () => { expect(result.errors.join("\n")).toContain("escapes the deck directory"); }); +test("deck-local packs resolve before installed packs and lock as relative paths", async () => { + const { root, deck, packs } = fixture({ + "deck/effects/motion/pack.json": JSON.stringify({ id: "motion", version: "0.0.1", hostApiVersion: 1, exports: { Pose: "./Pose.ts" } }), + "deck/effects/motion/Pose.ts": 'export function sample(x) { "worklet"; return x * 3; }', + }); + const lockFile = `${deck}.effects.lock.json`; + fs.writeFileSync(lockFile, JSON.stringify({ schemaVersion: 1, packs: { motion: { version: "1.0.0", sha256: "a".repeat(64) } } })); + let result = await compileDeck(deck, { effectPacksDirectory: packs }); + expect(result.success).toBe(true); + expect(result.dependencies).toContain(path.join(root, "deck/effects/motion/pack.json")); + const module = { exports: {} }; + Function("require", "module", "exports", result.code)(() => ({}), module, module.exports); + expect(module.exports.sample(21)).toBe(63); + + fs.rmSync(lockFile); + result = await compileDeck(deck, { effectPacksDirectory: path.join(root, "empty") }); + expect(result.success).toBe(true); + const lock = fs.readFileSync(lockFile, "utf8"); + expect(JSON.parse(lock).packs).toEqual({ motion: { path: "effects/motion" } }); + fs.writeFileSync(path.join(root, "deck/effects/motion/Pose.ts"), 'export function sample(x) { "worklet"; return x * 4; }'); + expect((await compileDeck(deck, { effectPacksDirectory: packs })).success).toBe(true); + expect(fs.readFileSync(lockFile, "utf8")).toBe(lock); + + fs.rmSync(path.join(root, "deck/effects"), { recursive: true }); + fs.writeFileSync(lockFile, JSON.stringify({ schemaVersion: 1, packs: { motion: { path: "../packs/motion" } } })); + result = await compileDeck(deck, { effectPacksDirectory: path.join(root, "empty") }); + expect(result.success).toBe(true); + expect(result.dependencies).toContain(path.join(packs, "motion/pack.json")); + + fs.writeFileSync(lockFile, JSON.stringify({ schemaVersion: 1, packs: { motion: { path: "../missing" } } })); + result = await compileDeck(deck, { effectPacksDirectory: packs }); + expect(result.success).toBe(false); + expect(result.missingEffectPacks).toEqual([]); + expect(result.errors.join("\n")).toContain("was not found at ../missing"); +}); + +test("installing catalog packs preserves local lock entries without consulting the catalog", () => { + const catalog = { schemaVersion: 1, packs: {}, effects: [] }; + const local = { schemaVersion: 1, packs: { world: { path: "effects/world" } } }; + expect(resolveInstallation(catalog, ["world"], local).lock).toEqual(local); +}); + test("manifests reject missing, incompatible, unexported and escaping entries", () => { const { packs, root } = fixture(); for (const ref of ["slides:missing/Pose", "slides:motion/Unknown", "slides:../motion/Pose", "slides:motion/../Pose", "slides:motion/__proto__"]) { diff --git a/packages/presentation/src/compiler/compileDeck.ts b/packages/presentation/src/compiler/compileDeck.ts index 7d619c28a..5f5856178 100644 --- a/packages/presentation/src/compiler/compileDeck.ts +++ b/packages/presentation/src/compiler/compileDeck.ts @@ -18,7 +18,7 @@ import { remarkWebviews } from "./remarkWebviews"; import type { CompileDeckResult } from "./types"; import { loadUniwindCompiler } from "./uniwindCompiler"; import { defaultEffectPacksDirectory, MissingEffectPack, resolveEffectPack } from "./effectPacks"; -import type { EffectLock } from "../effectPacks/schema"; +import { isLocalPack, type EffectLock } from "../effectPacks/schema"; import { effectLockPath, installedPacksDirectory, readEffectLock, writeEffectLock } from "../effectPacks/storage"; // Pass plugin modules rather than paths so the shipped compiler can be a single bundle. @@ -205,10 +205,11 @@ export function localDeckPlugin(entryPath: string, transitionDirectory?: string, try { const importerRoot = libraryRoots.get(args.importer); const pack = resolveEffectPack(args.path, options.effectPacksDirectory ?? defaultEffectPacksDirectory(), - importerRoot && packRoots.has(importerRoot) ? importerRoot : undefined, options.lock, verified); - if (pack.pin && options.resolvedPins) { + importerRoot && packRoots.has(importerRoot) ? importerRoot : undefined, options.lock, verified, path.dirname(entryPath)); + if (pack.local && options.resolvedPins) options.resolvedPins[pack.id] = pack.local; + else if (pack.pin && options.resolvedPins) { const previous = options.resolvedPins[pack.id]; - if (previous && (previous.version !== pack.pin.version || previous.sha256 !== pack.pin.sha256)) throw new Error(`Effect dependency conflict for "${pack.id}".`); + if (previous && (isLocalPack(previous) || previous.version !== pack.pin.version || previous.sha256 !== pack.pin.sha256)) throw new Error(`Effect dependency conflict for "${pack.id}".`); options.resolvedPins[pack.id] = pack.pin; } libraryRoots.set(pack.file, pack.root); diff --git a/packages/presentation/src/compiler/effectPacks.ts b/packages/presentation/src/compiler/effectPacks.ts index 04006e0d8..02dfae85b 100644 --- a/packages/presentation/src/compiler/effectPacks.ts +++ b/packages/presentation/src/compiler/effectPacks.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import { effectPackIncompatibility, validSha, validVersion, validatePin, type EffectLock, type PackPin } from "../effectPacks/schema"; +import { effectPackIncompatibility, isLocalPack, validSha, validVersion, validatePin, type EffectLock, type PackPin } from "../effectPacks/schema"; import { installedPacksDirectory, pinnedPackDirectory, verifyInstalledPack } from "../effectPacks/storage"; export function defaultEffectPacksDirectory() { @@ -10,29 +10,37 @@ export function defaultEffectPacksDirectory() { export class MissingEffectPack extends Error { constructor(public id: string, detail = "is not installed") { super(`Effect pack "${id}" ${detail}. Install the required effects before presenting.`); } } +/** A deck uses `/effects/` or its lockfile's relative path from source, before installed packs. */ +function localEffectPack(id: string, deckDirectory: string, locked?: EffectLock["packs"][string]) { + const relative = isLocalPack(locked) ? locked.path : path.join("effects", id); + if (fs.existsSync(path.join(deckDirectory, relative, "pack.json"))) return { path: relative }; + if (isLocalPack(locked)) throw new Error(`Local effect pack "${id}" was not found at ${locked.path}.`); +} export function resolveEffectPack(reference: string, directory: string, importerPackRoot?: string, lock?: EffectLock, - verified = new Set()) { + verified = new Set(), deckDirectory?: string) { const match = /^slides:([a-z][a-z0-9-]*)\/([A-Za-z][A-Za-z0-9-]*)$/.exec(reference); if (!match) throw new Error(`Invalid effect pack reference "${reference}".`); const [, id, entry] = match; - let pin: PackPin | undefined = lock?.packs[id]; - if (lock && !pin) throw new MissingEffectPack(id, "has no pinned version in this deck"); + const locked = lock?.packs[id]; + const local = deckDirectory ? localEffectPack(id, deckDirectory, locked) : undefined; + let pin: PackPin | undefined = local || isLocalPack(locked) ? undefined : locked; + if (lock && !local && !pin) throw new MissingEffectPack(id, "has no pinned version in this deck"); const importer = importerPackRoot && JSON.parse(fs.readFileSync(path.join(importerPackRoot, "pack.json"), "utf8")); const expected = importer && Object.hasOwn(importer.dependencies ?? {}, id) ? importer.dependencies[id] : undefined; if (importer && !expected && importer.id !== id) throw new Error(`Effect pack "${importer.id}" does not declare its dependency on "${id}".`); if (expected && !validVersion(expected)) throw new Error("Invalid effect dependency version."); const active = path.join(directory, id, "active.json"); - if (!pin && fs.existsSync(active)) pin = JSON.parse(fs.readFileSync(active, "utf8")); + if (!local && !pin && fs.existsSync(active)) pin = JSON.parse(fs.readFileSync(active, "utf8")); if (pin) validatePin(pin); // A default dependency belongs to its importing pack's exact version, even // when the user has independently updated that dependency's default. - if (!lock && expected && pin?.version !== expected) { + if (!local && !lock && expected && pin?.version !== expected) { const versions = path.join(directory, id, expected); const hashes = fs.existsSync(versions) ? fs.readdirSync(versions).filter(validSha) : []; if (hashes.length === 1) pin = { version: expected, sha256: hashes[0] }; else if (pin || hashes.length) throw new MissingEffectPack(id, `requires reinstalling version ${expected}`); } - const selected = pin ? pinnedPackDirectory(directory, id, pin) : path.join(directory, id); + const selected = local ? path.resolve(deckDirectory!, local.path) : pin ? pinnedPackDirectory(directory, id, pin) : path.join(directory, id); const manifest = path.join(selected, "pack.json"); if (!fs.existsSync(manifest)) throw new MissingEffectPack(id, pin ? `version ${pin.version} is not installed` : undefined); if (pin && !verified.has(selected)) { @@ -55,5 +63,5 @@ export function resolveEffectPack(reference: string, directory: string, importer if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative) || !fs.statSync(file).isFile()) { throw new Error(`Effect pack "${id}" entry "${entry}" escapes its pack directory.`); } - return { file, root, manifest, id, pin }; + return { file, root, manifest, id, pin, local }; } diff --git a/packages/presentation/src/effectPacks/schema.ts b/packages/presentation/src/effectPacks/schema.ts index 398b29bda..16088a82b 100644 --- a/packages/presentation/src/effectPacks/schema.ts +++ b/packages/presentation/src/effectPacks/schema.ts @@ -1,5 +1,8 @@ export type PackPin = { version: string; sha256: string }; -export type EffectLock = { schemaVersion: 1; packs: Record }; +/** A deck-relative pack folder, used from source rather than pinned to a catalog checksum. */ +export type LocalPackSource = { path: string }; +export type EffectLock = { schemaVersion: 1; packs: Record }; +export const isLocalPack = (entry: PackPin | LocalPackSource | undefined): entry is LocalPackSource => !!entry && "path" in entry; export type PackRelease = PackPin & { url: string; bytes: number; hostApiVersion: number; capabilities: string[]; dependencies: Record; exports: Record }; export type EffectListing = { slug: string; title: string; description: string; pack: string; entry: string; symbol: string; @@ -16,7 +19,11 @@ export function validatePin(pin: PackPin) { } export function validateLock(lock: EffectLock) { if (!lock || lock.schemaVersion !== 1 || !record(lock.packs)) throw new Error("Invalid effect lockfile."); - for (const [id, pin] of Object.entries(lock.packs)) { if (!validPackId(id)) throw new Error("Invalid pack ID."); validatePin(pin); } + for (const [id, pin] of Object.entries(lock.packs)) { + if (!validPackId(id)) throw new Error("Invalid pack ID."); + if (!isLocalPack(pin)) validatePin(pin); + else if (typeof pin.path !== "string" || !pin.path || pin.path.startsWith("/")) throw new Error("Local effect packs must use a deck-relative path."); + } return lock; } export function validateCatalog(catalog: EffectCatalog) { @@ -60,8 +67,10 @@ export function resolveInstallation(catalog: EffectCatalog, ids: string[], exist const pins = { ...existing.packs }; const releases = new Map(); const visiting = new Set(); function visit(id: string, required?: string) { if (!validPackId(id)) throw new Error("Invalid effect pack ID."); + const pin = requested[id] ?? pins[id]; + if (isLocalPack(pin)) return; const pack = catalog.packs[id]; if (!pack) throw new Error(`Effect pack "${id}" is not in the catalog.`); - const pin = requested[id] ?? pins[id]; if (pin) validatePin(pin); + if (pin) validatePin(pin); const version = pin?.version ?? required ?? pack.latest; if (required && version !== required) throw new Error(`Effect dependency conflict for "${id}": ${version} versus ${required}.`); const release = pack.versions[version]; diff --git a/packages/slides-effects/README.md b/packages/slides-effects/README.md index 35871a000..a008f139d 100644 --- a/packages/slides-effects/README.md +++ b/packages/slides-effects/README.md @@ -24,6 +24,9 @@ Decks resolve verified source snapshots from `~/Library/Application Support/Legend Slides/effects/`, including in a source checkout. To compile against this `packs/` directory while developing effects, set `LEGEND_SLIDES_EFFECT_PACKS` or pass `--effect-packs ` to the compiler. +A pack at `/effects//pack.json` is used from source before installed +packs, and edits recompile the deck. The lockfile records it as `{ "path": "effects/" }` +rather than a checksum; point that path elsewhere, as `apps/slides/examples/3d` does for `world`. Source is trusted code with the same host API access as a local deck component. The native app and shipped compiler omit these implementations. The gallery at diff --git a/packages/slides-effects/examples/world/example.mdx.effects.lock.json b/packages/slides-effects/examples/world/example.mdx.effects.lock.json new file mode 100644 index 000000000..baead833d --- /dev/null +++ b/packages/slides-effects/examples/world/example.mdx.effects.lock.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "packs": { + "world": { + "path": "../../packs/world" + } + } +} From ee38db9d934e7fb3c5b1b08a4d94eb257a0f9734 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 11:35:03 -0700 Subject: [PATCH 22/28] fix: render 3D worlds at device resolution with MSAA and clean shafts Native WebGPU drawables were sized in points and ignored the slide's display scale, so background worlds rendered at half the Retina resolution, and auto quality could halve that again. Back every surface at device pixels, as the web host already did. The render graph gains optional 4x MSAA targets: colour resolves once in the last render pass that writes it, and depth is read as texture_depth_multisampled_2d. World uses it for the HDR scene, depth and the mirror reflection (except at low quality), renders reflections at full scale so thin edges no longer break up, and ends with FXAA after tonemapping. Rings use 128 segments. Shafts render at half resolution with interleaved-gradient jitter, and fog reads them through a 3x3 tent, so they no longer look like grain. Auto quality now lowers resolution only when cadence collapses below ~40 fps, and never below 0.75. Per-frame uniform uploads reuse staging arrays instead of allocating. --- apps/slides/src/TypeGPUShader.tsx | 32 +++++++++------ .../__tests__/gpuResourceLifecycle.test.tsx | 2 +- apps/slides/src/gpuRenderGraph.ts | 35 +++++++++++++---- packages/presentation/src/GPUShader.tsx | 2 + packages/slides-effects/packs/world/README.md | 15 +++---- packages/slides-effects/packs/world/post.ts | 19 +++++---- packages/slides-effects/packs/world/render.ts | 39 +++++++++++++------ .../slides-effects/packs/world/resolution.ts | 5 ++- packages/slides-effects/tests/world.test.ts | 4 +- .../slides-effects/tests/worldLook.test.ts | 2 +- .../slides-effects/tests/worldRender.test.ts | 4 +- .../tests/worldResolution.test.ts | 28 +++++++------ 12 files changed, 124 insertions(+), 63 deletions(-) diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 695248852..d6a84b4fc 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -51,13 +51,8 @@ export function TypeGPUShader({ code, width: configuredWidth = 1696, height: con // Surface dimensions are independent of shader/device ownership. In particular, // live window resizing must not recompile pipelines or reset simulation buffers. useLayoutEffect(() => { - if (Platform.OS === "web") { - const context = canvas.current?.getContext("webgpu"); - if (context) resizeBrowserSurface(context, width, height, displayScale); - return; - } - const surface = canvas.current?.getNativeSurface(); - if (surface) { surface.width = width; surface.height = height; } + const surface = Platform.OS === "web" ? canvas.current?.getContext("webgpu")?.canvas : canvas.current?.getNativeSurface(); + if (surface) resizeSurface(surface, width, height, displayScale); }, [width, height, displayScale, resources]); useEffect(() => { if (!supported) return; @@ -74,8 +69,8 @@ export function TypeGPUShader({ code, width: configuredWidth = 1696, height: con ; } -function resizeBrowserSurface(context: RNCanvasContext, width: number, height: number, displayScale: number) { - const surface = context.canvas; +/** Back the drawable at device pixels, the slide's display scale times the screen's backing scale. */ +function resizeSurface(surface: { width: number; height: number }, width: number, height: number, displayScale: number) { const pixelWidth = Math.max(1, Math.round(width * displayScale * PixelRatio.get())); const pixelHeight = Math.max(1, Math.round(height * displayScale * PixelRatio.get())); if (surface.width !== pixelWidth) surface.width = pixelWidth; @@ -126,7 +121,7 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle if (!context) throw new Error("WebGPU surface unavailable"); // The library sizes from DOM bounds, which are zero during preparation. // Set the drawing buffer before transferring ownership to UIFrames. - if (Platform.OS === "web") resizeBrowserSurface(context, width, height, displayScale); + if (Platform.OS === "web") resizeSurface(context.canvas, width, height, displayScale); const format = navigator.gpu.getPreferredCanvasFormat(); if (renderGraph) { const clock = device.createBuffer({ size: 32, usage: 64 | 8 }); @@ -247,7 +242,10 @@ function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, fra const scale = renderScale?.get(); try { const time = samplePlayback(state, previewTime, clockMode); - device.queue.writeBuffer(clock, 0, new Float32Array([time, dt, frameIndex, state.stepIndex, aspect, brightness, samplePlayback(state, previewTime, "slide"), samplePlayback(state, previewTime, "step")])); + const values = scratch(8); + values[0] = time; values[1] = dt; values[2] = frameIndex; values[3] = state.stepIndex; values[4] = aspect; values[5] = brightness; + values[6] = samplePlayback(state, previewTime, "slide"); values[7] = samplePlayback(state, previewTime, "step"); + device.queue.writeBuffer(clock, 0, values); submitGPUFrame(r, frameIndex, particleCount, particleVertexCount, frameData, scale); if (!frozen) count.value += 1; if (!rendered.value) { rendered.value = true; runOnJS(onReady)(); } @@ -291,6 +289,14 @@ function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, fra } +/** writeBuffer copies synchronously, so surfaces on the UI runtime share one exact-length staging array per size. */ +function scratch(length: number) { + "worklet"; + const runtime = globalThis as { __gpuScratch?: Record> }; + const arrays = runtime.__gpuScratch ??= {}; + return arrays[length] ??= new Float32Array(length); +} + function destroyGPU(device: GPUDevice, alive: SharedValue, disposed: SharedValue) { "worklet"; alive.value = false; @@ -339,7 +345,9 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part const data = frameData?.value; if (r.frameData && data) { if (data.length * 4 > r.frameData.size) throw new Error(`Frame data has ${data.length} values; its buffer holds ${r.frameData.size / 4}`); - device.queue.writeBuffer(r.frameData, 0, new Float32Array(data)); + const values = scratch(data.length); + values.set(data); + device.queue.writeBuffer(r.frameData, 0, values); } const encoder = device.createCommandEncoder(); if (r.graph) { diff --git a/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx b/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx index 1f31cabd5..515fe2423 100644 --- a/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx +++ b/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx @@ -30,7 +30,7 @@ test.each(["macos", "web"])("persistent backgrounds retain GPU resources across name => mocks[name] ?? require(name), {}, options => { setups++; report = options.reportError; return () => { cleanups++; }; }, { gpu: {} }, ); - const expectedSize = () => platform === "web" ? { width: size.width * 2, height: size.height * 2 } : size; + const expectedSize = () => ({ width: size.width * 2, height: size.height * 2 }); await act(() => { tree = create(, {createNodeMock: () => ({getNativeSurface: () => surface, getContext: () => ({ canvas: surface })})}); }); expect(surface).toEqual(expectedSize()); for (let width = 101; width <= 120; width++) { diff --git a/apps/slides/src/gpuRenderGraph.ts b/apps/slides/src/gpuRenderGraph.ts index 1205db097..d09b19b06 100644 --- a/apps/slides/src/gpuRenderGraph.ts +++ b/apps/slides/src/gpuRenderGraph.ts @@ -7,7 +7,7 @@ struct GraphVertex { @builtin(position) position: vec4f, @location(0) uv: vec2f return GraphVertex(vec4f(p * 2.0 - 1.0, 0.0, 1.0), vec2f(p.x, 1.0 - p.y)); }`; -type PassResources = { pass: GPUPass; pipeline: GPUComputePipeline | GPURenderPipeline; layout: GPUBindGroupLayout; group?: GPUBindGroup }; +type PassResources = { pass: GPUPass; pipeline: GPUComputePipeline | GPURenderPipeline; layout: GPUBindGroupLayout; group?: GPUBindGroup; resolves: boolean[] }; export type RenderGraphResources = { targets: GPUTarget[]; textures: Record; buffers: Record; passes: PassResources[]; clock: GPUBuffer; uniforms: GPUBuffer; sampler: GPUSampler; @@ -31,6 +31,10 @@ function view(r: RenderGraphResources, value: string) { const [id, mip] = reference(value); return r.textures[id].createView({ baseMipLevel: mip, mipLevelCount: 1 }); } +function multisampled(r: { targets: GPUTarget[] }, value: string) { + "worklet"; + return value !== "screen" && (targetFor(r.targets, value).samples ?? 1) > 1; +} function output(pass: GPUPass) { "worklet"; if (pass.kind === "fullscreen") return pass.mip === undefined ? pass.output : pass.output + "@" + pass.mip; @@ -57,6 +61,7 @@ export async function prepareRenderGraph(device: GPUDevice, definition: GPURende } for (const target of targets) { if (!Number.isFinite(target.scale ?? 1) || (target.scale ?? 1) <= 0 || !Number.isInteger(target.mips ?? 1) || (target.mips ?? 1) < 1) throw new Error("Invalid GPU target dimensions: " + target.id); + if (![1, 4].includes(target.samples ?? 1) || ((target.samples ?? 1) > 1 && ((target.mips ?? 1) > 1 || target.format === "r32float"))) throw new Error("Invalid GPU target samples: " + target.id); } const r: RenderGraphResources = { targets, textures: {}, buffers: {}, passes: [], clock, uniforms, sampler: device.createSampler({ minFilter: "linear", magFilter: "linear", mipmapFilter: "linear" }), width: 0, height: 0, scale: 1 }; @@ -89,7 +94,8 @@ export async function prepareRenderGraph(device: GPUDevice, definition: GPURende entries.push({ binding: index + 3, visibility: writing ? 4 : 7, ...(r.buffers[id] ? { buffer: { type: writing ? "storage" as const : "read-only-storage" as const } } : writing ? { storageTexture: { access: "write-only" as const, format: target!.format } } - : { texture: { sampleType: target!.format === "depth32float" ? "depth" as const : target!.format === "r32float" ? "unfilterable-float" as const : "float" as const } }) }); + : { texture: { sampleType: target!.format === "depth32float" ? "depth" as const : target!.format === "r32float" ? "unfilterable-float" as const : "float" as const, + multisampled: target!.format === "depth32float" && (target!.samples ?? 1) > 1 } }) }); } const module = device.createShaderModule({ code: pass.code + (pass.kind === "fullscreen" ? graphVertex : "") }); const diagnostics = await module.getCompilationInfo(); @@ -109,9 +115,18 @@ export async function prepareRenderGraph(device: GPUDevice, definition: GPURende ...(blend && blend !== "none" ? { blend: { color: { srcFactor: "one" as const, dstFactor: blend === "over" ? "one-minus-src-alpha" as const : "one" as const, operation: "add" as const }, alpha: { srcFactor: "one" as const, dstFactor: "one-minus-src-alpha" as const, operation: "add" as const } } } : {}) })) }, ...(pass.kind === "render" && pass.depth ? { depthStencil: { format: targetFor(targets, pass.depth).format, depthWriteEnabled: true, depthCompare: pass.depthCompare ?? "less" } } : {}), + ...(pass.kind === "render" ? { multisample: { count: targetFor(targets, pass.color[0] ?? pass.depth!).samples ?? 1 } } : {}), }); } - r.passes.push({ pass, pipeline, layout }); + r.passes.push({ pass, pipeline, layout, resolves: [] }); + } + // A multisampled colour target resolves once, in the last render pass that writes it. + const resolved = new Set(); + for (let i = r.passes.length - 1; i >= 0; i--) { + const { pass } = r.passes[i]; + if (pass.kind !== "render") continue; + r.passes[i].resolves = pass.color.map(color => multisampled(r, color) && !resolved.has(color)); + for (const color of pass.color) resolved.add(color); } return r; } @@ -124,12 +139,16 @@ export function resizeRenderGraph(r: RenderGraphResources, device: GPUDevice, wi for (const target of r.targets) { const old = r.textures[target.id]; const [w, h] = dimensions(r, target.id, width, height, renderScale); - const mips = target.mips ?? 1; + const mips = target.mips ?? 1, samples = target.samples ?? 1, depth = target.format === "depth32float"; // Allocate enough base pixels for all declared mip levels, even in tiny previews. const minSize = Math.pow(2, mips - 1); r.textures[target.id] = device.createTexture({ size: [Math.max(minSize, w), Math.max(minSize, h)], format: target.format, mipLevelCount: mips, - usage: 16 | 4 | (target.format === "depth32float" ? 0 : 8) }); + sampleCount: depth ? samples : 1, usage: 16 | 4 | (depth || samples > 1 ? 0 : 8) }); old?.destroy(); + if (samples > 1 && !depth) { + r.textures[target.id + "#ms"]?.destroy(); + r.textures[target.id + "#ms"] = device.createTexture({ size: [w, h], format: target.format, sampleCount: samples, usage: 16 }); + } } for (const resource of r.passes) { const pass = resource.pass; @@ -162,8 +181,10 @@ export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, en } else { const colors = pass.kind === "render" ? pass.color : [output(pass)]; const clear = pass.kind === "render" ? pass.clear : undefined; - const command = encoder.beginRenderPass({ colorAttachments: colors.map(color => ({ view: color === "screen" ? screen.createView() : view(r, color), - clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" : "clear", storeOp: "store" })), + const command = encoder.beginRenderPass({ colorAttachments: colors.map((color, index) => color === "screen" || !multisampled(r, color) + ? { view: color === "screen" ? screen.createView() : view(r, color), clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" as const : "clear" as const, storeOp: "store" as const } + : { view: r.textures[color + "#ms"].createView(), resolveTarget: resource.resolves[index] ? view(r, color) : undefined, + clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" as const : "clear" as const, storeOp: resource.resolves[index] ? "discard" as const : "store" as const }), ...(pass.kind === "render" && pass.depth ? { depthStencilAttachment: { view: view(r, pass.depth), depthClearValue: 1, depthLoadOp: clear === false ? "load" as const : "clear" as const, depthStoreOp: "store" as const } } : {}) }); command.setPipeline(resource.pipeline as GPURenderPipeline); command.setBindGroup(0, resource.group!); // Dawn reports zero-count draws through a logging callback that calls the JS runtime from this UI thread, which can crash Hermes. diff --git a/packages/presentation/src/GPUShader.tsx b/packages/presentation/src/GPUShader.tsx index ca7b0931a..7185ce33f 100644 --- a/packages/presentation/src/GPUShader.tsx +++ b/packages/presentation/src/GPUShader.tsx @@ -29,6 +29,8 @@ export function GPUShader(props: GPUShaderProps) { export type GPUTarget = { id: string; format: "rgba16float" | "rgba8unorm" | "depth32float" | "r32float"; scale?: number; mips?: number; + /** 4 renders through a multisampled attachment; colour resolves into this target, depth is read as texture_depth_multisampled_2d. */ + samples?: 1 | 4; }; export type GPUBufferTarget = { id: string; size: number; init?: Float32Array }; /** Resource references accept `id@N` to select mip N; bindings follow reads, then writes. */ diff --git a/packages/slides-effects/packs/world/README.md b/packages/slides-effects/packs/world/README.md index 35ddd0f96..e8e309467 100644 --- a/packages/slides-effects/packs/world/README.md +++ b/packages/slides-effects/packs/world/README.md @@ -92,16 +92,17 @@ upload still scale with the instance count; the graph API targets 20–60 nodes. The pipeline is an instanced HDR scene with depth, optional planar reflections, height fog and volumetric lights, depth of field, a half-resolution bloom mip chain with a soft threshold knee and round 13-tap/tent filters, chromatic aberration, ACES exposure/tonemapping, sRGB encoding, vignette -and grain. `high` and `auto` use full-resolution scene targets and five +and grain, then FXAA. The drawable is backed at device pixels. `high`, `medium` and `auto` render +the scene through 4x MSAA colour and depth; `high` and `auto` use full-resolution scene targets and five bloom levels. `medium` uses 0.85 scene scale and five levels; `low` uses 0.65 scene scale and three levels. Output resolution stays unchanged. `renderScale` is a static multiplier greater than zero and at most one for every offscreen target. It combines with the preset and the per-effect scales; it does not rebuild pipelines. `auto` starts each slide with the high preset and adjusts an additional multiplier from 1 -down to 0.5 in 0.125 steps. It samples live playback callback intervals, waits -through startup and resizing, lowers resolution after sustained intervals above -18.5ms, and requires several healthy windows before raising it. Paused and +down to 0.75 in 0.125 steps. It samples live playback callback intervals, waits +through startup and resizing, lowers resolution only after sustained intervals above +25ms (a collapse below ~40 fps), and requires several healthy windows before raising it. Paused and outgoing frames keep the last live scale; preparation and previews use the explicit static scale. A retained world may resize once when leaving preparation. This policy changes @@ -187,9 +188,9 @@ viewport and fade near the finite floor boundary. The regular fog stage also fades the floor into the scene. DOF uses a half-resolution gather and depth-aware composite. Shafts integrate -point/spot scattering along the visible camera ray at quarter resolution. -Their fixed pixel jitter avoids synchronized integration bands; changing target -resolution changes that sampling pattern. +point/spot scattering along the visible camera ray at half resolution. +Interleaved-gradient jitter avoids synchronized integration bands, and fog applies them +through a 3x3 tent filter so the jitter never shows as grain. Static `scale` overrides are available on DOF, Floor (reflection resolution) and Shafts; DOF and Shafts also accept `samples`. These change pipeline topology. Animated focus, aperture, blur radius, lighting and effect strengths update uniforms. diff --git a/packages/slides-effects/packs/world/post.ts b/packages/slides-effects/packs/world/post.ts index 6eb2949e7..66d52ced8 100644 --- a/packages/slides-effects/packs/world/post.ts +++ b/packages/slides-effects/packs/world/post.ts @@ -18,14 +18,12 @@ export function shaftCode(bindings: string, lights: number, steps: number) { return `${bindings} @group(0) @binding(3) var depth:texture_depth_2d; ${depthCode()} -@fragment fn shafts(@location(0) uv:vec2f)->@location(0) vec4f { +@fragment fn shafts(@location(0) uv:vec2f,@builtin(position) position:vec4f)->@location(0) vec4f { let ray=worldRay(uv); let direction=normalize(ray); let distance=min(depthAt(uv),100.); let stride=distance/${steps}.; let density=max(uniforms[46],.0001); let anisotropy=clamp(uniforms[47],-.8,.8); - let pixel=vec2u(uv*vec2f(textureDimensions(depth))); - var seed=pixel.x*1973u+pixel.y*9277u+89173u; - seed=(seed^(seed>>16u))*2246822519u; seed=(seed^(seed>>13u))*3266489917u; - let jitter=f32((seed^(seed>>16u))&65535u)/65536.; + // Interleaved gradient noise spreads the step offsets evenly, so fog's 3x3 tent removes the banding without white-noise grain. + let jitter=fract(52.9829189*fract(dot(floor(position.xy),vec2f(.06711056,.00583715)))); var scattering=vec3f(0.); var opticalDepth=0.; for(var sampleIndex=0u;sampleIndex<${steps}u;sampleIndex++) { let z=(f32(sampleIndex)+jitter)*stride; let world=vector(0u)+ray*z; @@ -48,14 +46,21 @@ export function fogCode(bindings: string, shafts: boolean) { return `${bindings} @group(0) @binding(3) var source:texture_2d; @group(0) @binding(4) var depth:texture_depth_2d; -${shafts ? "@group(0) @binding(5) var shaftField:texture_2d;" : ""} +${shafts ? `@group(0) @binding(5) var shaftField:texture_2d; +fn shaftAt(uv:vec2f)->vec3f { + let texel=1./vec2f(textureDimensions(shaftField)); var sum=vec3f(0.); + for(var y=-1;y<=1;y++) { for(var x=-1;x<=1;x++) { + sum+=textureSampleLevel(shaftField,linearSampler,uv+vec2f(f32(x),f32(y))*texel,0.).rgb*f32((2-abs(x))*(2-abs(y))); + } } + return sum/16.; +}` : ""} ${depthCode()} @fragment fn fog(@location(0) uv:vec2f)->@location(0) vec4f { let distance=depthAt(uv); let world=vector(0u)+worldRay(uv)*distance; let heightDensity=exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); let fogAmount=1.-exp(-distance*uniforms[12]*heightDensity); var color=mix(textureSampleLevel(source,linearSampler,uv,0.).rgb,vector(20u),clamp(fogAmount,0.,1.)); - ${shafts ? "color+=textureSampleLevel(shaftField,linearSampler,uv,0.).rgb;" : ""} + ${shafts ? "color+=shaftAt(uv);" : ""} return vec4f(color,1.); }`; } diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index ba98b00aa..cbcee45e4 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -50,8 +50,8 @@ const mesh: Record = { normal=right*cos(uv.y*6.2831853)+up*sin(uv.y*6.2831853); point=2.*(1.-t)*t*control+t*t*endpoint+normal*uniforms[base+19u]; ` }, - ring: { count: 32 * 6 * 6, code: ` - let cell=vid/6u; let uv=(vec2f(f32(cell%32u),f32(cell/32u))+corner(vid))/vec2f(32.,6.); + ring: { count: 128 * 6 * 6, code: ` + let cell=vid/6u; let uv=(vec2f(f32(cell%128u),f32(cell/128u))+corner(vid))/vec2f(128.,6.); let a=uv.x*6.2831853; let b=uv.y*6.2831853; let radial=vec3f(cos(a),0.,sin(a)); normal=radial*cos(b)+vec3f(0.,sin(b),0.); point=radial*radius+normal*uniforms[base+19u]; @@ -248,9 +248,25 @@ const bloomComposite = `${worldBindings} @group(0) @binding(3) var source:texture_2d; @group(0) @binding(4) var bloom:texture_2d; @fragment fn bloomComposite(@location(0) uv:vec2f)->@location(0) vec4f { return vec4f(textureSampleLevel(source,linearSampler,uv,0.).rgb+textureSampleLevel(bloom,linearSampler,uv,0.).rgb*uniforms[9],1.); }`; -const copyCode = `${worldBindings} +// FXAA on the display-referred image catches what MSAA cannot: shading edges, the floor horizon and low-quality scenes. +const fxaaCode = `${worldBindings} @group(0) @binding(3) var source:texture_2d; -@fragment fn copyColor(@location(0) uv:vec2f)->@location(0) vec4f { return textureSampleLevel(source,linearSampler,uv,0.); }`; +fn fetch(uv:vec2f)->vec3f { return textureSampleLevel(source,linearSampler,uv,0.).rgb; } +fn luma(color:vec3f)->f32 { return dot(color,vec3f(.299,.587,.114)); } +@fragment fn fxaa(@location(0) uv:vec2f)->@location(0) vec4f { + let texel=1./vec2f(textureDimensions(source)); let center=fetch(uv); let m=luma(center); + let nw=luma(fetch(uv+vec2f(-1.,-1.)*texel)); let ne=luma(fetch(uv+vec2f(1.,-1.)*texel)); + let sw=luma(fetch(uv+vec2f(-1.,1.)*texel)); let se=luma(fetch(uv+vec2f(1.,1.)*texel)); + let lumaMin=min(m,min(min(nw,ne),min(sw,se))); let lumaMax=max(m,max(max(nw,ne),max(sw,se))); + if(lumaMax-lumaMinlumaMax),1.); +}`; type CustomDraw = CustomGeometry & { offset: number; count: number }; type SDFDraw = SDFGeometry & { offset: number; count: number }; @@ -289,8 +305,9 @@ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): const scale = topology.quality === "low" ? .65 : topology.quality === "medium" ? .85 : 1; const levels = topology.quality === "low" ? 3 : 5; const samples = topology.quality === "low" ? 8 : topology.quality === "medium" ? 12 : 16; + const msaa = topology.quality === "low" ? 1 : 4; const passes: GPUPass[] = []; - const targets: GPURenderGraphDefinition["targets"] = [{ id: "hdr", format: "rgba16float", scale }, { id: "depth", format: "depth32float", scale }, + const targets: GPURenderGraphDefinition["targets"] = [{ id: "hdr", format: "rgba16float", scale, samples: msaa }, { id: "depth", format: "depth32float", scale, samples: msaa }, { id: "bloom", format: "rgba16float", scale: scale * .5, mips: levels }, { id: "bloom-up", format: "rgba16float", scale: scale * .5, mips: levels }]; const background = topology.backgroundVector; const clear: [number, number, number, number] = [background[0], background[1], background[2], 1]; @@ -310,14 +327,14 @@ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): if (!draws && !cleared) add("empty", "box", 0, 0); }; if (topology.mirror) { - const reflectionScale = scale * (topology.reflectionScale ?? .5); - targets.push({ id: "reflect", format: "rgba16float", scale: reflectionScale }, { id: "reflectDepth", format: "depth32float", scale: reflectionScale }); + const reflectionScale = scale * (topology.reflectionScale ?? 1); + targets.push({ id: "reflect", format: "rgba16float", scale: reflectionScale, samples: msaa }, { id: "reflectDepth", format: "depth32float", scale: reflectionScale, samples: msaa }); renderScene(true); } if (topology.floor) passes.push({ kind: "render", id: "floor", code: floorCode(lights, !!topology.mirror), vertex: "floorVertex", fragment: "floorFragment", color: ["hdr"], depth: "depth", vertexCount: 3, clear, reads: topology.mirror ? ["reflect"] : [] }); renderScene(false, !!topology.floor); if (topology.shafts) { - targets.push({ id: "shafts", format: "rgba16float", scale: scale * (topology.shaftScale ?? .25) }); + targets.push({ id: "shafts", format: "rgba16float", scale: scale * (topology.shaftScale ?? .5) }); passes.push({ kind: "fullscreen", id: "shafts", code: shaftCode(worldBindings, lights, topology.shaftSamples ?? samples), fragment: "shafts", reads: ["depth"], output: "shafts" }); } let color = "hdr", customIndex = 0; @@ -360,10 +377,10 @@ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): custom(stage, false); } custom("tonemap", true); - const postTone = (topology.passes ?? []).some(item => item.after === "tonemap"); - fullscreen("tonemap", finalCode(worldBindings), "tonemap", [color], postTone ? "tonemapped" : "screen"); + fullscreen("tonemap", finalCode(worldBindings), "tonemap", [color], "tonemapped"); custom("tonemap", false); - if (postTone) fullscreen("screen", copyCode, "copyColor", [color], "screen"); + fullscreen("screen", fxaaCode, "fxaa", [color], "screen"); + if (msaa > 1) for (const pass of passes) if (pass.reads?.includes("depth")) pass.code = pass.code.replace("texture_depth_2d", "texture_depth_multisampled_2d"); const passIds = new Set(); for (const pass of passes) { if (passIds.has(pass.id)) throw new Error("World pass id is duplicated: " + pass.id); diff --git a/packages/slides-effects/packs/world/resolution.ts b/packages/slides-effects/packs/world/resolution.ts index b59076303..853642ef8 100644 --- a/packages/slides-effects/packs/world/resolution.ts +++ b/packages/slides-effects/packs/world/resolution.ts @@ -26,11 +26,12 @@ export function advanceResolution(state: ResolutionState, sample: ResolutionSamp if (elapsed < 1000) return { ...state, lastTimestamp: timestamp, elapsed, frames }; const mean = elapsed / frames; const healthyWindows = mean <= 17.5 ? Math.min(5, state.healthyWindows + 1) : 0; - const lower = mean > 18.5 && state.scale > 0.5; + // Ordinary misses keep full resolution; only a collapse below ~40 fps trades pixels for cadence. + const lower = mean > 25 && state.scale > 0.75; const raise = healthyWindows >= 5 && state.scale < 1; const change = timestamp >= state.cooldownUntil && (lower || raise); return { ...state, lastTimestamp: timestamp, elapsed: 0, frames: 0, - scale: change ? Math.max(0.5, Math.min(1, state.scale + (lower ? -0.125 : 0.125))) : state.scale, + scale: change ? Math.max(0.75, Math.min(1, state.scale + (lower ? -0.125 : 0.125))) : state.scale, healthyWindows: change ? 0 : healthyWindows, warmupUntil: change ? timestamp + 500 : state.warmupUntil, cooldownUntil: change ? timestamp + 3000 : state.cooldownUntil }; diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index d630aa0e6..b8b2316e9 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -150,7 +150,9 @@ test("HDR scene and five-level half-resolution bloom describe independent attach expect(render.map(pass => pass.instanceCount)).toEqual([1, 1, 3, 1]); expect(render[0].clear).toBeArray(); expect(render.slice(1).every(pass => pass.clear === false)).toBe(true); - expect(graph.passes.at(-1)).toMatchObject({ id: "tonemap", output: "screen" }); + expect(graph.passes.at(-1)).toMatchObject({ id: "screen", fragment: "fxaa", output: "screen" }); + expect(graph.targets.filter(target => target.samples === 4).map(target => target.id)).toEqual(["hdr", "depth"]); + expect(render.every(pass => pass.vertexCount > 0)).toBe(true); expect(graph.uniformsSize).toBe(sampleWorldUniforms(scene, clock(0)).length); expect(orbInstances({ count: 3, seed: 7 })).toEqual(orbInstances({ count: 3, seed: 7 })); expect(orbInstances({ count: 3, seed: 7 })).not.toEqual(orbInstances({ count: 3, seed: 8 })); diff --git a/packages/slides-effects/tests/worldLook.test.ts b/packages/slides-effects/tests/worldLook.test.ts index 16aa22223..64b4bd498 100644 --- a/packages/slides-effects/tests/worldLook.test.ts +++ b/packages/slides-effects/tests/worldLook.test.ts @@ -24,7 +24,7 @@ for (const quality of ["low", "medium", "high"] as const) { } const reflection = graph.targets.find(target => target.id === "reflect")!; const hdr = graph.targets.find(target => target.id === "hdr")!; - expect(reflection.scale).toBe(hdr.scale! * .5); + expect(reflection).toMatchObject({ scale: hdr.scale, samples: hdr.samples }); expect(graph.passes.filter(pass => pass.kind === "render")).toHaveLength(5); }); } diff --git a/packages/slides-effects/tests/worldRender.test.ts b/packages/slides-effects/tests/worldRender.test.ts index 4cdf52213..ff9afb678 100644 --- a/packages/slides-effects/tests/worldRender.test.ts +++ b/packages/slides-effects/tests/worldRender.test.ts @@ -18,10 +18,10 @@ for (const quality of ["low", "medium", "high"] as const) test(`all enabled post for (const id of ["reflection-sphere", "floor", "shafts", "dof-gather", "dof-composite", "bloom-composite", "tonemap"]) expect(ids).toContain(id); const scale = quality === "low" ? .65 : quality === "medium" ? .85 : 1; const target = (id: string) => graph.targets.find(target => target.id === id)!; - expect(target("reflect").scale).toBe(scale * .5); + expect(target("reflect").scale).toBe(scale); expect(target("reflectDepth").scale).toBe(target("reflect").scale); expect(target("dof").scale).toBe(scale * .5); - expect(target("shafts").scale).toBe(scale * .25); + expect(target("shafts").scale).toBe(scale * .5); expect(graph.uniformsSize).toBe(worldHeaderSize + worldInstanceSize); }); diff --git a/packages/slides-effects/tests/worldResolution.test.ts b/packages/slides-effects/tests/worldResolution.test.ts index f31ad6338..83d16d265 100644 --- a/packages/slides-effects/tests/worldResolution.test.ts +++ b/packages/slides-effects/tests/worldResolution.test.ts @@ -21,19 +21,23 @@ test("ordinary 60 Hz and 120 Hz cadence keeps full resolution", () => { } }); -test("sustained 50 fps steps down within bounds and healthy cadence recovers with hysteresis", () => { - const slow = frames(createResolutionState(), 20, 1000); - expect(slow.scales.slice(0, 60).every(scale => scale === 1)).toBe(true); - expect([...new Set(slow.scales)]).toEqual([1, 0.875, 0.75, 0.625, 0.5]); - expect(slow.state.scale).toBe(0.5); - expect(frames(createResolutionState(), 120, 200).state.scale).toBe(0.5); - const neutral = frames(slow.state, 18, 400); - expect(neutral.state.scale).toBe(0.5); +test("sustained 50 fps keeps full resolution", () => { + expect(frames(createResolutionState(), 20, 1000).scales.every(scale => scale === 1)).toBe(true); +}); + +test("collapsed cadence steps down to the 0.75 floor and healthy cadence recovers with hysteresis", () => { + const slow = frames(createResolutionState(), 30, 1000); + expect(slow.scales.slice(0, 40).every(scale => scale === 1)).toBe(true); + expect([...new Set(slow.scales)]).toEqual([1, 0.875, 0.75]); + expect(slow.state.scale).toBe(0.75); + expect(frames(createResolutionState(), 120, 200).state.scale).toBe(0.75); + const neutral = frames(slow.state, 20, 400); + expect(neutral.state.scale).toBe(0.75); const shortRecovery = frames(neutral.state, 1000 / 60, 240); - expect(shortRecovery.state.scale).toBe(0.5); + expect(shortRecovery.state.scale).toBe(0.75); const recovered = frames(shortRecovery.state, 1000 / 60, 1600); expect(recovered.state.scale).toBe(1); - expect(recovered.scales.every(scale => scale >= 0.5 && scale <= 1)).toBe(true); + expect(recovered.scales.every(scale => scale >= 0.75 && scale <= 1)).toBe(true); }); test("startup, isolated stalls, paused gaps and timestamp resets do not lower resolution", () => { @@ -57,7 +61,7 @@ test("startup, isolated stalls, paused gaps and timestamp resets do not lower re }); test("an isolated hitch preserves the current cooldown and recovery history", () => { - const state = frames(createResolutionState(), 20, 100).state; + const state = frames(createResolutionState(), 30, 70).state; expect(state.scale).toBe(0.875); expect(state.cooldownUntil).toBeGreaterThan(state.lastTimestamp!); const hitch = advanceResolution(state, { timestamp: state.lastTimestamp! + 500, playing: true, key: "scene" }); @@ -77,7 +81,7 @@ test("an isolated hitch preserves the current cooldown and recovery history", () test("adaptation never changes playback clocks across a real pause and resume", () => { let playback = createPlaybackState({ phase: "playing", slideKey: "clock", stepKey: "0", stepIndex: 0 }); let resolution = createResolutionState(); - for (let timestamp = 0; timestamp <= 10000; timestamp += 20) { + for (let timestamp = 0; timestamp <= 10000; timestamp += 40) { playback = advancePlayback(playback, timestamp); resolution = advanceResolution(resolution, { timestamp: playback.lastTimestamp, playing: playback.phase === "playing", key: playback.slideKey }); } From dba9fc743e78cf712f55568f60c4f1faad8cd65a Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 13:07:45 -0700 Subject: [PATCH 23/28] fix: keep slides split after comparisons in JSX props A `>` inside a JSX expression, such as `= 1} />`, ended the tag regex early, so the tag counted as an unclosed opening tag and every later `---` stayed inside the first slide. Scan tags with expression, brace and string state instead, skipping children expressions whose lines expression depth already covers. Notes scanning also stops resetting its tag state on `<` or `>` inside braces, so attribute strings after a comparison stay literal. --- .../src/__tests__/speakerNotesSource.test.ts | 28 +++++++- .../presentation/src/speakerNotesSource.ts | 68 ++++++++++++++++--- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/packages/presentation/src/__tests__/speakerNotesSource.test.ts b/packages/presentation/src/__tests__/speakerNotesSource.test.ts index a990a1ea9..cb77d6919 100644 --- a/packages/presentation/src/__tests__/speakerNotesSource.test.ts +++ b/packages/presentation/src/__tests__/speakerNotesSource.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { compileDeck } from "../compiler"; -import { updateSlideSpeakerNotes } from "../speakerNotesSource"; +import { getDeckSourceStructure, updateSlideSpeakerNotes } from "../speakerNotesSource"; describe("updateSlideSpeakerNotes", () => { test("updates only the target slide comment", () => { @@ -218,3 +218,29 @@ test("alternate separators keep notes edits bound to their own slide", () => { expect(updated).not.toContain(""); } }); + +test("comparison and arrow operators in JSX expressions keep later slides split", () => { + const blocks = [ + "= 1} />", + "", + " 1} />", + " item.label} />", + ' b"} hint={\'c >= d\'} note={`e => ${f > g ? "h" : "i"}`} />', + "= 1 ? 1 : 0, nested: { deep: step > 2 } }} />", + "= 1}\n render={() => ({ value: step > 2 })}\n/>", + "{step => step >= 1 && {step > 1 ? \"more > less\" : \"less\"}}", + "\n {items.filter(item => item.size >= 2).map(item => {item.label})}\n", + ]; + for (const block of blocks) { + const source = `# One\n\n${block}\n\n---\n\n# Two\n\n---\n\n# Three\n`; + const { slides } = getDeckSourceStructure(source); + expect(slides.map((slide) => source.slice(slide.contentStart, slide.end).trim())).toEqual([`# One\n\n${block}`, "# Two", "# Three"]); + } +}); + +test("attribute strings after a comparison stay literal", () => { + const source = '# One\n\n= 1} label="" />\n\n\n\n---\n\n# Two\n'; + const { comments, slides } = getDeckSourceStructure(source); + expect(comments.map((comment) => source.slice(comment.start, comment.end))).toEqual([""]); + expect(slides).toHaveLength(2); +}); diff --git a/packages/presentation/src/speakerNotesSource.ts b/packages/presentation/src/speakerNotesSource.ts index 72dc28cd4..79e28521e 100644 --- a/packages/presentation/src/speakerNotesSource.ts +++ b/packages/presentation/src/speakerNotesSource.ts @@ -95,8 +95,8 @@ function scanSource(lines: readonly SourceLine[], source: string) { if (ch === '"' || ch === "'" || ch === "`") { quote = ch; continue; } } if (ch === "\\") { at++; continue; } - if (ch === "<" && /^<\/?[A-Za-z]/.test(source.slice(at, at + 3))) tag = true; - if (ch === ">") tag = false; + if (!depth && ch === "<" && /^<\/?[A-Za-z]/.test(source.slice(at, at + 3))) tag = true; + if (!depth && ch === ">") tag = false; if (ch === "{") { depth++; continue; } if (ch === "}" && depth > 0) { depth--; continue; } if (depth > 0 || moduleCode) continue; @@ -126,6 +126,49 @@ function scanSource(lines: readonly SourceLine[], source: string) { return { fenced, comments, literals }; } +/** Offset after the `}` matching the `{` at `start`, skipping JS strings and comments. */ +function expressionEnd(content: string, start: number) { + let depth = 0; + for (let at = start; at < content.length; at++) { + const ch = content[at]; + if (ch === '"' || ch === "'" || ch === "`") { + for (at++; at < content.length && content[at] !== ch; at++) if (content[at] === "\\") at++; + } else if (ch === "/" && content[at + 1] === "*") { + at = content.indexOf("*/", at + 2) + 1; + if (at <= 0) break; + } else if (ch === "/" && content[at + 1] === "/") { + at = content.indexOf("\n", at); + if (at < 0) break; + } else if (ch === "{") depth++; + else if (ch === "}" && --depth === 0) return at + 1; + } + return content.length; +} + +const tagStart = /<(\/)?(?:[A-Za-z][\w.:-]*(?=[\s/>])|(?=>))/y; +const selfClose = /\/\s*>/y; + +/** A JSX tag at `start`, whose `>` may not be inside an attribute string or expression. */ +function tagAt(content: string, start: number) { + tagStart.lastIndex = start; + const opening = tagStart.exec(content); + if (!opening) return undefined; + const closing = Boolean(opening[1]); + for (let at = tagStart.lastIndex; at < content.length; at++) { + const ch = content[at]; + if (ch === "{") at = expressionEnd(content, at) - 1; + else if (ch === '"' || ch === "'") { + at = content.indexOf(ch, at + 1); + if (at < 0) return undefined; + } else if (ch === ">") return { closing, end: at + 1, selfClosing: false, start }; + else if (ch === "/") { + selfClose.lastIndex = at; + return selfClose.test(content) ? { closing, end: selfClose.lastIndex, selfClosing: !closing, start } : undefined; + } else if (ch === "<") return undefined; + } + return undefined; +} + function jsxDepths( source: string, lines: readonly SourceLine[], @@ -148,16 +191,19 @@ function jsxDepths( comments.forEach((comment) => mask(comment.start, comment.end)); const tags: Array<{ closing: boolean; end: number; selfClosing: boolean; start: number }> = []; - const tagPattern = /<(\/)?(?:[A-Za-z][\w.:-]*|)(?:\s[^<>]*?)?(\/)?\s*>/g; - let match: RegExpExecArray | null; const content = masked.join(""); - while ((match = tagPattern.exec(content))) { - tags.push({ - closing: Boolean(match[1]), - end: match.index + match[0].length, - selfClosing: Boolean(match[2]), - start: match.index, - }); + // Expressions are skipped whole: tags inside them balance there, and their lines + // are already excluded by expressionDepths. + for (let at = 0; at < content.length; at++) { + if (content[at] === "\\") at++; + else if (content[at] === "{") at = expressionEnd(content, at) - 1; + else if (content[at] === "<") { + const tag = tagAt(content, at); + if (tag) { + tags.push(tag); + at = tag.end - 1; + } + } } const events = tags.flatMap((tag) => tag.selfClosing From d567df85ff768801122280611091e1cca6189df4 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 21:17:42 -0700 Subject: [PATCH 24/28] fix: remove the pillar base seam and trim World post-pass cost A dashed cyan line ran along every pillar base in cinematic-title. The reflection pass kept faces lying exactly on the mirror plane, so each box's bottom face, seen at a grazing angle from the mirrored camera, picked up full rim light and showed through the floor as a broken line. Mesh reflections now discard at the floor plus .001, as SDFs already did. Opaque boxes also drop back faces, which removes the coplanar bottom face from the main pass and halves box fragment work. Cheaper post passes with the same output: - The fog's 3x3 shaft tent and the bloom upsample tent use four bilinear taps at half-texel offsets instead of nine. - The shaft march hoists the per-step segment length and replaces pow(x, 1.5) with x * sqrt(x). - The GPU frame callback writes its simulation epoch only when the slide, step or phase changes, not on every frame. --- apps/slides/src/TypeGPUShader.tsx | 4 +++- packages/slides-effects/packs/world/post.ts | 17 ++++++++--------- packages/slides-effects/packs/world/render.ts | 7 ++++--- .../slides-effects/tests/worldRender.test.ts | 7 +++++++ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index d6a84b4fc..15234ba10 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -237,7 +237,9 @@ function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, fra (state.phase === "playing" && previous.phase !== "playing"))) { count.value = 0; } - simulationEpoch.value = { phase: state.phase, slideKey: state.slideKey, stepKey: state.stepKey }; + if (previous.phase !== state.phase || previous.slideKey !== state.slideKey || previous.stepKey !== state.stepKey) { + simulationEpoch.value = { phase: state.phase, slideKey: state.slideKey, stepKey: state.stepKey }; + } const frameIndex = frozen ? Math.max(0, count.value - 1) : count.value; const scale = renderScale?.get(); try { diff --git a/packages/slides-effects/packs/world/post.ts b/packages/slides-effects/packs/world/post.ts index 66d52ced8..8851147d6 100644 --- a/packages/slides-effects/packs/world/post.ts +++ b/packages/slides-effects/packs/world/post.ts @@ -20,7 +20,7 @@ export function shaftCode(bindings: string, lights: number, steps: number) { ${depthCode()} @fragment fn shafts(@location(0) uv:vec2f,@builtin(position) position:vec4f)->@location(0) vec4f { let ray=worldRay(uv); let direction=normalize(ray); - let distance=min(depthAt(uv),100.); let stride=distance/${steps}.; + let distance=min(depthAt(uv),100.); let stride=distance/${steps}.; let segment=stride*length(ray); let density=max(uniforms[46],.0001); let anisotropy=clamp(uniforms[47],-.8,.8); // Interleaved gradient noise spreads the step offsets evenly, so fog's 3x3 tent removes the banding without white-noise grain. let jitter=fract(52.9829189*fract(dot(floor(position.xy),vec2f(.06711056,.00583715)))); @@ -28,15 +28,15 @@ ${depthCode()} for(var sampleIndex=0u;sampleIndex<${steps}u;sampleIndex++) { let z=(f32(sampleIndex)+jitter)*stride; let world=vector(0u)+ray*z; let localDensity=density*exp(clamp(-(world.y-uniforms[13])*uniforms[14],-8.,8.)); - let transmission=exp(-opticalDepth); opticalDepth+=localDensity*stride*length(ray); + let transmission=exp(-opticalDepth); opticalDepth+=localDensity*segment; for(var light=0u;light<${lights}u;light++) { let base=64u+light*12u; let delta=vector(base)-world; let lightDistance=max(length(delta),.001); let toward=delta/lightDistance; let radius=max(uniforms[base+7u],.001); let attenuation=pow(max(0.,1.-lightDistance/radius),2.)/(1.+lightDistance*lightDistance*.06); var cone=1.; let cosine=uniforms[base+11u]; if(cosine>-.99) { cone=smoothstep(cosine,min(.9999,cosine+.08),dot(-toward,normalize(vector(base+8u)))); } - let angle=dot(direction,toward); let phase=(1.-anisotropy*anisotropy)/pow(max(.01,1.+anisotropy*anisotropy-2.*anisotropy*angle),1.5); - scattering+=vector(base+4u)*uniforms[base+3u]*attenuation*cone*phase*transmission*localDensity*stride*length(ray); + let angle=dot(direction,toward); let falloff=max(.01,1.+anisotropy*anisotropy-2.*anisotropy*angle); let phase=(1.-anisotropy*anisotropy)/(falloff*sqrt(falloff)); + scattering+=vector(base+4u)*uniforms[base+3u]*attenuation*cone*phase*transmission*localDensity*segment; } } return vec4f(scattering*uniforms[45],1.); @@ -47,12 +47,11 @@ export function fogCode(bindings: string, shafts: boolean) { @group(0) @binding(3) var source:texture_2d; @group(0) @binding(4) var depth:texture_depth_2d; ${shafts ? `@group(0) @binding(5) var shaftField:texture_2d; +// The 3x3 tent as four bilinear taps at half-texel offsets. fn shaftAt(uv:vec2f)->vec3f { - let texel=1./vec2f(textureDimensions(shaftField)); var sum=vec3f(0.); - for(var y=-1;y<=1;y++) { for(var x=-1;x<=1;x++) { - sum+=textureSampleLevel(shaftField,linearSampler,uv+vec2f(f32(x),f32(y))*texel,0.).rgb*f32((2-abs(x))*(2-abs(y))); - } } - return sum/16.; + let texel=.5/vec2f(textureDimensions(shaftField)); var sum=vec3f(0.); + for(var i=0;i<4;i++) { sum+=textureSampleLevel(shaftField,linearSampler,uv+vec2f(f32(i&1)*2.-1.,f32(i>>1u)*2.-1.)*texel,0.).rgb; } + return sum*.25; }` : ""} ${depthCode()} @fragment fn fog(@location(0) uv:vec2f)->@location(0) vec4f { diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index cbcee45e4..2b5d4e8d5 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -180,8 +180,9 @@ struct SDFResult { @location(0) color:vec4f, @builtin(frag_depth) depth:f32 } }` : ` @fragment fn worldFragment(in:WorldVertex)->@location(0) vec4f { if(in.opacity<=.001) { discard; } - ${reflection ? "if(in.world.y=.999 && dot(normal,eye()-in.world)<0.) { discard; }" : ""} let material=${custom ? "shade(in.instance,in.world,normal,shapeContext())" : "Material(in.albedo,in.color*in.emissive,in.material.x,in.material.y)"}; let color=illuminate(in.world,normal,material,in.color,in.material.z,in.material.w); return vec4f(color*in.opacity,in.opacity); @@ -228,9 +229,9 @@ fn downsample13(uv:vec2f,karis:bool)->vec3f { let sum=box((j+k+l+m)*.25,.5,karis)+box((a+b+d+e)*.25,.125,karis)+box((b+c+e+f)*.25,.125,karis)+box((d+e+g+h)*.25,.125,karis)+box((e+f+h+i)*.25,.125,karis); return sum.rgb/sum.a; } +// The 3x3 tent as four bilinear taps at half-texel offsets. fn tent9(uv:vec2f)->vec3f { - return (tap(uv,vec2f(0.,0.))*4.+(tap(uv,vec2f(-1.,0.))+tap(uv,vec2f(1.,0.))+tap(uv,vec2f(0.,-1.))+tap(uv,vec2f(0.,1.)))*2. - +tap(uv,vec2f(-1.,-1.))+tap(uv,vec2f(1.,-1.))+tap(uv,vec2f(-1.,1.))+tap(uv,vec2f(1.,1.)))/16.; + return (tap(uv,vec2f(-.5,-.5))+tap(uv,vec2f(.5,-.5))+tap(uv,vec2f(-.5,.5))+tap(uv,vec2f(.5,.5)))*.25; } `; const downsample = `${filterCode} diff --git a/packages/slides-effects/tests/worldRender.test.ts b/packages/slides-effects/tests/worldRender.test.ts index ff9afb678..5623dc0c8 100644 --- a/packages/slides-effects/tests/worldRender.test.ts +++ b/packages/slides-effects/tests/worldRender.test.ts @@ -35,6 +35,13 @@ test("animated floor, light and focus changes preserve topology while static qua expect(graph.targets.find(target => target.id === "shafts")?.scale).toBe(.125); }); +test("opaque boxes drop back faces so a base resting on the floor cannot z-fight with it", () => { + const graph = buildWorldGraph(scene({ floor: { mirror: true }, instances: [primitive("box"), ...customInstances({ wgsl: quad, vertexCount: 6, count: 1 })] })); + const code = (id: string) => graph.passes.find(pass => pass.id === id)!.code; + for (const id of ["scene-box", "reflection-box"]) expect(code(id)).toContain("dot(normal,eye()-in.world)<0."); + expect(code("scene-custom-0")).not.toContain("dot(normal,eye()-in.world)<0."); +}); + test("opaque floor clears the scene before translucent geometry loads its color and depth", () => { const graph = buildWorldGraph(scene({ floor: { mirror: true }, instances: rowInstances({ id: "rows", count: 3 }) })); const draws = graph.passes.filter(pass => pass.kind === "render"); From 9010dc39374d5a89a5fc6deec097541977de9a83 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Fri, 30 Oct 2026 21:54:12 -0700 Subject: [PATCH 25/28] perf: reuse World uniforms and skip unchanged instance slots The 1000-voxel data-flow slide held 30 fps because every UI frame built a 32k-number array, sampled every voxel's position into an id-keyed record and recomputed all slots, then copied the array again for upload. World now samples into one Float32Array per prepared scene and uploads it directly. Instances whose tracks never animate are written once, voxels only when their collapse progress changes, positions are computed by id only for instances that edges or pulses read, and the counter source is indexed instead of searched. Test plan - world suites: 50/50, with a new test that reused sampling keeps one buffer and matches fresh sampling through a collapse (fails when voxel slots are treated as static) - scoped tsc on the changed sources and benchmark scripts: 0 diagnostics - native audience window, Debug build: data-flow 4 30.7 -> 59.9 fps on the first trace; stills unchanged --- apps/slides/src/TypeGPUShader.tsx | 8 +-- packages/presentation/src/GPUShader.tsx | 4 +- packages/slides-effects/packs/world/World.tsx | 2 +- packages/slides-effects/packs/world/scene.ts | 61 +++++++++++++++---- .../scripts/renderGraphBenchmark.ts | 2 +- packages/slides-effects/tests/world.test.ts | 15 +++++ .../tests/worldDataFlow.test.tsx | 22 ++++--- 7 files changed, 83 insertions(+), 31 deletions(-) diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 15234ba10..8268ae544 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -215,7 +215,7 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle return () => { cancelled = true; if (release) release(); else if (!transferred) device?.destroy(); }; } -function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, frameTimeIndex, displayScale = 1, width, height, aspect, brightness, particleCount, particleVertexCount = 6, clockMode = "step", frameData, previewTime, onError, onReady }: { resources: Resources; renderScale?: SharedValue; effectActive?: boolean; maxTime?:number; frameTimeIndex?:number; displayScale?:number; width: number; height: number; aspect: number; brightness: number; particleCount: number; particleVertexCount?: number; clockMode?: "slide" | "step"; frameData?: SharedValue; previewTime: number; onError(message: string): void; onReady(): void }) { +function UIFrames({ resources: r, renderScale, effectActive = true, maxTime, frameTimeIndex, displayScale = 1, width, height, aspect, brightness, particleCount, particleVertexCount = 6, clockMode = "step", frameData, previewTime, onError, onReady }: { resources: Resources; renderScale?: SharedValue; effectActive?: boolean; maxTime?:number; frameTimeIndex?:number; displayScale?:number; width: number; height: number; aspect: number; brightness: number; particleCount: number; particleVertexCount?: number; clockMode?: "slide" | "step"; frameData?: SharedValue>; previewTime: number; onError(message: string): void; onReady(): void }) { const active = usePresentationValue("isActive"); const preview = usePresentationValue("isPreview"); const preparing = usePresentationValue("isPreparing"); @@ -336,7 +336,7 @@ async function loadImageTexture(device: GPUDevice, uri: string) { } finally { data.dispose(); } } -function submitGPUFrame(r: Resources, frame: number, particleCount: number, particleVertexCount: number, frameData?: SharedValue, renderScale = 1) { +function submitGPUFrame(r: Resources, frame: number, particleCount: number, particleVertexCount: number, frameData?: SharedValue>, renderScale = 1) { "worklet"; const { device, context, clock, groups, compute, render, particles } = r; // Configure on UI after retirement of the previous owner, never during async setup. @@ -347,8 +347,8 @@ function submitGPUFrame(r: Resources, frame: number, particleCount: number, part const data = frameData?.value; if (r.frameData && data) { if (data.length * 4 > r.frameData.size) throw new Error(`Frame data has ${data.length} values; its buffer holds ${r.frameData.size / 4}`); - const values = scratch(data.length); - values.set(data); + let values = data; + if (!(values instanceof Float32Array)) { values = scratch(data.length); values.set(data); } device.queue.writeBuffer(r.frameData, 0, values); } const encoder = device.createCommandEncoder(); diff --git a/packages/presentation/src/GPUShader.tsx b/packages/presentation/src/GPUShader.tsx index 7185ce33f..e32a2aaee 100644 --- a/packages/presentation/src/GPUShader.tsx +++ b/packages/presentation/src/GPUShader.tsx @@ -6,7 +6,7 @@ export type GPUShaderProps = { particles?: boolean; particleCount?: number; particleVertexCount?: number; particleBlend?: "additive" | "over"; previewTime?: number; clock?: "slide" | "step"; image?: string; composite?: boolean; - frameData?: SharedValue; frameDataSize?: number; + frameData?: SharedValue>; frameDataSize?: number; active?: boolean; children?: ReactNode; maxTime?: number; frameTimeIndex?: number; renderGraph?: GPURenderGraphDefinition; @@ -49,7 +49,7 @@ export type GPUPass = export type GPURenderGraphDefinition = { targets: GPUTarget[]; buffers?: GPUBufferTarget[]; passes: GPUPass[] }; export type GPURenderGraphProps = Omit & - GPURenderGraphDefinition & { uniforms?: SharedValue; /** Float capacity, defaults to 4096. */ uniformsSize?: number }; + GPURenderGraphDefinition & { uniforms?: SharedValue>; /** Float capacity, defaults to 4096. */ uniformsSize?: number }; /** Group 0: Frame at 0, read-only float uniforms at 1, linear sampler at 2, * then reads and compute writes. Fullscreen vertex output has position and location-0 UV. */ export function GPURenderGraph({ targets, buffers, passes, uniforms, uniformsSize = 4096, ...props }: GPURenderGraphProps) { diff --git a/packages/slides-effects/packs/world/World.tsx b/packages/slides-effects/packs/world/World.tsx index 4c68f93de..b24cf1dbf 100644 --- a/packages/slides-effects/packs/world/World.tsx +++ b/packages/slides-effects/packs/world/World.tsx @@ -192,7 +192,7 @@ function WorldSceneRuntime({ width, height, background = false, quality = "auto" const clock = { ...state, slideTime: sampleClock(state, previewTime, "slide"), stepTime: sampleClock(state, previewTime, "step") }; const previous = state.phase === "preview" || state.phase === "preparing" ? null : origin.value.key === state.slideKey ? origin.value.pose : state.phase === "playing" ? entryPose?.value : null; - return sampleUniforms(prepared, clock, previous, enterDuration, effectiveScale.value); + return sampleUniforms(prepared, clock, previous, enterDuration, effectiveScale.value, true); }, [prepared, previewTime, enterDuration, entryPose, effectiveScale]); return ({ ...frame, value: focusDepth(frame.value, scene, clock, camera) })) }, clock); } -export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1, renderScale = 1): number[] { +type UniformCache = { values: Float32Array; keys: Float64Array }; +/** With reuse, every call for the same prepared scene returns one buffer and rewrites only slots whose inputs changed. */ +export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSample, previous?: CameraPose | null, enterDuration = 1, renderScale = 1, reuse = false): number[] | Float32Array { "worklet"; - const values = new Array(worldHeaderSize + scene.instances.length * worldInstanceSize).fill(0); + const count = scene.instances.length; + let cache: UniformCache | undefined; + if (reuse) { + const caches = (globalThis as { __worldUniforms?: WeakMap }).__worldUniforms ??= new WeakMap(); + cache = caches.get(scene); + if (!cache) { + cache = { values: new Float32Array(worldHeaderSize + count * worldInstanceSize), keys: new Float64Array(count).fill(NaN) }; + caches.set(scene, cache); + } + } + const values = cache?.values ?? new Array(worldHeaderSize + count * worldInstanceSize).fill(0); const objects: Record = {}; - for (const instance of scene.instances) if (instance.id) objects[instance.id] = instancePosition(instance, clock); + for (const instance of scene.instances) if (instance.anchor) objects[instance.id!] = instancePosition(instance, clock); const camera = sampleWorldCamera(scene, clock, previous, enterDuration); values[0] = camera.position[0]; values[1] = camera.position[1]; values[2] = camera.position[2]; values[3] = Math.tan(camera.fov * Math.PI / 360); values[4] = camera.target[0]; values[5] = camera.target[1]; values[6] = camera.target[2]; values[7] = camera.roll; @@ -377,19 +395,23 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl values[base + 11] = light.spot ? Math.cos(sampleTrack(light.angle, clock)) : -1; }); if (scene.counter) { - const voxel = scene.instances.find(instance => instance.voxel?.options.id === scene.counter!.source)!; values[53] = 1; - values[56] = Math.round(voxelRemaining(voxel, clock)); + values[56] = Math.round(voxelRemaining(scene.instances[scene.counterIndex!], clock)); values[57] = scene.counter.position?.[0] ?? 0.8; values[58] = scene.counter.position?.[1] ?? 0.12; values[59] = scene.counter.size ?? 0.07; const color = scene.counterColor!; values[60] = color[0]; values[61] = color[1]; values[62] = color[2]; } - for (let i = 0; i < scene.instances.length; i++) { + for (let i = 0; i < count; i++) { const instance = scene.instances[i]; + if (cache) { + const key = instance.motion === "static" ? 0 : instance.motion === "voxel" ? collapseProgress(instance.voxel!.options.collapse, clock) : NaN; + if (cache.keys[i] === key) continue; + cache.keys[i] = key; + } const index = worldHeaderSize + i * worldInstanceSize; - const position = instance.endpoints ? objects[instance.endpoints[0]] : instance.id ? objects[instance.id] : instancePosition(instance, clock); + const position = instance.endpoints ? objects[instance.endpoints[0]] : instance.anchor ? objects[instance.id!] : instancePosition(instance, clock); const offset = stageOffset(instance.stages, clock); const endpoint = instance.pulse?.tail ? objects[instance.pulse.options.id] : instance.endpoints ? objects[instance.endpoints[1]] - : [instance.end[0] + offset[0], instance.end[1] + offset[1], instance.end[2] + offset[2]]; + : !instance.stages?.length ? instance.end : [instance.end[0] + offset[0], instance.end[1] + offset[1], instance.end[2] + offset[2]]; const scale = sampleTrack(instance.scale, clock); let color = sampleTrack(instance.color, clock); const activation = instance.graph ? activationAt(instance.graph.node, instance.graph.nodes, instance.graph.activate, clock) : 1; @@ -420,9 +442,12 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl } values[index + 15] = curve; values[index + 19] = instance.thickness; values[index + 23] = instanceOpacity(instance, clock); - const material = instance.materialTracks!, sampledAlbedo = sampleTrack(material.albedo, clock); - const albedo = instance.material ? instance.graph ? mixVector(dim, sampledAlbedo, activation) : sampledAlbedo : [values[index + 4], values[index + 5], values[index + 6]]; - values[index + 24] = albedo[0]; values[index + 25] = albedo[1]; values[index + 26] = albedo[2]; + const material = instance.materialTracks!; + if (instance.material) { + const sampledAlbedo = sampleTrack(material.albedo, clock); + const albedo = instance.graph ? mixVector(dim, sampledAlbedo, activation) : sampledAlbedo; + values[index + 24] = albedo[0]; values[index + 25] = albedo[1]; values[index + 26] = albedo[2]; + } else { values[index + 24] = values[index + 4]; values[index + 25] = values[index + 5]; values[index + 26] = values[index + 6]; } values[index + 27] = sampleTrack(material.roughness, clock); values[index + 28] = sampleTrack(material.metal, clock); values[index + 29] = sampleTrack(material.rim, clock) * activation; values[index + 30] = activation; } @@ -431,7 +456,7 @@ export function sampleWorldUniforms(scene: PreparedWorldScene, clock: ClockSampl export type PreparedWorldScene = WorldScene & { exposureTrack: Track; fogTrack: Track; backgroundVector: Vec3; postTracks: { index: number; track: Track }[]; colorTracks: { index: number; track: Track }[]; - focusTrack?: Track; counterColor?: Vec3; + focusTrack?: Track; counterColor?: Vec3; counterIndex?: number; lightTracks: { position: Track; target: Track; color: Track; intensity: Track; range: Track; angle: Track; spot: boolean }[] }; export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { if ((scene.lights?.length ?? 0) > 4) throw new Error("World supports at most four point or spot lights"); @@ -488,6 +513,15 @@ export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { const order = (instance: Instance) => instance.custom ? `1:${instance.custom.wgsl}:${instance.custom.vertexCount}` : instance.sdf ? `2:${instance.sdf.wgsl}` : `0:${instance.kind}`; instances.sort((a, b) => { const x = order(a), y = order(b); return x < y ? -1 : x > y ? 1 : 0; }); + const anchorIds = new Set(instances.flatMap(instance => [...instance.endpoints ?? [], ...instance.pulse?.tail ? [instance.pulse.options.id] : []])); + const still = (track: Track) => track.frames.length === 0; + for (const instance of instances) { + if (instance.id && anchorIds.has(instance.id)) instance.anchor = true; + const material = instance.materialTracks!; + if ([instance.position, instance.size, instance.color, instance.emissive, instance.scale, material.albedo, material.roughness, material.metal, material.rim].every(still) && + (instance.stages ?? []).every(stage => still(stage.position) && still(stage.glow)) && + !instance.drift && !instance.pulse && !instance.spray && !instance.rows && !instance.graph && !instance.endpoints) instance.motion = instance.voxel?.options.collapse ? "voxel" : "static"; + } const fields: [number, Animated][] = [[9, scene.bloom?.intensity ?? 0], [10, scene.bloom?.threshold ?? 1], [11, scene.bloom?.radius ?? 0.8], [12, scene.fog?.density ?? 0], [13, scene.fog?.height ?? 0], [14, scene.fog?.falloff ?? 0.15], [15, scene.vignette ?? 0.25], [16, scene.grain ?? 0.015], [33, scene.dof ? scene.dof.aperture ?? 0.6 : 0], [34, scene.dof?.maxBlur ?? 12], [36, scene.floor?.y ?? -5], [37, scene.floor?.roughness ?? 0.06], @@ -498,6 +532,7 @@ export function prepareWorldScene(scene: WorldScene): PreparedWorldScene { focusTrack: scene.dof?.focus === undefined ? undefined : compileTrack(scene.dof.focus), colorTracks: [{ index: 40, track: compileTrack(scene.floor?.color ?? "#101827", colorVector) }, { index: 48, track: compileTrack(scene.rim?.color ?? "#ffffff", colorVector) }], counterColor: scene.counter ? colorVector(scene.counter.color ?? "#ffffff") : undefined, + counterIndex: scene.counter ? instances.findIndex(instance => instance.voxel?.options.id === scene.counter!.source) : undefined, lightTracks: (scene.lights ?? []).map(light => ({ position: compileTrack(light.position ?? [0, 8, 0]), target: compileTrack(light.target ?? [0, 0, 0]), color: compileTrack(light.color ?? "#ffffff", colorVector), intensity: compileTrack(light.intensity ?? 4), range: compileTrack(light.range ?? 25), angle: compileTrack(light.angle ?? 0.45), spot: light.kind === "spot" })), diff --git a/packages/slides-effects/scripts/renderGraphBenchmark.ts b/packages/slides-effects/scripts/renderGraphBenchmark.ts index fe7d326b2..221069e81 100644 --- a/packages/slides-effects/scripts/renderGraphBenchmark.ts +++ b/packages/slides-effects/scripts/renderGraphBenchmark.ts @@ -3,7 +3,7 @@ import { encodeRenderGraph, prepareRenderGraph } from "../../../apps/slides/src/ export type BenchmarkOptions = { definition: GPURenderGraphDefinition; - uniformsAt: (seconds: number, renderScale: number) => number[]; + uniformsAt: (seconds: number, renderScale: number) => ArrayLike; renderScaleAt?: (timestamp: number) => number; width: number; height: number; diff --git a/packages/slides-effects/tests/world.test.ts b/packages/slides-effects/tests/world.test.ts index b8b2316e9..326d8c4b4 100644 --- a/packages/slides-effects/tests/world.test.ts +++ b/packages/slides-effects/tests/world.test.ts @@ -294,6 +294,21 @@ test("voxel compaction keeps topology fixed and its counter matches the survivin expect(sampleWorldUniforms(scene, clock(0))[56]).toBe(512); }); +test("reused uniforms keep one buffer and match fresh sampling while static and collapsing slots are skipped", () => { + const scene = prepareWorldScene({ instances: [ + ...voxelInstances({ id: "cells", grid: [4, 4, 4], collapse: { step: 1, to: 0.2, duration: 2 } }), primitive("box", { id: "still", position: [2, 0, 0] }), + ...orbInstances({ count: 3, drift: 0.4 }), ...graphInstances({ data: tree("root", ["a"]), activate: [wave({ from: "root", step: 1 })] }), + ...pulseInstances({ id: "pulse", path: ["root", "a"], step: 1 }), + ], counter: { source: "cells" } }); + expect(scene.instances.filter(instance => instance.motion === "voxel")).toHaveLength(64); + expect(scene.instances.filter(instance => instance.anchor).map(instance => instance.id).sort()).toEqual(["node:a", "node:root", "pulse"]); + const reused = sampleWorldUniforms(scene, clock(0), undefined, 1, 1, true); + for (const sample of [clock(0.5), clock(0.5, 1, { 0: 0, 1: 0.5 }), clock(1.5, 1, { 0: 0, 1: 0.5 }), clock(4, 1, { 0: 0, 1: 0.5 }), clock(5, 1, { 0: 0, 1: 0.5 }), clock(0.2)]) { + expect(sampleWorldUniforms(scene, sample, undefined, 1, 1, true)).toBe(reused); + expect(reused).toEqual(Float32Array.from(sampleWorldUniforms(scene, sample))); + } +}); + test("removed voxels shrink in place, including collapse to zero and backwards sampling", () => { const scene = prepareWorldScene({ instances: voxelInstances({ id: "cells", grid: [5, 1, 1], collapse: { step: 1, to: 0.2, duration: 2, ease: "linear" } }), counter: { source: "cells" } }); let state = createPlaybackState({ phase: "playing", slideKey: "shrink", stepKey: "1", stepIndex: 1 }); diff --git a/packages/slides-effects/tests/worldDataFlow.test.tsx b/packages/slides-effects/tests/worldDataFlow.test.tsx index ae54cc2ad..c4751b1f5 100644 --- a/packages/slides-effects/tests/worldDataFlow.test.tsx +++ b/packages/slides-effects/tests/worldDataFlow.test.tsx @@ -12,8 +12,10 @@ import { colorVector, worldHeaderSize, worldInstanceSize } from "../packs/world/ const playback = { value: { phase: "playing", slideTime: 0, stepTime: 0, stepIndex: 0, stepOrigins: { 0: 0 } } }; let renderer, network; -const closeVector = (actual, expected) => expected.forEach((value, axis) => expect(actual[axis]).toBeCloseTo(value, 10)); -const values = index => renderer.root.findAllByType("world-gpu")[index].props.uniforms.value; +// World uploads float32 uniforms, so exact expectations round the same way. +const f32 = Math.fround, f32Vector = vector => vector.map(f32); +const closeVector = (actual, expected) => expected.forEach((value, axis) => expect(actual[axis]).toBeCloseTo(value, 5)); +const values = index => Array.from(renderer.root.findAllByType("world-gpu")[index].props.uniforms.value); const instances = index => { const data = values(index), result = []; for (let offset = worldHeaderSize; offset < data.length; offset += worldInstanceSize) result.push(data.slice(offset, offset + worldInstanceSize)); @@ -74,15 +76,15 @@ test("all five scenes fill the background and retain short overlay titles and na test("the 53-node cascade heats descendants then prunes to one branch while keeping dim geometry", () => { expect(layoutTree(network).nodes).toHaveLength(53); sample(0); - const nodes = () => instances(0).filter(item => item[3] === 0.24); - const edges = instances(0).filter(item => item[15] === 0.45); + const nodes = () => instances(0).filter(item => item[3] === f32(0.24)); + const edges = instances(0).filter(item => item[15] === f32(0.45)); expect(nodes()).toHaveLength(53); expect(edges).toHaveLength(52); for (const item of [...nodes(), ...edges]) { expect(item[23]).toBe(1); - expect(item.slice(4, 7)).toEqual(colorVector("#647b9a")); + expect(item.slice(4, 7)).toEqual(f32Vector(colorVector("#647b9a"))); } expect(values(0)[1]).toBeCloseTo(1.8); - expect(values(0)[9]).toBe(1.8); expect(values(0)[12]).toBe(0.035); + expect(values(0)[9]).toBe(f32(1.8)); expect(values(0)[12]).toBe(f32(0.035)); sample(0.5, 1); const partial = nodes().filter(item => item[30] > 0); expect(partial.length).toBeGreaterThan(1); expect(partial.length).toBeLessThan(53); @@ -100,17 +102,17 @@ test("the 53-node cascade heats descendants then prunes to one branch while keep test("the cyan orb follows one curved edge with a fainter trail and matching camera focus", () => { sample(2, 1); const head = instances(1).find(item => item[7] === 14); - const trail = instances(1).find(item => item[7] === 14 * 0.3); + const trail = instances(1).find(item => item[7] === f32(14 * 0.3)); expect(head, "orb head").toBeDefined(); expect(trail, "orb trail").toBeDefined(); - expect(head.slice(4, 7)).toEqual(colorVector("#22d3ee")); + expect(head.slice(4, 7)).toEqual(f32Vector(colorVector("#22d3ee"))); expect(head.slice(8, 11)).toEqual([1, 1, 1]); expect(trail[19]).toBeCloseTo(head[3] * 0.18); expect(trail.slice(12, 15)).toEqual(head.slice(0, 3)); expect(trail[15]).toBeGreaterThan(0); expect(values(1).slice(4, 7)).toEqual(head.slice(0, 3)); expect(values(1)[32]).toBeCloseTo(7); - expect(instances(1).filter(item => item[7] > 0 && item[3] === 0.2)).toHaveLength(2); + expect(instances(1).filter(item => item[7] > 0 && item[3] === f32(0.2))).toHaveLength(2); }); test("rows, stage flights and voxel reduction retain their clock-driven behavior", () => { @@ -125,5 +127,5 @@ test("rows, stage flights and voxel reduction retain their clock-driven behavior expect(values(4)[56]).toBe(1000); sample(2.4, 1); expect(values(4)[56]).toBe(200); - expect(instances(4).filter(item => item[3] > 0 && item[3] <= 0.17)).toHaveLength(200); + expect(instances(4).filter(item => item[3] > 0 && item[3] <= f32(0.17))).toHaveLength(200); }); From 88431136533acb92ad152e732a4f1d8cb29acc01 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Sat, 31 Oct 2026 02:43:02 -0700 Subject: [PATCH 26/28] fix: stop charging WebGPU textures to the Hermes heap react-native-webgpu reports each createTexture size as Hermes external memory, and Hades counts external bytes toward its 3 GiB heap limit. World's device-resolution MSAA targets across the presenter and audience windows pushed the UI worklet runtime past it, aborting with an OOM. Textures created by the device now opt out, like the canvas texture. --- patches/react-native-webgpu@0.8.5.patch | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/patches/react-native-webgpu@0.8.5.patch b/patches/react-native-webgpu@0.8.5.patch index 0d36becd4..2abae7dab 100644 --- a/patches/react-native-webgpu@0.8.5.patch +++ b/patches/react-native-webgpu@0.8.5.patch @@ -30,3 +30,18 @@ diff --git a/cpp/rnwgpu/SurfaceRegistry.h b/cpp/rnwgpu/SurfaceRegistry.h } SurfaceRegistry(const SurfaceRegistry &) = delete; +diff --git a/cpp/rnwgpu/api/GPUDevice.cpp b/cpp/rnwgpu/api/GPUDevice.cpp +--- a/cpp/rnwgpu/api/GPUDevice.cpp ++++ b/cpp/rnwgpu/api/GPUDevice.cpp +@@ -161,7 +161,10 @@ GPUDevice::createTexture(std::shared_ptr descriptor) { + throw std::runtime_error("Error with GPUTextureDescriptor"); + } + auto texture = _instance.CreateTexture(&desc); +- return std::make_shared(texture, descriptor->label.value_or("")); ++ // GPU memory is not Hermes heap: charging it as external memory lets a few ++ // full-screen render targets exhaust the runtime's heap limit and abort. ++ return std::make_shared(texture, descriptor->label.value_or(""), ++ false); + } + + std::shared_ptr GPUDevice::createShaderModule( From 8d083f66daabd0c5ff0171c675059d80842b7207 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Sat, 31 Oct 2026 07:14:37 -0700 Subject: [PATCH 27/28] fix: render native WebGPU at device pixels and keep MSAA draws on chip On macOS the drawable stayed at the view's size in points, so a fullscreen Retina audience window rendered 3D slides at 1710x1020 and upscaled them. getNativeSurface() and getContext() return new wrappers on every call, so setting a size on one never reached the context that renders. Each canvas now keeps one context, which both sizes the drawable and renders, on every platform. Consecutive render passes that load the same attachments now share one encoder pass, so tiled GPUs keep multisampled color and depth on chip instead of storing and reloading them between draws. --- apps/slides/src/TypeGPUShader.tsx | 22 +++++++----- .../src/__tests__/gpuRenderGraph.test.ts | 20 +++++++++-- .../__tests__/gpuResourceLifecycle.test.tsx | 13 ++++--- apps/slides/src/__tests__/gpuSetup.test.ts | 9 +++-- .../src/__tests__/typeGPUShader.test.ts | 2 +- apps/slides/src/gpuRenderGraph.ts | 34 +++++++++++++------ 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/apps/slides/src/TypeGPUShader.tsx b/apps/slides/src/TypeGPUShader.tsx index 8268ae544..3e74b7443 100644 --- a/apps/slides/src/TypeGPUShader.tsx +++ b/apps/slides/src/TypeGPUShader.tsx @@ -39,6 +39,12 @@ export function TypeGPUShader({ code, width: configuredWidth = 1696, height: con const height = background ? Math.max(1, Math.round(size.height)) : configuredHeight; const brightness = background ? intensity : 1; const canvas = useRef(null); + // Native getContext() returns a new wrapper per call, each with its own canvas size, so one wrapper both sizes and renders. + const context = useRef(null); + const surfaceContext = () => { + if (!context.current) context.current = canvas.current?.getContext("webgpu") ?? null; + return context.current; + }; const supported = typeof navigator !== "undefined" && !!navigator.gpu; const [resources, setResources] = useState(); const [preparedResources, setPreparedResources] = useState(); @@ -51,12 +57,12 @@ export function TypeGPUShader({ code, width: configuredWidth = 1696, height: con // Surface dimensions are independent of shader/device ownership. In particular, // live window resizing must not recompile pipelines or reset simulation buffers. useLayoutEffect(() => { - const surface = Platform.OS === "web" ? canvas.current?.getContext("webgpu")?.canvas : canvas.current?.getNativeSurface(); + const surface = surfaceContext()?.canvas; if (surface) resizeSurface(surface, width, height, displayScale); }, [width, height, displayScale, resources]); useEffect(() => { if (!supported) return; - return startShaderCanvas({ canvas, code, particles, particleCount, particleBlend, image, composite, width, height, displayScale, frameDataSize, renderGraph, + return startShaderCanvas({ canvas, surfaceContext, code, particles, particleCount, particleBlend, image, composite, width, height, displayScale, frameDataSize, renderGraph, reportError: caught => reportSlideError(caught, errorContext.current.slideIndex, errorContext.current.isPreview), setResources, setError }); }, [supported, code, particles, particleCount, particleBlend, image, composite, frameDataSize, renderGraph]); @@ -88,8 +94,8 @@ function requestShaderDevice(adapter: GPUAdapter): Promise { } // Setup stays outside React compilation; the UI renderer owns transferred handles. -function startShaderCanvas({ canvas, code, particles, particleCount, particleBlend = "additive", image, composite = false, width = 1696, height = 650, displayScale = 1, frameDataSize = 0, renderGraph, reportError, setResources, setError }: { - canvas: RefObject; code: string; renderGraph?: GPURenderGraphDefinition; +function startShaderCanvas({ canvas, surfaceContext, code, particles, particleCount, particleBlend = "additive", image, composite = false, width = 1696, height = 650, displayScale = 1, frameDataSize = 0, renderGraph, reportError, setResources, setError }: { + canvas: RefObject; surfaceContext(): RNCanvasContext | null; code: string; renderGraph?: GPURenderGraphDefinition; particles: boolean; particleCount: number; particleBlend?: "additive" | "over"; image?: string; composite?: boolean; width?: number; height?: number; displayScale?: number; frameDataSize?: number; reportError(error: unknown): void; setResources(resources: Resources | undefined): void; setError(error: string | undefined): void; @@ -117,11 +123,11 @@ function startShaderCanvas({ canvas, code, particles, particleCount, particleBle void device.lost.then(info => { if (!cancelled && info.reason !== "destroyed") setError("WebGPU device lost: " + info.message); }); - const context = canvas.current.getContext("webgpu"); + const context = surfaceContext(); if (!context) throw new Error("WebGPU surface unavailable"); - // The library sizes from DOM bounds, which are zero during preparation. - // Set the drawing buffer before transferring ownership to UIFrames. - if (Platform.OS === "web") resizeSurface(context.canvas, width, height, displayScale); + // The library sizes from view bounds in points (zero during preparation on web). + // Set the drawing buffer in device pixels before transferring ownership to UIFrames. + resizeSurface(context.canvas, width, height, displayScale); const format = navigator.gpu.getPreferredCanvasFormat(); if (renderGraph) { const clock = device.createBuffer({ size: 32, usage: 64 | 8 }); diff --git a/apps/slides/src/__tests__/gpuRenderGraph.test.ts b/apps/slides/src/__tests__/gpuRenderGraph.test.ts index 0814269db..c911300e4 100644 --- a/apps/slides/src/__tests__/gpuRenderGraph.test.ts +++ b/apps/slides/src/__tests__/gpuRenderGraph.test.ts @@ -80,6 +80,20 @@ test("empty draws and dispatches still run their passes without a zero-count GPU expect(g.commands[0][1].colorAttachments[0].loadOp).toBe("clear"); }); +test("consecutive draws into the same multisampled attachments share one pass; the last pass writing them resolves", async () => { + const g = gpu(); + const draw = (id, extra = {}) => ({ kind: "render", id, code: "", vertex: "vertex", fragment: "fragment", color: ["hdr"], depth: "depth", vertexCount: 6, clear: false, ...extra }); + const r = await prepareRenderGraph(g.device, { targets: [{ id: "hdr", format: "rgba16float", samples: 4 }, { id: "depth", format: "depth32float", samples: 4 }, { id: "glyphs", format: "rgba8unorm" }], + passes: [draw("boxes", { clear: [0, 0, 0, 1] }), draw("cards"), draw("labels", { reads: ["glyphs"] }), draw("overlay", { depth: undefined }), + { kind: "fullscreen", id: "screen", code: "", fragment: "final", reads: ["hdr"], output: "screen" }] }, "bgra8unorm", {}, {}); + encodeRenderGraph(r, g.device, g.encoder, { width: 64, height: 64, createView: () => "screen" }); + expect(g.commands.map(c => c[0])).toEqual(["render", "draw", "draw", "draw", "render", "draw", "render", "draw"]); + const [first, second] = g.commands.filter(c => c[0] === "render").map(c => c[1].colorAttachments[0]); + expect(first).toMatchObject({ loadOp: "clear", storeOp: "store", resolveTarget: undefined }); + expect(second).toMatchObject({ loadOp: "load", storeOp: "discard" }); + expect(second.resolveTarget).toBeDefined(); +}); + test("graph rejects invalid references and shader diagnostics before a frame can receive pipelines", async () => { const g = gpu(); await expect(prepareRenderGraph(g.device, { ...definition, passes: [{ ...definition.passes[3], reads: ["bloom@3"] }] }, "bgra8unorm", {}, {})).rejects.toThrow("Invalid GPU mip"); @@ -131,10 +145,10 @@ test("native callback copies retain one mutable graph across draws and resizing" g.device.queue.submit = () => {}; g.encoder.finish = () => ({}); const screen = { width: 1920, height: 1080, createView: () => "screen" }; - const context = { __gpuHandle: true, configure() {}, getCurrentTexture: () => screen, present() {} }; + const context = { __gpuHandle: true, canvas: { width: 1920, height: 1080 }, configure() {}, getCurrentTexture: () => screen, present() {} }; const makeMutable = value => ({ __shared: true, value }); const mocks = { - react: {}, "react-native": { Platform: { OS: "macos" } }, "@shopify/react-native-skia": {}, "react-native-webgpu": {}, + react: {}, "react-native": { Platform: { OS: "macos" }, PixelRatio: { get: () => 1 } }, "@shopify/react-native-skia": {}, "react-native-webgpu": {}, "@legend-apps/presentation": {}, "./slidesStore": {}, "./gpuRenderGraph": { prepareRenderGraph, encodeRenderGraph }, "react-native-reanimated": { makeMutable, runOnUI: fn => fn }, @@ -145,7 +159,7 @@ test("native callback copies retain one mutable graph across draws and resizing" ); let stop; const resources = await new Promise((resolve, reject) => { - stop = startShaderCanvas({ canvas: { current: { getContext: () => context } }, renderGraph: definition, code: "", particles: false, particleCount: 1, + stop = startShaderCanvas({ canvas: { current: {} }, surfaceContext: () => context, renderGraph: definition, code: "", particles: false, particleCount: 1, setResources: value => { if (value) resolve(value); }, setError: error => { if (error) reject(new Error(error)); }, reportError: reject }); }); // Worklet closures copy plain objects; shared values and native GPU handles retain identity. diff --git a/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx b/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx index 515fe2423..5f63a6365 100644 --- a/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx +++ b/apps/slides/src/__tests__/gpuResourceLifecycle.test.tsx @@ -17,7 +17,9 @@ test.each(["macos", "web"])("persistent backgrounds retain GPU resources across let index = 0, preview = false, setups = 0, cleanups = 0, report, tree; const errors = []; let size = {width: 100, height: 100}; - const surface = {width: 0, height: 0}; + // Like native, every getContext() returns a new wrapper with its own canvas size. + const contexts = []; + const surface = () => contexts.at(-1)?.canvas; const scale = React.createContext(1); const mocks = { react: React, "react-native": { View: "View", Text: "Text", Platform: { OS: platform }, PixelRatio: { get: () => 2 } }, @@ -31,13 +33,14 @@ test.each(["macos", "web"])("persistent backgrounds retain GPU resources across { gpu: {} }, ); const expectedSize = () => ({ width: size.width * 2, height: size.height * 2 }); - await act(() => { tree = create(, {createNodeMock: () => ({getNativeSurface: () => surface, getContext: () => ({ canvas: surface })})}); }); - expect(surface).toEqual(expectedSize()); + await act(() => { tree = create(, {createNodeMock: () => ({getNativeSurface: () => ({width: 0, height: 0}), getContext: () => { const context = { canvas: {width: 0, height: 0} }; contexts.push(context); return context; }})}); }); + expect(surface()).toEqual(expectedSize()); for (let width = 101; width <= 120; width++) { size = {width, height: width}; await act(() => tree.update()); - expect(surface).toEqual(expectedSize()); + expect(surface()).toEqual(expectedSize()); } + expect(contexts).toHaveLength(1); expect(setups).toBe(1); expect(cleanups).toBe(0); index = 3; preview = true; await act(() => tree.update()); @@ -50,7 +53,7 @@ test.each(["macos", "web"])("persistent backgrounds retain GPU resources across await act(() => tree.update()); const setupsBeforeResize = setups; await act(() => tree.update()); - expect(surface).toEqual({ width: 180, height: 180 }); + expect(surface()).toEqual({ width: 180, height: 180 }); expect(setups).toBe(setupsBeforeResize); } await act(() => tree.unmount()); diff --git a/apps/slides/src/__tests__/gpuSetup.test.ts b/apps/slides/src/__tests__/gpuSetup.test.ts index 18aec5a9c..e5f8c8771 100644 --- a/apps/slides/src/__tests__/gpuSetup.test.ts +++ b/apps/slides/src/__tests__/gpuSetup.test.ts @@ -24,16 +24,19 @@ async function setup(validationError, cancelDuringValidation = false) { createComputePipeline: () => { calls.push("pipeline"); return {}; }, createRenderPipeline: () => { calls.push("pipeline"); return {}; }, destroy: () => { calls.push("destroy"); done(); }, }; - const mocks = { react: {}, "react-native": { Platform: { OS: "macos" } }, "@shopify/react-native-skia": {}, "react-native-webgpu": {}, "@legend-apps/presentation": {}, "./slidesStore": {}, + const mocks = { react: {}, "react-native": { Platform: { OS: "macos" }, PixelRatio: { get: () => 2 } }, "@shopify/react-native-skia": {}, "react-native-webgpu": {}, "@legend-apps/presentation": {}, "./slidesStore": {}, "react-native-reanimated": { makeMutable: value => ({ value }), runOnUI: fn => fn } }; const start = new Function("require", "exports", "navigator", code + "\nreturn startShaderCanvas;")( name => mocks[name] ?? require(name), {}, { gpu: { requestAdapter: async () => ({ requestDevice: async () => device }), getPreferredCanvasFormat: () => "bgra8unorm" } }, ); - const stop = start({ canvas: { current: { getContext: () => ({ configure() {} }), getNativeSurface: () => ({}) } }, + const context = { configure() {}, canvas: { width: 100, height: 100 } }; + const stop = start({ canvas: { current: {} }, surfaceContext: () => context, code: "shader", width: 100, height: 100, particles: false, particleCount: 10, setResources: value => { resources.push(value); if (value) done(); }, setError: value => { if (value) errors.push(value); }, reportError: error => reports.push(error.message), }); while (!resolveScope) await Promise.resolve(); + // The rendering context's drawable is sized in device pixels, not the view's points. + expect(context.canvas).toEqual({ width: 200, height: 200 }); expect(calls.indexOf("listener")).toBeLessThan(calls.indexOf("pipeline")); expect(calls.indexOf("scope")).toBeLessThan(calls.indexOf("pipeline")); expect(resources).toEqual([undefined]); @@ -105,7 +108,7 @@ test.each(["web", "macos"])("canvas device handoff waits for deferred frame reti } }), }; pendingDevices.push(device); - const stop = startShaderCanvas({ canvas: { current: { getContext: () => context } }, code: "", particles: false, particleCount: 1, + const stop = startShaderCanvas({ canvas: { current: {} }, surfaceContext: () => context, code: "", particles: false, particleCount: 1, renderGraph: { targets: [], passes: [{ kind: "fullscreen", id: `screen-${id}`, code: "", fragment: "main", output: "screen" }] }, setResources: value => { resources.push(value); if (value) transferred.resolve(value); }, setError: error => { if (error) errors.push(error); }, reportError: error => errors.push(String(error)) }); await ready.promise; diff --git a/apps/slides/src/__tests__/typeGPUShader.test.ts b/apps/slides/src/__tests__/typeGPUShader.test.ts index dc6f82108..d0a895b50 100644 --- a/apps/slides/src/__tests__/typeGPUShader.test.ts +++ b/apps/slides/src/__tests__/typeGPUShader.test.ts @@ -303,7 +303,7 @@ test("offscreen browser setup sizes first and allocates simulation resources onl Object.defineProperty(globalThis,"navigator",{configurable:true,value:{gpu:{requestAdapter:async()=>({requestDevice:async()=>device}),getPreferredCanvasFormat:()=>"bgra8unorm"}}}); try { for (const simulation of [false,true]) await new Promise((resolve,reject)=>{ - start({canvas:{current:{getContext:()=>{surface.width=0;surface.height=0;return context;}}}, + start({canvas:{current:{}},surfaceContext:()=>{surface.width=0;surface.height=0;return context;}, code:simulation ? "@compute @workgroup_size(64) fn simulate() {}" : "",particles:false,particleCount:1,width:2016,height:1000,displayScale:0.5, setResources:r=>{if(r){assertSize();expect(r.configured.value).toBe(false);expect(configurations).toBe(0);expect(r.states.length).toBe(simulation ? 2 : 0);expect(r.groups.length).toBe(simulation ? 2 : 1);resolve();}},setError:error=>{if(error)reject(new Error(error));},reportError:reject}); }); diff --git a/apps/slides/src/gpuRenderGraph.ts b/apps/slides/src/gpuRenderGraph.ts index d09b19b06..06f9e26fa 100644 --- a/apps/slides/src/gpuRenderGraph.ts +++ b/apps/slides/src/gpuRenderGraph.ts @@ -167,8 +167,8 @@ export function resizeRenderGraph(r: RenderGraphResources, device: GPUDevice, wi export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, encoder: GPUCommandEncoder, screen: GPUTexture, renderScale = 1) { "worklet"; resizeRenderGraph(r, device, screen.width, screen.height, renderScale); - for (const resource of r.passes) { - const pass = resource.pass; + for (let index = 0; index < r.passes.length; index++) { + const resource = r.passes[index], pass = resource.pass; if (pass.kind === "compute") { const command = encoder.beginComputePass(); command.setPipeline(resource.pipeline as GPUComputePipeline); command.setBindGroup(0, resource.group!); @@ -181,16 +181,30 @@ export function encodeRenderGraph(r: RenderGraphResources, device: GPUDevice, en } else { const colors = pass.kind === "render" ? pass.color : [output(pass)]; const clear = pass.kind === "render" ? pass.clear : undefined; - const command = encoder.beginRenderPass({ colorAttachments: colors.map((color, index) => color === "screen" || !multisampled(r, color) + // Following draws that load the same attachments join this pass (validation already forbids reading them), so tiled GPUs keep MSAA samples on chip + // instead of storing and reloading them between draws; the batch resolves as its last draw would have. + let last = index; + while (pass.kind === "render" && last + 1 < r.passes.length) { + const next = r.passes[last + 1].pass; + if (next.kind !== "render" || next.clear !== false || next.depth !== pass.depth || next.color.length !== colors.length || + next.color.some((color, slot) => color !== colors[slot])) break; + last++; + } + const resolves = r.passes[last].resolves; + const command = encoder.beginRenderPass({ colorAttachments: colors.map((color, slot) => color === "screen" || !multisampled(r, color) ? { view: color === "screen" ? screen.createView() : view(r, color), clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" as const : "clear" as const, storeOp: "store" as const } - : { view: r.textures[color + "#ms"].createView(), resolveTarget: resource.resolves[index] ? view(r, color) : undefined, - clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" as const : "clear" as const, storeOp: resource.resolves[index] ? "discard" as const : "store" as const }), + : { view: r.textures[color + "#ms"].createView(), resolveTarget: resolves[slot] ? view(r, color) : undefined, + clearValue: clear || [0, 0, 0, 0], loadOp: clear === false ? "load" as const : "clear" as const, storeOp: resolves[slot] ? "discard" as const : "store" as const }), ...(pass.kind === "render" && pass.depth ? { depthStencilAttachment: { view: view(r, pass.depth), depthClearValue: 1, depthLoadOp: clear === false ? "load" as const : "clear" as const, depthStoreOp: "store" as const } } : {}) }); - command.setPipeline(resource.pipeline as GPURenderPipeline); command.setBindGroup(0, resource.group!); - // Dawn reports zero-count draws through a logging callback that calls the JS runtime from this UI thread, which can crash Hermes. - // An empty pass still clears its attachments. - const vertices = pass.kind === "render" ? pass.vertexCount : 3, instances = pass.kind === "render" ? pass.instanceCount ?? 1 : 1; - if (vertices > 0 && instances > 0) command.draw(vertices, instances); + for (; index <= last; index++) { + const draw = r.passes[index], drawPass = draw.pass; + command.setPipeline(draw.pipeline as GPURenderPipeline); command.setBindGroup(0, draw.group!); + // Dawn reports zero-count draws through a logging callback that calls the JS runtime from this UI thread, which can crash Hermes. + // An empty pass still clears its attachments. + const vertices = drawPass.kind === "render" ? drawPass.vertexCount : 3, instances = drawPass.kind === "render" ? drawPass.instanceCount ?? 1 : 1; + if (vertices > 0 && instances > 0) command.draw(vertices, instances); + } + index = last; command.end(); } } From 4d219f280439f0eab85de31a7e58673588ea1518 Mon Sep 17 00:00:00 2001 From: Jay Meistrich Date: Sat, 31 Oct 2026 07:47:19 -0700 Subject: [PATCH 28/28] perf: skip World MSAA where the drawable already supersamples With the native drawable now at device pixels, a fullscreen Retina audience window has 1.5 or more device pixels per slide unit. Auto quality drops MSAA there; FXAA and the analytic edges handle anti-aliasing. Thumbnails and explicit high quality keep 4x MSAA. --- packages/slides-effects/packs/world/World.tsx | 10 ++++++---- packages/slides-effects/packs/world/render.ts | 4 ++-- packages/slides-effects/tests/worldRender.test.ts | 8 ++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/slides-effects/packs/world/World.tsx b/packages/slides-effects/packs/world/World.tsx index b24cf1dbf..ebefa805e 100644 --- a/packages/slides-effects/packs/world/World.tsx +++ b/packages/slides-effects/packs/world/World.tsx @@ -1,6 +1,6 @@ -import { GPURenderGraph, PresentationObservableProvider, samplePlayback, useBackgroundSize, usePlayback, usePresentation$, usePresentationValue, useRuntimeProjection } from "@legend-apps/presentation"; -import { Children, Component, Fragment, isValidElement, useMemo, useState, type ReactNode } from "react"; -import { View } from "react-native"; +import { GPURenderGraph, PresentationCanvasScaleContext, PresentationObservableProvider, samplePlayback, useBackgroundSize, usePlayback, usePresentation$, usePresentationValue, useRuntimeProjection } from "@legend-apps/presentation"; +import { Children, Component, Fragment, isValidElement, useContext, useMemo, useState, type ReactNode } from "react"; +import { PixelRatio, View } from "react-native"; import { makeMutable, useAnimatedReaction, useDerivedValue, useSharedValue, type SharedValue } from "react-native-reanimated"; import type { CameraOptions, CameraPose } from "./camera"; import { buildWorldGraph, worldGraphTopology } from "./render"; @@ -138,7 +138,9 @@ function WorldSurface(props: WorldProps) { const width = background ? size.width : configuredWidth; const height = background ? size.height : configuredHeight; const { prepared, overlay } = useMemo(() => worldContent(children, quality, exposure, backgroundColor, scrim, postOrder), [children, quality, exposure, backgroundColor, scrim, postOrder]); - const topology = JSON.stringify(worldGraphTopology(prepared)); + // At 1.5 or more device pixels per slide unit the drawable already supersamples, so auto quality drops MSAA. + const supersampled = PixelRatio.get() * useContext(PresentationCanvasScaleContext) >= 1.5; + const topology = JSON.stringify({ ...worldGraphTopology(prepared), supersampled }); const [continuity] = useState(() => props.id ? retainedPose(props.id) : undefined); const origin = useSharedValue<{ key: string; pose: CameraPose | null }>({ key: "", pose: null }); const resolution = useSharedValue(createResolutionState()); diff --git a/packages/slides-effects/packs/world/render.ts b/packages/slides-effects/packs/world/render.ts index 2b5d4e8d5..652ad3876 100644 --- a/packages/slides-effects/packs/world/render.ts +++ b/packages/slides-effects/packs/world/render.ts @@ -273,7 +273,7 @@ type CustomDraw = CustomGeometry & { offset: number; count: number }; type SDFDraw = SDFGeometry & { offset: number; count: number }; export type WorldGraphTopology = { quality: WorldScene["quality"]; counts: Record; backgroundVector: Vec3; - floor?: boolean; mirror?: boolean; dof?: boolean; shafts?: boolean; lightCount?: number; + floor?: boolean; mirror?: boolean; dof?: boolean; shafts?: boolean; lightCount?: number; supersampled?: boolean; dofScale?: number; dofSamples?: number; reflectionScale?: number; shaftScale?: number; shaftSamples?: number; customs?: CustomDraw[]; sdfs?: SDFDraw[]; passes?: WorldPostPass[]; postOrder?: string[]; }; @@ -306,7 +306,7 @@ export function buildWorldGraph(scene: PreparedWorldScene | WorldGraphTopology): const scale = topology.quality === "low" ? .65 : topology.quality === "medium" ? .85 : 1; const levels = topology.quality === "low" ? 3 : 5; const samples = topology.quality === "low" ? 8 : topology.quality === "medium" ? 12 : 16; - const msaa = topology.quality === "low" ? 1 : 4; + const msaa = topology.quality === "low" || topology.quality === "auto" && topology.supersampled ? 1 : 4; const passes: GPUPass[] = []; const targets: GPURenderGraphDefinition["targets"] = [{ id: "hdr", format: "rgba16float", scale, samples: msaa }, { id: "depth", format: "depth32float", scale, samples: msaa }, { id: "bloom", format: "rgba16float", scale: scale * .5, mips: levels }, { id: "bloom-up", format: "rgba16float", scale: scale * .5, mips: levels }]; diff --git a/packages/slides-effects/tests/worldRender.test.ts b/packages/slides-effects/tests/worldRender.test.ts index 5623dc0c8..a5e6e3867 100644 --- a/packages/slides-effects/tests/worldRender.test.ts +++ b/packages/slides-effects/tests/worldRender.test.ts @@ -126,3 +126,11 @@ test("custom ids cannot shadow built-in passes or another custom pass", () => { { id: "scanlines", wgsl: scanlines, after: "fog" }, { id: "scanlines", wgsl: scanlines, after: "bloom" }, ] }))).toThrow("pass id is duplicated"); }); + +test("auto quality drops MSAA only where the drawable already supersamples the slide", () => { + const scene = prepareWorldScene({ quality: "auto", instances: [primitive("box")], floor: { mirror: true } }); + const samples = (topology: object) => buildWorldGraph({ ...worldGraphTopology(scene), ...topology }).targets.filter(target => target.samples === 4).map(target => target.id); + expect(samples({})).toEqual(["hdr", "depth", "reflect", "reflectDepth"]); + expect(samples({ supersampled: true })).toEqual([]); + expect(samples({ supersampled: true, quality: "high" })).toHaveLength(4); +});