diff --git a/packages/editor/src/components/editor/bake-thumbnail.tsx b/packages/editor/src/components/editor/bake-thumbnail.tsx new file mode 100644 index 000000000..a78fd8763 --- /dev/null +++ b/packages/editor/src/components/editor/bake-thumbnail.tsx @@ -0,0 +1,95 @@ +'use client' + +import { + computeHeroFraming, + createSnapshotPipeline, + GRID_LAYER, + heroCameraPose, + temporarilyHideNodeTypes, + useViewer, +} from '@pascal-app/viewer' +import { useThree } from '@react-three/fiber' +import { useEffect, useRef } from 'react' +import { PerspectiveCamera } from 'three' +import type { WebGPURenderer } from 'three/webgpu' +import { EDITOR_LAYER } from '../../lib/constants' + +export function BakeThumbnail({ + active, + onComplete, + onError, +}: { + active: boolean + onComplete: (blob: Blob, size: { w: number; h: number }) => void + onError: (message: string) => void +}) { + const renderer = useThree((state) => state.gl) + const scene = useThree((state) => state.scene) + const doneRef = useRef(false) + + useEffect(() => { + if (!(active && !doneRef.current)) return + doneRef.current = true + + const run = async () => { + const restoreNodeVisibility = temporarilyHideNodeTypes(['scan', 'guide', 'spawn']) + let pipeline: Awaited> = null + + try { + const framing = computeHeroFraming() + if (!framing) { + onError('scene has no framable content') + return + } + + const { width, height } = renderer.domElement + const aspect = width / height + const camera = new PerspectiveCamera(60, aspect, 0.1, 1000) + camera.layers.disable(EDITOR_LAYER) + camera.layers.disable(GRID_LAYER) + const pose = heroCameraPose({ + boxes: framing.boxes, + aim: framing.aim, + azimuthRad: framing.azimuthRad, + aspect, + }) + camera.position.set(pose.position[0], pose.position[1], pose.position[2]) + camera.lookAt(pose.target[0], pose.target[1], pose.target[2]) + camera.updateMatrixWorld() + + pipeline = await createSnapshotPipeline({ + renderer: renderer as unknown as WebGPURenderer, + scene, + camera, + }) + if (!pipeline) { + onError('thumbnail pipeline failed to build') + return + } + + pipeline.applyEnvironment({ + theme: useViewer.getState().sceneTheme, + transparent: false, + grade: true, + edges: useViewer.getState().edges, + camera, + }) + const { blob, outW, outH } = await pipeline.capture({ captureMode: 'standard' }) + onComplete(blob, { w: outW, h: outH }) + } catch (error) { + console.error( + '[bake-thumbnail]', + error instanceof Error ? (error.stack ?? error.message) : error, + ) + onError(error instanceof Error ? error.message : String(error)) + } finally { + pipeline?.dispose() + restoreNodeVisibility() + } + } + + void run() + }, [active, onComplete, onError, renderer, scene]) + + return null +} diff --git a/packages/editor/src/components/editor/thumbnail-generator.tsx b/packages/editor/src/components/editor/thumbnail-generator.tsx index 5a50ba5fe..e0be3b65f 100644 --- a/packages/editor/src/components/editor/thumbnail-generator.tsx +++ b/packages/editor/src/components/editor/thumbnail-generator.tsx @@ -1,47 +1,25 @@ 'use client' -import { emitter, sceneRegistry } from '@pascal-app/core' +import { emitter } from '@pascal-app/core' import { - backdropGradient, - deepSkyColor, + computeHeroFraming, + createSnapshotPipeline, GRID_LAYER, - getSceneTheme, - horizonHazeColor, - packNormalToRGB, - SSGI_PARAMS, + heroCameraPose, + type SnapshotPipeline, snapLevelsToTruePositions, - unpackRGBToNormal, + THUMBNAIL_HEIGHT, + THUMBNAIL_WIDTH, + temporarilyHideNodeTypes, useViewer, } from '@pascal-app/viewer' import type { CameraControls } from '@react-three/drei' import { useThree } from '@react-three/fiber' import { useCallback, useEffect, useRef } from 'react' import * as THREE from 'three' -import { UnsignedByteType } from 'three' -import { ssgi } from 'three/addons/tsl/display/SSGINode.js' -import { denoise } from 'three/examples/jsm/tsl/display/DenoiseNode.js' -import { fxaa } from 'three/examples/jsm/tsl/display/FXAANode.js' -import { - convertToTexture, - diffuseColor, - float, - mix, - mrt, - normalView, - output, - pass, - sample, - screenUV, - smoothstep, - uniform, - vec4, -} from 'three/tsl' -import { RenderPipeline, RenderTarget, type WebGPURenderer } from 'three/webgpu' +import type { WebGPURenderer } from 'three/webgpu' import { EDITOR_LAYER } from '../../lib/constants' -const THUMBNAIL_WIDTH = 1920 -const THUMBNAIL_HEIGHT = 1080 - export interface SnapshotCameraData { position: [number, number, number] target: [number, number, number] | null @@ -64,20 +42,7 @@ export const ThumbnailGenerator = ({ onThumbnailCapture }: ThumbnailGeneratorPro const onThumbnailCaptureRef = useRef(onThumbnailCapture) const thumbnailCameraRef = useRef(null) - const pipelineRef = useRef(null) - const renderTargetRef = useRef(null) - - // Backdrop compositing for scene snapshots (studio renders, project - // thumbnails): theme background + sky gradient, same world-ray math as the - // viewport backdrop in viewer's post-processing. Uniform-driven so the one - // cached pipeline serves both opaque and transparent (preset/item) captures. - const bgColorUniform = useRef(uniform(new THREE.Color('#ffffff'))) - const bgSkyUniform = useRef(uniform(new THREE.Color('#ffffff'))) - const bgSkyDeepUniform = useRef(uniform(new THREE.Color('#ffffff'))) - const bgHazeUniform = useRef(uniform(new THREE.Color('#ffffff'))) - const bgProjInvUniform = useRef(uniform(new THREE.Matrix4())) - const bgCamWorldUniform = useRef(uniform(new THREE.Matrix4())) - const bgMixUniform = useRef(uniform(1)) + const pipelineRef = useRef(null) useEffect(() => { onThumbnailCaptureRef.current = onThumbnailCapture @@ -93,116 +58,24 @@ export const ThumbnailGenerator = ({ onThumbnailCapture }: ThumbnailGeneratorPro let mounted = true const buildPipeline = async () => { - try { - if ((gl as any).init) await (gl as any).init() - if (!mounted) return - - // pass() handles MRT internally for all material types, including custom - // shaders — unlike renderer.setMRT() which crashes on non-NodeMaterials. - // pass() also respects camera.layers, so EDITOR_LAYER + GRID_LAYER objects are filtered. - const scenePass = pass(scene, cam) - scenePass.setMRT( - mrt({ - output, - diffuseColor, - normal: packNormalToRGB(normalView), - }), - ) - - const scenePassColor = scenePass.getTextureNode('output') - const scenePassDepth = scenePass.getTextureNode('depth') - const scenePassNormal = scenePass.getTextureNode('normal') - - scenePass.getTexture('diffuseColor').type = UnsignedByteType - scenePass.getTexture('normal').type = UnsignedByteType - - const sceneNormal = sample((uv) => unpackRGBToNormal(scenePassNormal.sample(uv))) - - const giPass = ssgi(scenePassColor, scenePassDepth, sceneNormal, cam as any) - giPass.sliceCount.value = SSGI_PARAMS.sliceCount - giPass.stepCount.value = SSGI_PARAMS.stepCount - giPass.radius.value = SSGI_PARAMS.radius - giPass.expFactor.value = SSGI_PARAMS.expFactor - giPass.thickness.value = SSGI_PARAMS.thickness - giPass.backfaceLighting.value = SSGI_PARAMS.backfaceLighting - giPass.aoIntensity.value = SSGI_PARAMS.aoIntensity - giPass.giIntensity.value = SSGI_PARAMS.giIntensity - giPass.useLinearThickness.value = SSGI_PARAMS.useLinearThickness - giPass.useScreenSpaceSampling.value = SSGI_PARAMS.useScreenSpaceSampling - giPass.useTemporalFiltering = SSGI_PARAMS.useTemporalFiltering - - // r185: SSGI's AO lives in its own single-channel texture (getAONode) - // rather than the alpha of one packed rgba texture. - const aoTexture = (giPass as any).getAONode() - const aoAsRgb = vec4(aoTexture.r, aoTexture.r, aoTexture.r, float(1)) - const denoisePass = denoise(aoAsRgb, scenePassDepth, sceneNormal, cam) - denoisePass.index.value = 0 - denoisePass.radius.value = 4 - - // Same far-field AO fade as the viewport pipeline — without it the - // horizon picks up a visible AO line in captures. - const aoFarFade = smoothstep( - float(0.9994), - float(0.9998), - scenePassDepth.sample(screenUV).r, - ) - const ao = mix((denoisePass as any).r, float(1), aoFarFade) - const sceneRgb = scenePassColor.rgb.mul(ao) - - // Per-pixel world ray from the capture camera → sky gradient above the - // horizon (dir.y = 0), flat background below — mirrors the viewport - // backdrop. bgMix 0 bypasses it and keeps the capture transparent. - const ndc = vec4( - screenUV.x.mul(2).sub(1), - float(1).sub(screenUV.y).mul(2).sub(1), - 1, - 1, - ) as any - const viewRay = (bgProjInvUniform.current as any).mul(ndc) - const worldDir = (bgCamWorldUniform.current as any) - .mul(vec4(viewRay.xyz, 0)) - .xyz.normalize() - const bgGradient = backdropGradient({ - dirY: worldDir.y, - background: bgColorUniform.current, - haze: bgHazeUniform.current, - sky: bgSkyUniform.current, - skyDeep: bgSkyDeepUniform.current, - }) - const alpha = scenePassColor.a - const finalOutput = vec4( - mix(sceneRgb, mix(bgGradient, sceneRgb, alpha), bgMixUniform.current), - mix(alpha, float(1), bgMixUniform.current), - ) - - // FXAA requires a texture node as input; convertToTexture renders finalOutput - // into an intermediate RT so FXAA can sample it with neighbour UV offsets. - const aaOutput = fxaa(convertToTexture(finalOutput)) - - const pipeline = new RenderPipeline(gl as unknown as WebGPURenderer) - pipeline.outputNode = aaOutput - pipelineRef.current = pipeline - - // Dedicated render target — pipeline outputs here instead of the canvas, - // so R3F's main render loop can never overwrite our capture. - const { width, height } = gl.domElement - renderTargetRef.current = new RenderTarget(width, height, { depthBuffer: true }) - } catch (error) { - console.error( - '[thumbnail] Failed to build post-processing pipeline, will use fallback render.', - error, - ) + const pipeline = await createSnapshotPipeline({ + renderer: gl as unknown as WebGPURenderer, + scene, + camera: cam, + }) + if (!mounted) { + pipeline?.dispose() + return } + pipelineRef.current = pipeline } - buildPipeline() + void buildPipeline() return () => { mounted = false pipelineRef.current?.dispose() pipelineRef.current = null - renderTargetRef.current?.dispose() - renderTargetRef.current = null } }, [gl, scene]) @@ -242,16 +115,15 @@ export const ThumbnailGenerator = ({ onThumbnailCapture }: ThumbnailGeneratorPro // uniforms below. thumbnailCamera.updateMatrixWorld() - const theme = getSceneTheme(useViewer.getState().sceneTheme) - bgColorUniform.current.value.set(theme.background) - bgSkyUniform.current.value.set(theme.backgroundSky ?? theme.background) - bgSkyDeepUniform.current.value.set(deepSkyColor(theme.backgroundSky ?? theme.background)) - bgHazeUniform.current.value.set( - horizonHazeColor(theme.backgroundSky ?? theme.background, theme.appearance), - ) - bgProjInvUniform.current.value.copy(thumbnailCamera.projectionMatrixInverse) - bgCamWorldUniform.current.value.copy(thumbnailCamera.matrixWorld) - bgMixUniform.current.value = transparent ? 0 : 1 + const pipeline = pipelineRef.current + pipeline?.applyEnvironment({ + theme: useViewer.getState().sceneTheme, + transparent, + grade: useViewer.getState().shading === 'rendered', + // Preset/item captures stay clean; scene captures mirror the canvas. + edges: transparent ? 'off' : useViewer.getState().edges, + camera: thumbnailCamera, + }) // Capture camera data for snapshot storage const pos = mainCamera.position @@ -286,166 +158,66 @@ export const ThumbnailGenerator = ({ onThumbnailCapture }: ThumbnailGeneratorPro // are registered. Spawn renders on SCENE_LAYER for occlusion, so the // thumbnail camera's layer mask can't filter it either. Returns a // function that restores the original visibility. - const restoreNodeVisibility = (() => { - const saved = new Map() - for (const type of ['scan', 'guide', 'spawn'] as const) { - const ids = sceneRegistry.byType[type]! - ids.forEach((id) => { - const node = sceneRegistry.nodes.get(id) - if (node) { - saved.set(node, node.visible) - node.visible = false - } + const restoreNodeVisibility = temporarilyHideNodeTypes(['scan', 'guide', 'spawn']) + + // Auto-save shots don't copy the user's mid-edit camera — they re-pose + // onto the same computed hero angle the published thumbnail uses, so a + // project's card never shows a half-zoomed working view. Measured after + // the level snap so stacked positions frame correctly. User-driven + // captures (captureMode set) keep the exact viewport pose. + if (snapLevels) { + const framing = computeHeroFraming() + if (framing) { + const pose = heroCameraPose({ + boxes: framing.boxes, + aim: framing.aim, + azimuthRad: framing.azimuthRad, + aspect: width / height, }) - } - return () => { - saved.forEach((wasVisible, node) => { - node.visible = wasVisible + thumbnailCamera.position.set(pose.position[0], pose.position[1], pose.position[2]) + thumbnailCamera.lookAt(pose.target[0], pose.target[1], pose.target[2]) + thumbnailCamera.updateMatrixWorld() + pipeline?.applyEnvironment({ + theme: useViewer.getState().sceneTheme, + transparent, + grade: useViewer.getState().shading === 'rendered', + edges: transparent ? 'off' : useViewer.getState().edges, + camera: thumbnailCamera, }) + cameraData.position = pose.position + cameraData.target = pose.target } - })() + } let blob: Blob - if (pipelineRef.current && renderTargetRef.current) { - const rt = renderTargetRef.current - - // Resize RT if the canvas dimensions changed - if (rt.width !== width || rt.height !== height) { - rt.setSize(width, height) - } - - const renderer = gl as unknown as WebGPURenderer + if (pipeline) { + let capturePromise: ReturnType // Notify other systems (wall cutouts, selection manager) to restore // their overrides before capture and re-apply them after. try { emitter.emit('thumbnail:before-capture', undefined) - ;(renderer as any).setClearAlpha(0) - renderer.setRenderTarget(rt) - pipelineRef.current.render() + capturePromise = pipeline.capture({ + captureMode, + cropRegion, + standardSize, + }) } finally { // Restore level positions, levelMode, and node visibility immediately // after the render — before the async GPU readback. Runs in `finally` // so a render failure can't leave helpers permanently hidden. - renderer.setRenderTarget(null) emitter.emit('thumbnail:after-capture', undefined) restoreLevels() restoreLevelMode?.() restoreNodeVisibility() } - // Read pixels from the RT asynchronously. - // WebGPU copyTextureToBuffer aligns each row to 256 bytes, so we must - // depad the rows before constructing ImageData. - const pixels = (await (renderer as any).readRenderTargetPixelsAsync( - rt, - 0, - 0, - width, - height, - )) as Uint8Array - - const actualBytesPerRow = width * 4 - const tightTotal = actualBytesPerRow * height - const paddedBytesPerRow = Math.ceil(actualBytesPerRow / 256) * 256 - // Two readback shapes to handle: - // - WebGPU (`copyTextureToBuffer`): top-down + 256-byte row padding - // when width*4 isn't already a multiple of 256. - // - WebGL2 fallback (iOS Chrome, etc.): tightly-packed but bottom-up - // (OpenGL framebuffer convention). - // `isWebGPURenderer` lies — it stays true even when the renderer - // falls back to the WebGL backend. Inspect the actual backend - // instead (presence of a GPU device, or backend constructor name). - const backend = (renderer as any).backend - const isWebGPU = - !!backend?.device || - backend?.isWebGPUBackend === true || - backend?.constructor?.name === 'WebGPUBackend' - let tightPixels: Uint8ClampedArray - if (isWebGPU) { - // WebGPU: depad rows if needed; orientation is already top-down. - if (paddedBytesPerRow === actualBytesPerRow) { - tightPixels = new Uint8ClampedArray( - pixels.buffer, - pixels.byteOffset, - Math.min(pixels.byteLength, tightTotal), - ) - } else { - tightPixels = new Uint8ClampedArray(tightTotal) - for (let row = 0; row < height; row++) { - tightPixels.set( - pixels.subarray( - row * paddedBytesPerRow, - row * paddedBytesPerRow + actualBytesPerRow, - ), - row * actualBytesPerRow, - ) - } - } - } else { - // WebGL2: tight buffer in bottom-up order — flip rows. - tightPixels = new Uint8ClampedArray(tightTotal) - for (let row = 0; row < height; row++) { - const srcStart = (height - 1 - row) * actualBytesPerRow - tightPixels.set( - pixels.subarray(srcStart, srcStart + actualBytesPerRow), - row * actualBytesPerRow, - ) - } - } - - const imageData = new ImageData( - tightPixels as unknown as Uint8ClampedArray, - width, - height, - ) - const srcCanvas = new OffscreenCanvas(width, height) - srcCanvas.getContext('2d')!.putImageData(imageData, 0, 0) - - let outW: number - let outH: number - - if (captureMode === 'viewport') { - outW = width - outH = height - const offscreen = new OffscreenCanvas(outW, outH) - offscreen.getContext('2d')!.drawImage(srcCanvas, 0, 0) - blob = await offscreen.convertToBlob({ type: 'image/png' }) - } else if (captureMode === 'area' && cropRegion) { - const sx = Math.round(cropRegion.x * width) - const sy = Math.round(cropRegion.y * height) - outW = Math.round(cropRegion.width * width) - outH = Math.round(cropRegion.height * height) - const offscreen = new OffscreenCanvas(outW, outH) - offscreen.getContext('2d')!.drawImage(srcCanvas, sx, sy, outW, outH, 0, 0, outW, outH) - blob = await offscreen.convertToBlob({ type: 'image/png' }) - } else { - // Standard: center-crop to the requested aspect (default 1920×1080) - const srcAspect = width / height - const dstAspect = standardW / standardH - let sx = 0, - sy = 0, - sWidth = width, - sHeight = height - if (srcAspect > dstAspect) { - sWidth = Math.round(height * dstAspect) - sx = Math.round((width - sWidth) / 2) - } else if (srcAspect < dstAspect) { - sHeight = Math.round(width / dstAspect) - sy = Math.round((height - sHeight) / 2) - } - outW = standardW - outH = standardH - const offscreen = new OffscreenCanvas(outW, outH) - offscreen - .getContext('2d')! - .drawImage(srcCanvas, sx, sy, sWidth, sHeight, 0, 0, outW, outH) - blob = await offscreen.convertToBlob({ type: 'image/png' }) - } + const result = await capturePromise + blob = result.blob if (captureMode !== undefined) cameraData.captureMode = captureMode - cameraData.resolution = { w: outW, h: outH } + cameraData.resolution = { w: result.outW, h: result.outH } } else { // Fallback: plain render directly to the canvas try { diff --git a/packages/editor/src/index.tsx b/packages/editor/src/index.tsx index c9a0c81e7..88d2890a3 100644 --- a/packages/editor/src/index.tsx +++ b/packages/editor/src/index.tsx @@ -26,6 +26,7 @@ export { default as Editor } from './components/editor' // surface uses the shorter, shell-friendly names from the unified // preset-system spec. export { BakeExporter } from './components/editor/bake-exporter' +export { BakeThumbnail } from './components/editor/bake-thumbnail' export { FirstPersonControls } from './components/editor/first-person-controls' export { FloatingActionMenu as FloatingMenu } from './components/editor/floating-action-menu' // Embed surface — the editor's real in-canvas affordances, so a host can mount diff --git a/packages/viewer/src/index.ts b/packages/viewer/src/index.ts index 8aba75c91..316d3a46c 100644 --- a/packages/viewer/src/index.ts +++ b/packages/viewer/src/index.ts @@ -74,6 +74,14 @@ export { SUBTRACTION, } from './lib/csg-utils' export type { EdgeMode } from './lib/edge-style' +export { + computeHeroFraming, + DEFAULT_FRAMING_EXCLUDED_TYPES, + type HeroFraming, + heroCameraPose, + temporarilyHideNodeTypes, + unionRegisteredNodeBounds, +} from './lib/hero-pose' export { applyIsolation, clearIsolation, @@ -119,6 +127,16 @@ export { SCENE_THEMES, type SceneTheme, } from './lib/scene-themes' +export { + createSnapshotPipeline, + type SnapshotCaptureMode, + type SnapshotCaptureResult, + type SnapshotCropRegion, + type SnapshotPipeline, + type SnapshotSize, + THUMBNAIL_HEIGHT, + THUMBNAIL_WIDTH, +} from './lib/snapshot-pipeline' export { getPascalTextureRef, type PascalTextureColorSpace, diff --git a/packages/viewer/src/lib/hero-pose.ts b/packages/viewer/src/lib/hero-pose.ts new file mode 100644 index 000000000..bdcd768d0 --- /dev/null +++ b/packages/viewer/src/lib/hero-pose.ts @@ -0,0 +1,275 @@ +import { sceneRegistry, useScene } from '@pascal-app/core' +import { Box3, type Object3D, Vector3 } from 'three' + +export const DEFAULT_FRAMING_EXCLUDED_TYPES = ['site', 'scan', 'guide', 'spawn'] as const + +export function heroCameraPose({ + boxes, + aspect, + aim, + fovDeg = 60, + azimuthRad = Math.PI / 4, + elevationRad = (13 * Math.PI) / 180, + padding = 1.03, + minDistance = 4, + frameShift = 0, +}: { + boxes: Box3 | readonly Box3[] + aspect: number + /** Where the camera looks (frame center). Defaults to the union-box center; + * pass the building's center to keep it dead-center while outlying boxes + * (lot plate, far palms) simply take asymmetric margin — the corner fit + * still guarantees every box stays in frame. */ + aim?: [number, number, number] + fovDeg?: number + azimuthRad?: number + elevationRad?: number + padding?: number + minDistance?: number + /** Fraction of the frustum half-height to drop the aim by. 0 keeps the aim + * (the building center) exactly at frame center. */ + frameShift?: number +}): { + position: [number, number, number] + target: [number, number, number] +} { + const list = Array.isArray(boxes) ? (boxes as readonly Box3[]) : [boxes as Box3] + const union = new Box3() + for (const box of list) union.union(box) + const center = aim ? new Vector3(aim[0], aim[1], aim[2]) : union.getCenter(new Vector3()) + const tanVertical = Math.tan(((fovDeg / 2) * Math.PI) / 180) + const tanHorizontal = tanVertical * aspect + + // Camera basis for the chosen azimuth/elevation: `dir` points from the + // target toward the camera, `forward` is the view direction. + const dir = new Vector3( + Math.sin(azimuthRad) * Math.cos(elevationRad), + Math.sin(elevationRad), + Math.cos(azimuthRad) * Math.cos(elevationRad), + ) + const forward = dir.clone().negate() + const right = new Vector3().crossVectors(forward, new Vector3(0, 1, 0)).normalize() + const up = new Vector3().crossVectors(right, forward) + + // Exact fit: for every corner of every per-node box, the minimal camera + // distance along `dir` that keeps it inside the frustum. Fitting the corners + // of the UNION box instead reads far too wide at a diagonal azimuth — the + // union AABB's extreme corners are the empty diamond tips nothing actually + // occupies. (A bounding-sphere fit is worse still for flat, spread scenes.) + const corner = new Vector3() + const offset = new Vector3() + let distance = minDistance + for (const box of list) { + for (const x of [box.min.x, box.max.x]) { + for (const y of [box.min.y, box.max.y]) { + for (const z of [box.min.z, box.max.z]) { + corner.set(x, y, z) + offset.subVectors(corner, center) + const lateral = offset.dot(right) + const vertical = offset.dot(up) + const depth = offset.dot(forward) + distance = Math.max( + distance, + (Math.abs(lateral) / tanHorizontal - depth) * padding, + (Math.abs(vertical) / tanVertical - depth) * padding, + ) + } + } + } + } + + // Reframe: slide aim and camera together along the view-up axis — pure + // composition shift, no perspective change. + const drop = up.clone().multiplyScalar(-frameShift * distance * tanVertical) + const target = center.clone().add(drop) + + return { + position: [ + target.x + dir.x * distance, + target.y + dir.y * distance, + target.z + dir.z * distance, + ], + target: [target.x, target.y, target.z], + } +} + +export function unionRegisteredNodeBounds({ + excludeTypes, +}: { + excludeTypes: readonly string[] +}): Box3 | null { + const excluded = new Set(excludeTypes) + const result = new Box3() + + for (const [type, ids] of Object.entries(sceneRegistry.byType)) { + if (excluded.has(type)) continue + + for (const id of ids) { + const object = sceneRegistry.nodes.get(id) + if (!object) continue + const bounds = new Box3().setFromObject(object) + if (!bounds.isEmpty()) result.union(bounds) + } + } + + return result.isEmpty() ? null : result +} + +function perNodeBoxes(excludeTypes: readonly string[]): Box3[] { + const excluded = new Set(excludeTypes) + const boxes: Box3[] = [] + for (const [type, ids] of Object.entries(sceneRegistry.byType)) { + if (excluded.has(type)) continue + for (const id of ids) { + const object = sceneRegistry.nodes.get(id) + if (!object) continue + const bounds = new Box3().setFromObject(object) + if (!bounds.isEmpty()) boxes.push(bounds) + } + } + return boxes +} + +/** + * Matches the `SiteNode` bootstrap polygon (a 30×30 square at the origin) — + * same rule as the editor's `computeSceneBoundsXZ`: an untouched default lot + * says nothing about the user's intent, so it shouldn't drive framing. + */ +function isDefaultSitePolygon(points: unknown[]): boolean { + if (points.length !== 4) return false + const expected: [number, number][] = [ + [-15, -15], + [15, -15], + [15, 15], + [-15, 15], + ] + for (let i = 0; i < 4; i++) { + const p = points[i] + const e = expected[i]! + if (!Array.isArray(p) || p.length < 2) return false + if (p[0] !== e[0] || p[1] !== e[1]) return false + } + return true +} + +/** Flat boxes for intentionally-shaped site plates, from scene DATA — the + * site's Object3D can't be measured (it carries the ±400 m horizon disc). */ +function sitePlateBoxes(): Box3[] { + const boxes: Box3[] = [] + for (const node of Object.values(useScene.getState().nodes)) { + if ((node as { type?: string }).type !== 'site') continue + const polygon = (node as { polygon?: { points?: unknown[] } }).polygon + const points = polygon?.points + if (!Array.isArray(points) || isDefaultSitePolygon(points)) continue + const box = new Box3() + for (const point of points) { + if (Array.isArray(point) && point.length >= 2) { + box.expandByPoint(new Vector3(Number(point[0]), 0, Number(point[1]))) + } + } + if (!box.isEmpty()) boxes.push(box) + } + return boxes +} + +/** Dominant wall direction folded to a 90°-periodic axis (length-weighted + * vector sum of 4θ), so the hero azimuth sits at a true 45° to the building's + * facades no matter how the user rotated their plan. */ +function dominantWallYaw(): number | null { + let vx = 0 + let vz = 0 + for (const node of Object.values(useScene.getState().nodes)) { + if ((node as { type?: string }).type !== 'wall') continue + const { start, end } = node as { start?: unknown; end?: unknown } + if (!(Array.isArray(start) && Array.isArray(end))) continue + const dx = Number(end[0]) - Number(start[0]) + const dz = Number(end[1]) - Number(start[1]) + const length = Math.hypot(dx, dz) + if (!(Number.isFinite(length) && length > 0.01)) continue + const theta = Math.atan2(dz, dx) + vx += length * Math.cos(4 * theta) + vz += length * Math.sin(4 * theta) + } + if (vx === 0 && vz === 0) return null + return Math.atan2(vz, vx) / 4 +} + +export type HeroFraming = { + /** Fit constraints — everything that must stay in frame. */ + boxes: Box3[] + /** Frame center: plate/structure center on XZ, building center on Y. */ + aim: [number, number, number] + /** 45° to the dominant facade (falls back to world 45°). */ + azimuthRad: number +} + +/** + * Framing for a scene hero shot. The base set is the built structure (walls, + * roofs, slabs, …) plus any intentionally-shaped site plate; an item joins + * only when it sits near that base — one palm at the far lot corner must not + * shrink the building. `building`/`level` container groups are skipped (their + * Object3D holds the whole subtree). The aim keeps the building dead-center: + * XZ from the plate+structure union, Y from the structure alone so the + * building — not the ground — is vertically centered. Scenes with no + * structure (pure furniture arrangements) fall back to framing every + * non-helper node. + */ +export function computeHeroFraming(): HeroFraming | null { + const structural = perNodeBoxes([...DEFAULT_FRAMING_EXCLUDED_TYPES, 'item', 'building', 'level']) + if (structural.length === 0) { + const all = perNodeBoxes([...DEFAULT_FRAMING_EXCLUDED_TYPES, 'building', 'level']) + if (all.length === 0) return null + const union = new Box3() + for (const box of all) union.union(box) + const center = union.getCenter(new Vector3()) + return { boxes: all, aim: [center.x, center.y, center.z], azimuthRad: Math.PI / 4 } + } + + const structuralUnion = new Box3() + for (const box of structural) structuralUnion.union(box) + + const plates = sitePlateBoxes() + const groundUnion = structuralUnion.clone() + for (const box of plates) groundUnion.union(box) + + const nearby = groundUnion + .clone() + .expandByVector(groundUnion.getSize(new Vector3()).multiplyScalar(0.15)) + + const boxes = [...structural, ...plates] + const itemIds = sceneRegistry.byType.item! + for (const id of itemIds) { + const object = sceneRegistry.nodes.get(id) + if (!object) continue + const bounds = new Box3().setFromObject(object) + if (!bounds.isEmpty() && nearby.intersectsBox(bounds)) boxes.push(bounds) + } + + const groundCenter = groundUnion.getCenter(new Vector3()) + const structuralCenter = structuralUnion.getCenter(new Vector3()) + const yaw = dominantWallYaw() + return { + boxes, + aim: [groundCenter.x, structuralCenter.y, groundCenter.z], + azimuthRad: Math.PI / 4 - (yaw ?? 0), + } +} + +export function temporarilyHideNodeTypes(types: readonly string[]): () => void { + const saved = new Map() + for (const type of types) { + const ids = sceneRegistry.byType[type]! + ids.forEach((id) => { + const node = sceneRegistry.nodes.get(id) + if (node) { + saved.set(node, node.visible) + node.visible = false + } + }) + } + return () => { + saved.forEach((wasVisible, node) => { + node.visible = wasVisible + }) + } +} diff --git a/packages/viewer/src/lib/snapshot-pipeline.ts b/packages/viewer/src/lib/snapshot-pipeline.ts new file mode 100644 index 000000000..44d8488c2 --- /dev/null +++ b/packages/viewer/src/lib/snapshot-pipeline.ts @@ -0,0 +1,379 @@ +import { type Camera, Color, Matrix4, type Scene, UnsignedByteType } from 'three' +import { ssgi } from 'three/addons/tsl/display/SSGINode.js' +import { denoise } from 'three/examples/jsm/tsl/display/DenoiseNode.js' +import { fxaa } from 'three/examples/jsm/tsl/display/FXAANode.js' +import { + convertToTexture, + diffuseColor, + float, + mix, + mrt, + normalView, + output, + pass, + sample, + saturation, + screenUV, + smoothstep, + uniform, + vec3, + vec4, +} from 'three/tsl' +import { RenderPipeline, RenderTarget, type WebGPURenderer } from 'three/webgpu' +import { GRADE_PARAMS, SSGI_PARAMS } from '../components/viewer/post-processing' +import { backdropGradient, deepSkyColor, horizonHazeColor } from './backdrop' +import { type EdgeMode, edgeColorFor, edgeOpacityScaleFor } from './edge-style' +import { inkedEdges } from './ink-edges' +import { getSceneTheme } from './scene-themes' +import { packNormalToRGB, unpackRGBToNormal } from './tsl-compat' + +export const THUMBNAIL_WIDTH = 1920 +export const THUMBNAIL_HEIGHT = 1080 + +export type SnapshotCaptureMode = 'standard' | 'viewport' | 'area' + +export type SnapshotCropRegion = { + x: number + y: number + width: number + height: number +} + +export type SnapshotSize = { + w: number + h: number +} + +export type SnapshotCaptureResult = { + blob: Blob + outW: number + outH: number +} + +export type SnapshotPipeline = { + applyEnvironment: ({ + theme, + transparent, + grade, + edges, + camera, + }: { + theme: string + transparent: boolean + grade: boolean + edges: EdgeMode + camera: Camera + }) => void + capture: ({ + captureMode, + cropRegion, + standardSize, + }: { + captureMode?: SnapshotCaptureMode + cropRegion?: SnapshotCropRegion + standardSize?: SnapshotSize + }) => Promise + dispose: () => void +} + +export async function createSnapshotPipeline({ + renderer, + scene, + camera, +}: { + renderer: WebGPURenderer + scene: Scene + camera: Camera +}): Promise { + try { + if ((renderer as any).init) await (renderer as any).init() + + // Backdrop compositing for scene snapshots (studio renders, project + // thumbnails): theme background + sky gradient, same world-ray math as the + // viewport backdrop in viewer's post-processing. Uniform-driven so the one + // cached pipeline serves both opaque and transparent (preset/item) captures. + const bgColorUniform = uniform(new Color('#ffffff')) + const bgSkyUniform = uniform(new Color('#ffffff')) + const bgSkyDeepUniform = uniform(new Color('#ffffff')) + const bgHazeUniform = uniform(new Color('#ffffff')) + const bgProjInvUniform = uniform(new Matrix4()) + const bgCamWorldUniform = uniform(new Matrix4()) + const bgMixUniform = uniform(1) + const gradeMixUniform = uniform(0) + const inkMixUniform = uniform(0) + const inkColorUniform = uniform(new Color('#1a1d24')) + const inkOpacityUniform = uniform(0.5) + const inkOpacityScaleUniform = uniform(1) + + // pass() handles MRT internally for all material types, including custom + // shaders — unlike renderer.setMRT() which crashes on non-NodeMaterials. + // pass() also respects camera.layers, so caller-disabled objects are filtered. + const scenePass = pass(scene, camera) + scenePass.setMRT( + mrt({ + output, + diffuseColor, + normal: packNormalToRGB(normalView), + }), + ) + + const scenePassColor = scenePass.getTextureNode('output') + const scenePassDepth = scenePass.getTextureNode('depth') + const scenePassNormal = scenePass.getTextureNode('normal') + + scenePass.getTexture('diffuseColor').type = UnsignedByteType + scenePass.getTexture('normal').type = UnsignedByteType + + const sceneNormal = sample((uv) => unpackRGBToNormal(scenePassNormal.sample(uv))) + + const giPass = ssgi(scenePassColor, scenePassDepth, sceneNormal, camera as any) + giPass.sliceCount.value = SSGI_PARAMS.sliceCount + giPass.stepCount.value = SSGI_PARAMS.stepCount + giPass.radius.value = SSGI_PARAMS.radius + giPass.expFactor.value = SSGI_PARAMS.expFactor + giPass.thickness.value = SSGI_PARAMS.thickness + giPass.backfaceLighting.value = SSGI_PARAMS.backfaceLighting + giPass.aoIntensity.value = SSGI_PARAMS.aoIntensity + giPass.giIntensity.value = SSGI_PARAMS.giIntensity + giPass.useLinearThickness.value = SSGI_PARAMS.useLinearThickness + giPass.useScreenSpaceSampling.value = SSGI_PARAMS.useScreenSpaceSampling + giPass.useTemporalFiltering = SSGI_PARAMS.useTemporalFiltering + + // r185: SSGI's AO lives in its own single-channel texture (getAONode) + // rather than the alpha of one packed rgba texture. + const aoTexture = (giPass as any).getAONode() + const aoAsRgb = vec4(aoTexture.r, aoTexture.r, aoTexture.r, float(1)) + const denoisePass = denoise(aoAsRgb, scenePassDepth, sceneNormal, camera) + denoisePass.index.value = 0 + denoisePass.radius.value = 4 + + // Same far-field AO fade as the viewport pipeline — without it the + // horizon picks up a visible AO line in captures. + const aoFarFade = smoothstep(float(0.9994), float(0.9998), scenePassDepth.sample(screenUV).r) + const ao = mix((denoisePass as any).r, float(1), aoFarFade) + const aoRgb = scenePassColor.rgb.mul(ao) + + // Ink edges, mirroring the viewport pipeline (AO → ink → grade) so + // captures carry the same soft/strong edge look the canvas shows. + // Uniform-gated like grade/backdrop: one cached pipeline serves all modes. + // Radius scales with render height: a supersampled capture (4K → 1080p) + // would otherwise halve the apparent line weight vs the viewport's 1px. + const inkRadius = Math.max(1, Math.round(renderer.domElement.height / 1080)) + const inkedRgb = inkedEdges({ + sceneRgb: aoRgb, + depthTex: scenePassDepth, + normalTex: scenePassNormal, + inkColor: inkColorUniform, + radius: inkRadius, + opacity: float(inkOpacityUniform).mul(inkOpacityScaleUniform), + }) + const ungradedSceneRgb = mix(aoRgb, inkedRgb, inkMixUniform) + const gradeRgb = (rgb: any) => + saturation(rgb.div(0.18).pow(vec3(GRADE_PARAMS.contrast)).mul(0.18), GRADE_PARAMS.saturation) + const sceneRgb = mix(ungradedSceneRgb, gradeRgb(ungradedSceneRgb), gradeMixUniform) + + // Per-pixel world ray from the capture camera → sky gradient above the + // horizon (dir.y = 0), flat background below — mirrors the viewport + // backdrop. bgMix 0 bypasses it and keeps the capture transparent. + const ndc = vec4(screenUV.x.mul(2).sub(1), float(1).sub(screenUV.y).mul(2).sub(1), 1, 1) as any + const viewRay = (bgProjInvUniform as any).mul(ndc) + const worldDir = (bgCamWorldUniform as any).mul(vec4(viewRay.xyz, 0)).xyz.normalize() + const ungradedBgGradient = backdropGradient({ + dirY: worldDir.y, + background: bgColorUniform, + haze: bgHazeUniform, + sky: bgSkyUniform, + skyDeep: bgSkyDeepUniform, + }) + const bgGradient = mix(ungradedBgGradient, gradeRgb(ungradedBgGradient), gradeMixUniform) + const alpha = scenePassColor.a + const finalOutput = vec4( + mix(sceneRgb, mix(bgGradient, sceneRgb, alpha), bgMixUniform), + mix(alpha, float(1), bgMixUniform), + ) + + // FXAA requires a texture node as input; convertToTexture renders finalOutput + // into an intermediate RT so FXAA can sample it with neighbour UV offsets. + const aaOutput = fxaa(convertToTexture(finalOutput)) + + const pipeline = new RenderPipeline(renderer) + pipeline.outputNode = aaOutput + + // Dedicated render target — pipeline outputs here instead of the canvas, + // so R3F's main render loop can never overwrite our capture. + const { width, height } = renderer.domElement + const renderTarget = new RenderTarget(width, height, { depthBuffer: true }) + + return { + applyEnvironment: ({ theme, transparent, grade, edges, camera: captureCamera }) => { + const sceneTheme = getSceneTheme(theme) + inkMixUniform.value = edges === 'off' ? 0 : 1 + inkOpacityUniform.value = edges === 'strong' ? 1 : 0.5 + inkColorUniform.value.set(edgeColorFor(sceneTheme.background)) + inkOpacityScaleUniform.value = edgeOpacityScaleFor(sceneTheme.background) + bgColorUniform.value.set(sceneTheme.background) + bgSkyUniform.value.set(sceneTheme.backgroundSky ?? sceneTheme.background) + bgSkyDeepUniform.value.set(deepSkyColor(sceneTheme.backgroundSky ?? sceneTheme.background)) + bgHazeUniform.value.set( + horizonHazeColor( + sceneTheme.backgroundSky ?? sceneTheme.background, + sceneTheme.appearance, + ), + ) + bgMixUniform.value = transparent ? 0 : 1 + gradeMixUniform.value = grade ? 1 : 0 + + // The capture camera never joins the scene graph, so its matrixWorld + // is only refreshed by the render itself — too late for the backdrop + // uniforms below. + captureCamera.updateMatrixWorld() + bgProjInvUniform.value.copy(captureCamera.projectionMatrixInverse) + bgCamWorldUniform.value.copy(captureCamera.matrixWorld) + }, + capture: async ({ captureMode, cropRegion, standardSize }) => { + const standardW = standardSize?.w ?? THUMBNAIL_WIDTH + const standardH = standardSize?.h ?? THUMBNAIL_HEIGHT + const { width: captureWidth, height: captureHeight } = renderer.domElement + + // Resize RT if the canvas dimensions changed + if (renderTarget.width !== captureWidth || renderTarget.height !== captureHeight) { + renderTarget.setSize(captureWidth, captureHeight) + } + + try { + ;(renderer as any).setClearAlpha(0) + renderer.setRenderTarget(renderTarget) + pipeline.render() + } finally { + renderer.setRenderTarget(null) + } + + // Let callers restore visibility and other capture policy immediately + // after the render, before the asynchronous GPU readback begins. + await Promise.resolve() + + // Read pixels from the RT asynchronously. + // WebGPU copyTextureToBuffer aligns each row to 256 bytes, so we must + // depad the rows before constructing ImageData. + const pixels = (await (renderer as any).readRenderTargetPixelsAsync( + renderTarget, + 0, + 0, + captureWidth, + captureHeight, + )) as Uint8Array + + const actualBytesPerRow = captureWidth * 4 + const tightTotal = actualBytesPerRow * captureHeight + const paddedBytesPerRow = Math.ceil(actualBytesPerRow / 256) * 256 + // Two readback shapes to handle: + // - WebGPU (`copyTextureToBuffer`): top-down + 256-byte row padding + // when width*4 isn't already a multiple of 256. + // - WebGL2 fallback (iOS Chrome, etc.): tightly-packed but bottom-up + // (OpenGL framebuffer convention). + // `isWebGPURenderer` lies — it stays true even when the renderer + // falls back to the WebGL backend. Inspect the actual backend + // instead (presence of a GPU device, or backend constructor name). + const backend = (renderer as any).backend + const isWebGPU = + !!backend?.device || + backend?.isWebGPUBackend === true || + backend?.constructor?.name === 'WebGPUBackend' + let tightPixels: Uint8ClampedArray + if (isWebGPU) { + // WebGPU: depad rows if needed; orientation is already top-down. + if (paddedBytesPerRow === actualBytesPerRow) { + tightPixels = new Uint8ClampedArray( + pixels.buffer, + pixels.byteOffset, + Math.min(pixels.byteLength, tightTotal), + ) + } else { + tightPixels = new Uint8ClampedArray(tightTotal) + for (let row = 0; row < captureHeight; row++) { + tightPixels.set( + pixels.subarray( + row * paddedBytesPerRow, + row * paddedBytesPerRow + actualBytesPerRow, + ), + row * actualBytesPerRow, + ) + } + } + } else { + // WebGL2: tight buffer in bottom-up order — flip rows. + tightPixels = new Uint8ClampedArray(tightTotal) + for (let row = 0; row < captureHeight; row++) { + const srcStart = (captureHeight - 1 - row) * actualBytesPerRow + tightPixels.set( + pixels.subarray(srcStart, srcStart + actualBytesPerRow), + row * actualBytesPerRow, + ) + } + } + + const imageData = new ImageData( + tightPixels as unknown as Uint8ClampedArray, + captureWidth, + captureHeight, + ) + const srcCanvas = new OffscreenCanvas(captureWidth, captureHeight) + srcCanvas.getContext('2d')!.putImageData(imageData, 0, 0) + + let outW: number + let outH: number + let blob: Blob + + if (captureMode === 'viewport') { + outW = captureWidth + outH = captureHeight + const offscreen = new OffscreenCanvas(outW, outH) + offscreen.getContext('2d')!.drawImage(srcCanvas, 0, 0) + blob = await offscreen.convertToBlob({ type: 'image/png' }) + } else if (captureMode === 'area' && cropRegion) { + const sx = Math.round(cropRegion.x * captureWidth) + const sy = Math.round(cropRegion.y * captureHeight) + outW = Math.round(cropRegion.width * captureWidth) + outH = Math.round(cropRegion.height * captureHeight) + const offscreen = new OffscreenCanvas(outW, outH) + offscreen.getContext('2d')!.drawImage(srcCanvas, sx, sy, outW, outH, 0, 0, outW, outH) + blob = await offscreen.convertToBlob({ type: 'image/png' }) + } else { + // Standard: center-crop to the requested aspect (default 1920×1080) + const srcAspect = captureWidth / captureHeight + const dstAspect = standardW / standardH + let sx = 0 + let sy = 0 + let sWidth = captureWidth + let sHeight = captureHeight + if (srcAspect > dstAspect) { + sWidth = Math.round(captureHeight * dstAspect) + sx = Math.round((captureWidth - sWidth) / 2) + } else if (srcAspect < dstAspect) { + sHeight = Math.round(captureWidth / dstAspect) + sy = Math.round((captureHeight - sHeight) / 2) + } + outW = standardW + outH = standardH + const offscreen = new OffscreenCanvas(outW, outH) + offscreen + .getContext('2d')! + .drawImage(srcCanvas, sx, sy, sWidth, sHeight, 0, 0, outW, outH) + blob = await offscreen.convertToBlob({ type: 'image/png' }) + } + + return { blob, outW, outH } + }, + dispose: () => { + pipeline.dispose() + renderTarget.dispose() + }, + } + } catch (error) { + console.error( + '[thumbnail] Failed to build post-processing pipeline, will use fallback render.', + error, + ) + return null + } +}