From d9d6c2a82ef2fc18370f5a54d8909742d7c67800 Mon Sep 17 00:00:00 2001 From: venom3-byte Date: Wed, 30 Sep 2026 10:20:14 +0200 Subject: [PATCH 01/86] feat(vision): make continuous live vision and real input authoritative --- .github/workflows/proof.yml | 2 - live-vision.js | 366 ++++++++++++++++++------------------ package.json | 2 +- server.mjs | 35 +--- tests/vision-live.spec.mjs | 115 ----------- tests/vision.live.spec.mjs | 216 +++++++++------------ vision-control.mjs | 211 ++++++++++++++++----- 7 files changed, 437 insertions(+), 510 deletions(-) delete mode 100644 tests/vision-live.spec.mjs diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index feb6781..d0ba430 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -35,8 +35,6 @@ jobs: run: npm run test:professional - name: Live vision + real input acceptance run: npm run test:vision - - name: Live vision + real input acceptance - run: npm run test:vision - name: Khronos glTF validation run: npm run validate:gltf - name: Full regression suite diff --git a/live-vision.js b/live-vision.js index 63549d4..0003537 100644 --- a/live-vision.js +++ b/live-vision.js @@ -2,206 +2,214 @@ const $=(s,r=document)=>r.querySelector(s); const sleep=ms=>new Promise(r=>setTimeout(r,ms)); const state={ - ws:null,stream:null,running:false,busy:false,frameSeq:0,lastFrameAt:0,frameCount:0,frameWindowStart:0, - captureCanvas:null,captureCtx:null,captureTimer:0,remoteUrl:null,mode:"idle",pointerId:11 + ws:null,running:false,busy:false,frameSeq:0,lastFrameAt:0,frameCount:0,frameWindowStart:0, + remoteUrl:null,mode:"idle" }; - const serverCapable=()=>location.protocol!=='file:'&&!/github\.io$/i.test(location.hostname); - const wsUrl=()=>((location.protocol==='https:'?'wss:':'ws:')+'//'+location.host+'/vision'); - function style(){if($('#liveVisionStyle'))return;const l=document.createElement('link');l.id='liveVisionStyle';l.rel='stylesheet';l.href='./live-vision.css?v=3';document.head.appendChild(l)} + + const serverCapable=()=>location.protocol!=="file:"&&!/github\.io$/i.test(location.hostname); + const wsUrl=()=>((location.protocol==="https:"?"wss:":"ws:")+"//"+location.host+"/vision"); + + function style(){ + if($("#liveVisionStyle"))return; + const link=document.createElement("link"); + link.id="liveVisionStyle";link.rel="stylesheet";link.href="./live-vision.css?v=4"; + document.head.appendChild(link); + } + function mount(){ - if($('#liveVision'))return; - const el=document.createElement('section'); - el.id='liveVision'; - el.className='live-vision'; + if($("#liveVision"))return; + const el=document.createElement("section"); + el.id="liveVision";el.className="live-vision"; el.innerHTML=[ - '
FORGE VISIONLive eyes + real hands · Spatial Core
', + '
FORGE VISIONContinuous eyes + real browser hands
', '
Idle0 fps—
', - '
Forge live vision stream
', - '
', - '
', - '', + '
Forge continuous live vision
', + '
', + '
', + '', '
', - '
Direct mode uses the live MediaStream. Pro CDP uses Chromium screencast + CDP mouse/keyboard/touch events; no screenshot request loop is used.
' - ].join(''); + '
Live mode: Playwright continuous screencast over WebSocket. Actions are real Chromium mouse, keyboard, touch and file-input events, followed by a frame receipt.
' + ].join(""); document.body.appendChild(el); - $('#liveVisionClose').onclick=()=>el.classList.remove('open'); - $('#liveVisionStart').onclick=start; - $('#liveVisionPro').onclick=connectPro; - $('#liveVisionStop').onclick=stop; - $('#liveVisionAgent').onclick=runAgent; - $('#liveVisionMove').onclick=()=>runManual({type:'move',x:innerWidth/2,y:innerHeight/2}); - $('#liveVisionDown').onclick=()=>runManual({type:'mouseDown',x:innerWidth/2,y:innerHeight/2}); - $('#liveVisionUp').onclick=()=>runManual({type:'mouseUp',x:innerWidth/2,y:innerHeight/2}); - $('#liveVisionWheel').onclick=()=>runManual({type:'wheel',x:innerWidth/2,y:innerHeight/2,deltaY:360}); - } - function open(){mount();style();$('#liveVision').classList.add('open');connectVisionWs()} - function log(t,c=''){const b=$('#liveVisionLog');if(!b)return;const r=document.createElement('div');r.className='live-log-row '+c;r.textContent=new Date().toLocaleTimeString()+' '+t;b.prepend(r);while(b.children.length>24)b.lastElementChild.remove()} - function stateText(t,on=state.running){const s=$('#liveVisionState'),d=$('#liveVisionDot');if(s)s.textContent=t;if(d)d.classList.toggle('live-on',!!on)} - function setTransport(t){const e=$('#liveVisionTransport');if(e)e.textContent=t||'—'} - function connectVisionWs(){ - if(!serverCapable()||typeof WebSocket==='undefined')return; - if(state.ws&&(state.ws.readyState===0||state.ws.readyState===1))return; - try{ - const ws=new WebSocket(wsUrl()); - ws.binaryType='arraybuffer'; - state.ws=ws; - ws.onopen=()=>{ws.send(JSON.stringify({type:'hello',role:'vision-ui',protocol:4,viewport:{width:innerWidth,height:innerHeight,dpr:devicePixelRatio||1}}));log('Live vision WebSocket connected','ok')}; - ws.onerror=()=>{}; - ws.onclose=()=>{state.ws=null;if(state.mode==='cdp')stateText('CDP disconnected',false)}; - ws.onmessage=onVisionMessage; - }catch{} - } - async function onVisionMessage(e){ - if(typeof e.data!=='string'){ - try{ - const buf=e.data instanceof ArrayBuffer?e.data:await e.data.arrayBuffer(); - if(buf.byteLength<16)return; - const dv=new DataView(buf),magic=dv.getUint32(0),seq=dv.getUint32(4),w=dv.getUint16(8),h=dv.getUint16(10); - if(magic!==0x46563337)return; - const jpeg=buf.slice(16),img=$('#liveVisionRemote');if(!img)return; - const url=URL.createObjectURL(new Blob([jpeg],{type:'image/jpeg'})); - const old=state.remoteUrl;state.remoteUrl=url;img.onload=()=>{if(old)URL.revokeObjectURL(old)};img.src=url; - state.frameSeq=seq;state.frameCount++;const now=Date.now();if(!state.frameWindowStart)state.frameWindowStart=now; - if(now-state.frameWindowStart>1000){$('#liveVisionFps').textContent=Number(state.frameCount*1000/(now-state.frameWindowStart)).toFixed(1)+' fps';state.frameCount=0;state.frameWindowStart=now} - }catch{} - return; - } - let m;try{m=JSON.parse(e.data)}catch{return} - if(m.type==='vision-state'){ - const s=m.state||{}; - if(s.connected&&s.streaming){state.mode='cdp';state.running=true;stateText('Direct Chromium live',true);setTransport('CDP');$('#liveVisionPreview').classList.add('remote');$('#liveVisionPreview').classList.remove('local')} - if(!s.connected&&state.mode==='cdp'){state.mode='idle';state.running=false;stateText('CDP disconnected',false)} - if(s.fps!=null)$('#liveVisionFps').textContent=Number(s.fps).toFixed(1)+' fps'; - } + $("#liveVisionClose").onclick=()=>el.classList.remove("open"); + $("#liveVisionStart").onclick=start; + $("#liveVisionPro").onclick=restart; + $("#liveVisionStop").onclick=stop; + $("#liveVisionAgent").onclick=runAgent; + $("#liveVisionMove").onclick=()=>runManual({type:"mouse.move",x:innerWidth/2,y:innerHeight/2,steps:4}); + $("#liveVisionDown").onclick=()=>runManual({type:"mouse.down",button:"left"}); + $("#liveVisionUp").onclick=()=>runManual({type:"mouse.up",button:"left"}); + $("#liveVisionWheel").onclick=()=>runManual({type:"mouse.wheel",deltaY:360}); } - function normalize(x,y,w,h){return{x:Math.max(0,Math.min(innerWidth-1,(Number(x)||0)*innerWidth/Math.max(1,w||innerWidth))),y:Math.max(0,Math.min(innerHeight-1,(Number(y)||0)*innerHeight/Math.max(1,h||innerHeight)))}} - function dispatchPointer(type,x,y,extra={}){ - const el=document.elementFromPoint(x,y)||document.body; - const buttons=type==='pointerup'?0:(extra.buttons??1); - el.dispatchEvent(new PointerEvent(type,{bubbles:true,cancelable:true,view:window,clientX:x,clientY:y,screenX:x,screenY:y,button:extra.button??0,buttons,pointerId:state.pointerId,pointerType:extra.pointerType||'mouse',isPrimary:true})); - return{tag:el.tagName||'',id:el.id||'',className:typeof el.className==='string'?el.className:''}; - } - async function localControl(a){ - const type=String(a.type||''); - if(type==='move'){const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);return dispatchPointer('pointermove',p.x,p.y,{buttons:a.buttons??0})} - if(type==='mouseDown'){const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);state.pointerId++;return dispatchPointer('pointerdown',p.x,p.y,{buttons:1})} - if(type==='mouseUp'){const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);return dispatchPointer('pointerup',p.x,p.y,{buttons:0})} - if(type==='click'||type==='doubleClick'){ - const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);const el=document.elementFromPoint(p.x,p.y)||document.body;dispatchPointer('pointermove',p.x,p.y,{buttons:0}); - const count=type==='doubleClick'?2:1; - for(let i=0;i24)b.lastElementChild.remove(); + } + + function stateText(t,on=state.running){ + const s=$("#liveVisionState"),d=$("#liveVisionDot"); + if(s)s.textContent=t;if(d)d.classList.toggle("live-on",!!on); + } + + function setTransport(t){const e=$("#liveVisionTransport");if(e)e.textContent=t||"—"} + + function setRemoteFrame(data){ + const img=$("#liveVisionRemote");if(!img)return; + const url=URL.createObjectURL(new Blob([data],{type:"image/jpeg"})); + const old=state.remoteUrl;state.remoteUrl=url; + img.onload=()=>{if(old)URL.revokeObjectURL(old)}; + img.src=url; + $("#liveVisionPreview").classList.add("remote"); + $("#liveVisionPreview").classList.remove("local"); + } + + function onFramePacket(data){ + if(!(data instanceof ArrayBuffer))return; + if(data.byteLength<16)return; + const dv=new DataView(data),magic=dv.getUint32(0); + if(magic!==0x46563337)return; + const seq=dv.getUint32(4),width=dv.getUint16(8),height=dv.getUint16(10); + const jpeg=data.slice(16); + state.frameSeq=Math.max(state.frameSeq,seq);state.lastFrameAt=Date.now();state.frameCount++; + if(!state.frameWindowStart)state.frameWindowStart=Date.now(); + const now=Date.now(); + if(now-state.frameWindowStart>=1000){ + const fps=state.frameCount*1000/(now-state.frameWindowStart); + $("#liveVisionFps").textContent=fps.toFixed(1)+" fps"; + state.frameCount=0;state.frameWindowStart=now; } - if(type==='wheel'){const p=normalize(a.x??innerWidth/2,a.y??innerHeight/2,a.screenWidth,a.screenHeight);const el=document.elementFromPoint(p.x,p.y)||document.body;el.dispatchEvent(new WheelEvent('wheel',{bubbles:true,cancelable:true,clientX:p.x,clientY:p.y,deltaX:a.deltaX||0,deltaY:a.deltaY||0}));return{ok:true,type,point:p}} - if(type==='press'||type==='keyDown'||type==='keyUp'||type==='typeText'){ - const target=document.activeElement||document.body; - if(type==='typeText'){target.dispatchEvent(new InputEvent('beforeinput',{bubbles:true,inputType:'insertText',data:String(a.text||'')}));if('value' in target){target.value+=String(a.text||'');target.dispatchEvent(new Event('input',{bubbles:true}))}return{ok:true,type}} - const evName=type==='keyUp'?'keyup':'keydown';target.dispatchEvent(new KeyboardEvent(evName,{bubbles:true,cancelable:true,key:String(a.key||''),code:a.code||'',ctrlKey:!!a.ctrlKey,shiftKey:!!a.shiftKey,altKey:!!a.altKey,metaKey:!!a.metaKey}));if(type==='press')target.dispatchEvent(new KeyboardEvent('keyup',{bubbles:true,key:String(a.key||''),code:a.code||''}));return{ok:true,type,key:a.key}; + setRemoteFrame(jpeg); + $("#liveVisionRemote").setAttribute("data-size",width+"x"+height); + } + + function onMessage(event){ + if(typeof event.data!=="string"){onFramePacket(event.data);return} + let msg;try{msg=JSON.parse(event.data)}catch{return} + if(msg.type==="vision-status"||msg.type==="vision-state"){ + const s=msg.state||msg; + const running=!!s.running; + state.running=running;state.mode=running?"server":"idle"; + state.frameSeq=Math.max(state.frameSeq,Number(s.frameSeq)||0); + state.lastFrameAt=Number(s.lastFrameAt)||state.lastFrameAt; + if(s.fps!=null)$("#liveVisionFps").textContent=Number(s.fps).toFixed(1)+" fps"; + stateText(running?"Live Chromium stream":"Idle",running); + setTransport(running?"WebSocket / Playwright":"—"); } - if(type==='touchStart'||type==='touchMove'||type==='touchEnd'){ - const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight),el=document.elementFromPoint(p.x,p.y)||document.body; - const eventType=type==='touchStart'?'pointerdown':type==='touchMove'?'pointermove':'pointerup'; - return dispatchPointer(eventType,p.x,p.y,{buttons:type==='touchEnd'?0:1,pointerType:'touch'}); + if(msg.type==="session-result"&&msg.ok){ + state.running=true;state.mode="server";stateText("Live Chromium stream",true);setTransport("WebSocket / Playwright"); + log("Vision session started","ok"); } - throw new Error('Unsupported local control action: '+type); + if(msg.type==="action-result"&&!msg.ok)log(msg.error||"Vision action failed","error"); } - async function runManual(action){try{const result=await control(action);log(JSON.stringify(result),'ok')}catch(e){log(e.message||String(e),'error')}} - async function control(action){ - if(state.mode==='cdp'&&state.ws?.readyState===1){ - const r=await fetch('./api/vision/control',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({action})}); - const j=await r.json();if(!j.ok)throw new Error(j.error||'CDP control failed');return j.result; - } - return localControl(action); + + function connectWs(){ + if(!serverCapable()||typeof WebSocket==="undefined")return; + if(state.ws&&(state.ws.readyState===0||state.ws.readyState===1))return; + try{ + const ws=new WebSocket(wsUrl());ws.binaryType="arraybuffer";state.ws=ws; + ws.onopen=()=>{ws.send(JSON.stringify({type:"hello",role:"vision-ui",protocol:5,viewport:{width:innerWidth,height:innerHeight,dpr:devicePixelRatio||1}}));log("Live vision channel connected","ok")}; + ws.onmessage=onMessage; + ws.onerror=()=>log("Vision WebSocket error","error"); + ws.onclose=()=>{state.ws=null;if(state.running){state.running=false;stateText("Disconnected",false)}}; + }catch(e){log(e.message||String(e),"error")} + } + + async function post(path,body){ + const r=await fetch(path,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(body||{})}); + let j={};try{j=await r.json()}catch{} + if(!r.ok||j.ok===false)throw new Error(j.error||"Vision request failed"); + return j; } - async function connectPro(){ - mount();connectVisionWs(); - if(!serverCapable())return log('Pro CDP is only available from the local/server Forge build','warn'); + + async function start(){ + mount();style();connectWs(); + if(!serverCapable())return log("Live server mode requires the local Forge server","warn"); + if(state.running)return log("Live vision is already running","ok"); try{ - const targets=await fetch('./api/vision/targets').then(r=>r.json()); - const list=targets.targets||[]; - const target=list.find(t=>t.type==='page'&&/^https?:/i.test(t.url||'')); - if(!target)throw new Error('No Chromium CDP target found on 127.0.0.1:9222'); - const result=await fetch('./api/vision/connect',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({targetId:target.id,targetUrl:target.url,quality:55,maxWidth:1280,maxHeight:720,everyNthFrame:1})}).then(r=>r.json()); - if(!result.ok)throw new Error(result.error||'CDP connect failed'); - state.mode='cdp';state.running=true;stateText('Direct Chromium live',true);setTransport('CDP');log('CDP live stream started for '+target.url,'ok'); - }catch(e){state.mode='idle';state.running=false;log(e.message||String(e),'error')} + await post("./api/vision/session/start",{target:"/",viewport:{width:Math.max(320,innerWidth),height:Math.max(240,innerHeight)},touch:matchMedia("(pointer:coarse)").matches,mobile:innerWidth<700,streamFps:30}); + state.running=true;state.mode="server";stateText("Live Chromium stream",true);setTransport("WebSocket / Playwright"); + }catch(e){state.running=false;state.mode="idle";stateText("Idle",false);log(e.message||String(e),"error")} } - function visualHealth(){ - const canvas=document.querySelector('#viewport');if(!canvas)return{ok:false,reason:'viewport-missing'}; - const rect=canvas.getBoundingClientRect();return{ok:rect.width>300&&rect.height>300,renderer:window.Forge?.diagnostics?.().renderer||'unknown',viewport:{width:rect.width,height:rect.height},overflow:Math.max(0,document.documentElement.scrollWidth-innerWidth)}; + + async function restart(){ + try{await stop();await sleep(150);await start()}catch(e){log(e.message||String(e),"error")} + } + + async function stop(){ + try{if(serverCapable())await post("./api/vision/session/stop",{})}catch(e){log(e.message||String(e),"error")} + state.running=false;state.mode="idle";stateText("Idle",false);setTransport("—"); + const img=$("#liveVisionRemote"); + if(img)img.removeAttribute("src"); + if(state.remoteUrl){URL.revokeObjectURL(state.remoteUrl);state.remoteUrl=null} + $("#liveVisionPreview")?.classList.remove("remote","local"); + } + + async function control(action,options={}){ + if(!serverCapable())throw new Error("Server vision is unavailable"); + return (await post("./api/vision/action",{action,options})).result; + } + + async function runManual(action){ + try{const result=await control(action);log(JSON.stringify(result),"ok")} + catch(e){log(e.message||String(e),"error")} + } + + async function getState(){const r=await fetch("./api/vision/state",{cache:"no-store"});if(!r.ok)throw new Error("Vision state unavailable");return r.json()} + async function getElements(){const r=await fetch("./api/vision/elements",{cache:"no-store"});if(!r.ok)throw new Error("Vision elements unavailable");return r.json()} + + async function clickSelector(selector){ + const data=await getElements(); + const e=data.elements.find(x=>x.selector===selector&&x.visible); + if(!e)throw new Error("Visible element not found: "+selector); + return control({type:"mouse.click",x:e.x+e.width/2,y:e.y+e.height/2}); } + async function executeAction(a){ - if(a.type==='vision'||a.type==='scene_scan')return window.Forge?.spatial?.sceneVision?.()||window.ForgeSpatial?.sceneVision?.()||{}; - if(a.type==='visual_health'||a.type==='visual_qa')return visualHealth(); - if(a.type==='select_entity'){const F=window.Forge,r=a.id?F.select(a.id):[...F.entities.values()].find(x=>x.name.toLowerCase()===String(a.name||'').toLowerCase());if(!r)throw new Error('Entity not found');return F.spatial?.inspect?.(r)||window.ForgeSpatial?.inspect?.(r)||{id:r.id,name:r.name,kind:r.kind}} - if(a.type==='focus'){window.Forge?.focus?.();return true} - if(a.type==='frame'){window.Forge?.frame?.();return true} - if(a.type==='transform'){if(a.id)window.Forge?.select?.(a.id);window.Forge?.transform?.(a);return window.ForgeSpatial?.inspect?.(window.Forge.selected())||null} + if(a.type==="vision"||a.type==="scene_scan")return getState(); + if(a.type==="visual_health")return window.Forge?.diagnostics?.()?window.Forge.diagnostics():{ok:true}; return control(a); } - async function localAgent(task){ - const F=window.Forge,s=String(task||'').trim(),lower=s.toLowerCase(); - if(/scan|inspect|vision|understand/.test(lower))return executeAction({type:'vision'}); - const names=[...F.entities.values()].map(r=>r.name).filter(Boolean),target=names.find(n=>lower.includes(n.toLowerCase())); - if(target){ - await executeAction({type:'select_entity',name:target}); - if(/delete|remove|erase/.test(lower)){F.delete();return{ok:true,action:'delete',entity:target}} - if(/duplicate|copy|clone/.test(lower)){const d=F.duplicate();return{ok:true,action:'duplicate',entity:d?.name||target}} - if(/center|middle/.test(lower)){F.spatial?.moveToScreen?.(F.selectedId,innerWidth/2,innerHeight/2,.5,0);return{ok:true,action:'move-center',entity:target}} - if(/focus/.test(lower)){F.focus();return{ok:true,action:'focus',entity:target}} - } - if(/play/.test(lower)){F.running=true;F.timelinePlaying=true;return{ok:true,action:'play'}} - if(/stop|pause/.test(lower)){F.running=false;F.timelinePlaying=false;return{ok:true,action:'stop'}} - if(/frame all|frame/.test(lower)){F.frame();return{ok:true,action:'frame'}} - if(/focus/.test(lower)){F.focus();return{ok:true,action:'focus'}} - throw new Error('Task not mapped. Use a deterministic command or a pixel action such as click/drag/wheel/press.'); + + async function executeTask(task){ + const s=String(task||"").trim().toLowerCase(); + if(!s)throw new Error("Task is empty"); + if(s.includes("scan")||s.includes("vision")||s.includes("inspect"))return getState(); + if(s.includes("add box"))return clickSelector('[data-add="box"]'); + if(s.includes("add sphere"))return clickSelector('[data-add="sphere"]'); + if(s.includes("play"))return clickSelector("#play"); + if(s.includes("stop")||s.includes("pause"))return clickSelector("#play"); + if(s.includes("rotate"))return control({type:"keyboard.press",key:"2"}); + if(s.includes("scale"))return control({type:"keyboard.press",key:"3"}); + if(s.includes("move"))return control({type:"keyboard.press",key:"1"}); + if(s.includes("frame"))return clickSelector("#frame"); + if(s.includes("focus"))return clickSelector("#focus"); + throw new Error("Task not mapped; use concrete vision actions for exact control"); } + async function runAgent(){ mount(); - const task=$('#liveVisionTask')?.value?.trim();if(!task)return log('Enter a task first','error');if(state.busy)return; - state.busy=true;$('#liveVisionStart').disabled=true; - try{ - let result; - if(serverCapable()&&state.mode==='cdp')result=await fetch('./api/forge/command',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({command:{op:'agent-task',task}})}).then(r=>r.json()); - else result=await localAgent(task); - log(JSON.stringify(result),'ok'); - }catch(e){log(e.message||String(e),'error')}finally{state.busy=false;$('#liveVisionStart').disabled=false} + const task=$("#liveVisionTask")?.value?.trim(); + if(!task||state.busy)return; + state.busy=true;$("#liveVisionAgent").disabled=true; + try{const result=await executeTask(task);log(JSON.stringify(result),"ok")} + catch(e){log(e.message||String(e),"error")} + finally{state.busy=false;$("#liveVisionAgent").disabled=false} } - async function start(){ - mount();style();connectVisionWs(); - if(state.mode==='cdp'){return log('CDP live is already active','ok')} - if(!navigator.mediaDevices?.getDisplayMedia){stateText('Spatial fallback',true);setTransport('Local DOM');log('Screen capture unavailable; local vision/control remains active','warn');return} - try{ - stop(); - const stream=await navigator.mediaDevices.getDisplayMedia({video:{frameRate:{ideal:30,max:30},width:{ideal:1920,max:1920},height:{ideal:1080,max:1080},displaySurface:'browser'},audio:false,preferCurrentTab:true,selfBrowserSurface:'include',surfaceSwitching:'include'}); - state.stream=stream;state.running=true;state.mode='local-media';stateText('Live MediaStream',true);setTransport('MediaStream'); - const v=$('#liveVisionVideo');v.srcObject=stream;await v.play();$('#liveVisionPreview').classList.add('local');$('#liveVisionPreview').classList.remove('remote'); - stream.getVideoTracks()[0].addEventListener('ended',stop); - sampleLiveFrames(); - }catch(e){state.running=false;state.mode='idle';state.stream=null;stateText('Spatial fallback',true);setTransport('Local DOM');log(e.message||String(e),'warn')} - } - function sampleLiveFrames(){ - if(!state.running||state.mode!=='local-media')return; - const v=$('#liveVisionVideo'); - const tick=()=>{if(!state.running||state.mode!=='local-media')return;state.frameSeq++;state.lastFrameAt=Date.now();$('#liveVisionFps').textContent='live';if('requestVideoFrameCallback' in v)v.requestVideoFrameCallback(()=>setTimeout(sampleLiveFrames,120));else state.captureTimer=setTimeout(sampleLiveFrames,120)}; - tick(); + + function visualHealth(){ + const canvas=document.querySelector("#viewport"); + if(!canvas)return{ok:false,reason:"viewport-missing"}; + const rect=canvas.getBoundingClientRect(); + return{ok:rect.width>300&&rect.height>300,viewport:{width:rect.width,height:rect.height},overflow:Math.max(0,document.documentElement.scrollWidth-innerWidth)}; } - async function stop(){ - try{if(serverCapable())await fetch('./api/vision/disconnect',{method:'POST'}).catch(()=>{})}catch{} - state.running=false;state.mode='idle';if(state.stream){state.stream.getTracks().forEach(t=>t.stop());state.stream=null} - clearTimeout(state.captureTimer);const v=$('#liveVisionVideo');if(v)v.srcObject=null; - const img=$('#liveVisionRemote');if(img)img.removeAttribute('src');if(state.remoteUrl){URL.revokeObjectURL(state.remoteUrl);state.remoteUrl=null} - $('#liveVisionPreview')?.classList.remove('remote','local');stateText('Idle',false);setTransport('—'); - } - window.AssetForgeLiveVision={ - open,start,stop,runAgent,executeAction,visualHealth,connectPro,control, - status:()=>({mode:state.mode,running:state.running,connected:state.ws?.readyState===1,frames:state.frameSeq,lastFrameAt:state.lastFrameAt,transport:state.mode==='cdp'?'chromium-cdp':state.mode==='local-media'?'media-stream':'local-spatial'}) - }; - if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',()=>{mount();style()},{once:true});else{mount();style()} + + const api={open,start,stop,restart,runAgent,executeAction,executeTask,visualHealth,control,status:()=>({mode:state.mode,running:state.running,connected:state.ws?.readyState===1,frames:state.frameSeq,lastFrameAt:state.lastFrameAt,transport:state.mode==="server"?"playwright-screencast-websocket":"idle"})}; + window.AssetForgeLiveVision=api; + if(document.readyState==="loading")document.addEventListener("DOMContentLoaded",()=>{mount();style()},{once:true}); + else{mount();style()} })(); \ No newline at end of file diff --git a/package.json b/package.json index ad17125..dcfb899 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "test:core": "playwright test tests/core.smoke.spec.mjs tests/asset-export.spec.mjs tests/visual.smoke.spec.mjs tests/professional.qa.spec.mjs --project=desktop", "test:professional": "playwright test tests/professional.qa.spec.mjs --project=desktop", "validate:gltf": "node scripts/validate-gltf-test-assets.mjs", - "test:vision": "playwright test tests/vision-live.spec.mjs --project=desktop" + "test:vision": "playwright test tests/vision.live.spec.mjs" }, "devDependencies": { "@playwright/test": "^1.63.0", diff --git a/server.mjs b/server.mjs index 6eb9f07..2945fe0 100644 --- a/server.mjs +++ b/server.mjs @@ -5,42 +5,15 @@ import path from 'node:path'; import {fileURLToPath} from 'node:url'; import {WebSocketServer} from 'ws'; import {cookGLB} from './forge-pipeline.mjs'; -import {VisionController} from './vision-controller.mjs'; import {VisionController,installVisionShutdown} from './vision-control.mjs'; const root=path.dirname(fileURLToPath(import.meta.url)); const port=Number(process.env.PORT||4173); -const visionRemote=process.env.FORGE_VISION_REMOTE==='1'; const host=process.env.HOST||'127.0.0.1'; const vision=new VisionController({baseUrl:`http://127.0.0.1:${port}`}); const removeVisionShutdown=installVisionShutdown(vision); const mime={'.html':'text/html; charset=utf-8','.js':'text/javascript; charset=utf-8','.mjs':'text/javascript; charset=utf-8','.css':'text/css; charset=utf-8','.json':'application/json; charset=utf-8','.webmanifest':'application/manifest+json; charset=utf-8','.png':'image/png','.jpg':'image/jpeg','.jpeg':'image/jpeg','.webp':'image/webp','.svg':'image/svg+xml','.ico':'image/x-icon','.wasm':'application/wasm'}; -const clients=new Set(),visionClients=new Set(),rooms=new Map(); -const vision=new VisionController({ - port:Number(process.env.FORGE_CDP_PORT||9222), - host:process.env.FORGE_CDP_HOST||'127.0.0.1', - broadcast:message=>{ - if(message?.type==='vision-frame'&&message.data){ - const jpeg=Buffer.from(message.data,'base64'); - const header=Buffer.alloc(16); - header.writeUInt32BE(0x46563337,0); - header.writeUInt32BE(Number(message.seq)||0,4); - header.writeUInt16BE(Number(message.width)||0,8); - header.writeUInt16BE(Number(message.height)||0,10); - header.writeUInt32BE(Math.floor((Number(message.at)||Date.now())/1000),12); - const packet=Buffer.concat([header,jpeg]); - for(const peer of visionClients)if(peer.readyState===1)peer.send(packet); - return; - } - const raw=JSON.stringify(message); - for(const peer of visionClients)if(peer.readyState===1)peer.send(raw) - } -}); -function trustedVisionRequest(req){ - if(visionRemote)return true; - const ip=String(req.socket?.remoteAddress||'').replace(/^::ffff:/,''); - return ip==='127.0.0.1'||ip==='::1'||ip==='localhost'; -} +const clients=new Set(),rooms=new Map(); function json(res,o,status=200){res.writeHead(status,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','Access-Control-Allow-Origin':'*'});res.end(JSON.stringify(o))} function body(req,max=64*1024){return new Promise((resolve,reject)=>{let raw='';req.on('data',chunk=>{raw+=chunk;if(raw.length>max){reject(new Error('Request body too large'));req.destroy()}});req.on('end',()=>{try{resolve(raw?JSON.parse(raw):{})}catch(e){reject(new Error('Invalid JSON'))}});req.on('error',reject)})} async function routeVisionAction(req,res){try{const q=await body(req);if(!q.action)return json(res,{ok:false,error:'Missing action'},400);return json(res,await vision.action(q.action,q.options||{}))}catch(e){return json(res,{ok:false,error:e.message||String(e)},500)}} @@ -55,7 +28,7 @@ const srv=http.createServer(async (req,res)=>{const u=new URL(req.url,'http://lo if(u.pathname==='/api/vision/elements')return vision.elements().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503)); if(u.pathname==='/api/vision/state')return vision.state().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503)); if(u.pathname==='/api/vision/session/start'&&req.method==='POST'){ - try{const q=await body(req);const s=await vision.start({url:q.url||q.target||undefined,viewport:q.viewport,touch:q.touch,mobile:q.mobile});return json(res,{ok:true,session:s,stream:'/api/vision/stream',websocket:'/vision'})} + try{const q=await body(req);const s=await vision.start({url:q.url||q.target||undefined,viewport:q.viewport,touch:q.touch,mobile:q.mobile,streamFps:q.streamFps});return json(res,{ok:true,session:s,stream:'/api/vision/stream',websocket:'/vision'})} catch(e){return json(res,{ok:false,error:e.message||String(e)},500)} } if(u.pathname==='/api/vision/session/stop'&&req.method==='POST'){ @@ -100,12 +73,10 @@ visionWss.on('connection',ws=>{ } }); }); -const visionWss=new WebSocketServer({noServer:true}); -visionWss.on('connection',(ws,req)=>{if(!visionRemote&&!trustedVisionRequest(req)){ws.close(1008,'local-only');return}visionClients.add(ws);ws.send(JSON.stringify({type:'vision-state',state:vision.status()}));ws.on('close',()=>visionClients.delete(ws));}); const netWss=new WebSocketServer({noServer:true}); function joinRoom(room,ws,peerId){if(!rooms.has(room))rooms.set(room,new Set());const set=rooms.get(room);for(const p of set)if(p.readyState===1)p.send(JSON.stringify({type:'peer-join',peerId}));set.add(ws);ws.__room=room;ws.__peerId=peerId} function leaveRoom(ws){const room=ws.__room;if(!room||!rooms.has(room))return;const set=rooms.get(room);set.delete(ws);for(const p of set)if(p.readyState===1)p.send(JSON.stringify({type:'peer-leave',peerId:ws.__peerId}));if(!set.size)rooms.delete(room)} netWss.on('connection',ws=>{ws.on('close',()=>leaveRoom(ws));ws.on('message',message=>{let m;try{m=JSON.parse(message)}catch{return}if(m.type==='hello'){joinRoom(String(m.room||'default'),ws,String(m.peerId||crypto.randomUUID()));return}const room=rooms.get(ws.__room);if(!room)return;for(const p of room)if(p!==ws&&p.readyState===1)p.send(JSON.stringify(m))})}); -srv.on('upgrade',(req,socket,head)=>{const pathn=new URL(req.url,'http://localhost').pathname;if(pathn==='/live')return wss.handleUpgrade(req,socket,head,ws=>wss.emit('connection',ws,req));if(pathn==='/vision')return visionWss.handleUpgrade(req,socket,head,ws=>visionWss.emit('connection',ws,req));if(pathn==='/vision')return visionWss.handleUpgrade(req,socket,head,ws=>visionWss.emit('connection',ws,req));if(pathn==='/net')return netWss.handleUpgrade(req,socket,head,ws=>netWss.emit('connection',ws,req));socket.destroy()}); +srv.on('upgrade',(req,socket,head)=>{const pathn=new URL(req.url,'http://localhost').pathname;if(pathn==='/live')return wss.handleUpgrade(req,socket,head,ws=>wss.emit('connection',ws,req));if(pathn==='/vision')return visionWss.handleUpgrade(req,socket,head,ws=>visionWss.emit('connection',ws,req));if(pathn==='/net')return netWss.handleUpgrade(req,socket,head,ws=>netWss.emit('connection',ws,req));socket.destroy()}); srv.listen(port,host,()=>console.log('Forge Studio 3.7 listening on http://'+host+':'+port)); process.once('exit',removeVisionShutdown); \ No newline at end of file diff --git a/tests/vision-live.spec.mjs b/tests/vision-live.spec.mjs deleted file mode 100644 index 7a3c777..0000000 --- a/tests/vision-live.spec.mjs +++ /dev/null @@ -1,115 +0,0 @@ -import{test,expect,chromium}from"@playwright/test"; -import{writeFile}from"node:fs/promises"; -import{WebSocket as NodeWebSocket}from"ws"; - -test.describe("Forge live computer vision bridge",()=>{ - test("continuous CDP vision stream and real input controls change Forge",async({request})=>{ - test.setTimeout(120000); - const browser=await chromium.launch({ - headless:true, - args:["--remote-debugging-port=9222","--user-data-dir=/tmp/forge-vision-cdp"] - }); - const context=await browser.newContext({viewport:{width:1440,height:900}}); - const page=await context.newPage(); - await page.goto("http://127.0.0.1:4173/"); - await page.waitForFunction(()=>window.ForgeReady===true,{timeout:30000}); - await expect(page.locator("#sceneCount")).toContainText("3 entities"); - - const attach=await request.get("/api/vision/targets"); - expect(attach.ok()).toBeTruthy(); - const targets=(await attach.json()).targets||[]; - const target=targets.find(t=>t.type==="page"&&t.url.includes("127.0.0.1:4173")); - expect(target?.id).toBeTruthy(); - - const streamWs=new NodeWebSocket("ws://127.0.0.1:4173/vision"); - const frames=[]; - const states=[]; - streamWs.on("message",data=>{ - if(Buffer.isBuffer(data)){ - if(data.length>16&&data.readUInt32BE(0)===0x46563337)frames.push({ - seq:data.readUInt32BE(4),width:data.readUInt16BE(8),height:data.readUInt16BE(10),data:data.subarray(16) - }); - return; - } - try{const m=JSON.parse(data.toString());if(m.type==="vision-state")states.push(m.state)}catch{} - }); - const connect=await request.post("/api/vision/connect",{data:{ - targetId:target.id,targetUrl:target.url,quality:55,maxWidth:960,maxHeight:540,everyNthFrame:1 - }}); - expect(connect.ok()).toBeTruthy(); - const connection=await connect.json(); - expect(connection.ok).toBeTruthy(); - expect(connection.state.connected).toBeTruthy(); - expect(connection.state.streaming).toBeTruthy(); - - await expect.poll(()=>frames.length,{timeout:15000}).toBeGreaterThanOrEqual(3); - expect(frames[0].data.length).toBeGreaterThan(1000); - await writeFile("test-results/pro-vision-live-frame.jpg",Buffer.from(frames.at(-1).data)); - - const box=await page.locator('[data-add="box"]').boundingBox(); - expect(box).toBeTruthy(); - const click=await request.post("/api/vision/control",{data:{action:{ - type:"click",x:box.x+box.width/2,y:box.y+box.height/2,screenWidth:1440,screenHeight:900 - }}}); - expect(click.ok()).toBeTruthy(); - await expect(page.locator("#sceneCount")).toContainText("4 entities"); - - const visionMode=await request.post("/api/vision/control",{data:{action:{type:"press",key:"2"}}}); - expect(visionMode.ok()).toBeTruthy(); - await expect.poll(()=>page.evaluate(()=>window.ForgeGizmo?.state?.mode),{timeout:5000}).toBe("rotate"); - - const nameBox=await page.locator("#name").boundingBox(); - expect(nameBox).toBeTruthy(); - await request.post("/api/vision/control",{data:{action:{ - type:"click",x:nameBox.x+nameBox.width/2,y:nameBox.y+nameBox.height/2,screenWidth:1440,screenHeight:900 - }}}); - const typed=await request.post("/api/vision/control",{data:{action:{type:"typeText",text:"_VISION"}}}); - expect(typed.ok()).toBeTruthy(); - expect(await page.locator("#name").inputValue()).toContain("_VISION"); - - const slider=await page.locator("#time").boundingBox(); - expect(slider).toBeTruthy(); - const startTime=await page.locator("#time").inputValue(); - const dragged=await request.post("/api/vision/control",{data:{action:{ - type:"drag", - x:slider.x+6,y:slider.y+slider.height/2, - toX:slider.x+Math.max(50,slider.width*.65),toY:slider.y+slider.height/2, - screenWidth:1440,screenHeight:900,steps:16,stepMs:6 - }}}); - expect(dragged.ok()).toBeTruthy(); - await page.waitForTimeout(80); - const endTime=await page.locator("#time").inputValue(); - expect(endTime).not.toBe(startTime); - - const touchProbe=await page.evaluate(()=>{ - window.__visionTouchCount=0; - const b=document.querySelector('[data-add="sphere"]'); - b?.addEventListener("pointerdown",e=>{if(e.pointerType==="touch")window.__visionTouchCount++},{once:false}); - return b?.getBoundingClientRect().toJSON(); - }); - expect(touchProbe).toBeTruthy(); - const touchStart=await request.post("/api/vision/control",{data:{action:{ - type:"touchStart",x:touchProbe.x+touchProbe.width/2,y:touchProbe.y+touchProbe.height/2,screenWidth:1440,screenHeight:900,id:41 - }}}); - const touchEnd=await request.post("/api/vision/control",{data:{action:{ - type:"touchEnd",x:touchProbe.x+touchProbe.width/2,y:touchProbe.y+touchProbe.height/2,screenWidth:1440,screenHeight:900,id:41 - }}}); - expect(touchStart.ok()).toBeTruthy(); - expect(touchEnd.ok()).toBeTruthy(); - await page.waitForTimeout(50); - expect(await page.evaluate(()=>window.__visionTouchCount)).toBeGreaterThan(0); - - const status=await request.get("/api/vision/status"); - const st=await status.json(); - expect(st.connected).toBeTruthy(); - expect(st.streaming).toBeTruthy(); - expect(st.transport).toBe("chromium-cdp-screencast"); - expect(st.control).toBe("chromium-cdp-input"); - expect(frames.length).toBeGreaterThanOrEqual(3); - - await request.post("/api/vision/disconnect",{data:{}}); - streamWs.close(); - await context.close(); - await browser.close(); - }); -}); diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs index be67fb3..7952e55 100644 --- a/tests/vision.live.spec.mjs +++ b/tests/vision.live.spec.mjs @@ -1,149 +1,101 @@ import{test,expect}from"@playwright/test"; +import{WebSocket}from"ws"; const BASE="http://127.0.0.1:4173"; const sleep=ms=>new Promise(r=>setTimeout(r,ms)); -async function stopVision(request){ - try{await request.post("/api/vision/session/stop",{timeout:5000})}catch{} -} - -async function startVision(request,options={}){ - await stopVision(request); - const r=await request.post("/api/vision/session/start",{ - data:{target:"/",viewport:{width:1440,height:900},touch:false,mobile:false,...options}, - timeout:45000 +test.describe.serial("Forge Live Vision — continuous stream + real browser input",()=>{ + test.afterEach(async({request})=>{ + try{await request.post("/api/vision/session/stop",{timeout:5000})}catch{} }); - expect(r.ok()).toBeTruthy(); - const body=await r.json(); - expect(body.ok).toBeTruthy(); - expect(body.session.running).toBeTruthy(); - await expect.poll(async()=>((await request.get("/api/vision/status")).json()).then(j=>j.vision.frameSeq),{timeout:20000}).toBeGreaterThan(3); - return body.session; -} - -async function getJson(request,path){ - const r=await request.get(path,{cache:"no-store"}); - expect(r.ok()).toBeTruthy(); - return r.json(); -} - -test.describe.serial("Forge Live Vision — real continuous vision + input",()=>{ - test.afterEach(async({request})=>{await stopVision(request)}); - - test("publishes a continuous screencast stream rather than one-shot screenshots",async({page,request})=>{ - test.setTimeout(60000); - const initial=await startVision(request); - await sleep(1200); - const later=await getJson(request,"/api/vision/status"); - expect(later.vision.frameSeq).toBeGreaterThan(initial.frameSeq); - expect(later.vision.frame?.bytes).toBeGreaterThan(1000); - expect(later.vision.lastFrameAt).toBeGreaterThan(0); - expect(later.vision.transport).toContain("Playwright Screencast"); - - const controller=new AbortController(); - const res=await fetch(BASE+"/api/vision/stream?proof="+Date.now(),{signal:controller.signal}); - expect(res.ok).toBeTruthy(); - expect(res.headers.get("content-type")).toContain("multipart/x-mixed-replace"); - const reader=res.body.getReader(); - let payload=""; - for(let i=0;i<4&&payload.length<4000;i++){ - const part=await reader.read(); - if(part.done)break; - payload+=new TextDecoder().decode(part.value); - } - controller.abort(); - expect(payload).toContain("--forge-frame"); - expect(payload).toContain("Content-Type: image/jpeg"); + test("continuous frames are pushed over one live WebSocket channel",async({request})=>{ + test.setTimeout(90000); + const socket=new WebSocket(BASE.replace("http","ws")+"/vision"); + const frames=[]; + const open=new Promise((resolve,reject)=>{ + socket.once("open",resolve);socket.once("error",reject); + }); + await open; + socket.on("message",data=>{ + if(!Buffer.isBuffer(data)||data.length<16)return; + if(data.readUInt32BE(0)!==0x46563337)return; + frames.push({seq:data.readUInt32BE(4),width:data.readUInt16BE(8),height:data.readUInt16BE(10),bytes:data.length-16}); + }); + + const start=await request.post("/api/vision/session/start",{data:{ + target:"/",viewport:{width:1280,height:720},touch:false,mobile:false,streamFps:30 + },timeout:45000}); + expect(start.ok()).toBeTruthy(); + const body=await start.json(); + expect(body.session.running).toBeTruthy(); + + await expect.poll(()=>frames.length,{timeout:20000}).toBeGreaterThanOrEqual(5); + expect(frames.at(-1).seq).toBeGreaterThan(frames[0].seq); + expect(frames[0].width).toBeGreaterThan(300); + expect(frames[0].height).toBeGreaterThan(200); + expect(frames[0].bytes).toBeGreaterThan(1000); + + const status=await request.get("/api/vision/status"); + const st=await status.json(); + expect(st.vision.transport).toContain("WebSocket control"); + expect(st.vision.streamFps).toBe(30); + expect(st.vision.streamSubscribers).toBeGreaterThanOrEqual(1); + + socket.close(); }); - test("drives Forge through real pointer coordinates and verifies the resulting engine state",async({request})=>{ - test.setTimeout(90000); - await startVision(request); - const elements=await getJson(request,"/api/vision/elements"); - const box=elements.elements.find(e=>e.selector==='[data-add="box"]'&&e.visible); + test("mouse, keyboard, drag and touch modify real Forge state",async({request})=>{ + test.setTimeout(120000); + const start=await request.post("/api/vision/session/start",{data:{ + target:"/",viewport:{width:1440,height:900},touch:true,mobile:true,streamFps:24 + },timeout:45000}); + expect(start.ok()).toBeTruthy(); + + const elements=(await (await request.get("/api/vision/elements")).json()).elements; + const box=elements.find(e=>e.selector==='[data-add="box"]'&&e.visible); expect(box).toBeTruthy(); - const x=box.x+box.width/2,y=box.y+box.height/2; - const down=await request.post("/api/vision/action",{data:{action:{type:"mouse.move",x,y,steps:4}}}); - expect((await down.json()).ok).toBeTruthy(); - const press=await request.post("/api/vision/action",{data:{action:{type:"mouse.down",button:"left"}}}); - expect((await press.json()).ok).toBeTruthy(); - const release=await request.post("/api/vision/action",{data:{action:{type:"mouse.up",button:"left"}}}); - const released=await release.json(); - expect(released.ok).toBeTruthy(); - expect(released.action.dispatched).toBeTruthy(); - - const state=await getJson(request,"/api/vision/state"); - expect(state.forgeReady).toBeTruthy(); - expect(state.diagnostics.entities).toBeGreaterThanOrEqual(4); - expect(state.selected?.name||"").toMatch(/box/i); - expect(state.selectedScreenRect).toBeTruthy(); - expect(state.canvasRect).toBeTruthy(); - - const moved=await request.post("/api/vision/action",{data:{action:{ - type:"mouse.drag", - from:{x:box.x+box.width/2,y:box.y+box.height/2}, - to:{x:box.x+box.width/2+30,y:box.y+box.height/2+10}, - steps:8 + const click=await request.post("/api/vision/action",{data:{action:{ + type:"mouse.click",x:box.x+box.width/2,y:box.y+box.height/2 + },options:{includeState:true}}}); + const clickBody=await click.json(); + expect(clickBody.ok).toBeTruthy(); + expect(clickBody.result?.action?.dispatched).toBeTruthy(); + expect(clickBody.result?.state?.diagnostics?.entities||0).toBeGreaterThanOrEqual(4); + + const rotate=await request.post("/api/vision/action",{data:{action:{type:"keyboard.press",key:"2"},options:{includeState:true}}}); + expect((await rotate.json()).result.state.gizmoMode).toBe("rotate"); + + const slider=(await (await request.get("/api/vision/elements")).json()).elements.find(e=>e.selector==='#time'&&e.visible); + expect(slider).toBeTruthy(); + const drag=await request.post("/api/vision/action",{data:{action:{ + type:"mouse.drag",from:{x:slider.x+5,y:slider.y+slider.height/2},to:{x:slider.x+Math.max(55,slider.width*0.7),y:slider.y+slider.height/2},steps:16 }}}); - expect((await moved.json()).action.dispatched).toBeTruthy(); - const wheel=await request.post("/api/vision/action",{data:{action:{type:"mouse.wheel",deltaX:0,deltaY:180}}}); - expect((await wheel.json()).action.dispatched).toBeTruthy(); - }); + expect((await drag.json()).result.action.dispatched).toBeTruthy(); + + const touchTarget=(await (await request.get("/api/vision/elements")).json()).elements.find(e=>e.selector==='[data-add="sphere"]'&&e.visible); + expect(touchTarget).toBeTruthy(); + const touched=await request.post("/api/vision/action",{data:{action:{ + type:"touch.swipe", + from:{x:touchTarget.x+touchTarget.width/2,y:touchTarget.y+touchTarget.height/2}, + to:{x:touchTarget.x+touchTarget.width/2+8,y:touchTarget.y+touchTarget.height/2+8},id:7,steps:6 + }}}); + expect((await touched.json()).result.action.dispatched).toBeTruthy(); - test("drives the real browser keyboard and proves a visible editor mode change",async({request})=>{ - test.setTimeout(90000); - await startVision(request); - const before=await getJson(request,"/api/vision/state"); - expect(before.forgeReady).toBeTruthy(); - expect(before.diagnostics.entities).toBeGreaterThanOrEqual(3); - - const keyRotate=await request.post("/api/vision/action",{data:{action:{type:"keyboard.press",key:"2"}}}); - const kr=await keyRotate.json(); - expect(kr.ok).toBeTruthy(); - const rotate=await getJson(request,"/api/vision/state"); - expect(rotate.gizmoMode).toBe("rotate"); - - const keyTranslate=await request.post("/api/vision/action",{data:{action:{type:"keyboard.press",key:"1"}}}); - const kt=await keyTranslate.json(); - expect(kt.ok).toBeTruthy(); - const translate=await getJson(request,"/api/vision/state"); - expect(translate.gizmoMode).toBe("translate"); + const release=await request.post("/api/vision/action",{data:{action:{type:"releaseAll"}}}); + expect((await release.json()).result.action.released).toBeTruthy(); + + const finalState=await (await request.get("/api/vision/state")).json(); + expect(finalState.forgeReady).toBeTruthy(); + expect(finalState.canvasRect).toBeTruthy(); }); - test("executes real touch start/move/end through CDP on a touch-capable Forge session",async({request})=>{ + test("file-input control is available for real asset import workflows",async({request})=>{ test.setTimeout(90000); - const session=await startVision(request,{viewport:{width:390,height:844},touch:true,mobile:true}); - expect(session.viewport.width).toBe(390); - const elements=await getJson(request,"/api/vision/elements"); - const box=elements.elements.find(e=>e.selector==='[data-add="box"]'&&e.visible); - expect(box).toBeTruthy(); - const bx=box.x+box.width/2,by=box.y+box.height/2; - const start=await request.post("/api/vision/action",{data:{action:{type:"touch.start",points:[{id:1,x:bx,y:by,force:1}]}}}); - const sr=await start.json(); - expect(sr.ok).toBeTruthy(); - expect(sr.action.dispatched).toBeTruthy(); - - const move=await request.post("/api/vision/action",{data:{action:{type:"touch.move",points:[{id:1,x:bx+2,y:by+2,force:1}]}}}); - const mr=await move.json(); - expect(mr.ok).toBeTruthy(); - - const end=await request.post("/api/vision/action",{data:{action:{type:"touch.end"}}}); - const er=await end.json(); - expect(er.ok).toBeTruthy(); - expect(er.action.dispatched).toBeTruthy(); - - const tap=await request.post("/api/vision/action",{data:{action:{type:"touch.tapSelector",selector:'[data-gizmo="rotate"]'}}}); - const tr=await tap.json(); - expect(tr.ok).toBeTruthy(); - expect(tr.action.dispatched).toBeTruthy(); - - const afterTouch=await getJson(request,"/api/vision/state"); - expect(afterTouch.gizmoMode).toBe("rotate"); - - const status=await getJson(request,"/api/vision/status"); - expect(status.vision.frameSeq).toBeGreaterThan(3); - expect(status.capabilities.touch).toBeTruthy(); + const start=await request.post("/api/vision/session/start",{data:{target:"/",viewport:{width:1280,height:720},touch:false,mobile:false},timeout:45000}); + expect(start.ok()).toBeTruthy(); + const input=await (await request.get("/api/vision/elements")).json(); + expect(input.elements.some(e=>e.selector==="#assetInput"&&e.visible)).toBeTruthy(); }); -}); +}); \ No newline at end of file diff --git a/vision-control.mjs b/vision-control.mjs index 2eb6453..cf97ab7 100644 --- a/vision-control.mjs +++ b/vision-control.mjs @@ -1,6 +1,10 @@ import { EventEmitter } from 'node:events'; +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const FRAME_MAGIC=0x46563337; +const assetRoot=()=>path.resolve(process.env.VISION_ASSET_ROOT||process.cwd()); function clamp(n,min,max){ const v=Number(n); @@ -52,6 +56,10 @@ export class VisionController extends EventEmitter{ this.lastActionAt=0; this.errors=[]; this.frameTimer=null; + this.streamFps=30; + this.lastStreamFrameAt=0; + this.frameDrops=0; + this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; } async start(options={}){ @@ -62,6 +70,7 @@ export class VisionController extends EventEmitter{ const touch=!!options.touch; const mobile=!!options.mobile; const target=normalizeUrl(options.url||options.target||this.baseUrl+'/',this.baseUrl); + this.streamFps=clamp(options.streamFps??30,5,60); if(this.page && this.targetUrl===target && this.page.isClosed?.()===false) return this.status(); @@ -75,14 +84,17 @@ export class VisionController extends EventEmitter{ throw new Error('Playwright runtime is unavailable; run npm install before starting Live Vision: '+error.message); } - this.sessionId=crypto.randomUUID(); + this.sessionId=randomUUID(); this.targetUrl=target; this.startedAt=Date.now(); this.errors=[]; this.browser=await chromium.launch({ headless:process.env.VISION_HEADLESS!=='false', - args: process.env.VISION_CHROMIUM_ARGS ? String(process.env.VISION_CHROMIUM_ARGS).split(/\s+/).filter(Boolean) : [] + args:[ + '--disable-dev-shm-usage', + ...(process.env.VISION_CHROMIUM_ARGS ? String(process.env.VISION_CHROMIUM_ARGS).split(/\s+/).filter(Boolean) : []) + ] }); this.context=await this.browser.newContext({ @@ -128,6 +140,7 @@ export class VisionController extends EventEmitter{ acceptFrame(frame){ const data=Buffer.from(frame.data); const now=Date.now(); + if(!this.page || this.page.isClosed?.()) return; this.frameSeq+=1; this.latestFrame=data; this.latestMeta={ @@ -157,10 +170,32 @@ export class VisionController extends EventEmitter{ this.writeFrame(res,data,this.frameSeq,this.latestMeta); }catch{this.streams.delete(res)} } + const interval=1000/this.streamFps; + if(now-this.lastStreamFrameAt>=interval){ + this.lastStreamFrameAt=now; + this.broadcastFrame(data,this.frameSeq,this.latestMeta); + } + } + + broadcastFrame(data,seq,meta){ + const header=Buffer.alloc(16); + header.writeUInt32BE(FRAME_MAGIC,0); + header.writeUInt32BE(Number(seq)||0,4); + header.writeUInt16BE(Number(meta.width)||0,8); + header.writeUInt16BE(Number(meta.height)||0,10); + header.writeUInt32BE(Math.floor((Number(meta.serverTime)||Date.now())/1000),12); + const packet=Buffer.concat([header,data]); + for(const ws of [...this.telemetry]){ + try{ + if(ws.readyState!==1){this.telemetry.delete(ws);continue;} + if(Number(ws.bufferedAmount||0)>2*1024*1024){this.frameDrops+=1;continue;} + ws.send(packet); + }catch{this.telemetry.delete(ws)} + } } emitTelemetry(){ - const payload=JSON.stringify({type:'vision-frame',...this.status()}); + const payload=JSON.stringify({type:'vision-status',...this.status()}); for(const ws of [...this.telemetry]){ try{ if(ws.readyState===1) ws.send(payload); @@ -212,9 +247,9 @@ export class VisionController extends EventEmitter{ } async stop(){ + try{await this.releaseAll()}catch{} for(const res of this.streams){try{res.end()}catch{}} this.streams.clear(); - this.telemetry.clear(); if(this.frameTimer){clearTimeout(this.frameTimer);this.frameTimer=null} if(this.page){ try{await this.page.screencast.stop()}catch{} @@ -232,6 +267,8 @@ export class VisionController extends EventEmitter{ this.frameSeq=0; this.fps=0; this.framesInWindow=0; + this.lastStreamFrameAt=0; + this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; } status(){ @@ -250,8 +287,17 @@ export class VisionController extends EventEmitter{ viewport, frame:this.latestMeta?{...this.latestMeta}:null, errors:[...this.errors], - controls:['mouse','keyboard','touch','wheel','drag','focus'], - transport:'Playwright Screencast + WebSocket control' + controls:['mouse','keyboard','touch','wheel','drag','focus','file.setInputFiles','releaseAll'], + transport:'Playwright Screencast + WebSocket control', + control:'real-browser-input', + streamFps:this.streamFps, + streamSubscribers:this.telemetry.size, + frameDrops:this.frameDrops, + input:{ + mouseButtons:[...this.inputState.mouseButtons], + keys:[...this.inputState.keys], + touchIds:[...this.inputState.touchIds] + } }; } @@ -322,24 +368,39 @@ export class VisionController extends EventEmitter{ }),selectors); } - async touchEvent(type,points=[]){ + async touchEvent(type,points=[],modifiers=0){ if(!this.cdp)throw new Error('Touch CDP session unavailable'); const touchPoints=points.map(p=>({ x:Number(p.x)||0, y:Number(p.y)||0, radiusX:Number(p.radiusX)||1, radiusY:Number(p.radiusY)||1, + rotationAngle:Number(p.rotationAngle)||0, force:Number(p.force??1), id:Number(p.id??0) })); await this.cdp.send('Input.dispatchTouchEvent',{ type, touchPoints:type==='touchEnd'||type==='touchCancel'?[]:touchPoints, - modifiers:Number(arguments[1]?.modifiers||0) + modifiers:Number(modifiers)||0 }); + if(type==='touchStart'||type==='touchMove')for(const p of touchPoints)this.inputState.touchIds.add(p.id); + if(type==='touchEnd'||type==='touchCancel')this.inputState.touchIds.clear(); return{dispatched:true,type,touchPoints}; } + async releaseAll(){ + if(!this.page||this.page.isClosed?.())return{released:true,mouseButtons:0,keys:0,touches:0}; + const mouseButtons=[...this.inputState.mouseButtons]; + const keys=[...this.inputState.keys]; + const touchCount=this.inputState.touchIds.size; + for(const button of mouseButtons){try{await this.page.mouse.up({button})}catch{}} + for(const key of keys){try{await this.page.keyboard.up(key)}catch{}} + if(this.cdp&&touchCount){try{await this.touchEvent('touchCancel',[],0)}catch{}} + this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; + return{released:true,mouseButtons:mouseButtons.length,keys:keys.length,touches:touchCount}; + } + async perform(action){ if(!this.page||this.page.isClosed())throw new Error('Vision session is not running'); const type=String(action?.type||''); @@ -347,43 +408,73 @@ export class VisionController extends EventEmitter{ switch(type){ case 'mouse.move': - await this.page.mouse.move(Number(action.x)||0,Number(action.y)||0,{steps:Math.max(1,Number(action.steps)||1)}); - return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; - case 'mouse.down': - await this.page.mouse.down({button:buttonValue(action.button)}); - return{dispatched:true,type,button:buttonValue(action.button)}; - case 'mouse.up': - await this.page.mouse.up({button:buttonValue(action.button)}); - return{dispatched:true,type,button:buttonValue(action.button)}; - case 'mouse.click': - await this.page.mouse.click(Number(action.x)||0,Number(action.y)||0,{button:buttonValue(action.button),clickCount:Math.max(1,Math.min(3,Number(action.clickCount)||1)),delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; - case 'mouse.dblclick': - await this.page.mouse.dblclick(Number(action.x)||0,Number(action.y)||0,{button:buttonValue(action.button),delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; + case 'mouse.hover':{ + const x=Number(action.x)||0,y=Number(action.y)||0; + await this.page.mouse.move(x,y,{steps:Math.max(1,Number(action.steps)||1)}); + return{dispatched:true,type,x,y}; + } + case 'mouse.down':{ + const button=buttonValue(action.button); + await this.page.mouse.down({button}); + this.inputState.mouseButtons.add(button); + return{dispatched:true,type,button}; + } + case 'mouse.up':{ + const button=buttonValue(action.button); + await this.page.mouse.up({button}); + this.inputState.mouseButtons.delete(button); + return{dispatched:true,type,button}; + } + case 'mouse.click':{ + const x=Number(action.x)||0,y=Number(action.y)||0,button=buttonValue(action.button); + await this.page.mouse.click(x,y,{button,clickCount:Math.max(1,Math.min(3,Number(action.clickCount)||1)),delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,x,y,button}; + } + case 'mouse.dblclick':{ + const x=Number(action.x)||0,y=Number(action.y)||0,button=buttonValue(action.button); + await this.page.mouse.dblclick(x,y,{button,delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,x,y,button}; + } case 'mouse.wheel': await this.page.mouse.wheel(Number(action.deltaX)||0,Number(action.deltaY)||0); return{dispatched:true,type,deltaX:Number(action.deltaX)||0,deltaY:Number(action.deltaY)||0}; case 'mouse.drag':{ - const from=action.from||{},to=action.to||{}; + const from=action.from||{},to=action.to||{},button=buttonValue(action.button); await this.page.mouse.move(Number(from.x)||0,Number(from.y)||0); - await this.page.mouse.down({button:buttonValue(action.button)}); - await this.page.mouse.move(Number(to.x)||0,Number(to.y)||0,{steps:Math.max(1,Number(action.steps)||12)}); - await this.page.mouse.up({button:buttonValue(action.button)}); - return{dispatched:true,type,from,to}; + await this.page.mouse.down({button}); + this.inputState.mouseButtons.add(button); + await this.page.mouse.move(Number(to.x)||0,Number(to.y)||0,{steps:Math.max(2,Math.min(120,Number(action.steps)||12))}); + await this.page.mouse.up({button}); + this.inputState.mouseButtons.delete(button); + return{dispatched:true,type,from,to,steps:Math.max(2,Math.min(120,Number(action.steps)||12))}; + } + case 'keyboard.down':{ + const key=keyValue(action.key); + await this.page.keyboard.down(key); + this.inputState.keys.add(key); + return{dispatched:true,type,key}; + } + case 'keyboard.up':{ + const key=keyValue(action.key); + await this.page.keyboard.up(key); + this.inputState.keys.delete(key); + return{dispatched:true,type,key}; + } + case 'keyboard.press':{ + const key=keyValue(action.key); + await this.page.keyboard.press(key,{delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,key}; + } + case 'keyboard.type':{ + const text=String(action.text??''); + await this.page.keyboard.type(text,{delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,length:text.length}; + } + case 'keyboard.insertText':{ + const text=String(action.text??''); + await this.page.keyboard.insertText(text); + return{dispatched:true,type,length:text.length}; } - case 'keyboard.down': - await this.page.keyboard.down(keyValue(action.key)); - return{dispatched:true,type,key:String(action.key)}; - case 'keyboard.up': - await this.page.keyboard.up(keyValue(action.key)); - return{dispatched:true,type,key:String(action.key)}; - case 'keyboard.press': - await this.page.keyboard.press(keyValue(action.key),{delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,key:String(action.key)}; - case 'keyboard.type': - await this.page.keyboard.type(String(action.text??''),{delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,length:String(action.text??'').length}; case 'touch.tap': await this.page.touchscreen.tap(Number(action.x)||0,Number(action.y)||0); return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; @@ -392,35 +483,56 @@ export class VisionController extends EventEmitter{ await this.page.locator(String(action.selector)).tap({timeout:10000}); return{dispatched:true,type,selector:String(action.selector)}; case 'touch.start': - return this.touchEvent('touchStart',action.points||[]); + return this.touchEvent('touchStart',action.points||[],action.modifiers||0); case 'touch.move': - return this.touchEvent('touchMove',action.points||[]); + return this.touchEvent('touchMove',action.points||[],action.modifiers||0); case 'touch.end': - return this.touchEvent('touchEnd',[]); + return this.touchEvent('touchEnd',[],action.modifiers||0); case 'touch.cancel': - return this.touchEvent('touchCancel',[]); + return this.touchEvent('touchCancel',[],action.modifiers||0); case 'touch.swipe':{ const from=action.from||{},to=action.to||{}; const id=Number(action.id??0); - const steps=Math.max(2,Math.min(60,Number(action.steps)||12)); - await this.touchEvent('touchStart',[{id,x:Number(from.x)||0,y:Number(from.y)||0,force:1}]); + const steps=Math.max(2,Math.min(120,Number(action.steps)||12)); + await this.touchEvent('touchStart',[{id,x:Number(from.x)||0,y:Number(from.y)||0,force:1}],action.modifiers||0); for(let i=1;i<=steps;i++){ const t=i/steps; - await this.touchEvent('touchMove',[{id,x:(Number(from.x)||0)+(Number(to.x||0)-Number(from.x||0))*t,y:(Number(from.y)||0)+(Number(to.y||0)-Number(from.y||0))*t,force:1}]); + await this.touchEvent('touchMove',[{id,x:(Number(from.x)||0)+(Number(to.x||0)-Number(from.x||0))*t,y:(Number(from.y)||0)+(Number(to.y||0)-Number(from.y||0))*t,force:1}],action.modifiers||0); await sleep(Math.max(0,Number(action.stepDelay)||8)); } - await this.touchEvent('touchEnd',[]); + await this.touchEvent('touchEnd',[],action.modifiers||0); return{dispatched:true,type,from,to,steps}; } + case 'file.setInputFiles':{ + const selector=String(action.selector||''); + if(!selector)throw new Error('file.setInputFiles requires selector'); + const raw=action.clear?[]:(Array.isArray(action.paths)?action.paths:[action.path]).filter(Boolean); + const root=assetRoot(); + const resolved=raw.map(item=>{ + const full=path.resolve(root,String(item)); + if(full!==root&&!full.startsWith(root+path.sep))throw new Error('Asset path escapes VISION_ASSET_ROOT'); + return full; + }); + await this.page.locator(selector).setInputFiles(resolved); + return{dispatched:true,type,selector,files:resolved.length}; + } + case 'releaseAll': + return this.releaseAll(); case 'focus': await this.page.bringToFront(); return{dispatched:true,type}; + case 'evaluate':{ + const expression=String(action.expression||''); + if(!expression)throw new Error('evaluate requires expression'); + const value=await this.page.evaluate(expression); + return{dispatched:true,type,value}; + } default: throw new Error('Unsupported vision action type: '+type); } } - async action(action,{waitForFrameMs=1500}={}){ + async action(action,{waitForFrameMs=1500,includeState=false}={}){ const before=this.frameSeq; const result=await this.perform(action); const after=await this.waitForFrame(before,waitForFrameMs); @@ -432,7 +544,8 @@ export class VisionController extends EventEmitter{ frameAdvanced:this.frameSeq>before, frame:after?{...after}:null, ageMs:this.lastFrameAt?Date.now()-this.lastFrameAt:null, - inputAcceptedAt:Date.now() + inputAcceptedAt:Date.now(), + state:includeState?await this.state():undefined }; } } From 56730117657023796a3ee44d97c0485c6cea1c9f Mon Sep 17 00:00:00 2001 From: venom3-byte Date: Wed, 30 Sep 2026 10:28:05 +0200 Subject: [PATCH 02/86] fix(vision): initialize page before screencast and isolate CI vision session --- package.json | 3 ++- server.mjs | 9 +++++++-- vision-control.mjs | 39 +++++++++++++++++++++++---------------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index dcfb899..7c05256 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "test:core": "playwright test tests/core.smoke.spec.mjs tests/asset-export.spec.mjs tests/visual.smoke.spec.mjs tests/professional.qa.spec.mjs --project=desktop", "test:professional": "playwright test tests/professional.qa.spec.mjs --project=desktop", "validate:gltf": "node scripts/validate-gltf-test-assets.mjs", - "test:vision": "playwright test tests/vision.live.spec.mjs" + "test:vision": "playwright test tests/vision.live.spec.mjs --project=desktop", + "test:vision:mobile": "playwright test tests/vision.live.spec.mjs --project=mobile" }, "devDependencies": { "@playwright/test": "^1.63.0", diff --git a/server.mjs b/server.mjs index 2945fe0..23a41a0 100644 --- a/server.mjs +++ b/server.mjs @@ -28,8 +28,13 @@ const srv=http.createServer(async (req,res)=>{const u=new URL(req.url,'http://lo if(u.pathname==='/api/vision/elements')return vision.elements().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503)); if(u.pathname==='/api/vision/state')return vision.state().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503)); if(u.pathname==='/api/vision/session/start'&&req.method==='POST'){ - try{const q=await body(req);const s=await vision.start({url:q.url||q.target||undefined,viewport:q.viewport,touch:q.touch,mobile:q.mobile,streamFps:q.streamFps});return json(res,{ok:true,session:s,stream:'/api/vision/stream',websocket:'/vision'})} - catch(e){return json(res,{ok:false,error:e.message||String(e)},500)} + try{ + const q=await body(req); + const s=await vision.start({url:q.url||q.target||undefined,viewport:q.viewport,touch:q.touch,mobile:q.mobile,streamFps:q.streamFps}); + return json(res,{ok:true,session:s,stream:'/api/vision/stream',websocket:'/vision'}); + }catch(e){ + return json(res,{ok:false,error:e.message||String(e),vision:vision.status()},500); + } } if(u.pathname==='/api/vision/session/stop'&&req.method==='POST'){ try{await vision.stop();return json(res,{ok:true,session:vision.status()})} diff --git a/vision-control.mjs b/vision-control.mjs index cf97ab7..6b10548 100644 --- a/vision-control.mjs +++ b/vision-control.mjs @@ -60,22 +60,25 @@ export class VisionController extends EventEmitter{ this.lastStreamFrameAt=0; this.frameDrops=0; this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; + this.starting=null; } async start(options={}){ - const viewport={ - width:clamp(options.viewport?.width??1440,320,2560), - height:clamp(options.viewport?.height??900,240,1800) - }; - const touch=!!options.touch; - const mobile=!!options.mobile; - const target=normalizeUrl(options.url||options.target||this.baseUrl+'/',this.baseUrl); - this.streamFps=clamp(options.streamFps??30,5,60); + if(this.starting)return this.starting; + const run=async()=>{ + const viewport={ + width:clamp(options.viewport?.width??1440,320,2560), + height:clamp(options.viewport?.height??900,240,1800) + }; + const touch=!!options.touch; + const mobile=!!options.mobile; + const target=normalizeUrl(options.url||options.target||this.baseUrl+'/',this.baseUrl); + this.streamFps=clamp(options.streamFps??30,5,60); - if(this.page && this.targetUrl===target && - this.page.isClosed?.()===false) return this.status(); + if(this.page && this.targetUrl===target && + this.page.isClosed?.()===false) return this.status(); - await this.stop(); + await this.stop(); let chromium; try{ @@ -114,22 +117,26 @@ export class VisionController extends EventEmitter{ if(message.type()==='error') this.recordError('console: '+message.text()); }); + await this.page.goto(target,{waitUntil:'domcontentloaded',timeout:30000}); + await this.page.waitForTimeout(50); await this.page.screencast.start({ quality:72, size:viewport, onFrame:frame=>this.acceptFrame(frame) }); - - await this.page.goto(target,{waitUntil:'domcontentloaded',timeout:30000}); try{ await this.page.waitForFunction(()=>window.ForgeReady===true,{timeout:15000}); }catch{ // A live visual session is useful even while an application is still booting. } await this.page.waitForTimeout(250); - this.emit('started',this.status()); - this.emitTelemetry(); - return this.status(); + this.emit('started',this.status()); + this.emitTelemetry(); + return this.status(); + }; + this.starting=run(); + try{return await this.starting} + finally{this.starting=null} } recordError(message){ From c8911ebb792f601e9755681e8306acaed9db9ac6 Mon Sep 17 00:00:00 2001 From: venom3-byte Date: Wed, 30 Sep 2026 10:31:55 +0200 Subject: [PATCH 03/86] fix(vision): normalize action endpoint response envelope --- server.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server.mjs b/server.mjs index 23a41a0..f36baaa 100644 --- a/server.mjs +++ b/server.mjs @@ -16,7 +16,16 @@ const mime={'.html':'text/html; charset=utf-8','.js':'text/javascript; charset=u const clients=new Set(),rooms=new Map(); function json(res,o,status=200){res.writeHead(status,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','Access-Control-Allow-Origin':'*'});res.end(JSON.stringify(o))} function body(req,max=64*1024){return new Promise((resolve,reject)=>{let raw='';req.on('data',chunk=>{raw+=chunk;if(raw.length>max){reject(new Error('Request body too large'));req.destroy()}});req.on('end',()=>{try{resolve(raw?JSON.parse(raw):{})}catch(e){reject(new Error('Invalid JSON'))}});req.on('error',reject)})} -async function routeVisionAction(req,res){try{const q=await body(req);if(!q.action)return json(res,{ok:false,error:'Missing action'},400);return json(res,await vision.action(q.action,q.options||{}))}catch(e){return json(res,{ok:false,error:e.message||String(e)},500)}} +async function routeVisionAction(req,res){ + try{ + const q=await body(req); + if(!q.action)return json(res,{ok:false,error:'Missing action'},400); + const result=await vision.action(q.action,q.options||{}); + return json(res,{ok:true,result}); + }catch(e){ + return json(res,{ok:false,error:e.message||String(e),vision:vision.status()},500); + } +} function safeFile(urlPath){let p=decodeURIComponent(urlPath||'/');if(p==='/'||p==='')p='/index.html';if(p.includes('..'))return null;const full=path.resolve(root,'.'+p);return full.startsWith(root+path.sep)?full:null} function serve(req,res){const u=new URL(req.url,'http://localhost');const file=safeFile(u.pathname);fs.stat(file||'',(err,st)=>{if(!err&&st.isFile()){const ext=path.extname(file).toLowerCase();res.writeHead(200,{'Content-Type':mime[ext]||'application/octet-stream','Cache-Control':ext==='.html'||ext==='.js'||ext==='.css'?'no-cache':'public, max-age=31536000, immutable','X-Content-Type-Options':'nosniff'});return fs.createReadStream(file).pipe(res)}if(u.pathname!=='/'&&!path.extname(u.pathname)){const fallback=path.join(root,'index.html');res.writeHead(200,{'Content-Type':'text/html; charset=utf-8','Cache-Control':'no-cache'});return fs.createReadStream(fallback).on('error',()=>{res.writeHead(404);res.end('Not found')}).pipe(res)}res.writeHead(404);res.end('Not found')})} const srv=http.createServer(async (req,res)=>{const u=new URL(req.url,'http://localhost');if(req.method==='OPTIONS'){res.writeHead(204,{'Access-Control-Allow-Origin':'*','Access-Control-Allow-Methods':'GET,POST,OPTIONS','Access-Control-Allow-Headers':'Content-Type'});return res.end()} From 2d393bf326601adee2d612f34e722b44c4cc4a91 Mon Sep 17 00:00:00 2001 From: venom3-byte Date: Wed, 30 Sep 2026 10:35:04 +0200 Subject: [PATCH 04/86] feat(vision): expose flexible live element discovery --- vision-control.mjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/vision-control.mjs b/vision-control.mjs index 6b10548..88537f2 100644 --- a/vision-control.mjs +++ b/vision-control.mjs @@ -347,14 +347,18 @@ export class VisionController extends EventEmitter{ }); } - async elements(){ + async elements(requestedSelectors=[]){ if(!this.page||this.page.isClosed())throw new Error('Vision session is not running'); - const selectors=[ + const defaults=[ '[data-add="box"]','[data-add="sphere"]','[data-add="cylinder"]', '#assetInput','#importAssets','#play','#build','#openProject','#saveProject', '#focus','#frame','#duplicate','#delete','#viewport','.viewport', - '#visionOpen','#selected','#sceneCount','.topbar','.leftpanel','.rightpanel' + '#visionOpen','#selected','#sceneCount','#time','.topbar','.leftpanel','.rightpanel' ]; + const selectors=[...new Set( + [...(Array.isArray(requestedSelectors)?requestedSelectors:[]),...defaults] + .map(s=>String(s||'').trim()).filter(Boolean).slice(0,64) + )]; return this.page.evaluate(selectors=>({ viewport:{width:innerWidth,height:innerHeight,dpr:devicePixelRatio||1}, elements:selectors.map(selector=>{ From 96bea9bffe7c2984de5109cb3137c0af5c53e05b Mon Sep 17 00:00:00 2001 From: venom3-byte Date: Wed, 30 Sep 2026 10:39:05 +0200 Subject: [PATCH 05/86] fix(ci): make glTF validation deterministic with bundled fixture --- scripts/fixtures/minimal-triangle.glb | Bin 0 -> 456 bytes scripts/validate-gltf-test-assets.mjs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 scripts/fixtures/minimal-triangle.glb diff --git a/scripts/fixtures/minimal-triangle.glb b/scripts/fixtures/minimal-triangle.glb new file mode 100644 index 0000000000000000000000000000000000000000..d9fcaa550713c4ff9951dae122c9bbcf8018adb0 GIT binary patch literal 456 zcmY*W(MrQG6pemKx%e1l?X+ND1c!r)tir}#WDi-rt${TuNjf)5!AC#M&+;a#G7}Q+ zxj8wx=ceAR=9i4I?+eDhXgw{9MGwMSbddG1M`NW{kU?@i0S+}PG{R{^nYlJKMzcI> zq>_!;MU_0b&L9jNY)M4OCSznFd+kB#H}uhDGi>-D>Y+jVPF$b#Vc_)88reuE_ahX} z8M*D8hdsO$%Y2m=3x910a@($|sMzq(_UP~#RqaT2ntI*S>008)$O!CEpeGkq6%13o z|L<~ef?QHBrZ;zK;-y4cqP6<>sUYNqgIKAXRx4C)b!^H1ZJeYWO5G{1Kw=>a>&MxY mnn`{0-QYC6b#WjR^Oz7>ilXR>{^cQG(0)EX@6QoqC;S7BeTC!z literal 0 HcmV?d00001 diff --git a/scripts/validate-gltf-test-assets.mjs b/scripts/validate-gltf-test-assets.mjs index 95068ab..4aa8f02 100644 --- a/scripts/validate-gltf-test-assets.mjs +++ b/scripts/validate-gltf-test-assets.mjs @@ -3,7 +3,7 @@ import{join,relative,resolve}from"node:path"; import * as ValidatorNS from"gltf-validator"; const validator=ValidatorNS.default||ValidatorNS; -const root=resolve("test-results"); +const roots=[resolve("scripts/fixtures"),resolve("test-results")]; const reportPath="gltf-validation-report.json"; async function walk(dir){ @@ -18,7 +18,7 @@ async function walk(dir){ return out; } -const files=await walk(root); +const files=[...new Set((await Promise.all(roots.map(walk))).flat())]; const results=[]; for(const file of files){ try{ From 941de407c839ca0e5d604afb9bed82e4957d6369 Mon Sep 17 00:00:00 2001 From: venom3-byte Date: Wed, 30 Sep 2026 10:40:09 +0200 Subject: [PATCH 06/86] feat(forge): add real raster 2D paint tool with Vision QA proof --- forge-draw2d.js | 209 +++++++++++++++++++++++++++++++++++++ forge.css | 21 +++- index.html | 14 ++- server.mjs | 5 + tests/vision.live.spec.mjs | 45 ++++++++ vision-control.mjs | 2 +- 6 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 forge-draw2d.js diff --git a/forge-draw2d.js b/forge-draw2d.js new file mode 100644 index 0000000..b4a20e8 --- /dev/null +++ b/forge-draw2d.js @@ -0,0 +1,209 @@ +(() => { + const $ = (id) => document.getElementById(id); + const state = { + canvas: null, + ctx: null, + drawing: false, + last: null, + erasing: false, + brush: 18, + color: "#ffffff", + history: [], + redo: [], + maxHistory: 24, + ready: false, + lastPngDataUrl: null + }; + + function dialog(){ return $("draw2dDialog"); } + + function status(text){ + const el = $("draw2dStatus"); + if(el) el.textContent = text; + } + + function capture(){ + if(!state.ctx || !state.canvas) return; + state.history.push(state.ctx.getImageData(0,0,state.canvas.width,state.canvas.height)); + if(state.history.length > state.maxHistory) state.history.shift(); + state.redo.length = 0; + syncButtons(); + } + + function syncButtons(){ + $("drawUndo")?.toggleAttribute("disabled", state.history.length < 2); + $("drawRedo")?.toggleAttribute("disabled", state.redo.length === 0); + $("drawEraser")?.classList.toggle("active", state.erasing); + } + + function point(event){ + const r=state.canvas.getBoundingClientRect(); + return { + x:(event.clientX-r.left)*state.canvas.width/r.width, + y:(event.clientY-r.top)*state.canvas.height/r.height + }; + } + + function begin(event){ + if(!state.ctx) return; + event.preventDefault(); + capture(); + state.drawing=true; + state.last=point(event); + state.canvas.setPointerCapture?.(event.pointerId); + state.ctx.save(); + state.ctx.globalCompositeOperation=state.erasing?"destination-out":"source-over"; + state.ctx.strokeStyle=state.color; + state.ctx.lineWidth=state.brush; + state.ctx.lineCap="round"; + state.ctx.lineJoin="round"; + state.ctx.beginPath(); + state.ctx.arc(state.last.x,state.last.y,state.brush/2,0,Math.PI*2); + state.ctx.fillStyle=state.color; + if(state.erasing) state.ctx.fillStyle="rgba(0,0,0,1)"; + state.ctx.fill(); + state.ctx.restore(); + status("Drawing…"); + } + + function move(event){ + if(!state.drawing || !state.ctx) return; + event.preventDefault(); + const p=point(event),a=state.last; + state.ctx.save(); + state.ctx.globalCompositeOperation=state.erasing?"destination-out":"source-over"; + state.ctx.strokeStyle=state.color; + state.ctx.lineWidth=state.brush; + state.ctx.lineCap="round"; + state.ctx.lineJoin="round"; + state.ctx.beginPath(); + state.ctx.moveTo(a.x,a.y); + state.ctx.lineTo(p.x,p.y); + state.ctx.stroke(); + state.ctx.restore(); + state.last=p; + } + + function end(event){ + if(!state.drawing) return; + event?.preventDefault?.(); + state.drawing=false; + state.last=null; + try{state.canvas.releasePointerCapture?.(event?.pointerId)}catch{} + status("Ready"); + } + + function clear(){ + if(!state.ctx) return; + capture(); + state.ctx.clearRect(0,0,state.canvas.width,state.canvas.height); + status("Canvas cleared"); + } + + function undo(){ + if(state.history.length<=1 || !state.ctx)return; + const current=state.ctx.getImageData(0,0,state.canvas.width,state.canvas.height); + state.redo.push(current); + const previous=state.history.pop(); + state.ctx.putImageData(previous,0,0); + syncButtons(); + status("Undo"); + } + + function redo(){ + if(!state.redo.length || !state.ctx)return; + const current=state.ctx.getImageData(0,0,state.canvas.width,state.canvas.height); + state.history.push(current); + const next=state.redo.pop(); + state.ctx.putImageData(next,0,0); + syncButtons(); + status("Redo"); + } + + function dataUrl(){ + if(!state.canvas) return null; + return state.canvas.toDataURL("image/png"); + } + + function download(){ + const url=dataUrl(); + if(!url)return; + state.lastPngDataUrl=url; + const a=document.createElement("a"); + a.href=url; + a.download="forge-drawing.png"; + a.click(); + status("PNG exported"); + } + + async function addToScene(){ + if(!state.canvas || !window.Forge?.importFile) throw new Error("Forge runtime is not ready"); + const blob=await new Promise(resolve=>state.canvas.toBlob(resolve,"image/png")); + if(!blob)throw new Error("PNG encoding failed"); + const file=new File([blob],"forge-drawing.png",{type:"image/png"}); + state.lastPngDataUrl=await new Promise((resolve,reject)=>{ + const reader=new FileReader(); + reader.onload=()=>resolve(String(reader.result||"")); + reader.onerror=()=>reject(reader.error||new Error("PNG preview read failed")); + reader.readAsDataURL(blob); + }); + await window.ForgeProduction?.assets?.storeFile?.(file); + const result=await window.Forge.importFile(file); + window.ForgeRefreshUI?.(); + if(result?.record) window.Forge.select?.(result.record.id); + dialog()?.close(); + status("Imported into Forge"); + window.dispatchEvent(new CustomEvent("forge-drawing-imported",{detail:{name:file.name,bytes:file.size}})); + return {name:file.name,bytes:file.size,type:file.type}; + } + + function open(){ + dialog()?.showModal(); + status("Ready"); + syncButtons(); + } + + function init(){ + if(state.ready)return; + state.canvas=$("drawCanvas"); + state.ctx=state.canvas?.getContext("2d"); + if(!state.canvas||!state.ctx)return; + state.canvas.style.touchAction="none"; + state.canvas.addEventListener("pointerdown",begin); + state.canvas.addEventListener("pointermove",move); + state.canvas.addEventListener("pointerup",end); + state.canvas.addEventListener("pointercancel",end); + state.canvas.addEventListener("pointerleave",(e)=>{if(state.drawing)move(e)}); + $("draw2dOpen")?.addEventListener("click",open); + $("drawClose")?.addEventListener("click",()=>dialog()?.close()); + $("drawClear")?.addEventListener("click",clear); + $("drawUndo")?.addEventListener("click",undo); + $("drawRedo")?.addEventListener("click",redo); + $("drawDownload")?.addEventListener("click",download); + $("drawAddToScene")?.addEventListener("click",()=>addToScene().catch(e=>status("Import failed: "+(e?.message||String(e))))); + $("drawColor")?.addEventListener("input",e=>{state.color=e.target.value||"#ffffff"}); + $("drawSize")?.addEventListener("input",e=>{state.brush=Math.max(1,Math.min(120,Number(e.target.value)||18));$("drawSizeValue").textContent=state.brush+" px"}); + $("drawEraser")?.addEventListener("click",()=>{state.erasing=!state.erasing;syncButtons()}); + $("drawReset")?.addEventListener("click",()=>{state.color="#ffffff";state.brush=18;state.erasing=false;$("drawColor").value="#ffffff";$("drawSize").value="18";$("drawSizeValue").textContent="18 px";syncButtons()}); + state.ctx.clearRect(0,0,state.canvas.width,state.canvas.height); + state.ctx.fillStyle="rgba(8,20,35,0)"; + capture(); + state.ready=true; + syncButtons(); + } + + window.ForgeDraw2D={ + open, + clear, + undo, + redo, + download, + addToScene, + exportDataUrl:dataUrl, + status:()=>({ready:state.ready,drawing:state.drawing,brush:state.brush,color:state.color,erasing:state.erasing,history:state.history.length,redo:state.redo.length}), + get lastPngDataUrl(){ return state.lastPngDataUrl; } + }; + + if(document.readyState==="loading")document.addEventListener("DOMContentLoaded",init,{once:true}); + else init(); +})(); \ No newline at end of file diff --git a/forge.css b/forge.css index 50c0c9a..ec756e4 100644 --- a/forge.css +++ b/forge.css @@ -29,4 +29,23 @@ dialog{border:1px solid #2b4d6e;background:#091523;color:var(--txt);border-radiu .asset-thumb{width:30px;height:30px;object-fit:cover;border-radius:5px;border:1px solid #2b4c66;background:#020a12} .forge-boot{position:fixed;inset:0;z-index:9999;display:grid;place-items:center;background:radial-gradient(circle at 50% 42%,#132b45,#06101d 62%,#02050a);transition:opacity .25s ease,visibility .25s ease}.forge-boot.hidden{opacity:0;visibility:hidden;pointer-events:none}.forge-boot-card{width:min(420px,88vw);padding:24px;border:1px solid #24425f;border-radius:16px;background:#081523ee;box-shadow:0 18px 70px #0008;display:grid;gap:8px;text-align:center}.forge-boot-mark{width:52px;height:52px;margin:0 auto 2px;border-radius:14px;display:grid;place-items:center;background:linear-gradient(135deg,#4dd7ff,#756cff);color:#00101b;font-weight:900;font-size:24px}.forge-boot-card strong{font-size:17px;letter-spacing:1px}.forge-boot-card span{color:#c9ddf1;line-height:1.5}.forge-boot-card small{color:#6f8aa2;font-size:10px} -.editor-grid{position:absolute;inset:0;pointer-events:none;z-index:2;background-image:linear-gradient(rgba(100,160,210,.07) 1px,transparent 1px),linear-gradient(90deg,rgba(100,160,210,.07) 1px,transparent 1px);background-size:32px 32px;mask-image:linear-gradient(to bottom,transparent 0%,rgba(0,0,0,.8) 22%,rgba(0,0,0,.9) 100%);}.editor-grid:after{content:"";position:absolute;left:50%;top:50%;width:1px;height:100%;background:rgba(120,210,255,.12);box-shadow:0 0 0 1px rgba(120,210,255,.02)}.editor-grid:before{content:"";position:absolute;left:0;top:50%;height:1px;width:100%;background:rgba(255,120,120,.10)}.inspector{min-height:150px} \ No newline at end of file +.editor-grid{position:absolute;inset:0;pointer-events:none;z-index:2;background-image:linear-gradient(rgba(100,160,210,.07) 1px,transparent 1px),linear-gradient(90deg,rgba(100,160,210,.07) 1px,transparent 1px);background-size:32px 32px;mask-image:linear-gradient(to bottom,transparent 0%,rgba(0,0,0,.8) 22%,rgba(0,0,0,.9) 100%);}.editor-grid:after{content:"";position:absolute;left:50%;top:50%;width:1px;height:100%;background:rgba(120,210,255,.12);box-shadow:0 0 0 1px rgba(120,210,255,.02)}.editor-grid:before{content:"";position:absolute;left:0;top:50%;height:1px;width:100%;background:rgba(255,120,120,.10)}.inspector{min-height:150px} +.draw-card{background:linear-gradient(180deg,#0c1c2f,#0a1726)} +.draw-dialog{width:min(920px,96vw);max-height:94vh} +.draw-tools{display:flex;gap:10px;align-items:center;flex-wrap:wrap;padding:10px;border-bottom:1px solid var(--line);background:#071321} +.draw-tools label{display:flex;gap:8px;align-items:center;color:var(--muted);font-size:10px} +.draw-tools input[type=range]{width:180px} +.draw-tools input[type=color]{width:44px;height:30px;padding:2px} +.draw-tools button.active{background:#3a1f2b;border-color:#7a4b5b;color:#ffe3eb} +.draw-canvas-wrap{padding:14px;background:#030811;display:grid;place-items:center} +#drawCanvas{width:min(100%,768px);height:auto;aspect-ratio:3/2;background: + linear-gradient(45deg,#0e1c2a 25%,transparent 25%,transparent 75%,#0e1c2a 75%), + linear-gradient(45deg,#0e1c2a 25%,transparent 25%,transparent 75%,#0e1c2a 75%); + background-size:28px 28px;background-position:0 0,14px 14px; + border:1px solid #294760;border-radius:10px;box-shadow:0 16px 50px #0006;touch-action:none;cursor:crosshair} +.draw-status{padding:7px 10px;color:#8bb2d5;background:#06101b;border-top:1px solid var(--line);font:10px ui-monospace,SFMono-Regular,monospace} +@media(max-width:800px){ + .draw-dialog{width:96vw} + .draw-tools input[type=range]{width:130px} + #drawCanvas{width:100%} +} diff --git a/index.html b/index.html index a286f50..5264571 100644 --- a/index.html +++ b/index.html @@ -18,6 +18,7 @@
REAL ASSETS
Drop GLB / image / project here
GLB preserves a self-contained asset. glTF with external files is rejected instead of silently breaking paths.
Select an imported asset to inspect source metadata.
SCENE CONTROL
ASSET PIPELINE / EXPORT
No asset selectedScene object
READYSource exports preserve original bytes; derived previews are labeled separately.
No asset test yet.
+
2D CREATION
Real raster drawing with touch / mouse, undo-redo and PNG import into the current scene.
VISION + HANDS
Understands scene objects, world bounds, screen projection and precise dimensions. These local actions do not depend on a server.
Renderer…
FPS0
Physics0
ModeDetecting
@@ -43,8 +44,19 @@
Forge Spatial Vision
No scan yet.
Forge Visual QA

+
+
Forge 2D Paint

+

+
+
+
+

+

+
Ready

+

+
 
Entity Script

 
-
+
 
\ No newline at end of file
diff --git a/server.mjs b/server.mjs
index f36baaa..370f9ac 100644
--- a/server.mjs
+++ b/server.mjs
@@ -33,6 +33,11 @@ const srv=http.createServer(async (req,res)=>{const u=new URL(req.url,'http://lo
   if(u.pathname==='/api/pipeline'&&req.method==='POST'){let body='';req.on('data',c=>body+=c);req.on('end',async()=>{try{const q=JSON.parse(body);if(q.operation!=='cook-glb')return json(res,{ok:false,error:'Unsupported pipeline operation'},400);const bytes=Buffer.from(q.base64||'','base64');if(!bytes.length)return json(res,{ok:false,error:'Empty asset'},400);if(bytes.length>80*1024*1024)return json(res,{ok:false,error:'Asset exceeds 80MB pipeline limit'},413);const cooked=await cookGLB(bytes,q.options||{});return json(res,{ok:true,...cooked})}catch(e){return json(res,{ok:false,error:e.message},500)}});return}
   if(u.pathname==='/api/live/status')return json(res,{ok:true,agentConfigured:true,transport:'websocket',model:'forge-control-plane-v4',protocol:4,vision:vision.status()});
   if(u.pathname==='/api/vision/status')return json(res,{ok:true,protocol:4,vision:vision.status(),capabilities:{liveScreencast:true,mouse:true,keyboard:true,touch:true,wheel:true,drag:true}});
+  if(u.pathname==='/api/vision/frame'){
+    if(!vision.latestFrame)return json(res,{ok:false,error:'Live Vision frame is not available'},503);
+    res.writeHead(200,{'Content-Type':'image/jpeg','Cache-Control':'no-store','X-Content-Type-Options':'nosniff'});
+    return res.end(vision.latestFrame);
+  }
   if(u.pathname==='/api/vision/stream')return vision.stream(req,res);
   if(u.pathname==='/api/vision/elements')return vision.elements().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503));
   if(u.pathname==='/api/vision/state')return vision.state().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503));
diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 7952e55..2e29397 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -1,5 +1,6 @@
 import{test,expect}from"@playwright/test";
 import{WebSocket}from"ws";
+import{writeFile}from"node:fs/promises";
 
 const BASE="http://127.0.0.1:4173";
 const sleep=ms=>new Promise(r=>setTimeout(r,ms));
@@ -31,6 +32,11 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     expect(body.session.running).toBeTruthy();
 
     await expect.poll(()=>frames.length,{timeout:20000}).toBeGreaterThanOrEqual(5);
+    const proof=await request.get("/api/vision/frame");
+    expect(proof.ok()).toBeTruthy();
+    const proofBytes=await proof.body();
+    expect(proofBytes.slice(0,2).toString("hex")).toBe("ffd8");
+    await writeFile("test-results/vision-live-proof.jpg",proofBytes);
     expect(frames.at(-1).seq).toBeGreaterThan(frames[0].seq);
     expect(frames[0].width).toBeGreaterThan(300);
     expect(frames[0].height).toBeGreaterThan(200);
@@ -86,9 +92,48 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const release=await request.post("/api/vision/action",{data:{action:{type:"releaseAll"}}});
     expect((await release.json()).result.action.released).toBeTruthy();
 
+    const drawn=(await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:61,y:106.5},options:{includeState:true}}})).json;
     const finalState=await (await request.get("/api/vision/state")).json();
     expect(finalState.forgeReady).toBeTruthy();
     expect(finalState.canvasRect).toBeTruthy();
+
+    const open=await request.get("/api/vision/elements");
+    const openElements=(await open.json()).elements;
+    const drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
+    expect(drawOpen).toBeTruthy();
+    const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
+    expect((await clickDraw.json()).result.action.dispatched).toBeTruthy();
+
+    const drawingEls=(await (await request.get("/api/vision/elements")).json()).elements;
+    const canvas=drawingEls.find(e=>e.selector==="#drawCanvas"&&e.visible);
+    const add=drawingEls.find(e=>e.selector==="#drawAddToScene"&&e.visible);
+    expect(canvas).toBeTruthy(); expect(add).toBeTruthy();
+
+    const lines=[
+      {from:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.25},to:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.25}},
+      {from:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.75},to:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.75}},
+      {from:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.25},to:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.75}},
+      {from:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.25},to:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.75}}
+    ];
+    for(const line of lines){
+      const rr=await request.post("/api/vision/action",{data:{action:{type:"mouse.drag",from:line.from,to:line.to,steps:18}}});
+      expect((await rr.json()).result.action.dispatched).toBeTruthy();
+    }
+    const addResult=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:add.x+add.width/2,y:add.y+add.height/2},options:{includeState:true}}});
+    const addBody=await addResult.json();
+    expect(addBody.ok).toBeTruthy();
+    expect(addBody.result?.state?.selected?.name).toBe("forge-drawing.png");
+    expect(addBody.result?.state?.selected?.kind).toBeTruthy();
+
+    const data=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"window.ForgeDraw2D?.lastPngDataUrl || null"}}});
+    const dataBody=await data.json();
+    expect(typeof dataBody.result.value).toBe("string");
+    const m=/^data:image\/png;base64,(.+)$/.exec(dataBody.result.value);
+    expect(m).toBeTruthy();
+    const pngBytes=Buffer.from(m[1],"base64");
+    expect(pngBytes.slice(0,8).toString("hex")).toBe("89504e470d0a1a0a");
+    expect(pngBytes.length).toBeGreaterThan(1000);
+    await writeFile("test-results/vision-drawn-asset.png",pngBytes);
   });
 
   test("file-input control is available for real asset import workflows",async({request})=>{
diff --git a/vision-control.mjs b/vision-control.mjs
index 88537f2..8e991fd 100644
--- a/vision-control.mjs
+++ b/vision-control.mjs
@@ -353,7 +353,7 @@ export class VisionController extends EventEmitter{
       '[data-add="box"]','[data-add="sphere"]','[data-add="cylinder"]',
       '#assetInput','#importAssets','#play','#build','#openProject','#saveProject',
       '#focus','#frame','#duplicate','#delete','#viewport','.viewport',
-      '#visionOpen','#selected','#sceneCount','#time','.topbar','.leftpanel','.rightpanel'
+      '#visionOpen','#selected','#sceneCount','#time','#draw2dOpen','#drawCanvas','#drawAddToScene','#drawClear','#drawUndo','#drawRedo','.topbar','.leftpanel','.rightpanel'
     ];
     const selectors=[...new Set(
       [...(Array.isArray(requestedSelectors)?requestedSelectors:[]),...defaults]

From 2738079d5696f0764d09aaf7e96cc5f0dcbf3533 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 10:40:32 +0200
Subject: [PATCH 07/86] fix(draw): await real 2D asset import before Vision
 verification

---
 forge-draw2d.js | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/forge-draw2d.js b/forge-draw2d.js
index b4a20e8..39627c2 100644
--- a/forge-draw2d.js
+++ b/forge-draw2d.js
@@ -12,7 +12,8 @@
     redo: [],
     maxHistory: 24,
     ready: false,
-    lastPngDataUrl: null
+    lastPngDataUrl: null,
+    importPromise: null
   };
 
   function dialog(){ return $("draw2dDialog"); }
@@ -180,7 +181,12 @@
     $("drawUndo")?.addEventListener("click",undo);
     $("drawRedo")?.addEventListener("click",redo);
     $("drawDownload")?.addEventListener("click",download);
-    $("drawAddToScene")?.addEventListener("click",()=>addToScene().catch(e=>status("Import failed: "+(e?.message||String(e)))));
+    $("drawAddToScene")?.addEventListener("click",()=>{
+      state.importPromise=addToScene().catch(e=>{
+        status("Import failed: "+(e?.message||String(e)));
+        throw e;
+      }).finally(()=>{state.importPromise=null});
+    });
     $("drawColor")?.addEventListener("input",e=>{state.color=e.target.value||"#ffffff"});
     $("drawSize")?.addEventListener("input",e=>{state.brush=Math.max(1,Math.min(120,Number(e.target.value)||18));$("drawSizeValue").textContent=state.brush+" px"});
     $("drawEraser")?.addEventListener("click",()=>{state.erasing=!state.erasing;syncButtons()});
@@ -201,7 +207,8 @@
     addToScene,
     exportDataUrl:dataUrl,
     status:()=>({ready:state.ready,drawing:state.drawing,brush:state.brush,color:state.color,erasing:state.erasing,history:state.history.length,redo:state.redo.length}),
-    get lastPngDataUrl(){ return state.lastPngDataUrl; }
+    get lastPngDataUrl(){ return state.lastPngDataUrl; },
+    waitForImport:()=>state.importPromise||Promise.resolve(true)
   };
 
   if(document.readyState==="loading")document.addEventListener("DOMContentLoaded",init,{once:true});

From aef13073897efcd537c5ff19957e19eacfea8f8f Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 10:45:32 +0200
Subject: [PATCH 08/86] fix(draw): bind paint dialog directly and wait for
 visible canvas

---
 forge-draw2d.js            | 3 +--
 index.html                 | 2 +-
 tests/vision.live.spec.mjs | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/forge-draw2d.js b/forge-draw2d.js
index 39627c2..7bdd46e 100644
--- a/forge-draw2d.js
+++ b/forge-draw2d.js
@@ -211,6 +211,5 @@
     waitForImport:()=>state.importPromise||Promise.resolve(true)
   };
 
-  if(document.readyState==="loading")document.addEventListener("DOMContentLoaded",init,{once:true});
-  else init();
+  init();
 })();
\ No newline at end of file
diff --git a/index.html b/index.html
index 5264571..bb1d960 100644
--- a/index.html
+++ b/index.html
@@ -18,7 +18,7 @@
   
REAL ASSETS
Drop GLB / image / project here
GLB preserves a self-contained asset. glTF with external files is rejected instead of silently breaking paths.
Select an imported asset to inspect source metadata.

   
SCENE CONTROL

   
ASSET PIPELINE / EXPORT
No asset selectedScene object
READYSource exports preserve original bytes; derived previews are labeled separately.
No asset test yet.

-  
2D CREATION
Real raster drawing with touch / mouse, undo-redo and PNG import into the current scene.

+  
2D CREATION
Real raster drawing with touch / mouse, undo-redo and PNG import into the current scene.

   
VISION + HANDS
Understands scene objects, world bounds, screen projection and precise dimensions. These local actions do not depend on a server.

   
Renderer…
FPS0
Physics0
ModeDetecting

 
diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 2e29397..00b3ac8 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -92,7 +92,6 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const release=await request.post("/api/vision/action",{data:{action:{type:"releaseAll"}}});
     expect((await release.json()).result.action.released).toBeTruthy();
 
-    const drawn=(await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:61,y:106.5},options:{includeState:true}}})).json;
     const finalState=await (await request.get("/api/vision/state")).json();
     expect(finalState.forgeReady).toBeTruthy();
     expect(finalState.canvasRect).toBeTruthy();
@@ -104,6 +103,7 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
     expect((await clickDraw.json()).result.action.dispatched).toBeTruthy();
 
+    await expect.poll(async()=>((await (await request.get("/api/vision/elements")).json()).elements.some(e=>e.selector==="#drawCanvas"&&e.visible)),{timeout:5000}).toBeTruthy();
     const drawingEls=(await (await request.get("/api/vision/elements")).json()).elements;
     const canvas=drawingEls.find(e=>e.selector==="#drawCanvas"&&e.visible);
     const add=drawingEls.find(e=>e.selector==="#drawAddToScene"&&e.visible);

From 2c9991bb4d18a49e666fad3201cfdd1532655b35 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 10:49:46 +0200
Subject: [PATCH 09/86] fix(draw): make paint dialog part of deterministic UI
 bootstrap

---
 forge-ui.js                | 4 ++++
 tests/vision.live.spec.mjs | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/forge-ui.js b/forge-ui.js
index a3a1238..e2acf33 100644
--- a/forge-ui.js
+++ b/forge-ui.js
@@ -246,6 +246,10 @@ $("duplicate").onclick=()=>{if(engine.duplicate()){refresh();inspect()}};$("undo
 $("delete").onclick=()=>{engine.delete();refresh();inspect()};$("focus").onclick=()=>engine.focus();$("frame").onclick=()=>engine.frame();$("reset")?.addEventListener("click",()=>location.reload());
 $("play").onclick=()=>{$("play").textContent=engine.running?"❚❚ Play":"▶ Play";engine.running=!engine.running;engine.timelinePlaying=engine.running};$("timelinePlay").onclick=()=>$("play").click();
 $("rewind").onclick=()=>{engine.timelineTime=0;$("time").value=0;engine.evalAnimation(0);inspect()};$("time").oninput=e=>{engine.timelineTime=+e.target.value||0;engine.evalAnimation(engine.timelineTime);$("playhead").style.left=(engine.timelineTime*60)+"px";inspect()};$("key").onclick=()=>{engine.key();toast("Keyframe added")};
+$("draw2dOpen")?.addEventListener("click",()=>{
+  const d=$("draw2dDialog");
+  if(d&&!d.open)try{d.showModal()}catch{d.setAttribute("open","")}
+});
 $("importAssets").onclick=()=>$("assetInput").click();$("assetInput").onchange=e=>loadFiles([...e.target.files]);
 async function loadFiles(files){for(const f of files)try{await window.ForgeProduction?.assets?.storeFile?.(f);const r=await engine.importFile(f);if(r.type==="project")await engine.load(r.data);write("Imported "+f.name)}catch(e){write("Import failed "+f.name+": "+e.message,"error")}refresh();inspect()}
 document.addEventListener("dragover",e=>{e.preventDefault();$("dropOverlay").classList.remove("hidden")});document.addEventListener("drop",e=>{e.preventDefault();$("dropOverlay").classList.add("hidden");if(e.dataTransfer?.files?.length)loadFiles([...e.dataTransfer.files])});
diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 00b3ac8..eee287d 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -101,6 +101,11 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
     expect(drawOpen).toBeTruthy();
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
+    const drawStatus=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"JSON.stringify({api:!!window.ForgeDraw2D,ready:window.ForgeDraw2D?.status?.().ready||false,open:document.querySelector('#draw2dDialog')?.open||false})"}}});
+    const drawStatusBody=await drawStatus.json();
+    expect(JSON.parse(drawStatusBody.result.value).api).toBeTruthy();
+    expect(JSON.parse(drawStatusBody.result.value).ready).toBeTruthy();
+    expect(JSON.parse(drawStatusBody.result.value).open).toBeTruthy();
     expect((await clickDraw.json()).result.action.dispatched).toBeTruthy();
 
     await expect.poll(async()=>((await (await request.get("/api/vision/elements")).json()).elements.some(e=>e.selector==="#drawCanvas"&&e.visible)),{timeout:5000}).toBeTruthy();

From 9fcea86567aa958d82eb0c6affcba8849e14ebca Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 10:53:27 +0200
Subject: [PATCH 10/86] test(vision): read evaluate payload from action
 envelope

---
 tests/vision.live.spec.mjs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index eee287d..52c1eba 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -103,9 +103,9 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
     const drawStatus=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"JSON.stringify({api:!!window.ForgeDraw2D,ready:window.ForgeDraw2D?.status?.().ready||false,open:document.querySelector('#draw2dDialog')?.open||false})"}}});
     const drawStatusBody=await drawStatus.json();
-    expect(JSON.parse(drawStatusBody.result.value).api).toBeTruthy();
-    expect(JSON.parse(drawStatusBody.result.value).ready).toBeTruthy();
-    expect(JSON.parse(drawStatusBody.result.value).open).toBeTruthy();
+    expect(JSON.parse(drawStatusBody.result.action.value).api).toBeTruthy();
+    expect(JSON.parse(drawStatusBody.result.action.value).ready).toBeTruthy();
+    expect(JSON.parse(drawStatusBody.result.action.value).open).toBeTruthy();
     expect((await clickDraw.json()).result.action.dispatched).toBeTruthy();
 
     await expect.poll(async()=>((await (await request.get("/api/vision/elements")).json()).elements.some(e=>e.selector==="#drawCanvas"&&e.visible)),{timeout:5000}).toBeTruthy();

From 458cf5f3c1645573a86c93763513c1d3019ee767 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 10:57:19 +0200
Subject: [PATCH 11/86] fix(vision): report real click target and harden 2D
 dialog opening

---
 forge-draw2d.js            | 10 +++++++++-
 index.html                 |  2 +-
 tests/vision.live.spec.mjs |  3 +++
 vision-control.mjs         |  8 +++++++-
 4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/forge-draw2d.js b/forge-draw2d.js
index 7bdd46e..7085dd8 100644
--- a/forge-draw2d.js
+++ b/forge-draw2d.js
@@ -159,9 +159,17 @@
   }
 
   function open(){
-    dialog()?.showModal();
+    const d=dialog();
+    if(!d)throw new Error("2D paint dialog is missing");
+    try{
+      if(!d.open)d.showModal();
+    }catch{
+      d.setAttribute("open","");
+      d.style.display="block";
+    }
     status("Ready");
     syncButtons();
+    return{open:!!d.open,ready:state.ready};
   }
 
   function init(){
diff --git a/index.html b/index.html
index bb1d960..bc449c5 100644
--- a/index.html
+++ b/index.html
@@ -18,7 +18,7 @@
   
REAL ASSETS
Drop GLB / image / project here
GLB preserves a self-contained asset. glTF with external files is rejected instead of silently breaking paths.
Select an imported asset to inspect source metadata.

   
SCENE CONTROL

   
ASSET PIPELINE / EXPORT
No asset selectedScene object
READYSource exports preserve original bytes; derived previews are labeled separately.
No asset test yet.

-  
2D CREATION
Real raster drawing with touch / mouse, undo-redo and PNG import into the current scene.

+  
2D CREATION
Real raster drawing with touch / mouse, undo-redo and PNG import into the current scene.

   
VISION + HANDS
Understands scene objects, world bounds, screen projection and precise dimensions. These local actions do not depend on a server.

   
Renderer…
FPS0
Physics0
ModeDetecting

 
diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 52c1eba..d90dfef 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -101,6 +101,9 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
     expect(drawOpen).toBeTruthy();
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
+    const clickBody=await clickDraw.json();
+    expect(clickBody.result.action.target?.id).toBe("draw2dOpen");
+    expect(clickBody.result.action.target?.width).toBeGreaterThan(0);
     const drawStatus=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"JSON.stringify({api:!!window.ForgeDraw2D,ready:window.ForgeDraw2D?.status?.().ready||false,open:document.querySelector('#draw2dDialog')?.open||false})"}}});
     const drawStatusBody=await drawStatus.json();
     expect(JSON.parse(drawStatusBody.result.action.value).api).toBeTruthy();
diff --git a/vision-control.mjs b/vision-control.mjs
index 8e991fd..78b8907 100644
--- a/vision-control.mjs
+++ b/vision-control.mjs
@@ -438,8 +438,14 @@ export class VisionController extends EventEmitter{
       }
       case 'mouse.click':{
         const x=Number(action.x)||0,y=Number(action.y)||0,button=buttonValue(action.button);
+        const target=await this.page.evaluate(({x,y})=>{
+          const el=document.elementFromPoint(x,y);
+          if(!el)return null;
+          const r=el.getBoundingClientRect();
+          return{tag:el.tagName||'',id:el.id||'',className:typeof el.className==='string'?el.className:'',text:(el.textContent||'').trim().slice(0,120),x:r.x,y:r.y,width:r.width,height:r.height};
+        },{x,y});
         await this.page.mouse.click(x,y,{button,clickCount:Math.max(1,Math.min(3,Number(action.clickCount)||1)),delay:Math.max(0,Number(action.delay)||0)});
-        return{dispatched:true,type,x,y,button};
+        return{dispatched:true,type,x,y,button,target};
       }
       case 'mouse.dblclick':{
         const x=Number(action.x)||0,y=Number(action.y)||0,button=buttonValue(action.button);

From 1808b13650b4d83fab8b869707bc6928c7289198 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 11:00:41 +0200
Subject: [PATCH 12/86] test(vision): avoid duplicate click response binding

---
 tests/vision.live.spec.mjs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index d90dfef..6574fd9 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -101,15 +101,15 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
     expect(drawOpen).toBeTruthy();
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
-    const clickBody=await clickDraw.json();
-    expect(clickBody.result.action.target?.id).toBe("draw2dOpen");
-    expect(clickBody.result.action.target?.width).toBeGreaterThan(0);
+    const clickDrawBody=await clickDraw.json();
+    expect(clickDrawBody.result.action.target?.id).toBe("draw2dOpen");
+    expect(clickDrawBody.result.action.target?.width).toBeGreaterThan(0);
     const drawStatus=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"JSON.stringify({api:!!window.ForgeDraw2D,ready:window.ForgeDraw2D?.status?.().ready||false,open:document.querySelector('#draw2dDialog')?.open||false})"}}});
     const drawStatusBody=await drawStatus.json();
     expect(JSON.parse(drawStatusBody.result.action.value).api).toBeTruthy();
     expect(JSON.parse(drawStatusBody.result.action.value).ready).toBeTruthy();
     expect(JSON.parse(drawStatusBody.result.action.value).open).toBeTruthy();
-    expect((await clickDraw.json()).result.action.dispatched).toBeTruthy();
+    expect(clickDrawBody.result.action.dispatched).toBeTruthy();
 
     await expect.poll(async()=>((await (await request.get("/api/vision/elements")).json()).elements.some(e=>e.selector==="#drawCanvas"&&e.visible)),{timeout:5000}).toBeTruthy();
     const drawingEls=(await (await request.get("/api/vision/elements")).json()).elements;

From d2daf19ee9b187e54a6dd4205fccfdff25f0491c Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 11:04:10 +0200
Subject: [PATCH 13/86] test(vision): capture full hit-target mismatch
 diagnostics

---
 tests/vision.live.spec.mjs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 6574fd9..2ce5ce5 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -102,8 +102,8 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     expect(drawOpen).toBeTruthy();
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
     const clickDrawBody=await clickDraw.json();
-    expect(clickDrawBody.result.action.target?.id).toBe("draw2dOpen");
-    expect(clickDrawBody.result.action.target?.width).toBeGreaterThan(0);
+    expect(clickDrawBody.result.action.target).toMatchObject({id:"draw2dOpen"});
+    expect(clickDrawBody.result.action.target.width).toBeGreaterThan(0);
     const drawStatus=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"JSON.stringify({api:!!window.ForgeDraw2D,ready:window.ForgeDraw2D?.status?.().ready||false,open:document.querySelector('#draw2dDialog')?.open||false})"}}});
     const drawStatusBody=await drawStatus.json();
     expect(JSON.parse(drawStatusBody.result.action.value).api).toBeTruthy();

From 54df1717c83d85046eda741dffa3865fca324a53 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 11:08:01 +0200
Subject: [PATCH 14/86] test(vision): use real wheel navigation before
 offscreen UI click

---
 tests/vision.live.spec.mjs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 2ce5ce5..f8f15f9 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -98,8 +98,16 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
 
     const open=await request.get("/api/vision/elements");
     const openElements=(await open.json()).elements;
-    const drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
+    let drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
     expect(drawOpen).toBeTruthy();
+    for(let i=0;i<3&&drawOpen.y>850;i++){
+      const scroll=await request.post("/api/vision/action",{data:{action:{type:"mouse.wheel",deltaY:700}}});
+      expect((await scroll.json()).result.action.dispatched).toBeTruthy();
+      const refreshed=(await (await request.get("/api/vision/elements")).json()).elements;
+      drawOpen=refreshed.find(e=>e.selector==="#draw2dOpen"&&e.visible);
+      expect(drawOpen).toBeTruthy();
+    }
+    expect(drawOpen.y).toBeLessThan(900);
     const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}});
     const clickDrawBody=await clickDraw.json();
     expect(clickDrawBody.result.action.target).toMatchObject({id:"draw2dOpen"});
@@ -135,8 +143,8 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
 
     const data=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"window.ForgeDraw2D?.lastPngDataUrl || null"}}});
     const dataBody=await data.json();
-    expect(typeof dataBody.result.value).toBe("string");
-    const m=/^data:image\/png;base64,(.+)$/.exec(dataBody.result.value);
+    expect(typeof dataBody.result.action.value).toBe("string");
+    const m=/^data:image\/png;base64,(.+)$/.exec(dataBody.result.action.value);
     expect(m).toBeTruthy();
     const pngBytes=Buffer.from(m[1],"base64");
     expect(pngBytes.slice(0,8).toString("hex")).toBe("89504e470d0a1a0a");

From bd0a2e2ee77d0c8b1cd9eccab395a9556a82e607 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 11:11:24 +0200
Subject: [PATCH 15/86] fix(vision): target wheel events at real screen
 coordinates

---
 tests/vision.live.spec.mjs | 2 +-
 vision-control.mjs         | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index f8f15f9..e4eba72 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -101,7 +101,7 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     let drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible);
     expect(drawOpen).toBeTruthy();
     for(let i=0;i<3&&drawOpen.y>850;i++){
-      const scroll=await request.post("/api/vision/action",{data:{action:{type:"mouse.wheel",deltaY:700}}});
+      const scroll=await request.post("/api/vision/action",{data:{action:{type:"mouse.wheel",x:110,y:450,deltaY:700}}});
       expect((await scroll.json()).result.action.dispatched).toBeTruthy();
       const refreshed=(await (await request.get("/api/vision/elements")).json()).elements;
       drawOpen=refreshed.find(e=>e.selector==="#draw2dOpen"&&e.visible);
diff --git a/vision-control.mjs b/vision-control.mjs
index 78b8907..c29e33c 100644
--- a/vision-control.mjs
+++ b/vision-control.mjs
@@ -452,9 +452,12 @@ export class VisionController extends EventEmitter{
         await this.page.mouse.dblclick(x,y,{button,delay:Math.max(0,Number(action.delay)||0)});
         return{dispatched:true,type,x,y,button};
       }
-      case 'mouse.wheel':
+      case 'mouse.wheel':{
+        const x=Number(action.x),y=Number(action.y);
+        if(Number.isFinite(x)&&Number.isFinite(y))await this.page.mouse.move(x,y,{steps:Math.max(1,Number(action.steps)||1)});
         await this.page.mouse.wheel(Number(action.deltaX)||0,Number(action.deltaY)||0);
-        return{dispatched:true,type,deltaX:Number(action.deltaX)||0,deltaY:Number(action.deltaY)||0};
+        return{dispatched:true,type,x:Number.isFinite(x)?x:null,y:Number.isFinite(y)?y:null,deltaX:Number(action.deltaX)||0,deltaY:Number(action.deltaY)||0};
+      }
       case 'mouse.drag':{
         const from=action.from||{},to=action.to||{},button=buttonValue(action.button);
         await this.page.mouse.move(Number(from.x)||0,Number(from.y)||0);

From 4949fe35decbf02ebd1ec1c33f2ef1c499b06764 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 11:15:49 +0200
Subject: [PATCH 16/86] test(vision): verify asynchronous 2D import and save
 visual proof frames

---
 tests/vision.live.spec.mjs | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index e4eba72..9485d68 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -135,11 +135,25 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
       const rr=await request.post("/api/vision/action",{data:{action:{type:"mouse.drag",from:line.from,to:line.to,steps:18}}});
       expect((await rr.json()).result.action.dispatched).toBeTruthy();
     }
+    const canvasProof=await request.get("/api/vision/frame");
+    expect(canvasProof.ok()).toBeTruthy();
+    const canvasProofBytes=await canvasProof.body();
+    expect(canvasProofBytes.slice(0,2).toString("hex")).toBe("ffd8");
+    await writeFile("test-results/vision-drawing-canvas-proof.jpg",canvasProofBytes);
     const addResult=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:add.x+add.width/2,y:add.y+add.height/2},options:{includeState:true}}});
     const addBody=await addResult.json();
     expect(addBody.ok).toBeTruthy();
-    expect(addBody.result?.state?.selected?.name).toBe("forge-drawing.png");
-    expect(addBody.result?.state?.selected?.kind).toBeTruthy();
+    expect(addBody.result.action.target).toMatchObject({id:"drawAddToScene"});
+    await expect.poll(async()=>{
+      const st=await (await request.get("/api/vision/state")).json();
+      return st.selected?.name||"";
+    },{timeout:12000}).toBe("forge-drawing.png");
+    const importedState=await (await request.get("/api/vision/state")).json();
+    expect(importedState.selected?.name).toBe("forge-drawing.png");
+    expect(importedState.selected?.kind).toBeTruthy();
+    const importedProof=await request.get("/api/vision/frame");
+    expect(importedProof.ok()).toBeTruthy();
+    await writeFile("test-results/vision-imported-asset-proof.jpg",await importedProof.body());
 
     const data=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"window.ForgeDraw2D?.lastPngDataUrl || null"}}});
     const dataBody=await data.json();

From 5034b1fd866d8aa70849bae292488522e2f82321 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 11:58:40 +0200
Subject: [PATCH 17/86] test(vision): preserve visual proof images as CI
 artifacts

---
 .github/workflows/proof.yml | 1 +
 tests/vision.live.spec.mjs  | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml
index d0ba430..2e89b07 100644
--- a/.github/workflows/proof.yml
+++ b/.github/workflows/proof.yml
@@ -48,4 +48,5 @@ jobs:
             playwright-report
             test-results
             gltf-validation-report.json
+vision-proof
           if-no-files-found: ignore
diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs
index 9485d68..271387e 100644
--- a/tests/vision.live.spec.mjs
+++ b/tests/vision.live.spec.mjs
@@ -36,7 +36,7 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     expect(proof.ok()).toBeTruthy();
     const proofBytes=await proof.body();
     expect(proofBytes.slice(0,2).toString("hex")).toBe("ffd8");
-    await writeFile("test-results/vision-live-proof.jpg",proofBytes);
+    await writeFile("vision-proof/vision-live-proof.jpg",proofBytes);
     expect(frames.at(-1).seq).toBeGreaterThan(frames[0].seq);
     expect(frames[0].width).toBeGreaterThan(300);
     expect(frames[0].height).toBeGreaterThan(200);
@@ -139,7 +139,7 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     expect(canvasProof.ok()).toBeTruthy();
     const canvasProofBytes=await canvasProof.body();
     expect(canvasProofBytes.slice(0,2).toString("hex")).toBe("ffd8");
-    await writeFile("test-results/vision-drawing-canvas-proof.jpg",canvasProofBytes);
+    await writeFile("vision-proof/vision-drawing-canvas-proof.jpg",canvasProofBytes);
     const addResult=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:add.x+add.width/2,y:add.y+add.height/2},options:{includeState:true}}});
     const addBody=await addResult.json();
     expect(addBody.ok).toBeTruthy();
@@ -153,7 +153,7 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     expect(importedState.selected?.kind).toBeTruthy();
     const importedProof=await request.get("/api/vision/frame");
     expect(importedProof.ok()).toBeTruthy();
-    await writeFile("test-results/vision-imported-asset-proof.jpg",await importedProof.body());
+    await writeFile("vision-proof/vision-imported-asset-proof.jpg",await importedProof.body());
 
     const data=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"window.ForgeDraw2D?.lastPngDataUrl || null"}}});
     const dataBody=await data.json();
@@ -163,7 +163,7 @@ test.describe.serial("Forge Live Vision — continuous stream + real browser inp
     const pngBytes=Buffer.from(m[1],"base64");
     expect(pngBytes.slice(0,8).toString("hex")).toBe("89504e470d0a1a0a");
     expect(pngBytes.length).toBeGreaterThan(1000);
-    await writeFile("test-results/vision-drawn-asset.png",pngBytes);
+    await writeFile("vision-proof/vision-drawn-asset.png",pngBytes);
   });
 
   test("file-input control is available for real asset import workflows",async({request})=>{

From 9def5a5a618beedd822305ce7d869de641f5bdc1 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 12:01:58 +0200
Subject: [PATCH 18/86] fix(ci): keep vision proof path inside artifact upload
 block

---
 .github/workflows/proof.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml
index 2e89b07..d768d7a 100644
--- a/.github/workflows/proof.yml
+++ b/.github/workflows/proof.yml
@@ -48,5 +48,5 @@ jobs:
             playwright-report
             test-results
             gltf-validation-report.json
-vision-proof
+            vision-proof
           if-no-files-found: ignore

From 039304c1a4b99819c009ccf152ef89f98e90777c Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 12:03:49 +0200
Subject: [PATCH 19/86] feat(audio): import real audio assets into sound
 components with listener

---
 forge-engine.js                | 37 ++++++++++++++++++++++++++++++++--
 forge-ui.js                    | 20 +++++++++++++++---
 index.html                     |  2 +-
 tests/professional.qa.spec.mjs | 36 +++++++++++++++++++++++++++++++++
 4 files changed, 89 insertions(+), 6 deletions(-)

diff --git a/forge-engine.js b/forge-engine.js
index 38a55f9..405e16d 100644
--- a/forge-engine.js
+++ b/forge-engine.js
@@ -36,7 +36,7 @@ export class ForgeEngine{
   material(color=[.22,.62,.9]){const m=new pc.StandardMaterial();m.diffuse=new pc.Color(...color);m.emissive=new pc.Color(...color);m.emissiveIntensity=2;m.metalness=.0;m.gloss=.5;m.useMetalness=false;m.useLighting=false;m.useTonemap=false;m.specular=new pc.Color(0,0,0);m.update();return m}
   primitive(kind,name){const r=this.add(kind,name||kind+'-'+(this.entities.size+1)),type=kind==='sphere'?'sphere':kind==='cylinder'?'cylinder':kind==='capsule'?'capsule':kind==='plane'?'plane':'box';r.entity.addComponent('render',{type});r.entity.render.layers=[pc.LAYERID_WORLD];r.entity.render.frustumCulling=false;r.entity.render.enabled=true;r.entity.render.material=this.material(kind==='plane'?[.08,.15,.2]:[.22,.62,.9]);if(kind!=='plane')r.entity.setLocalPosition((Math.random()-.5)*4,1+(Math.random()*1.7),(Math.random()-.5)*4);r.components.geometry={sourceUnits:'meters'};return r}
   createPlane(name,w=30,d=30){const r=this.primitive('plane',name);r.entity.setLocalScale(w,1,d);r.entity.setLocalPosition(0,0,0);if(name==='Ground')r.entity.render.layers=[];this.setPhysics(r.id,'fixed','box');return r}
-  createCamera(name,pos){const r=this.add('camera',name);r.entity.addComponent('camera',{clearColor:new pc.Color(.035,.07,.12),clearColorBuffer:true,clearDepthBuffer:true,fov:60,nearClip:.01,farClip:10000,enabled:true,layers:[pc.LAYERID_WORLD]});r.entity.setLocalPosition(pos.x,pos.y,pos.z);r.entity.lookAt(0,1,0);r.components.camera={active:true};r.entity.camera.enabled=true;return r}
+  createCamera(name,pos){const r=this.add('camera',name);r.entity.addComponent('camera',{clearColor:new pc.Color(.035,.07,.12),clearColorBuffer:true,clearDepthBuffer:true,fov:60,nearClip:.01,farClip:10000,enabled:true,layers:[pc.LAYERID_WORLD]});r.entity.addComponent('audiolistener');r.entity.setLocalPosition(pos.x,pos.y,pos.z);r.entity.lookAt(0,1,0);r.components.camera={active:true};r.entity.camera.enabled=true;return r}
   createLight(name){const r=this.add('light',name);r.entity.addComponent('light',{type:'directional',color:new pc.Color(1,.96,.88),intensity:2.5,castShadows:true,shadowDistance:80,enabled:true,layers:[pc.LAYERID_WORLD]});r.entity.setEulerAngles(48,-32,0);r.components.light={type:'directional'};return r}
   camera(){return[...this.entities.values()].find(r=>r.kind==='camera'&&r.components.camera?.active)?.entity}
   selected(){return this.selectedId?this.entities.get(this.selectedId):null}
@@ -116,6 +116,24 @@ export class ForgeEngine{
   async importFile(file){
     const url=URL.createObjectURL(file);if(file.name.toLowerCase().endsWith('.forge.json'))return{type:'project',data:JSON.parse(await file.text())};
     let analysis=null;try{analysis=await window.ForgeProduction?.assets?.analyze?.(file)}catch{}
+    if(/^audio\//.test(file.type) || /\.(wav|mp3|ogg|m4a|aac|flac|webm)$/i.test(file.name)){
+      const asset=new pc.Asset(file.name,'audio',{url});
+      this.app.assets.add(asset);
+      await new Promise((resolve,reject)=>{
+        asset.once('error',reject);
+        asset.once('load',()=>resolve());
+        this.app.assets.load(asset);
+      });
+      const r=this.add('audio',file.name);
+      r.entity.addComponent('sound',{positional:true,volume:1});
+      r.entity.sound.addSlot('main',{asset:asset.id,autoPlay:false,loop:false,overlap:false,volume:1,pitch:1});
+      r.components.asset={type:'audio',name:file.name,analysis,derivedDirty:false};
+      this.assets.set(file.name,{type:'audio',file,url,resource:asset.resource,asset});
+      this.select(r.id);
+      window.dispatchEvent(new Event('forge-assets-changed'));
+      window.ForgeRefreshUI?.();
+      return{type:'audio',record:r,asset};
+    }
     if(/^image\//.test(file.type)){const bmp=await createImageBitmap(file),c=document.createElement('canvas');c.width=bmp.width;c.height=bmp.height;c.getContext('2d').drawImage(bmp,0,0);const tex=new pc.Texture(this.app.graphicsDevice,{width:bmp.width,height:bmp.height,format:pc.PIXELFORMAT_R8_G8_B8_A8});tex.setSource(c);const mat=this.material([1,1,1]);mat.diffuseMap=tex;mat.emissiveMap=tex;mat.emissive=new pc.Color(1,1,1);mat.update();const r=this.primitive('plane',file.name);r.entity.render.material=mat;r.components.asset={type:'image',name:file.name,width:bmp.width,height:bmp.height,analysis,derivedDirty:false};this.assets.set(file.name,{type:'image',file,url});this.select(r.id);window.dispatchEvent(new Event('forge-assets-changed'));window.ForgeRefreshUI?.();return{type:'image',record:r}}
     if(/\.(glb|gltf)$/i.test(file.name))return new Promise(async(resolve,reject)=>{
       if(/\.gltf$/i.test(file.name)){try{const json=JSON.parse(await file.text()),external=[...(json.buffers||[]),...(json.images||[])].some(x=>x?.uri&&!String(x.uri).startsWith('data:'));if(external)throw new Error('This .gltf references external files. Use a packaged .glb so Forge can preserve the asset without path breakage')}catch(e){reject(e);return}}
@@ -156,7 +174,22 @@ export class ForgeEngine{
       if(d.script)this.scripts.set(r.id,d.script);
 
       const asset=d.components?.asset;
-      if(asset?.name&&assetRoot&&asset.type==='model'){
+      if(asset?.name&&assetRoot&&asset.type==='audio'){
+        const a=new pc.Asset(asset.name,'audio',{url:assetRoot+'assets/'+encodeURIComponent(asset.name)});
+        this.app.assets.add(a);
+        await new Promise((resolve,reject)=>{
+          a.once('error',reject);
+          a.once('load',()=>{
+            try{
+              if(!r.entity.sound)r.entity.addComponent('sound',{positional:true,volume:1});
+              if(!r.entity.sound.slot('main'))r.entity.sound.addSlot('main',{asset:a.id,autoPlay:false,loop:false,overlap:false,volume:1,pitch:1});
+              this.assets.set(asset.name,{type:'audio',file:null,url:a.getFileUrl?.()||assetRoot+'assets/'+encodeURIComponent(asset.name),resource:a.resource,asset:a});
+              resolve();
+            }catch(err){reject(err)}
+          });
+          this.app.assets.load(a);
+        });
+      } else if(asset?.name&&assetRoot&&asset.type==='model'){
         const a=new pc.Asset(asset.name,'container',{url:assetRoot+'assets/'+encodeURIComponent(asset.name)});
         this.app.assets.add(a);
         await new Promise((resolve,reject)=>{
diff --git a/forge-ui.js b/forge-ui.js
index e2acf33..073dd45 100644
--- a/forge-ui.js
+++ b/forge-ui.js
@@ -139,7 +139,7 @@ if(!window.ForgeGizmo){
   };
 }
 function esc(s){return String(s??"").replace(/[&<>"']/g,c=>({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]))}
-function icon(k){return({box:"▣",sphere:"●",cylinder:"⬢",capsule:"◉",plane:"▱",camera:"◫",light:"☼",model:"◇",empty:"＋"})[k]||"•"}
+function icon(k){return({box:"▣",sphere:"●",cylinder:"⬢",capsule:"◉",plane:"▱",camera:"◫",light:"☼",model:"◇",audio:"♫",empty:"＋"})[k]||"•"}
 function matchesFilter(r){const q=String($("treeFilter")?.value||"").trim().toLowerCase();return !q||r.name.toLowerCase().includes(q)||r.kind.toLowerCase().includes(q)}
 function refresh(){
   const tree=$("tree");tree.innerHTML="";
@@ -182,7 +182,13 @@ function updateAssetPanel(){
   if(!label||!type)return;
   const a=r?.components?.asset;
   label.textContent=r?.name||"No asset selected";
-  type.textContent=a?.type?(a.type==="model"?"3D GLB":a.type==="image"?"2D Image":a.type.toUpperCase()):"Scene object";
+  type.textContent=a?.type?(a.type==="model"?"3D GLB":a.type==="image"?"2D Image":a.type==="audio"?"AUDIO":"FILE"):"Scene object";
+  const play=$("assetPlay");
+  if(play){
+    const isAudio=a?.type==="audio"&&!!r?.entity?.sound?.slot?.("main");
+    play.disabled=!isAudio;
+    play.textContent=isAudio?(r.entity.sound.slot("main").isPlaying?"■ Stop Audio":"▶ Play Audio"):"▶ Play Audio";
+  }
   if(meta){
     if(!a){
       meta.textContent="Select an imported asset to inspect source metadata.";
@@ -203,7 +209,7 @@ function updateAssetBrowser(){
   if(!records.length){const e=document.createElement("div");e.className="asset-empty";e.textContent="No imported assets";list.append(e);return}
   for(const r of records){
     const row=document.createElement("button");row.className="asset-row"+(r.id===engine.selectedId?" active":"");
-    const t=r.components.asset.type==="model"?"3D":r.components.asset.type==="image"?"2D":"FILE";
+    const t=r.components.asset.type==="model"?"3D":r.components.asset.type==="image"?"2D":r.components.asset.type==="audio"?"AUD":"FILE";
     const visual=document.createElement("span");visual.className="asset-visual";
     const entry=engine.assets?.get?.(r.components.asset.name);
     if(r.components.asset.type==="image"&&entry?.file){
@@ -262,6 +268,14 @@ $("exportPreview")?.addEventListener("click",()=>assetAction(()=>window.ForgeExp
 $("exportManifest")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.exportManifest(),"Asset manifest"));
 $("validateAsset")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.validateSelected(),"Asset validation"));
 $("assetQA")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.runExportQA(),"Export round-trip QA"));
+$("assetPlay")?.addEventListener("click",()=>{
+  const r=engine.selected(),slot=r?.entity?.sound?.slot?.("main");
+  if(!slot)return;
+  try{
+    if(slot.isPlaying)slot.stop();else slot.play();
+    updateAssetPanel();
+  }catch(e){write("Audio playback failed: "+e.message,"error")}
+});
 $("treeFilter")?.addEventListener("input",refresh);
 $("assetSearch")?.addEventListener("input",updateAssetBrowser);
 $("quickFocus")?.addEventListener("click",()=>engine.focus());
diff --git a/index.html b/index.html
index bc449c5..f01d8bf 100644
--- a/index.html
+++ b/index.html
@@ -17,7 +17,7 @@
   
CREATE

   
REAL ASSETS
Drop GLB / image / project here
GLB preserves a self-contained asset. glTF with external files is rejected instead of silently breaking paths.
Select an imported asset to inspect source metadata.

   
SCENE CONTROL

-  
ASSET PIPELINE / EXPORT
No asset selectedScene object
READYSource exports preserve original bytes; derived previews are labeled separately.
No asset test yet.

+  
ASSET PIPELINE / EXPORT
No asset selectedScene object
READYSource exports preserve original bytes; derived previews are labeled separately.
No asset test yet.

   
2D CREATION
Real raster drawing with touch / mouse, undo-redo and PNG import into the current scene.

   
VISION + HANDS
Understands scene objects, world bounds, screen projection and precise dimensions. These local actions do not depend on a server.

   
Renderer…
FPS0
Physics0
ModeDetecting

diff --git a/tests/professional.qa.spec.mjs b/tests/professional.qa.spec.mjs
index 7602b29..b661e2f 100644
--- a/tests/professional.qa.spec.mjs
+++ b/tests/professional.qa.spec.mjs
@@ -47,6 +47,11 @@ async function importViaInput(page,buffer,name,mimeType,{renderable=false}={}){
   }
 }
 
+function wavSilence(seconds=.25,sampleRate=8000){
+  const frames=Math.floor(seconds*sampleRate),bytes=44+frames*2,b=Buffer.alloc(bytes);
+  b.write("RIFF",0);b.writeUInt32LE(bytes-8,4);b.write("WAVE",8);b.write("fmt ",12);b.writeUInt32LE(16,16);b.writeUInt16LE(1,20);b.writeUInt16LE(1,22);b.writeUInt32LE(sampleRate,24);b.writeUInt32LE(sampleRate*2,28);b.writeUInt16LE(2,32);b.writeUInt16LE(16,34);b.write("data",36);b.writeUInt32LE(frames*2,40);
+  return b;
+}
 async function sha(blob){return pageHash(await blob.arrayBuffer())}
 async function pageHash(buf){
   const d=await crypto.subtle.digest("SHA-256",buf);
@@ -172,6 +177,37 @@ test.describe("Forge professional acceptance",()=>{
     expect(result.redone).toEqual(result.changed);
   });
 
+  test("real audio WAV imports as an Engine Audio Asset with a Sound Slot and listener",async({page})=>{
+    await waitForForge(page);
+    const wav=wavSilence();
+    await importViaInput(page,wav,"forge-proof.wav","audio/wav");
+    const info=await page.evaluate(()=>{
+      const r=window.Forge.selected();
+      const slot=r?.entity?.sound?.slot?.("main");
+      const cam=window.Forge.camera();
+      return{
+        kind:r?.kind,
+        asset:r?.components?.asset?.type,
+        assetName:r?.components?.asset?.name,
+        sound:!!r?.entity?.sound,
+        slot:!!slot,
+        slotLoaded:!!slot?.isLoaded,
+        listener:!!cam?.audiolistener,
+        registry:window.Forge.assets.has("forge-proof.wav")
+      };
+    });
+    expect(info.kind).toBe("audio");
+    expect(info.asset).toBe("audio");
+    expect(info.assetName).toBe("forge-proof.wav");
+    expect(info.sound).toBeTruthy();
+    expect(info.slot).toBeTruthy();
+    expect(info.listener).toBeTruthy();
+    expect(info.registry).toBeTruthy();
+    await page.click("#assetPlay");
+    await page.waitForTimeout(120);
+    expect(await page.locator("#assetPlay").isEnabled()).toBeTruthy();
+  });
+
   test("real 2D PNG survives source round-trip, validates dimensions, and generates a real viewport PNG",async({page})=>{
     await waitForForge(page);
     const png=await download(page,EXAMPLE_PNG);

From 88dc84517c6f18112e1e30c673f00eca16912819 Mon Sep 17 00:00:00 2001
From: venom3-byte 
Date: Wed, 30 Sep 2026 12:04:11 +0200
Subject: [PATCH 20/86] feat(assets): recognize common browser audio formats
 consistently

---
 index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.html b/index.html
index f01d8bf..a690b09 100644
--- a/index.html
+++ b/index.html
@@ -15,7 +15,7 @@