diff --git a/.github/workflows/platform-build.yml b/.github/workflows/platform-build.yml new file mode 100644 index 0000000..ac500cc --- /dev/null +++ b/.github/workflows/platform-build.yml @@ -0,0 +1,92 @@ +name: Forge Platform Builds +on: + push: + branches: [vision-live-direct-fix2-2026-09-30] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: forge-platform-${{ github.ref }} + cancel-in-progress: true +jobs: + android: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + - name: Prepare Web runtime + run: node scripts/package-web-project.mjs --project packaging/sample-project.forge.json + - name: Install Capacitor + working-directory: packaging/android + run: npm install + - name: Create Android project + working-directory: packaging/android + run: npx cap add android + - name: Sync Web runtime + working-directory: packaging/android + run: npx cap sync android + - name: Build Debug APK + working-directory: packaging/android/android + shell: bash + run: ./gradlew assembleDebug + - name: Upload Android APK + uses: actions/upload-artifact@v4 + with: + name: forge-android-debug-apk + path: packaging/android/android/app/build/outputs/apk/debug/*.apk + if-no-files-found: error + + windows: + runs-on: windows-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Prepare Web runtime + run: node scripts/package-web-project.mjs --project packaging/sample-project.forge.json + - name: Install Electron Forge + working-directory: packaging/windows + run: npm install + - name: Build Windows package + working-directory: packaging/windows + run: npm run make + - name: Upload Windows package + uses: actions/upload-artifact@v4 + with: + name: forge-windows-package + path: packaging/windows/out/make/** + if-no-files-found: error + + packaging-proof: + runs-on: ubuntu-latest + needs: [android, windows] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Install test dependencies + run: npm install --no-audit --no-fund + - name: Install Chromium + run: npx playwright install --with-deps chromium + - name: Prepare Web runtime + run: node scripts/package-web-project.mjs --project packaging/sample-project.forge.json + - name: Validate packaged runtime manifest + run: node -e "const fs=require('fs'); const p=JSON.parse(fs.readFileSync('dist/forge-web/project.forge.json','utf8')); if(p.format!=='forge-project'||!fs.existsSync('dist/forge-web/index.html')||!fs.existsSync('dist/forge-web/build-info.json')) process.exit(1); console.log('Packaging manifest OK')" + - name: Browser boot proof for packaged runtime + run: npx playwright test tests/packaging-runtime.spec.mjs --project=desktop + - name: Upload packaged runtime proof + uses: actions/upload-artifact@v4 + with: + name: forge-packaged-runtime-proof + path: test-results/packaged-runtime-proof.png + if-no-files-found: error diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index feb6781..9a156a5 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -20,7 +20,15 @@ jobs: with: node-version: 22 - run: npm install - - run: npx playwright install --with-deps chromium + - name: Prepare browser runtime + shell: bash + run: | + if command -v google-chrome >/dev/null 2>&1; then + echo "PLAYWRIGHT_CHANNEL=chrome" >> "$GITHUB_ENV" + google-chrome --version + else + npx playwright install --with-deps chromium + fi - run: node scripts/validate-syntax.mjs - name: Upload syntax report if: always() @@ -35,8 +43,6 @@ jobs: run: npm run test:professional - name: Live vision + real input acceptance run: npm run test:vision - - name: Live vision + real input acceptance - run: npm run test:vision - name: Khronos glTF validation run: npm run validate:gltf - name: Full regression suite @@ -50,4 +56,5 @@ jobs: playwright-report test-results gltf-validation-report.json + vision-proof if-no-files-found: ignore diff --git a/favicon.svg b/favicon.svg new file mode 100644 index 0000000..2e36606 --- /dev/null +++ b/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/forge-draw2d.js b/forge-draw2d.js new file mode 100644 index 0000000..7085dd8 --- /dev/null +++ b/forge-draw2d.js @@ -0,0 +1,223 @@ +(() => { + const $ = (id) => document.getElementById(id); + const state = { + canvas: null, + ctx: null, + drawing: false, + last: null, + erasing: false, + brush: 18, + color: "#ffffff", + history: [], + redo: [], + maxHistory: 24, + ready: false, + lastPngDataUrl: null, + importPromise: null + }; + + function dialog(){ return $("draw2dDialog"); } + + function status(text){ + const el = $("draw2dStatus"); + if(el) el.textContent = text; + } + + function capture(){ + if(!state.ctx || !state.canvas) return; + state.history.push(state.ctx.getImageData(0,0,state.canvas.width,state.canvas.height)); + if(state.history.length > state.maxHistory) state.history.shift(); + state.redo.length = 0; + syncButtons(); + } + + function syncButtons(){ + $("drawUndo")?.toggleAttribute("disabled", state.history.length < 2); + $("drawRedo")?.toggleAttribute("disabled", state.redo.length === 0); + $("drawEraser")?.classList.toggle("active", state.erasing); + } + + function point(event){ + const r=state.canvas.getBoundingClientRect(); + return { + x:(event.clientX-r.left)*state.canvas.width/r.width, + y:(event.clientY-r.top)*state.canvas.height/r.height + }; + } + + function begin(event){ + if(!state.ctx) return; + event.preventDefault(); + capture(); + state.drawing=true; + state.last=point(event); + state.canvas.setPointerCapture?.(event.pointerId); + state.ctx.save(); + state.ctx.globalCompositeOperation=state.erasing?"destination-out":"source-over"; + state.ctx.strokeStyle=state.color; + state.ctx.lineWidth=state.brush; + state.ctx.lineCap="round"; + state.ctx.lineJoin="round"; + state.ctx.beginPath(); + state.ctx.arc(state.last.x,state.last.y,state.brush/2,0,Math.PI*2); + state.ctx.fillStyle=state.color; + if(state.erasing) state.ctx.fillStyle="rgba(0,0,0,1)"; + state.ctx.fill(); + state.ctx.restore(); + status("Drawing…"); + } + + function move(event){ + if(!state.drawing || !state.ctx) return; + event.preventDefault(); + const p=point(event),a=state.last; + state.ctx.save(); + state.ctx.globalCompositeOperation=state.erasing?"destination-out":"source-over"; + state.ctx.strokeStyle=state.color; + state.ctx.lineWidth=state.brush; + state.ctx.lineCap="round"; + state.ctx.lineJoin="round"; + state.ctx.beginPath(); + state.ctx.moveTo(a.x,a.y); + state.ctx.lineTo(p.x,p.y); + state.ctx.stroke(); + state.ctx.restore(); + state.last=p; + } + + function end(event){ + if(!state.drawing) return; + event?.preventDefault?.(); + state.drawing=false; + state.last=null; + try{state.canvas.releasePointerCapture?.(event?.pointerId)}catch{} + status("Ready"); + } + + function clear(){ + if(!state.ctx) return; + capture(); + state.ctx.clearRect(0,0,state.canvas.width,state.canvas.height); + status("Canvas cleared"); + } + + function undo(){ + if(state.history.length<=1 || !state.ctx)return; + const current=state.ctx.getImageData(0,0,state.canvas.width,state.canvas.height); + state.redo.push(current); + const previous=state.history.pop(); + state.ctx.putImageData(previous,0,0); + syncButtons(); + status("Undo"); + } + + function redo(){ + if(!state.redo.length || !state.ctx)return; + const current=state.ctx.getImageData(0,0,state.canvas.width,state.canvas.height); + state.history.push(current); + const next=state.redo.pop(); + state.ctx.putImageData(next,0,0); + syncButtons(); + status("Redo"); + } + + function dataUrl(){ + if(!state.canvas) return null; + return state.canvas.toDataURL("image/png"); + } + + function download(){ + const url=dataUrl(); + if(!url)return; + state.lastPngDataUrl=url; + const a=document.createElement("a"); + a.href=url; + a.download="forge-drawing.png"; + a.click(); + status("PNG exported"); + } + + async function addToScene(){ + if(!state.canvas || !window.Forge?.importFile) throw new Error("Forge runtime is not ready"); + const blob=await new Promise(resolve=>state.canvas.toBlob(resolve,"image/png")); + if(!blob)throw new Error("PNG encoding failed"); + const file=new File([blob],"forge-drawing.png",{type:"image/png"}); + state.lastPngDataUrl=await new Promise((resolve,reject)=>{ + const reader=new FileReader(); + reader.onload=()=>resolve(String(reader.result||"")); + reader.onerror=()=>reject(reader.error||new Error("PNG preview read failed")); + reader.readAsDataURL(blob); + }); + await window.ForgeProduction?.assets?.storeFile?.(file); + const result=await window.Forge.importFile(file); + window.ForgeRefreshUI?.(); + if(result?.record) window.Forge.select?.(result.record.id); + dialog()?.close(); + status("Imported into Forge"); + window.dispatchEvent(new CustomEvent("forge-drawing-imported",{detail:{name:file.name,bytes:file.size}})); + return {name:file.name,bytes:file.size,type:file.type}; + } + + function open(){ + const d=dialog(); + if(!d)throw new Error("2D paint dialog is missing"); + try{ + if(!d.open)d.showModal(); + }catch{ + d.setAttribute("open",""); + d.style.display="block"; + } + status("Ready"); + syncButtons(); + return{open:!!d.open,ready:state.ready}; + } + + function init(){ + if(state.ready)return; + state.canvas=$("drawCanvas"); + state.ctx=state.canvas?.getContext("2d"); + if(!state.canvas||!state.ctx)return; + state.canvas.style.touchAction="none"; + state.canvas.addEventListener("pointerdown",begin); + state.canvas.addEventListener("pointermove",move); + state.canvas.addEventListener("pointerup",end); + state.canvas.addEventListener("pointercancel",end); + state.canvas.addEventListener("pointerleave",(e)=>{if(state.drawing)move(e)}); + $("draw2dOpen")?.addEventListener("click",open); + $("drawClose")?.addEventListener("click",()=>dialog()?.close()); + $("drawClear")?.addEventListener("click",clear); + $("drawUndo")?.addEventListener("click",undo); + $("drawRedo")?.addEventListener("click",redo); + $("drawDownload")?.addEventListener("click",download); + $("drawAddToScene")?.addEventListener("click",()=>{ + state.importPromise=addToScene().catch(e=>{ + status("Import failed: "+(e?.message||String(e))); + throw e; + }).finally(()=>{state.importPromise=null}); + }); + $("drawColor")?.addEventListener("input",e=>{state.color=e.target.value||"#ffffff"}); + $("drawSize")?.addEventListener("input",e=>{state.brush=Math.max(1,Math.min(120,Number(e.target.value)||18));$("drawSizeValue").textContent=state.brush+" px"}); + $("drawEraser")?.addEventListener("click",()=>{state.erasing=!state.erasing;syncButtons()}); + $("drawReset")?.addEventListener("click",()=>{state.color="#ffffff";state.brush=18;state.erasing=false;$("drawColor").value="#ffffff";$("drawSize").value="18";$("drawSizeValue").textContent="18 px";syncButtons()}); + state.ctx.clearRect(0,0,state.canvas.width,state.canvas.height); + state.ctx.fillStyle="rgba(8,20,35,0)"; + capture(); + state.ready=true; + syncButtons(); + } + + window.ForgeDraw2D={ + open, + clear, + undo, + redo, + download, + addToScene, + exportDataUrl:dataUrl, + status:()=>({ready:state.ready,drawing:state.drawing,brush:state.brush,color:state.color,erasing:state.erasing,history:state.history.length,redo:state.redo.length}), + get lastPngDataUrl(){ return state.lastPngDataUrl; }, + waitForImport:()=>state.importPromise||Promise.resolve(true) + }; + + init(); +})(); \ No newline at end of file diff --git a/forge-engine.js b/forge-engine.js index 38a55f9..37a7d4d 100644 --- a/forge-engine.js +++ b/forge-engine.js @@ -16,6 +16,14 @@ function rayTriangle(ro,rd,a,b,c){ const v=q.cross(e1).dot(rd)*inv;if(v<0||u+v>1)return null;const t=e2.dot(q.cross(e1))*inv;return t>=0?t:null } + +const IMAGE_EXT_RE=/\.(png|jpe?g|webp|avif|gif|bmp|tga|tif|tiff)$/i; +const AUDIO_EXT_RE=/\.(wav|mp3|ogg|m4a|aac|flac|webm)$/i; +const MODEL_DIRECT_RE=/\.(glb|gltf)$/i; +const MODEL_CONVERT_RE=/\.(fbx|obj|dae|3ds|stl|ply|off|3mf|dxf|ase|b3d|lwo|lxo|m3d|md2|md3|md5mesh|ms3d|smd|vta|x|x3d)$/i; +const SOURCE_ONLY_EXT_RE=/\.(ttf|ttc|otf|wasm|json|xml|csv|txt|css|js|mjs|glsl|vert|frag|shader|zip|rar|7z|pdf|blend|usd|usda|usdc|usdz)$/i; +const extOf=file=>String(file?.name||"").toLowerCase().match(/\.([^.]+)$/)?.[1]||""; +const isModelFile=file=>MODEL_DIRECT_RE.test(String(file?.name||""))||MODEL_CONVERT_RE.test(String(file?.name||"")); export class ForgeEngine{ constructor(canvas,log=()=>{}){ this.canvas=canvas;this.log=log;this.app=null;this.root=null;this.entities=new Map();this.selectedId=null;this.assets=new Map(); @@ -28,7 +36,7 @@ export class ForgeEngine{ this.app.setCanvasFillMode(pc.FILLMODE_NONE);this.app.setCanvasResolution(pc.RESOLUTION_AUTO);this.app.scene.gammaCorrection=pc.GAMMA_SRGB;this.app.scene.toneMapping=pc.TONEMAP_ACES;this.app.scene.physicalUnits=true;this.app.scene.ambientLightColor=new pc.Color(.18,.22,.28);this.app.scene.ambientLightColor=new pc.Color(.18,.22,.28); this.root=new pc.Entity('ForgeScene');this.app.root.addChild(this.root); await RAPIER.init();this.rapier=RAPIER;this.world=new RAPIER.World({x:0,y:-9.81,z:0});this.eventQueue=new RAPIER.EventQueue(true); - this.createCamera('Main Camera',{x:7,y:5,z:9});this.createLight('Sun');this.createPlane('Ground',30,30); + this.createCamera('Main Camera',{x:7,y:5,z:9});this.createLight('Sun');this.createPlane('Ground',30,30);this.app.scene.ambientLight=new pc.Color(.16,.18,.22);this.app.scene.exposure=1.15; this.app.on('update',dt=>this.update(dt));this.canvas.addEventListener('pointerdown',e=>this.pick(e));window.addEventListener('resize',()=>this.app.resizeCanvas());this.app.resizeCanvas();this.app.autoRender=true;this.app.start();this.app.render?.();this.log('Forge Engine 3.0 online');return this } rec(name,kind,e){const id=crypto.randomUUID();e.__forge={id,kind,name,components:{}};const r={id,kind,name,entity:e,components:e.__forge.components};this.entities.set(id,r);return r} @@ -36,16 +44,27 @@ export class ForgeEngine{ material(color=[.22,.62,.9]){const m=new pc.StandardMaterial();m.diffuse=new pc.Color(...color);m.emissive=new pc.Color(...color);m.emissiveIntensity=2;m.metalness=.0;m.gloss=.5;m.useMetalness=false;m.useLighting=false;m.useTonemap=false;m.specular=new pc.Color(0,0,0);m.update();return m} primitive(kind,name){const r=this.add(kind,name||kind+'-'+(this.entities.size+1)),type=kind==='sphere'?'sphere':kind==='cylinder'?'cylinder':kind==='capsule'?'capsule':kind==='plane'?'plane':'box';r.entity.addComponent('render',{type});r.entity.render.layers=[pc.LAYERID_WORLD];r.entity.render.frustumCulling=false;r.entity.render.enabled=true;r.entity.render.material=this.material(kind==='plane'?[.08,.15,.2]:[.22,.62,.9]);if(kind!=='plane')r.entity.setLocalPosition((Math.random()-.5)*4,1+(Math.random()*1.7),(Math.random()-.5)*4);r.components.geometry={sourceUnits:'meters'};return r} createPlane(name,w=30,d=30){const r=this.primitive('plane',name);r.entity.setLocalScale(w,1,d);r.entity.setLocalPosition(0,0,0);if(name==='Ground')r.entity.render.layers=[];this.setPhysics(r.id,'fixed','box');return r} - createCamera(name,pos){const r=this.add('camera',name);r.entity.addComponent('camera',{clearColor:new pc.Color(.035,.07,.12),clearColorBuffer:true,clearDepthBuffer:true,fov:60,nearClip:.01,farClip:10000,enabled:true,layers:[pc.LAYERID_WORLD]});r.entity.setLocalPosition(pos.x,pos.y,pos.z);r.entity.lookAt(0,1,0);r.components.camera={active:true};r.entity.camera.enabled=true;return r} + createCamera(name,pos){const r=this.add('camera',name);r.entity.addComponent('camera',{clearColor:new pc.Color(.035,.07,.12),clearColorBuffer:true,clearDepthBuffer:true,fov:60,nearClip:.01,farClip:10000,enabled:true,layers:[pc.LAYERID_WORLD]});r.entity.addComponent('audiolistener');r.entity.setLocalPosition(pos.x,pos.y,pos.z);r.entity.lookAt(0,1,0);r.components.camera={active:true};r.entity.camera.enabled=true;return r} createLight(name){const r=this.add('light',name);r.entity.addComponent('light',{type:'directional',color:new pc.Color(1,.96,.88),intensity:2.5,castShadows:true,shadowDistance:80,enabled:true,layers:[pc.LAYERID_WORLD]});r.entity.setEulerAngles(48,-32,0);r.components.light={type:'directional'};return r} camera(){return[...this.entities.values()].find(r=>r.kind==='camera'&&r.components.camera?.active)?.entity} selected(){return this.selectedId?this.entities.get(this.selectedId):null} select(id){this.selectedId=id;window.dispatchEvent(new CustomEvent('forge-selection',{detail:this.selected()}));return this.selected()} setView(mode){ - const cam=this.camera();if(!cam?.camera)return false;const target=this.selected()?.entity.getPosition()||new pc.Vec3(0,1,0),dist=Math.max(6,target.distance(cam.getPosition())||10); - const positions={front:new pc.Vec3(0,target.y,target.z+dist),back:new pc.Vec3(0,target.y,target.z-dist),left:new pc.Vec3(-dist,target.y,target.z),right:new pc.Vec3(dist,target.y,target.z),top:new pc.Vec3(target.x,target.y+dist,target.z),bottom:new pc.Vec3(target.x,target.y-dist,target.z)}; - if(mode==='perspective'){this.viewMode='perspective';cam.camera.projection=pc.PROJECTION_PERSPECTIVE;cam.setPosition(target.x+dist*.75,target.y+dist*.5,target.z+dist);cam.lookAt(target)} - else if(positions[mode]){this.viewMode=mode;cam.camera.projection=pc.PROJECTION_ORTHOGRAPHIC;cam.camera.orthoHeight=Math.max(.5,dist*.9);cam.setPosition(positions[mode]);cam.lookAt(target)} + const cam=this.camera();if(!cam?.camera)return false; + const selected=this.selected(),inspected=selected&&this.spatial?.inspect?.(selected); + const target=selected?.entity.getPosition()||new pc.Vec3(0,1,0); + const radius=Math.max(.5,Number(inspected?.world?.radius||0)||0.5); + const dist=Math.max(3,radius*3.2); + const positions={front:new pc.Vec3(target.x,target.y,target.z+dist),back:new pc.Vec3(target.x,target.y,target.z-dist),left:new pc.Vec3(target.x-dist,target.y,target.z),right:new pc.Vec3(target.x+dist,target.y,target.z),top:new pc.Vec3(target.x,target.y+dist,target.z),bottom:new pc.Vec3(target.x,target.y-dist,target.z)}; + if(mode==='perspective'){ + this.viewMode='perspective';cam.camera.projection=pc.PROJECTION_PERSPECTIVE; + cam.setPosition(target.x+dist*.75,target.y+dist*.5,target.z+dist);cam.lookAt(target) + }else if(positions[mode]){ + this.viewMode=mode;cam.camera.projection=pc.PROJECTION_ORTHOGRAPHIC; + cam.camera.orthoHeight=Math.max(.5,radius*2.35); + cam.setPosition(positions[mode]);cam.lookAt(target) + } + this.app?.resizeCanvas?.(); return true } applyMaterial(params={}){ @@ -113,16 +132,290 @@ export class ForgeEngine{ evalAnimation(t){for(const[id,a]of this.keyframes){const r=this.entities.get(id);if(!r||!a.length)continue;let k=a[0];for(let i=0;i=a[i].time&&t<=a[i+1].time){const A=a[i],B=a[i+1],f=(t-A.time)/Math.max(.0001,B.time-A.time);k={p:A.p.map((v,j)=>v+(B.p[j]-v)*f),r:A.r.map((v,j)=>v+(B.r[j]-v)*f),s:A.s.map((v,j)=>v+(B.s[j]-v)*f)};break}r.entity.setLocalPosition(...k.p);r.entity.setLocalEulerAngles(...k.r);r.entity.setLocalScale(...k.s)}} attachScript(id,code){this.scripts.set(id,code)} runScripts(dt){for(const[id,code]of this.scripts){const r=this.entities.get(id);if(!r)continue;try{new Function('api',code)({entity:r.entity,time:this.timelineTime,dt,engine:this})}catch(e){this.log('Script error '+e.message,'error')}}} - async importFile(file){ - const url=URL.createObjectURL(file);if(file.name.toLowerCase().endsWith('.forge.json'))return{type:'project',data:JSON.parse(await file.text())}; - let analysis=null;try{analysis=await window.ForgeProduction?.assets?.analyze?.(file)}catch{} - if(/^image\//.test(file.type)){const bmp=await createImageBitmap(file),c=document.createElement('canvas');c.width=bmp.width;c.height=bmp.height;c.getContext('2d').drawImage(bmp,0,0);const tex=new pc.Texture(this.app.graphicsDevice,{width:bmp.width,height:bmp.height,format:pc.PIXELFORMAT_R8_G8_B8_A8});tex.setSource(c);const mat=this.material([1,1,1]);mat.diffuseMap=tex;mat.emissiveMap=tex;mat.emissive=new pc.Color(1,1,1);mat.update();const r=this.primitive('plane',file.name);r.entity.render.material=mat;r.components.asset={type:'image',name:file.name,width:bmp.width,height:bmp.height,analysis,derivedDirty:false};this.assets.set(file.name,{type:'image',file,url});this.select(r.id);window.dispatchEvent(new Event('forge-assets-changed'));window.ForgeRefreshUI?.();return{type:'image',record:r}} - if(/\.(glb|gltf)$/i.test(file.name))return new Promise(async(resolve,reject)=>{ - if(/\.gltf$/i.test(file.name)){try{const json=JSON.parse(await file.text()),external=[...(json.buffers||[]),...(json.images||[])].some(x=>x?.uri&&!String(x.uri).startsWith('data:'));if(external)throw new Error('This .gltf references external files. Use a packaged .glb so Forge can preserve the asset without path breakage')}catch(e){reject(e);return}} - const asset=new pc.Asset(file.name,'container',{url});this.app.assets.add(asset);asset.once('error',reject);asset.once('load',()=>{try{const e=asset.resource.instantiateRenderEntity({castShadows:true,receiveShadows:true});e.name=file.name.replace(/\.[^.]+$/,'');this.root.addChild(e);const r=this.rec(e.name,'model',e);r.components.asset={type:'model',name:file.name,analysis:analysis||null,sourceUnits:'meters',importScale:1,derivedDirty:false};const clips=this.attachAnimations(e,asset.resource,analysis?.animationNames||[]);if(clips.length)r.components.animation={clips,playing:true};this.assets.set(file.name,{type:'model',file,url,resource:asset.resource});window.dispatchEvent(new Event('forge-assets-changed'));this.select(r.id);window.ForgeRefreshUI?.();requestAnimationFrame(()=>{requestAnimationFrame(()=>{window.ForgeSpatial?.inspect?.(r);resolve({type:'model',record:r})})})}catch(err){reject(err)}});this.app.assets.load(asset) + importedModelBounds(root){ + const min=new pc.Vec3(Infinity,Infinity,Infinity),max=new pc.Vec3(-Infinity,-Infinity,-Infinity); + root.syncHierarchy?.(); + root.forEach(n=>{ + for(const mi of n.render?.meshInstances||[]){ + const a=mi.aabb,mn=a?.getMin?.()||a?.min,mx=a?.getMax?.()||a?.max; + if(!mn||!mx)continue; + min.x=Math.min(min.x,mn.x);min.y=Math.min(min.y,mn.y);min.z=Math.min(min.z,mn.z); + max.x=Math.max(max.x,mx.x);max.y=Math.max(max.y,mx.y);max.z=Math.max(max.z,mx.z); + } + }); + if(!Number.isFinite(min.x))return null; + return{min,max,size:new pc.Vec3(max.x-min.x,max.y-min.y,max.z-min.z),center:new pc.Vec3((min.x+max.x)/2,(min.y+max.y)/2,(min.z+max.z)/2)}; + } + normalizeImportedModel(root,targetSize=2.4){ + const b=this.importedModelBounds(root);if(!b)return{ok:false,scale:1}; + const extent=Math.max(b.size.x,b.size.y,b.size.z),scale=extent>1e-6?targetSize/extent:1; + root.setLocalScale(scale,scale,scale); + root.setLocalPosition(-b.center.x*scale,targetSize*.5-b.center.y*scale,-b.center.z*scale); + root.syncHierarchy?.(); + return{ok:true,scale,sourceSize:[b.size.x,b.size.y,b.size.z],targetSize}; + } + prepareImportedModelPreview(root){ + let textured=0,previewAdjusted=0,cullAdjusted=0; + root.forEach(n=>{ + for(const mi of n.render?.meshInstances||[]){ + const mat=mi.material; + if(!mat)continue; + const preview=mat.clone?.()||this.material([.62,.68,.76]); + const hasMap=!!mat.diffuseMap; + if(hasMap){ + textured++; + preview.diffuse=new pc.Color(1,1,1); + preview.diffuseMap=mat.diffuseMap; + preview.emissiveMap=mat.diffuseMap; + preview.emissive=new pc.Color(1,1,1); + preview.emissiveIntensity=.75; + }else{ + preview.diffuse=new pc.Color(.62,.68,.76); + preview.emissive=new pc.Color(.10,.12,.16); + preview.emissiveIntensity=.35; + } + preview.useLighting=false; + preview.useMetalness=false; + preview.useTonemap=false; + if("cull" in preview)preview.cull=pc.CULLFACE_NONE; + if("blendType" in preview)preview.blendType=pc.BLEND_NONE; + if("opacity" in preview)preview.opacity=1; + if("alphaTest" in preview)preview.alphaTest=0; + preview.update?.(); + mi.material=preview; + previewAdjusted++; + if(preview.cull===pc.CULLFACE_NONE)cullAdjusted++; + } }); - return{type:'asset',name:file.name,analysis} + const normalized=this.normalizeImportedModel(root,2.4); + root.__forgeViewportPreview={textured,previewAdjusted,cullAdjusted,normalized}; + return root.__forgeViewportPreview; } + + async sha256(file){ + const digest=await crypto.subtle.digest("SHA-256",await file.arrayBuffer()); + return [...new Uint8Array(digest)].map(x=>x.toString(16).padStart(2,"0")).join(""); + } + async fileToDataUri(file){ + const bytes=new Uint8Array(await file.arrayBuffer()); + let binary=""; + for(let i=0;i[f.name.replaceAll("\\","/").split("/").at(-1).toLowerCase(),f])); + const patch=async(items)=>{ + for(const item of items||[]){ + const uri=item?.uri; + if(!uri||String(uri).startsWith("data:"))continue; + const clean=decodeURIComponent(String(uri).split(/[?#]/)[0]).replaceAll("\\","/").split("/").at(-1).toLowerCase(); + const dep=byName.get(clean); + if(!dep)throw new Error("Missing glTF dependency: "+uri+" — select the complete model package."); + item.uri=await this.fileToDataUri(dep); + } + }; + await patch(json.buffers); + await patch(json.images); + return new File([JSON.stringify(json)],file.name,{type:"model/gltf+json"}); + } + async convertLegacyModelToGLB(file,bundle=[]){ + const files=[file,...(bundle||[])].filter(Boolean); + const unique=[...new Map(files.map(f=>[f.name.replaceAll("\\","/"),f])).values()]; + if(!window.__ForgeAssimpJS){ + if(!window.__ForgeAssimpLoading){ + window.__ForgeAssimpLoading=new Promise((resolve,reject)=>{ + if(typeof window.assimpjs==="function"){resolve(window.assimpjs);return} + const s=document.createElement("script"); + s.src="https://cdn.jsdelivr.net/npm/assimpjs@0.0.10/dist/assimpjs.js"; + s.async=true; + s.onload=()=>resolve(window.assimpjs); + s.onerror=()=>reject(new Error("AssimpJS loader failed")); + document.head.appendChild(s); + }).then(factory=>{ + if(typeof factory!=="function")throw new Error("AssimpJS global API unavailable"); + return factory(); + }); + } + window.__ForgeAssimpJS=await window.__ForgeAssimpLoading; + } + const ajs=window.__ForgeAssimpJS; + const list=new ajs.FileList(); + for(const source of unique)list.AddFile(source.name.replaceAll("\\","/"),new Uint8Array(await source.arrayBuffer())); + const result=ajs.ConvertFileList(list,"glb2"); + if(!result?.IsSuccess?.()||!result?.FileCount?.())throw new Error("Legacy 3D conversion failed for ."+extOf(file)+": "+String(result?.GetErrorCode?.()||"unknown AssimpJS error")); + const out=result.GetFile(0); + return new File([out.GetContent()],file.name.replace(/.[^.]+$/,"")+".glb",{type:"model/gltf-binary"}); + } + async sanitizeGlbPreview(file){ + const name=String(file?.name||"").toLowerCase(); + if(!name.endsWith(".glb"))return{file,sanitized:false,removedExtensions:[]}; + const bytes=new Uint8Array(await file.arrayBuffer()); + if(bytes.length<20)return{file,sanitized:false,removedExtensions:[]}; + const dv=new DataView(bytes.buffer,bytes.byteOffset,bytes.byteLength); + if(dv.getUint32(0,true)!==0x46546c67||dv.getUint32(4,true)!==2)return{file,sanitized:false,removedExtensions:[]}; + const total=dv.getUint32(8,true); + let off=12,json=null,chunks=[]; + while(off+8<=Math.min(total,bytes.length)){ + const length=dv.getUint32(off,true),type=dv.getUint32(off+4,true),start=off+8,end=start+length; + if(end>bytes.length)break; + const chunk=bytes.slice(start,end); + chunks.push({type,bytes:chunk}); + if(type===0x4e4f534a)json=JSON.parse(new TextDecoder().decode(chunk).trim()); + off=end; + } + if(!json)return{file,sanitized:false,removedExtensions:[]}; + const unsupported=new Set([ + "KHR_materials_clearcoat","KHR_materials_emissive_strength","KHR_materials_iridescence", + "KHR_materials_transmission","KHR_materials_variants","KHR_texture_transform" + ]); + const used=[...(json.extensionsUsed||[])].filter(x=>unsupported.has(x)); + const required=[...(json.extensionsRequired||[])].filter(x=>unsupported.has(x)); + if(required.length)throw new Error("GLB requires unsupported glTF extensions: "+required.join(", ")); + if(!used.length)return{file,sanitized:false,removedExtensions:[]}; + const strip=o=>{ + if(!o||typeof o!=="object")return; + if(Array.isArray(o)){for(const v of o)strip(v);return} + if(o.extensions&&typeof o.extensions==="object"){ + for(const k of unsupported)o.extensions[k]===undefined||delete o.extensions[k]; + if(!Object.keys(o.extensions).length)delete o.extensions; + } + for(const v of Object.values(o))strip(v); + }; + strip(json); + json.extensionsUsed=(json.extensionsUsed||[]).filter(x=>!unsupported.has(x)); + json.extensionsRequired=(json.extensionsRequired||[]).filter(x=>!unsupported.has(x)); + if(!json.extensionsUsed.length)delete json.extensionsUsed; + if(!json.extensionsRequired.length)delete json.extensionsRequired; + const encoder=new TextEncoder(); + const jb=encoder.encode(JSON.stringify(json)); + const pad4=n=>(n+3)&~3; + const jp=new Uint8Array(pad4(jb.length));jp.set(jb);jp.fill(0x20,jb.length); + const outChunks=[{type:0x4e4f534a,bytes:jp},...chunks.filter(c=>c.type!==0x4e4f534a).map(c=>({type:c.type,bytes:new Uint8Array(c.bytes)}))]; + const totalLength=12+outChunks.reduce((n,c)=>n+8+c.bytes.length,0); + const out=new Uint8Array(totalLength),od=new DataView(out.buffer);od.setUint32(0,0x46546c67,true);od.setUint32(4,2,true);od.setUint32(8,totalLength,true); + let w=12; + for(const c of outChunks){od.setUint32(w,c.bytes.length,true);od.setUint32(w+4,c.type,true);out.set(c.bytes,w+8);w+=8+c.bytes.length} + return{file:new File([out],file.name,{type:"model/gltf-binary"}),sanitized:true,removedExtensions:used}; + } + + async importFile(file,options={}){ + if(!file||!file.name)throw new Error("Forge import requires a File-like source."); + const lower=String(file.name).toLowerCase(); + if(lower.endsWith(".forge.json"))return{type:"project",data:JSON.parse(await file.text())}; + + const url=URL.createObjectURL(file); + let analysis=null,sourceSha256=null; + try{analysis=await window.ForgeProduction?.assets?.analyze?.(file)}catch{} + try{sourceSha256=await this.sha256(file)}catch{} + try{await window.ForgeProduction?.assets?.storeFile?.(file)}catch(e){this.log("Asset vault store failed for "+file.name+": "+e.message,"error")} + const sourceEntry={type:"file",file,url,analysis,sourceSha256,sourcePreserved:true,extension:extOf(file),sourceOnly:true,state:"loading"}; + this.assets.set(file.name,sourceEntry); + + if(/^audio\//.test(file.type)||AUDIO_EXT_RE.test(lower)){ + try{ + let asset=null; + await new Promise((resolve,reject)=>{ + this.app.assets.loadFromUrlAndFilename(url,file.name,"audio",(err,loaded)=>{ + if(err){reject(err);return} + asset=loaded;resolve(); + }); + }); + const r=this.add("audio",file.name); + r.entity.addComponent("sound",{positional:true,volume:1}); + r.entity.sound.addSlot("main",{asset:asset.id,autoPlay:false,loop:false,overlap:false,volume:1,pitch:1}); + r.components.asset={type:"audio",name:file.name,analysis,sourceSha256,sourcePreserved:true,derivedDirty:false}; + this.assets.set(file.name,{type:"audio",file,url,resource:asset.resource,asset,analysis,sourceSha256,sourcePreserved:true,state:"ready"}); + this.select(r.id);window.dispatchEvent(new Event("forge-assets-changed"));window.ForgeRefreshUI?.(); + return{type:"audio",record:r,asset}; + }catch(error){ + this.log("Audio preview unavailable for "+file.name+"; preserving source asset: "+error.message,"error"); + } + } + + if(/^image\//.test(file.type)||IMAGE_EXT_RE.test(lower)){ + try{ + const bmp=await createImageBitmap(file); + const c=document.createElement("canvas");c.width=bmp.width;c.height=bmp.height;c.getContext("2d").drawImage(bmp,0,0); + const tex=new pc.Texture(this.app.graphicsDevice,{width:bmp.width,height:bmp.height,format:pc.PIXELFORMAT_R8_G8_B8_A8}); + tex.setSource(c); + const mat=new pc.StandardMaterial(); + mat.diffuse=new pc.Color(1,1,1);mat.diffuseMap=tex; + mat.emissive=new pc.Color(1,1,1);mat.emissiveMap=tex;mat.emissiveIntensity=.35; + mat.useLighting=false;mat.useTonemap=false;mat.cull=pc.CULLFACE_NONE;mat.blendType=pc.BLEND_NONE;mat.opacity=1;mat.alphaTest=0;mat.update(); + const r=this.primitive("plane",file.name); + const h=2.6,w=Math.max(.25,h*(bmp.width/Math.max(1,bmp.height))); + r.entity.setLocalEulerAngles(90,0,0);r.entity.setLocalScale(w,1,h);r.entity.setLocalPosition(0,h*.5,0); + r.entity.render.material=mat;r.entity.render.frustumCulling=false; + r.components.asset={type:"image",name:file.name,width:bmp.width,height:bmp.height,analysis,sourceUnits:"pixels",sourceSha256,sourcePreserved:true,presentation:"upright-2d-preview",pixelsPerWorldUnit:bmp.height/h,derivedDirty:false}; + this.assets.set(file.name,{type:"image",file,url,analysis,sourceSha256,sourcePreserved:true,state:"ready"});this.select(r.id);this.setView("front");this.frame(); + bmp.close?.();window.dispatchEvent(new Event("forge-assets-changed"));window.ForgeRefreshUI?.(); + return{type:"image",record:r}; + }catch(error){ + this.log("Raster preview unavailable for "+file.name+"; preserving source asset: "+error.message,"error"); + } + } + + if(isModelFile(file)){ + if(MODEL_CONVERT_RE.test(lower)&&!MODEL_DIRECT_RE.test(lower)){ + const converted=await this.convertLegacyModelToGLB(file,options?.files||[file]); + const convertedResult=await this.importFile(converted,{internalConversion:true}); + const record=convertedResult.record,internalEntry=this.assets.get(converted.name); + this.assets.delete(converted.name); + this.assets.set(file.name,{type:"model",file,url,resource:internalEntry?.resource,converted:true,convertedFrom:file.name,analysis,sourceSha256,sourcePreserved:true,state:"ready"}); + record.name=file.name.replace(/\.[^.]+$/,"");record.entity.name=record.name; + const previous=record.components.asset||{}; + record.components.asset={...previous,type:"model",name:file.name,analysis:analysis||null,sourceUnits:"source-native",sourceSha256,sourcePreserved:true,converted:true,convertedFrom:file.name,converter:"AssimpJS",derivedDirty:false,importScale:previous.importScale||1,normalized:!!previous.normalized,viewportPreview:previous.viewportPreview||null}; + this.select(record.id);window.dispatchEvent(new Event("forge-assets-changed"));window.ForgeRefreshUI?.(); + return{type:"model",record,converted:true}; + } + + return new Promise(async(resolve,reject)=>{ + try{ + let loadFile=file; + if(/\.gltf$/i.test(file.name)){ + const json=JSON.parse(await file.text()); + const hasExternal=[...(json.buffers||[]),...(json.images||[])].some(x=>x?.uri&&!String(x.uri).startsWith("data:")); + loadFile=hasExternal?await this.packageGltfBundle(file,options?.files||[file]):file; + } + const previewSource=await this.sanitizeGlbPreview(loadFile); + const loadUrl=URL.createObjectURL(previewSource.file); + const asset=new pc.Asset(file.name,"container",{url:loadUrl}); + this.app.assets.add(asset); + asset.once("error",err=>{URL.revokeObjectURL(loadUrl);reject(err)}); + asset.once("load",()=>{ + try{ + const e=asset.resource.instantiateRenderEntity({castShadows:true,receiveShadows:true}); + e.name=file.name.replace(/\.[^.]+$/,"");this.root.addChild(e); + const preview=this.prepareImportedModelPreview(e),r=this.rec(e.name,"model",e); + r.components.asset={ + type:"model",name:file.name,analysis:analysis||null,sourceUnits:"source-native", + sourceSha256,sourcePreserved:true,viewportSanitized:previewSource.sanitized===true, + sanitizedExtensions:previewSource.removedExtensions||[],previewInput:loadFile.name, + importScale:preview.normalized?.scale||1,normalized:!!preview.normalized?.ok, + derivedDirty:false,viewportPreview:preview + }; + const clips=this.attachAnimations(e,asset.resource,analysis?.animationNames||[]);if(clips.length)r.components.animation={clips,playing:true}; + this.assets.set(file.name,{type:"model",file,url,resource:asset.resource,asset,viewportFile:previewSource.file,analysis,sourceSha256,sourcePreserved:true,state:"ready"}); + URL.revokeObjectURL(loadUrl); + window.dispatchEvent(new Event("forge-assets-changed"));window.ForgeRefreshUI?.(); + requestAnimationFrame(()=>requestAnimationFrame(()=>{this.select(r.id);window.ForgeSpatial?.inspect?.(r);resolve({type:"model",record:r})})); + }catch(err){URL.revokeObjectURL(loadUrl);reject(err)} + }); + this.app.assets.load(asset); + }catch(err){reject(err)} + }); + } + + const entry=Object.assign(sourceEntry,{type:"file",state:"ready"}); + this.assets.set(file.name,entry); + window.dispatchEvent(new Event("forge-assets-changed"));window.ForgeRefreshUI?.(); + return{type:"asset",name:file.name,analysis,sourceOnly:true,entry}; + } + attachAnimations(entity,resource,names=[]){const tracks=resource?.animations||[];if(!tracks.length)return[];try{if(!entity.anim)entity.addComponent('anim',{activate:true,speed:1});const clips=[];for(let i=0;i{const p=r.entity.getLocalPosition(),q=r.entity.getLocalEulerAngles(),s=r.entity.getLocalScale();return{id:r.id,name:r.name,kind:r.kind,transform:{p:[p.x,p.y,p.z],r:[q.x,q.y,q.z],s:[s.x,s.y,s.z]},components:r.components,keyframes:this.keyframes.get(r.id)||[],script:this.scripts.get(r.id)||null}})}} @@ -156,7 +449,22 @@ export class ForgeEngine{ if(d.script)this.scripts.set(r.id,d.script); const asset=d.components?.asset; - if(asset?.name&&assetRoot&&asset.type==='model'){ + if(asset?.name&&assetRoot&&asset.type==='audio'){ + const a=new pc.Asset(asset.name,'audio',{url:assetRoot+'assets/'+encodeURIComponent(asset.name)}); + this.app.assets.add(a); + await new Promise((resolve,reject)=>{ + a.once('error',reject); + a.once('load',()=>{ + try{ + if(!r.entity.sound)r.entity.addComponent('sound',{positional:true,volume:1}); + if(!r.entity.sound.slot('main'))r.entity.sound.addSlot('main',{asset:a.id,autoPlay:false,loop:false,overlap:false,volume:1,pitch:1}); + this.assets.set(asset.name,{type:'audio',file:null,url:a.getFileUrl?.()||assetRoot+'assets/'+encodeURIComponent(asset.name),resource:a.resource,asset:a}); + resolve(); + }catch(err){reject(err)} + }); + this.app.assets.load(a); + }); + } else if(asset?.name&&assetRoot&&asset.type==='model'){ const a=new pc.Asset(asset.name,'container',{url:assetRoot+'assets/'+encodeURIComponent(asset.name)}); this.app.assets.add(a); await new Promise((resolve,reject)=>{ diff --git a/forge-production.js b/forge-production.js index 6a36a95..d4836db 100644 --- a/forge-production.js +++ b/forge-production.js @@ -9,18 +9,18 @@ class AssetPipeline { } async analyze(file) { - const base = { - id: crypto.randomUUID(), - name: file.name, - type: file.type || "application/octet-stream", - bytes: file.size, - modified: file.lastModified || Date.now() - }; - if (/^image\//.test(file.type)) return Object.assign(base, await this.analyzeImage(file)); - if (/\.glb$/i.test(file.name)) return Object.assign(base, await this.analyzeGLB(file)); - if (/\.gltf$/i.test(file.name)) return Object.assign(base, { kind:"gltf", status:"package-required", warnings:["GLTF may reference external files; prefer GLB or import the full package."], optimization:[] }); - if (/^audio\//.test(file.type) || /\.(wav|mp3|ogg|m4a)$/i.test(file.name)) return Object.assign(base,{kind:"audio",status:"ready",optimization:["Normalize loudness","Generate preview waveform","Create streaming/short-clip variants"]}); - return Object.assign(base,{kind:"file",status:"ready",optimization:[]}); + const base={id:crypto.randomUUID(),name:file.name,type:file.type||"application/octet-stream",bytes:file.size,modified:file.lastModified||Date.now(),extension:String(file.name).split(".").pop()?.toLowerCase()||""}; + const n=String(file.name).toLowerCase(); + const model=/\.(glb|gltf|fbx|obj|dae|3ds|stl|ply|off|3mf|dxf|ase|b3d|lwo|lxo|m3d|md2|md3|md5mesh|ms3d|smd|vta|x|x3d)$/i.test(n); + if(/^image\//.test(file.type)||/\.(png|jpe?g|webp|avif|gif|bmp|tga|tif|tiff)$/i.test(n))return Object.assign(base,await this.analyzeImage(file)); + if(/\.glb$/i.test(n))return Object.assign(base,await this.analyzeGLB(file)); + if(/\.gltf$/i.test(n))return Object.assign(base,{kind:"gltf",status:"package-or-single-file",warnings:["External buffers/textures are automatically packaged when supplied together."],optimization:["Prefer GLB for portable delivery"]}); + if(model)return Object.assign(base,{kind:"model-source",status:"conversion-capable",optimization:["Convert to GLB for native runtime","Preserve original source for re-export","Inspect materials, skeletons, animations and topology after conversion"]}); + if(/^audio\//.test(file.type)||/\.(wav|mp3|ogg|m4a|aac|flac|webm)$/i.test(n))return Object.assign(base,{kind:"audio",status:"ready",optimization:["Generate streaming and short-clip variants","Analyze loudness and loop points"]}); + if(/\.(ttf|ttc|otf)$/i.test(n))return Object.assign(base,{kind:"font",status:"source-ready",optimization:["Generate MSDF/SDF runtime atlas"]}); + if(/\.(glsl|vert|frag|shader)$/i.test(n))return Object.assign(base,{kind:"shader",status:"source-ready",optimization:["Validate shader source","Generate platform variants"]}); + if(/\.(json|xml|csv|txt|css|js|mjs|wasm)$/i.test(n))return Object.assign(base,{kind:"data",status:"source-ready",optimization:["Validate syntax/schema before runtime use"]}); + return Object.assign(base,{kind:"source",status:"source-ready",optimization:["Preserve exact source bytes and provenance"]}); } async analyzeImage(file) { @@ -197,14 +197,14 @@ const assets=new AssetPipeline(),world=new WorldSystems(),graph=new ForgeGraph() window.ForgeProduction={assets,world,graph,profiler}; const style=document.createElement("style"); -style.textContent=".forge-prod{position:fixed;right:14px;bottom:14px;z-index:90;display:flex;gap:5px;flex-wrap:wrap;max-width:430px}.forge-prod button{border:1px solid #29455f;background:#081727;color:#e8f1ff;border-radius:9px;padding:8px 10px;box-shadow:0 8px 26px #0007}.forge-modal{width:min(900px,94vw);border:1px solid #2d4e6d;background:#091523;color:#e8f1ff;border-radius:15px;padding:0}.forge-modal::backdrop{background:#000b;backdrop-filter:blur(8px)}.fm-head{display:flex;justify-content:space-between;padding:10px;border-bottom:1px solid #1b304a}.fm-body{padding:12px;max-height:70vh;overflow:auto}.fm-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.fm-card{border:1px solid #1b304a;background:#0b1727;border-radius:10px;padding:10px}.fm-card h4{margin:0 0 7px}.fm-actions{display:flex;gap:6px;flex-wrap:wrap;margin:8px 0}.fm-table{width:100%;border-collapse:collapse;font-size:11px}.fm-table td,.fm-table th{padding:6px;border-bottom:1px solid #162a40;text-align:left}.fm-node{position:relative;border:1px solid #315a7e;background:#0d1f33;border-radius:8px;padding:8px;margin:5px 0}.fm-code{white-space:pre-wrap;font:11px ui-monospace,monospace;background:#06101b;padding:9px;border-radius:8px}.fm-good{color:#48e28a}.fm-warn{color:#ffca66}.fm-bad{color:#ff7f8d}@media(max-width:700px){.forge-prod{left:8px;right:8px;bottom:8px}.fm-grid{grid-template-columns:1fr}}"; +style.textContent=".forge-modal{width:min(900px,94vw);border:1px solid #2d4e6d;background:#091523;color:#e8f1ff;border-radius:15px;padding:0}.forge-modal::backdrop{background:#000b;backdrop-filter:blur(8px)}.fm-head{display:flex;justify-content:space-between;padding:10px;border-bottom:1px solid #1b304a}.fm-body{padding:12px;max-height:70vh;overflow:auto}.fm-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.fm-card{border:1px solid #1b304a;background:#0b1727;border-radius:10px;padding:10px}.fm-card h4{margin:0 0 7px}.fm-actions{display:flex;gap:6px;flex-wrap:wrap;margin:8px 0}.fm-table{width:100%;border-collapse:collapse;font-size:11px}.fm-table td,.fm-table th{padding:6px;border-bottom:1px solid #162a40;text-align:left}.fm-node{position:relative;border:1px solid #315a7e;background:#0d1f33;border-radius:8px;padding:8px;margin:5px 0}.fm-code{white-space:pre-wrap;font:11px ui-monospace,monospace;background:#06101b;padding:9px;border-radius:8px}.fm-good{color:#48e28a}.fm-warn{color:#ffca66}.fm-bad{color:#ff7f8d}@media(max-width:700px){.fm-grid{grid-template-columns:1fr}}"; document.head.appendChild(style); function modal(title,body){ const d=document.createElement("dialog");d.className="forge-modal";d.innerHTML='
'+title+'
'+body+'
';d.querySelector("[data-x]").onclick=()=>d.close();document.body.appendChild(d);d.addEventListener("close",()=>d.remove(),{once:true});d.showModal();return d; } function openAssets(){ - const d=modal("Forge Asset Lab",'

Production import

Use Analyze to audit geometry, textures, animations, alpha, dimensions and optimization recommendations.

Registry

'); + const d=modal("Forge Asset Lab",'

Production import

Use Analyze to audit geometry, textures, animations, alpha, dimensions and optimization recommendations.

Registry

'); const render=()=>{const box=d.querySelector("#fmRegistry"),list=assets.all();box.innerHTML=list.length?list.map(a=>'
'+a.name+'
'+a.kind+' · '+(a.width?a.width+"×"+a.height:"")+" · "+(a.triangles?a.triangles+" triangles":"")+'
'+(a.optimization||[]).join(" · ")+'
').join(""):'Registry is empty.'}; d.querySelector("#fmAnalyze").onclick=async()=>{const fs=[...d.querySelector("#fmFiles").files];const r=await assets.process(fs);render();d.querySelector("#fmHint").textContent=r.map(x=>x.name+": "+x.status).join(" | ")||"No files selected"}; const cook=document.createElement("button");cook.textContent="Cook GLB + LOD";cook.onclick=async()=>{const f=d.querySelector("#fmFiles").files[0];if(!f||!/\.glb$/i.test(f.name)){d.querySelector("#fmHint").textContent="Select a GLB first";return}try{const bytes=new Uint8Array(await f.arrayBuffer());let bin="";for(let i=0;ix.charCodeAt(0)));z.file("cook-manifest.json",JSON.stringify(data.manifest,null,2));const blob=await z.generateAsync({type:"blob",compression:"DEFLATE"});const a=document.createElement("a");a.download=f.name.replace(/\.glb$/i,"")+"_cooked.zip";a.href=URL.createObjectURL(blob);a.click();d.querySelector("#fmHint").textContent="Cooked package: LOD0/LOD50/LOD80/LOD95 + manifest";toast("GLB cook completed")}catch(e){d.querySelector("#fmHint").textContent="Cook failed: "+e.message}};d.querySelector(".fm-actions").append(cook); @@ -355,9 +355,34 @@ function open2D(){ d.querySelector("#spriteImport2D").onclick=async()=>{const fs=[...d.querySelector("#spriteFiles2D").files];if(!fs.length)return;const r=await window.Forge2D.spriteFromFile(fs[0],"Sprite");Forge.select(r.id);refreshHierarchy();toast("Sprite imported")}; d.querySelector("#twoDState").textContent=JSON.stringify(window.Forge2D?.status?.(),null,2); } -function addBar(){ - const bar=document.createElement("div");bar.className="forge-prod"; - const buttons=[["Asset Lab",openAssets],["Material",openMaterial],["Shader",openShader],["Animation",openAnimation],["Anim Graph",openAnimationGraph],["Sequencer",openCinematics],["Render",openRender],["VFX",openVFX],["2D",open2D],["World / PCG",openPCG],["Logic Graph",openGraph],["Profiler",openProfiler],["Vision",()=>window.AssetForgeLiveVision?.open()],["Runtime",openRuntime],["Build Web",openBuild]]; - buttons.forEach(([t,f])=>{const b=document.createElement("button");b.textContent=t;b.onclick=f;bar.append(b)});document.body.append(bar); +function openEditors(){ + const d=modal("Forge Editors",'
'+[ + ["Asset Lab",openAssets,"Import, inspect and process source assets."], + ["Material Lab",openMaterial,"PBR material parameters and presets."], + ["Shader Lab",openShader,"Cross-platform shader presets and diagnostics."], + ["Animation",openAnimation,"Imported clips and playback."], + ["Animation Graph + Rig",openAnimationGraph,"State graphs and rig profiles."], + ["Sequencer",openCinematics,"Transform tracks and keyframes."], + ["Render",openRender,"Quality profiles and material graphs."], + ["VFX",openVFX,"Runtime particle/VFX controls."], + ["2D",open2D,"Raster/sprite authoring and import."], + ["World / PCG",openPCG,"Terrain and world generation systems."], + ["Logic Graph",openGraph,"Gameplay logic graph."], + ["Profiler",openProfiler,"Runtime performance diagnostics."], + ["Vision",()=>window.AssetForgeLiveVision?.open?.(),"Live browser vision and real input control."], + ["Runtime",openRuntime,"Game runtime, navigation, saves and networking."], + ["Build Web",openBuild,"Portable web runtime build."] + ].map(([t,f,desc])=>'

'+t+'

'+desc+'
').join("")+'
'); + const map=new Map([ + ["Asset Lab",openAssets],["Material Lab",openMaterial],["Shader Lab",openShader],["Animation",openAnimation], + ["Animation Graph + Rig",openAnimationGraph],["Sequencer",openCinematics],["Render",openRender],["VFX",openVFX], + ["2D",open2D],["World / PCG",openPCG],["Logic Graph",openGraph],["Profiler",openProfiler], + ["Vision",()=>window.AssetForgeLiveVision?.open?.()],["Runtime",openRuntime],["Build Web",openBuild] + ]); + d.querySelectorAll("[data-editor-tool]").forEach(b=>b.onclick=()=>{ + const fn=map.get(b.dataset.editorTool);if(!fn)return; + d.close();setTimeout(()=>{try{fn()}catch(e){toast("Editor failed: "+e.message)}},0); + }); + return d; } -addBar(); +window.ForgeProduction.openEditors=openEditors; diff --git a/forge-realcar-preview.js b/forge-realcar-preview.js new file mode 100644 index 0000000..03c44d6 --- /dev/null +++ b/forge-realcar-preview.js @@ -0,0 +1,174 @@ +let state=null; + +async function modules(){ + const THREE=await import("three"); + const {GLTFLoader}=await import("https://cdn.jsdelivr.net/npm/three@0.185.1/examples/jsm/loaders/GLTFLoader.js"); + const {OrbitControls}=await import("https://cdn.jsdelivr.net/npm/three@0.185.1/examples/jsm/controls/OrbitControls.js"); + const {RoomEnvironment}=await import("https://cdn.jsdelivr.net/npm/three@0.185.1/examples/jsm/environments/RoomEnvironment.js"); + return{THREE,GLTFLoader,OrbitControls,RoomEnvironment}; +} + +function disposeObject(THREE,obj){ + obj?.traverse?.(node=>{ + if(!node.isMesh)return; + node.geometry?.dispose?.(); + const materials=Array.isArray(node.material)?node.material:[node.material]; + for(const m of materials){ + for(const key of ["map","normalMap","roughnessMap","metalnessMap","aoMap","emissiveMap","clearcoatMap","clearcoatNormalMap","clearcoatRoughnessMap"]){ + if(m?.[key]?.dispose) m[key].dispose(); + } + m?.dispose?.(); + } + }); +} + +export async function openRealCarPreview({file,name="Realistic Car",meta={}}){ + const host=document.getElementById("forgeAssetPreview"); + const canvas=document.getElementById("forgeAssetPreviewCanvas"); + const close=document.getElementById("forgeAssetPreviewClose"); + if(!host||!canvas)throw new Error("Forge 3D asset preview host is missing"); + if(state){try{state.dispose()}catch{}state=null} + + const {THREE,GLTFLoader,OrbitControls,RoomEnvironment}=await modules(); + const renderer=new THREE.WebGLRenderer({canvas,antialias:true,alpha:false,preserveDrawingBuffer:true}); + renderer.setPixelRatio(Math.min(window.devicePixelRatio||1,2)); + renderer.outputColorSpace=THREE.SRGBColorSpace; + renderer.toneMapping=THREE.ACESFilmicToneMapping; + renderer.toneMappingExposure=1.08; + renderer.shadowMap.enabled=true; + renderer.shadowMap.type=THREE.PCFSoftShadowMap; + + const scene=new THREE.Scene(); + scene.background=new THREE.Color("#07111d"); + + const camera=new THREE.PerspectiveCamera(38,1,.01,500); + const controls=new OrbitControls(camera,canvas); + controls.enableDamping=true; + controls.dampingFactor=.07; + controls.minDistance=2.4; + controls.maxDistance=16; + controls.autoRotate=true; + controls.autoRotateSpeed=.65; + controls.target.set(0,1,0); + + const pmrem=new THREE.PMREMGenerator(renderer); + const environment=new RoomEnvironment(renderer); + scene.environment=pmrem.fromScene(environment,.04).texture; + + const key=new THREE.DirectionalLight(0xffffff,3.4); + key.position.set(5,8,6); + key.castShadow=true; + key.shadow.mapSize.set(2048,2048); + key.shadow.camera.near=.1; + key.shadow.camera.far=40; + scene.add(key); + + const fill=new THREE.DirectionalLight(0x8ab8ff,1.4); + fill.position.set(-6,4,-4); + scene.add(fill); + + const rim=new THREE.DirectionalLight(0xffd6aa,1.6); + rim.position.set(1,5,-8); + scene.add(rim); + + const floor=new THREE.Mesh( + new THREE.PlaneGeometry(28,28), + new THREE.MeshStandardMaterial({color:0x111a26,roughness:.34,metalness:.25}) + ); + floor.rotation.x=-Math.PI/2; + floor.receiveShadow=true; + scene.add(floor); + + const sourceBytes=new Uint8Array(await file.arrayBuffer()); + const loader=new GLTFLoader(); + const gltf=await new Promise((resolve,reject)=>loader.parse(sourceBytes.buffer,"",resolve,reject)); + const root=gltf.scene; + root.traverse(node=>{ + if(!node.isMesh)return; + node.castShadow=true; + node.receiveShadow=true; + const mats=Array.isArray(node.material)?node.material:[node.material]; + for(const m of mats){ + if("envMapIntensity" in m)m.envMapIntensity=1.15; + if("metalness" in m&&m.metalness<0)m.metalness=.0; + if("roughness" in m&&m.roughness<=0)m.roughness=.18; + } + }); + + const box=new THREE.Box3().setFromObject(root); + const size=new THREE.Vector3(),center=new THREE.Vector3(); + box.getSize(size);box.getCenter(center); + const longest=Math.max(size.x,size.y,size.z); + const scale=longest>0?4.8/longest:1; + root.scale.setScalar(scale); + root.position.set(-center.x*scale,-box.min.y*scale,-center.z*scale); + root.rotation.y=-.35; + scene.add(root); + + const normalizedBox=new THREE.Box3().setFromObject(root); + const normalizedSize=new THREE.Vector3(); + normalizedBox.getSize(normalizedSize); + const radius=Math.max(normalizedSize.x,normalizedSize.z)*.62; + camera.position.set(radius*1.28,Math.max(1.5,normalizedSize.y*.62),radius*1.28); + camera.lookAt(0,Math.max(.7,normalizedSize.y*.36),0); + controls.target.set(0,Math.max(.7,normalizedSize.y*.36),0); + + const resize=()=>{ + const rect=host.getBoundingClientRect(); + const w=Math.max(1,rect.width),h=Math.max(1,rect.height); + renderer.setSize(w,h,false); + camera.aspect=w/h; + camera.updateProjectionMatrix(); + }; + const onResize=()=>resize(); + window.addEventListener("resize",onResize); + host.classList.remove("hidden"); + resize(); + + let raf=0; + const loop=()=>{ + controls.update(); + renderer.render(scene,camera); + raf=requestAnimationFrame(loop); + }; + loop(); + + const stats={name,sourceBytes:sourceBytes.byteLength,meshes:0,vertices:0,triangles:0,materials:0,scale}; + const mats=new Set(); + root.traverse(node=>{ + if(!node.isMesh)return; + stats.meshes++; + const pos=node.geometry?.attributes?.position; + if(pos)stats.vertices+=pos.count; + const idx=node.geometry?.index; + stats.triangles+=idx?idx.count/3:Math.floor((pos?.count||0)/3); + const list=Array.isArray(node.material)?node.material:[node.material]; + for(const m of list)if(m)mats.add(m); + }); + stats.materials=mats.size; + + const setText=()=>{ + const text=document.getElementById("forgeAssetPreviewMeta"); + if(text)text.textContent=`${name} · ${stats.vertices.toLocaleString()} vertices · ${Math.round(stats.triangles).toLocaleString()} triangles · ${stats.materials} materials`; + }; + setText(); + + const dispose=()=>{ + cancelAnimationFrame(raf); + window.removeEventListener("resize",onResize); + controls.dispose(); + pmrem.dispose(); + environment.dispose?.(); + disposeObject(THREE,root); + floor.geometry.dispose();floor.material.dispose(); + renderer.dispose(); + host.classList.add("hidden"); + state=null; + }; + close?.addEventListener("click",dispose,{once:true}); + state={dispose,renderer,scene,camera,controls,stats}; + window.ForgeDemoAssets=Object.assign(window.ForgeDemoAssets||{},{ + previewReady:true,previewStats:stats,closeCarPreview:dispose + }); + return{stats,dispose}; +} diff --git a/forge-runtime.js b/forge-runtime.js index 9c60a46..fce0d29 100644 --- a/forge-runtime.js +++ b/forge-runtime.js @@ -13,18 +13,50 @@ class InputSystem{ snapshot(){return Object.fromEntries([...this.bindings].map(([k])=>[k,this.isDown(k)]))} mountMobileControls(){ - if(document.getElementById("forgeMobileControls"))return; - const root=document.createElement("div");root.id="forgeMobileControls";Object.assign(root.style,{position:"fixed",inset:"0",zIndex:"110",pointerEvents:"none",display:"none"}); - root.innerHTML='
'; - document.body.appendChild(root); - const joy=root.querySelector("#forgeJoy"),knob=root.querySelector("#forgeKnob");let active=false,cx=0,cy=0; - const move=e=>{if(!active)return;const r=joy.getBoundingClientRect(),dx=e.clientX-(r.left+r.width/2),dy=e.clientY-(r.top+r.height/2),m=45,l=Math.min(m,Math.hypot(dx,dy)),a=Math.atan2(dy,dx);const x=Math.cos(a)*l/m,y=Math.sin(a)*l/m;knob.style.transform="translate("+Math.round(x*36)+"px,"+Math.round(y*36)+"px)";this.virtualMove={x:x,z:-y}}; - const end=()=>{active=false;this.virtualMove={x:0,z:0};knob.style.transform=""}; - joy.addEventListener("pointerdown",e=>{active=true;joy.setPointerCapture(e.pointerId);move(e)});joy.addEventListener("pointermove",move);joy.addEventListener("pointerup",end);joy.addEventListener("pointercancel",end); - for(const b of root.querySelectorAll("[data-mobile]")){const a=b.dataset.mobile;b.addEventListener("pointerdown",e=>{e.preventDefault();this.down.add(a)});b.addEventListener("pointerup",()=>this.down.delete(a));b.addEventListener("pointercancel",()=>this.down.delete(a));b.addEventListener("pointerleave",()=>this.down.delete(a))} - const show=()=>{root.style.display=(navigator.maxTouchPoints>0||matchMedia("(pointer:coarse)").matches)?"block":"none"};show();addEventListener("resize",show); - } -} + let root=null; + const editorMode=!!document.getElementById("play"); + const editorPlayActive=()=>document.body?.dataset?.forgeRuntimeActive==="1"||document.documentElement?.dataset?.forgeRuntimeActive==="1"; + const isTouch=()=>navigator.maxTouchPoints>0||matchMedia("(pointer:coarse)").matches; + + const destroy=()=>{ + if(!root)return; + try{root.remove()}catch{} + root=null; + this.virtualMove={x:0,z:0}; + }; + + const create=()=>{ + if(root)return; + root=document.createElement("div"); + root.id="forgeMobileControls"; + Object.assign(root.style,{position:"fixed",inset:"0",zIndex:"110",pointerEvents:"none",display:"none"}); + root.innerHTML='
'; + document.body.appendChild(root); + const joy=root.querySelector("#forgeJoy"),knob=root.querySelector("#forgeKnob");let active=false; + const move=e=>{if(!active)return;const r=joy.getBoundingClientRect(),dx=e.clientX-(r.left+r.width/2),dy=e.clientY-(r.top+r.height/2),m=45,l=Math.min(m,Math.hypot(dx,dy)),a=Math.atan2(dy,dx);const x=Math.cos(a)*l/m,y=Math.sin(a)*l/m;knob.style.transform="translate("+Math.round(x*36)+"px,"+Math.round(y*36)+"px)";this.virtualMove={x,z:-y}}; + const end=()=>{active=false;this.virtualMove={x:0,z:0};knob.style.transform=""}; + joy.addEventListener("pointerdown",e=>{active=true;joy.setPointerCapture(e.pointerId);move(e)}); + joy.addEventListener("pointermove",move);joy.addEventListener("pointerup",end);joy.addEventListener("pointercancel",end); + for(const b of root.querySelectorAll("[data-mobile]")){const a=b.dataset.mobile;b.addEventListener("pointerdown",e=>{e.preventDefault();this.down.add(a)});b.addEventListener("pointerup",()=>this.down.delete(a));b.addEventListener("pointercancel",()=>this.down.delete(a));b.addEventListener("pointerleave",()=>this.down.delete(a))} + }; + + const show=()=>{ + const standaloneRuntime=!editorMode; + const visible=(standaloneRuntime||editorPlayActive())&&isTouch(); + if(visible){ + create(); + if(root)root.style.display="block"; + }else if(editorMode){ + destroy(); + }else if(root){ + root.style.display="none"; + } + }; + + addEventListener("resize",show); + addEventListener("forge-runtime-visibility",show); + show(); + }} class NavigationSystem{ constructor(){this.cell=.75;this.width=80;this.height=80;this.blocked=new Uint8Array(this.width*this.height)} diff --git a/forge-ui.js b/forge-ui.js index a3a1238..b2fbd75 100644 --- a/forge-ui.js +++ b/forge-ui.js @@ -139,7 +139,7 @@ if(!window.ForgeGizmo){ }; } function esc(s){return String(s??"").replace(/[&<>"']/g,c=>({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]))} -function icon(k){return({box:"▣",sphere:"●",cylinder:"⬢",capsule:"◉",plane:"▱",camera:"◫",light:"☼",model:"◇",empty:"+"})[k]||"•"} +function icon(k){return({box:"▣",sphere:"●",cylinder:"⬢",capsule:"◉",plane:"▱",camera:"◫",light:"☼",model:"◇",audio:"♫",empty:"+"})[k]||"•"} function matchesFilter(r){const q=String($("treeFilter")?.value||"").trim().toLowerCase();return !q||r.name.toLowerCase().includes(q)||r.kind.toLowerCase().includes(q)} function refresh(){ const tree=$("tree");tree.innerHTML=""; @@ -182,7 +182,13 @@ function updateAssetPanel(){ if(!label||!type)return; const a=r?.components?.asset; label.textContent=r?.name||"No asset selected"; - type.textContent=a?.type?(a.type==="model"?"3D GLB":a.type==="image"?"2D Image":a.type.toUpperCase()):"Scene object"; + type.textContent=a?.type?(a.type==="model"?"3D Model":a.type==="image"?"2D Image":a.type==="audio"?"AUDIO":a.type==="file"?"SOURCE FILE":"FILE"):"Scene object"; + const play=$("assetPlay"); + if(play){ + const isAudio=a?.type==="audio"&&!!r?.entity?.sound?.slot?.("main"); + play.disabled=!isAudio; + play.textContent=isAudio?(r.entity.sound.slot("main").isPlaying?"■ Stop Audio":"▶ Play Audio"):"▶ Play Audio"; + } if(meta){ if(!a){ meta.textContent="Select an imported asset to inspect source metadata."; @@ -191,7 +197,9 @@ function updateAssetPanel(){ const bytes=Number(entry?.file?.size||0); const mime=entry?.file?.type||"application/octet-stream"; const dims=a.width&&a.height?(" · "+a.width+"×"+a.height+"px"):""; - meta.textContent="SOURCE · "+a.name+" · "+bytes.toLocaleString()+" bytes · "+mime+dims; + const license=a.license?(" · "+a.license):""; + const source=a.remoteSource?(" · REMOTE SOURCE"):(" · LOCAL SOURCE"); + meta.textContent="SOURCE · "+a.name+" · "+bytes.toLocaleString()+" bytes · "+mime+dims+license+source; } } } @@ -199,30 +207,32 @@ function updateAssetBrowser(){ const list=$("assetList");if(!list)return; const q=String($("assetSearch")?.value||"").trim().toLowerCase(); list.innerHTML=""; - const records=[...engine.entities.values()].filter(r=>r.components?.asset?.name).filter(r=>!q||r.name.toLowerCase().includes(q)||String(r.components.asset.type).toLowerCase().includes(q)); + const sceneRecords=[...engine.entities.values()].filter(r=>r.components?.asset?.name).map(r=>({scene:r,asset:r.components.asset,entry:engine.assets?.get?.(r.components.asset.name)})); + const sourceRecords=[...engine.assets.entries()].filter(([,v])=>v?.sourceOnly).map(([name,entry])=>({scene:null,asset:{name,type:entry.type||"file"},entry})); + const records=[...sceneRecords,...sourceRecords].filter(x=>!q||String(x.asset.name).toLowerCase().includes(q)||String(x.asset.type).toLowerCase().includes(q)); if(!records.length){const e=document.createElement("div");e.className="asset-empty";e.textContent="No imported assets";list.append(e);return} - for(const r of records){ - const row=document.createElement("button");row.className="asset-row"+(r.id===engine.selectedId?" active":""); - const t=r.components.asset.type==="model"?"3D":r.components.asset.type==="image"?"2D":"FILE"; + for(const item of records){ + const r=item.scene,a=item.asset,entry=item.entry; + const row=document.createElement("button");row.className="asset-row"+(r&&r.id===engine.selectedId?" active":""); + const t=a.type==="model"?"3D":a.type==="image"?"2D":a.type==="audio"?"AUD":a.type==="source"?"SRC":"FILE"; const visual=document.createElement("span");visual.className="asset-visual"; - const entry=engine.assets?.get?.(r.components.asset.name); - if(r.components.asset.type==="image"&&entry?.file){ + if(a.type==="image"&&entry?.file){ if(!entry.previewUrl){try{entry.previewUrl=URL.createObjectURL(entry.file)}catch{}} - if(entry.previewUrl){ - const img=document.createElement("img");img.className="asset-thumb";img.alt="";img.src=entry.previewUrl;visual.append(img); - }else{ - const ic=document.createElement("span");ic.className="asset-icon";ic.textContent=t;visual.append(ic); - } - }else{ - const ic=document.createElement("span");ic.className="asset-icon";ic.textContent=t;visual.append(ic); - } + if(entry.previewUrl){const img=document.createElement("img");img.className="asset-thumb";img.alt="";img.src=entry.previewUrl;visual.append(img)} + else{const ic=document.createElement("span");ic.className="asset-icon";ic.textContent=t;visual.append(ic)} + }else{const ic=document.createElement("span");ic.className="asset-icon";ic.textContent=t;visual.append(ic)} row.append(visual); - const nameNode=document.createElement("span");nameNode.className="asset-name";nameNode.textContent=r.name;row.append(nameNode); - const kindNode=document.createElement("span");kindNode.className="asset-kind";kindNode.textContent=r.kind;row.append(kindNode); - row.onclick=()=>{engine.select(r.id);refresh();inspect();engine.focus()}; - list.append(row) + const nameNode=document.createElement("span");nameNode.className="asset-name";nameNode.textContent=a.name;row.append(nameNode); + const kindNode=document.createElement("span");kindNode.className="asset-kind";kindNode.textContent=r?r.kind:(entry?.extension?entry.extension.toUpperCase():"SOURCE");row.append(kindNode); + row.title=r?"Open in scene":"Source asset preserved; select a renderable instance to edit"; + row.onclick=()=>{ + if(r){engine.select(r.id);refresh();inspect();engine.focus()} + else{const bytes=Number(entry?.file?.size||0);const hash=entry?.sourceSha256||"pending";write("Source asset preserved · "+a.name+" · "+bytes.toLocaleString()+" bytes · SHA-256 "+hash);toast("Source asset preserved — ready for a compatible importer")} + }; + list.append(row); } } + function apply(){ const r=engine.selected();if(!r)return; engine.transform({x:+$("px").value,y:+$("py").value,z:+$("pz").value,rx:+$("rx").value,ry:+$("ry").value,rz:+$("rz").value,sx:+$("sx").value,sy:+$("sy").value,sz:+$("sz").value}); @@ -244,12 +254,122 @@ document.querySelectorAll("[data-add]").forEach(b=>b.onclick=()=>{ $("apply").onclick=apply;["px","py","pz","rx","ry","rz","sx","sy","sz","name"].forEach(id=>$(id).onchange=apply); $("duplicate").onclick=()=>{if(engine.duplicate()){refresh();inspect()}};$("undo")?.addEventListener("click",()=>{if(engine.undo()){refresh();inspect()}});$("redo")?.addEventListener("click",()=>{if(engine.redo()){refresh();inspect()}}); $("delete").onclick=()=>{engine.delete();refresh();inspect()};$("focus").onclick=()=>engine.focus();$("frame").onclick=()=>engine.frame();$("reset")?.addEventListener("click",()=>location.reload()); -$("play").onclick=()=>{$("play").textContent=engine.running?"❚❚ Play":"▶ Play";engine.running=!engine.running;engine.timelinePlaying=engine.running};$("timelinePlay").onclick=()=>$("play").click(); +const setForgeRuntimeActive=active=>{document.body.dataset.forgeRuntimeActive=active?"1":"0";document.documentElement.dataset.forgeRuntimeActive=active?"1":"0";window.dispatchEvent(new Event("forge-runtime-visibility"))}; +setForgeRuntimeActive(false); +$("play").onclick=()=>{$("play").textContent=engine.running?"❚❚ Play":"▶ Play";engine.running=!engine.running;engine.timelinePlaying=engine.running;setForgeRuntimeActive(engine.running)};$("timelinePlay").onclick=()=>$("play").click(); $("rewind").onclick=()=>{engine.timelineTime=0;$("time").value=0;engine.evalAnimation(0);inspect()};$("time").oninput=e=>{engine.timelineTime=+e.target.value||0;engine.evalAnimation(engine.timelineTime);$("playhead").style.left=(engine.timelineTime*60)+"px";inspect()};$("key").onclick=()=>{engine.key();toast("Keyframe added")}; +$("draw2dOpen")?.addEventListener("click",()=>{ + const d=$("draw2dDialog"); + if(d&&!d.open)try{d.showModal()}catch{d.setAttribute("open","")} +}); +const REALISTIC_CAR_URL="https://sceneview.github.io/models/platforms/CarConcept.glb"; +async function importRealisticCar(){ + const button=$("importRealCar"); + const previous=button?.textContent; + const controller=new AbortController(); + const timer=setTimeout(()=>controller.abort(),45000); + try{ + if(button){button.disabled=true;button.textContent="🚘 Loading high-fidelity car…"} + const response=await fetch(REALISTIC_CAR_URL,{mode:"cors",cache:"no-store",signal:controller.signal}); + if(!response.ok)throw new Error("High-fidelity car download failed: HTTP "+response.status); + const blob=await response.blob(); + if(blob.size<5_000_000)throw new Error("High-fidelity car asset is unexpectedly small ("+blob.size+" bytes)"); + const file=new File([blob],"Forge-CarConcept-Khronos.glb",{type:"model/gltf-binary"}); + await window.ForgeProduction?.assets?.storeFile?.(file); + const result=await engine.importFile(file,{remoteSource:REALISTIC_CAR_URL}); + if(result?.type!=="model"||!result.record)throw new Error("High-fidelity car did not produce a native Forge 3D model"); + const record=result.record,entry=engine.assets.get(file.name); + const sourceSha256=await forgeExportProxy.sha256(file); + record.components.asset.remoteSource=REALISTIC_CAR_URL; + record.components.asset.sourceSha256=sourceSha256; + record.components.asset.license="CC BY 4.0"; + record.components.asset.licenseUrl="https://creativecommons.org/licenses/by/4.0/"; + record.components.asset.attribution="Darmstadt Graphics Group GmbH + Eric Chadwick (glTF conversion/optimization); Unity Fan (original Sketchfab model, CC0)"; + record.components.asset.catalogSource="KhronosGroup/glTF-Sample-Assets — CarConcept"; + record.components.asset.qualityProfile="high-fidelity-production-car-concept"; + if(entry)Object.assign(entry,{remoteSource:REALISTIC_CAR_URL,sourceSha256,license:"CC BY 4.0",catalogSource:"KhronosGroup/glTF-Sample-Assets — CarConcept"}); + engine.select(record.id); + engine.focus(); + refresh();inspect(); + toast("High-fidelity 3D car imported into Forge viewport"); + write("Imported high-fidelity glTF car into the native Forge 3D viewport"); + return result; + }catch(error){ + if(error?.name==="AbortError")throw new Error("High-fidelity car download timed out after 45s"); + throw error; + }finally{ + clearTimeout(timer); + if(button){button.disabled=false;button.textContent=previous||"🚘 Import high-quality car"} + } +} +window.ForgeDemoAssets={importRealisticCar,REALISTIC_CAR_URL}; +$("importRealCar")?.addEventListener("click",()=>importRealisticCar().catch(e=>write("High-fidelity car import failed: "+e.message,"error"))); $("importAssets").onclick=()=>$("assetInput").click();$("assetInput").onchange=e=>loadFiles([...e.target.files]); -async function loadFiles(files){for(const f of files)try{await window.ForgeProduction?.assets?.storeFile?.(f);const r=await engine.importFile(f);if(r.type==="project")await engine.load(r.data);write("Imported "+f.name)}catch(e){write("Import failed "+f.name+": "+e.message,"error")}refresh();inspect()} +async function fileDataUri(file){ + const bytes=new Uint8Array(await file.arrayBuffer()); + let binary=""; + const chunk=0x8000; + for(let i=0;i{const e=file.name.toLowerCase().split(".").pop();return({png:"image/png",jpg:"image/jpeg",jpeg:"image/jpeg",webp:"image/webp",avif:"image/avif",gif:"image/gif",bmp:"image/bmp",wav:"audio/wav",mp3:"audio/mpeg",ogg:"audio/ogg"}[e]||"application/octet-stream")})(); + return "data:"+mime+";base64,"+btoa(binary); +} +async function packageGltfWithSelectedFiles(gltf,files){ + const json=JSON.parse(await gltf.text()); + const byName=new Map(files.map(f=>[f.name.replaceAll("\\","/").split("/").at(-1).toLowerCase(),f])); + const patchUris=async group=>{ + for(const item of group||[]){ + const uri=item?.uri; + if(!uri||String(uri).startsWith("data:"))continue; + const clean=decodeURIComponent(String(uri).split(/[?#]/)[0]).replaceAll("\\","/").split("/").at(-1).toLowerCase(); + const f=byName.get(clean); + if(!f)throw new Error("Missing glTF dependency: "+uri); + item.uri=await fileDataUri(f); + } + }; + await patchUris(json.buffers); + await patchUris(json.images); + return new File([JSON.stringify(json)],gltf.name,{type:"model/gltf+json"}); +} +async function loadFiles(files){ + const list=[...(files||[])].filter(Boolean); + for(const f of list)try{await window.ForgeProduction?.assets?.storeFile?.(f)}catch(e){write("Asset vault store failed "+f.name+": "+e.message,"error")} + const gltf=list.find(f=>/\.gltf$/i.test(f.name)); + if(gltf){ + try{ + const packaged=await packageGltfWithSelectedFiles(gltf,list); + const r=await engine.importFile(packaged); + if(r.type==="project")await engine.load(r.data); + write("Imported packaged glTF "+gltf.name); + refresh();inspect(); + return; + }catch(e){ + write("glTF package import failed; source files remain in Content Browser: "+e.message,"error"); + } + } + const legacyModels=list.filter(f=>/\.(fbx|obj|dae|3ds)$/i.test(f.name)); + const legacyDependencyNames=new Set(list.filter(f=>/\.(mtl|tga|dds)$/i.test(f.name)).map(f=>f.name)); + for(const f of legacyModels){ + try{await engine.importFile(f,{files:list});write("Imported and converted "+f.name+" to an internal GLB representation")} + catch(e){write("Legacy 3D conversion failed "+f.name+": "+e.message,"error")} + } + for(const f of list){ + if(gltf&&f===gltf)continue; + if(legacyModels.includes(f))continue; + if(legacyDependencyNames.has(f.name))continue; + if(gltf&&f===gltf)continue; + try{ + const r=await engine.importFile(f); + if(r.type==="project")await engine.load(r.data); + write("Imported "+f.name); + }catch(e){ + write("Import failed "+f.name+": "+e.message,"error"); + } + } + refresh();inspect(); +} document.addEventListener("dragover",e=>{e.preventDefault();$("dropOverlay").classList.remove("hidden")});document.addEventListener("drop",e=>{e.preventDefault();$("dropOverlay").classList.add("hidden");if(e.dataTransfer?.files?.length)loadFiles([...e.dataTransfer.files])}); $("saveProject").onclick=()=>{window.ForgeProject.download();toast("Complete Forge project saved")};$("openProject").onclick=()=>$("projectInput").click();$("projectInput").onchange=async e=>{const f=e.target.files[0];if(!f)return;try{await window.ForgeProject.load(JSON.parse(await f.text()));refresh();inspect();toast("Complete Forge project loaded")}catch(err){write("Project load failed: "+err.message,"error")}}; +$("openEditors")?.addEventListener("click",()=>window.ForgeProduction?.openEditors?.()); $("build").onclick=async()=>{if(window.ForgeProduction?.build){try{const r=await window.ForgeProduction.build();if(r?.url){location.href=r.url;return}}catch(e){write("Build failed: "+e.message,"error")}}window.ForgeProject.download("forge-project-3.5.forge.json");toast("Portable Forge project exported")}; async function assetAction(fn,label){try{const r=await fn();$("assetReport").textContent=JSON.stringify(r,null,2);$("assetBadge").textContent=r.ok?"PASS":"CHECK";$("assetBadge").className="qa-badge "+(r.ok?"pass":"warn");toast(label+(r.ok?" — PASS":" — inspect report"))}catch(e){$("assetBadge").textContent="FAIL";$("assetBadge").className="qa-badge fail";$("assetReport").textContent=JSON.stringify({ok:false,error:e.message},null,2);toast(label+" — "+e.message)}} $("exportSource")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.exportSource(),"Source export")); @@ -258,6 +378,14 @@ $("exportPreview")?.addEventListener("click",()=>assetAction(()=>window.ForgeExp $("exportManifest")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.exportManifest(),"Asset manifest")); $("validateAsset")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.validateSelected(),"Asset validation")); $("assetQA")?.addEventListener("click",()=>assetAction(()=>window.ForgeExport.runExportQA(),"Export round-trip QA")); +$("assetPlay")?.addEventListener("click",()=>{ + const r=engine.selected(),slot=r?.entity?.sound?.slot?.("main"); + if(!slot)return; + try{ + if(slot.isPlaying)slot.stop();else slot.play(); + updateAssetPanel(); + }catch(e){write("Audio playback failed: "+e.message,"error")} +}); $("treeFilter")?.addEventListener("input",refresh); $("assetSearch")?.addEventListener("input",updateAssetBrowser); $("quickFocus")?.addEventListener("click",()=>engine.focus()); diff --git a/forge.css b/forge.css index 50c0c9a..1c3e3f8 100644 --- a/forge.css +++ b/forge.css @@ -29,4 +29,35 @@ dialog{border:1px solid #2b4d6e;background:#091523;color:var(--txt);border-radiu .asset-thumb{width:30px;height:30px;object-fit:cover;border-radius:5px;border:1px solid #2b4c66;background:#020a12} .forge-boot{position:fixed;inset:0;z-index:9999;display:grid;place-items:center;background:radial-gradient(circle at 50% 42%,#132b45,#06101d 62%,#02050a);transition:opacity .25s ease,visibility .25s ease}.forge-boot.hidden{opacity:0;visibility:hidden;pointer-events:none}.forge-boot-card{width:min(420px,88vw);padding:24px;border:1px solid #24425f;border-radius:16px;background:#081523ee;box-shadow:0 18px 70px #0008;display:grid;gap:8px;text-align:center}.forge-boot-mark{width:52px;height:52px;margin:0 auto 2px;border-radius:14px;display:grid;place-items:center;background:linear-gradient(135deg,#4dd7ff,#756cff);color:#00101b;font-weight:900;font-size:24px}.forge-boot-card strong{font-size:17px;letter-spacing:1px}.forge-boot-card span{color:#c9ddf1;line-height:1.5}.forge-boot-card small{color:#6f8aa2;font-size:10px} -.editor-grid{position:absolute;inset:0;pointer-events:none;z-index:2;background-image:linear-gradient(rgba(100,160,210,.07) 1px,transparent 1px),linear-gradient(90deg,rgba(100,160,210,.07) 1px,transparent 1px);background-size:32px 32px;mask-image:linear-gradient(to bottom,transparent 0%,rgba(0,0,0,.8) 22%,rgba(0,0,0,.9) 100%);}.editor-grid:after{content:"";position:absolute;left:50%;top:50%;width:1px;height:100%;background:rgba(120,210,255,.12);box-shadow:0 0 0 1px rgba(120,210,255,.02)}.editor-grid:before{content:"";position:absolute;left:0;top:50%;height:1px;width:100%;background:rgba(255,120,120,.10)}.inspector{min-height:150px} \ No newline at end of file +.editor-grid{position:absolute;inset:0;pointer-events:none;z-index:2;background-image:linear-gradient(rgba(100,160,210,.07) 1px,transparent 1px),linear-gradient(90deg,rgba(100,160,210,.07) 1px,transparent 1px);background-size:32px 32px;mask-image:linear-gradient(to bottom,transparent 0%,rgba(0,0,0,.8) 22%,rgba(0,0,0,.9) 100%);}.editor-grid:after{content:"";position:absolute;left:50%;top:50%;width:1px;height:100%;background:rgba(120,210,255,.12);box-shadow:0 0 0 1px rgba(120,210,255,.02)}.editor-grid:before{content:"";position:absolute;left:0;top:50%;height:1px;width:100%;background:rgba(255,120,120,.10)}.inspector{min-height:150px} +.draw-card{background:linear-gradient(180deg,#0c1c2f,#0a1726)} +.draw-dialog{width:min(920px,96vw);max-height:94vh} +.draw-tools{display:flex;gap:10px;align-items:center;flex-wrap:wrap;padding:10px;border-bottom:1px solid var(--line);background:#071321} +.draw-tools label{display:flex;gap:8px;align-items:center;color:var(--muted);font-size:10px} +.draw-tools input[type=range]{width:180px} +.draw-tools input[type=color]{width:44px;height:30px;padding:2px} +.draw-tools button.active{background:#3a1f2b;border-color:#7a4b5b;color:#ffe3eb} +.draw-canvas-wrap{padding:14px;background:#030811;display:grid;place-items:center} +#drawCanvas{width:min(100%,768px);height:auto;aspect-ratio:3/2;background: + linear-gradient(45deg,#0e1c2a 25%,transparent 25%,transparent 75%,#0e1c2a 75%), + linear-gradient(45deg,#0e1c2a 25%,transparent 25%,transparent 75%,#0e1c2a 75%); + background-size:28px 28px;background-position:0 0,14px 14px; + border:1px solid #294760;border-radius:10px;box-shadow:0 16px 50px #0006;touch-action:none;cursor:crosshair} +.draw-status{padding:7px 10px;color:#8bb2d5;background:#06101b;border-top:1px solid var(--line);font:10px ui-monospace,SFMono-Regular,monospace} +@media(max-width:800px){ + .draw-dialog{width:96vw} + .draw-tools input[type=range]{width:130px} + #drawCanvas{width:100%} +} + +/* Forge 4.0 editor organization: advanced tools are modal/dock-like, never overlaid on the viewport. */ +.forge-modal{box-shadow:0 28px 90px #0009;max-width:min(980px,96vw)} +.forge-modal .fm-card{min-height:118px} +.forge-modal .fm-card h4{font-size:12px;color:#e9f4ff} +.forge-modal .subtle{font-size:10px;color:#7f96ae;line-height:1.45} +#openEditors{border-color:#31506d;background:#0b1828} +#openEditors:hover{background:#10253a} +@media(max-width:800px){ + .forge-modal{width:min(96vw,620px);max-height:88vh} + .forge-modal .fm-grid{grid-template-columns:1fr} +} diff --git a/index.html b/index.html index a286f50..c3a054e 100644 --- a/index.html +++ b/index.html @@ -4,25 +4,34 @@ -Forge Studio 3.7 — Spatial Game & App Forge - +Forge Studio 4.0 — Spatial Game & App Forge + +
F
Forge StudioLoading engine, spatial core and editor modules…Preparing deterministic workspace
-
F
FORGEStudio 3.6 · Spatial Core
+
F
FORGEStudio 4.0 · Spatial Core
-
+
-
0 entitiesNone
+
0 entitiesNone
ANIMATION / SIMULATION
@@ -43,8 +52,19 @@
Forge Spatial Vision
No scan yet.
Forge Visual QA
+ +
Forge 2D Paint
+
+ + + +
+
+
Ready
+
+
Entity Script
- + \ No newline at end of file diff --git a/live-vision.js b/live-vision.js index 63549d4..0003537 100644 --- a/live-vision.js +++ b/live-vision.js @@ -2,206 +2,214 @@ const $=(s,r=document)=>r.querySelector(s); const sleep=ms=>new Promise(r=>setTimeout(r,ms)); const state={ - ws:null,stream:null,running:false,busy:false,frameSeq:0,lastFrameAt:0,frameCount:0,frameWindowStart:0, - captureCanvas:null,captureCtx:null,captureTimer:0,remoteUrl:null,mode:"idle",pointerId:11 + ws:null,running:false,busy:false,frameSeq:0,lastFrameAt:0,frameCount:0,frameWindowStart:0, + remoteUrl:null,mode:"idle" }; - const serverCapable=()=>location.protocol!=='file:'&&!/github\.io$/i.test(location.hostname); - const wsUrl=()=>((location.protocol==='https:'?'wss:':'ws:')+'//'+location.host+'/vision'); - function style(){if($('#liveVisionStyle'))return;const l=document.createElement('link');l.id='liveVisionStyle';l.rel='stylesheet';l.href='./live-vision.css?v=3';document.head.appendChild(l)} + + const serverCapable=()=>location.protocol!=="file:"&&!/github\.io$/i.test(location.hostname); + const wsUrl=()=>((location.protocol==="https:"?"wss:":"ws:")+"//"+location.host+"/vision"); + + function style(){ + if($("#liveVisionStyle"))return; + const link=document.createElement("link"); + link.id="liveVisionStyle";link.rel="stylesheet";link.href="./live-vision.css?v=4"; + document.head.appendChild(link); + } + function mount(){ - if($('#liveVision'))return; - const el=document.createElement('section'); - el.id='liveVision'; - el.className='live-vision'; + if($("#liveVision"))return; + const el=document.createElement("section"); + el.id="liveVision";el.className="live-vision"; el.innerHTML=[ - '
FORGE VISIONLive eyes + real hands · Spatial Core
', + '
FORGE VISIONContinuous eyes + real browser hands
', '
Idle0 fps—
', - '
Forge live vision stream
', - '
', - '
', - '', + '
Forge continuous live vision
', + '
', + '
', + '', '
', - '
Direct mode uses the live MediaStream. Pro CDP uses Chromium screencast + CDP mouse/keyboard/touch events; no screenshot request loop is used.
' - ].join(''); + '
Live mode: Playwright continuous screencast over WebSocket. Actions are real Chromium mouse, keyboard, touch and file-input events, followed by a frame receipt.
' + ].join(""); document.body.appendChild(el); - $('#liveVisionClose').onclick=()=>el.classList.remove('open'); - $('#liveVisionStart').onclick=start; - $('#liveVisionPro').onclick=connectPro; - $('#liveVisionStop').onclick=stop; - $('#liveVisionAgent').onclick=runAgent; - $('#liveVisionMove').onclick=()=>runManual({type:'move',x:innerWidth/2,y:innerHeight/2}); - $('#liveVisionDown').onclick=()=>runManual({type:'mouseDown',x:innerWidth/2,y:innerHeight/2}); - $('#liveVisionUp').onclick=()=>runManual({type:'mouseUp',x:innerWidth/2,y:innerHeight/2}); - $('#liveVisionWheel').onclick=()=>runManual({type:'wheel',x:innerWidth/2,y:innerHeight/2,deltaY:360}); - } - function open(){mount();style();$('#liveVision').classList.add('open');connectVisionWs()} - function log(t,c=''){const b=$('#liveVisionLog');if(!b)return;const r=document.createElement('div');r.className='live-log-row '+c;r.textContent=new Date().toLocaleTimeString()+' '+t;b.prepend(r);while(b.children.length>24)b.lastElementChild.remove()} - function stateText(t,on=state.running){const s=$('#liveVisionState'),d=$('#liveVisionDot');if(s)s.textContent=t;if(d)d.classList.toggle('live-on',!!on)} - function setTransport(t){const e=$('#liveVisionTransport');if(e)e.textContent=t||'—'} - function connectVisionWs(){ - if(!serverCapable()||typeof WebSocket==='undefined')return; - if(state.ws&&(state.ws.readyState===0||state.ws.readyState===1))return; - try{ - const ws=new WebSocket(wsUrl()); - ws.binaryType='arraybuffer'; - state.ws=ws; - ws.onopen=()=>{ws.send(JSON.stringify({type:'hello',role:'vision-ui',protocol:4,viewport:{width:innerWidth,height:innerHeight,dpr:devicePixelRatio||1}}));log('Live vision WebSocket connected','ok')}; - ws.onerror=()=>{}; - ws.onclose=()=>{state.ws=null;if(state.mode==='cdp')stateText('CDP disconnected',false)}; - ws.onmessage=onVisionMessage; - }catch{} - } - async function onVisionMessage(e){ - if(typeof e.data!=='string'){ - try{ - const buf=e.data instanceof ArrayBuffer?e.data:await e.data.arrayBuffer(); - if(buf.byteLength<16)return; - const dv=new DataView(buf),magic=dv.getUint32(0),seq=dv.getUint32(4),w=dv.getUint16(8),h=dv.getUint16(10); - if(magic!==0x46563337)return; - const jpeg=buf.slice(16),img=$('#liveVisionRemote');if(!img)return; - const url=URL.createObjectURL(new Blob([jpeg],{type:'image/jpeg'})); - const old=state.remoteUrl;state.remoteUrl=url;img.onload=()=>{if(old)URL.revokeObjectURL(old)};img.src=url; - state.frameSeq=seq;state.frameCount++;const now=Date.now();if(!state.frameWindowStart)state.frameWindowStart=now; - if(now-state.frameWindowStart>1000){$('#liveVisionFps').textContent=Number(state.frameCount*1000/(now-state.frameWindowStart)).toFixed(1)+' fps';state.frameCount=0;state.frameWindowStart=now} - }catch{} - return; - } - let m;try{m=JSON.parse(e.data)}catch{return} - if(m.type==='vision-state'){ - const s=m.state||{}; - if(s.connected&&s.streaming){state.mode='cdp';state.running=true;stateText('Direct Chromium live',true);setTransport('CDP');$('#liveVisionPreview').classList.add('remote');$('#liveVisionPreview').classList.remove('local')} - if(!s.connected&&state.mode==='cdp'){state.mode='idle';state.running=false;stateText('CDP disconnected',false)} - if(s.fps!=null)$('#liveVisionFps').textContent=Number(s.fps).toFixed(1)+' fps'; - } + $("#liveVisionClose").onclick=()=>el.classList.remove("open"); + $("#liveVisionStart").onclick=start; + $("#liveVisionPro").onclick=restart; + $("#liveVisionStop").onclick=stop; + $("#liveVisionAgent").onclick=runAgent; + $("#liveVisionMove").onclick=()=>runManual({type:"mouse.move",x:innerWidth/2,y:innerHeight/2,steps:4}); + $("#liveVisionDown").onclick=()=>runManual({type:"mouse.down",button:"left"}); + $("#liveVisionUp").onclick=()=>runManual({type:"mouse.up",button:"left"}); + $("#liveVisionWheel").onclick=()=>runManual({type:"mouse.wheel",deltaY:360}); } - function normalize(x,y,w,h){return{x:Math.max(0,Math.min(innerWidth-1,(Number(x)||0)*innerWidth/Math.max(1,w||innerWidth))),y:Math.max(0,Math.min(innerHeight-1,(Number(y)||0)*innerHeight/Math.max(1,h||innerHeight)))}} - function dispatchPointer(type,x,y,extra={}){ - const el=document.elementFromPoint(x,y)||document.body; - const buttons=type==='pointerup'?0:(extra.buttons??1); - el.dispatchEvent(new PointerEvent(type,{bubbles:true,cancelable:true,view:window,clientX:x,clientY:y,screenX:x,screenY:y,button:extra.button??0,buttons,pointerId:state.pointerId,pointerType:extra.pointerType||'mouse',isPrimary:true})); - return{tag:el.tagName||'',id:el.id||'',className:typeof el.className==='string'?el.className:''}; - } - async function localControl(a){ - const type=String(a.type||''); - if(type==='move'){const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);return dispatchPointer('pointermove',p.x,p.y,{buttons:a.buttons??0})} - if(type==='mouseDown'){const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);state.pointerId++;return dispatchPointer('pointerdown',p.x,p.y,{buttons:1})} - if(type==='mouseUp'){const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);return dispatchPointer('pointerup',p.x,p.y,{buttons:0})} - if(type==='click'||type==='doubleClick'){ - const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight);const el=document.elementFromPoint(p.x,p.y)||document.body;dispatchPointer('pointermove',p.x,p.y,{buttons:0}); - const count=type==='doubleClick'?2:1; - for(let i=0;i24)b.lastElementChild.remove(); + } + + function stateText(t,on=state.running){ + const s=$("#liveVisionState"),d=$("#liveVisionDot"); + if(s)s.textContent=t;if(d)d.classList.toggle("live-on",!!on); + } + + function setTransport(t){const e=$("#liveVisionTransport");if(e)e.textContent=t||"—"} + + function setRemoteFrame(data){ + const img=$("#liveVisionRemote");if(!img)return; + const url=URL.createObjectURL(new Blob([data],{type:"image/jpeg"})); + const old=state.remoteUrl;state.remoteUrl=url; + img.onload=()=>{if(old)URL.revokeObjectURL(old)}; + img.src=url; + $("#liveVisionPreview").classList.add("remote"); + $("#liveVisionPreview").classList.remove("local"); + } + + function onFramePacket(data){ + if(!(data instanceof ArrayBuffer))return; + if(data.byteLength<16)return; + const dv=new DataView(data),magic=dv.getUint32(0); + if(magic!==0x46563337)return; + const seq=dv.getUint32(4),width=dv.getUint16(8),height=dv.getUint16(10); + const jpeg=data.slice(16); + state.frameSeq=Math.max(state.frameSeq,seq);state.lastFrameAt=Date.now();state.frameCount++; + if(!state.frameWindowStart)state.frameWindowStart=Date.now(); + const now=Date.now(); + if(now-state.frameWindowStart>=1000){ + const fps=state.frameCount*1000/(now-state.frameWindowStart); + $("#liveVisionFps").textContent=fps.toFixed(1)+" fps"; + state.frameCount=0;state.frameWindowStart=now; } - if(type==='wheel'){const p=normalize(a.x??innerWidth/2,a.y??innerHeight/2,a.screenWidth,a.screenHeight);const el=document.elementFromPoint(p.x,p.y)||document.body;el.dispatchEvent(new WheelEvent('wheel',{bubbles:true,cancelable:true,clientX:p.x,clientY:p.y,deltaX:a.deltaX||0,deltaY:a.deltaY||0}));return{ok:true,type,point:p}} - if(type==='press'||type==='keyDown'||type==='keyUp'||type==='typeText'){ - const target=document.activeElement||document.body; - if(type==='typeText'){target.dispatchEvent(new InputEvent('beforeinput',{bubbles:true,inputType:'insertText',data:String(a.text||'')}));if('value' in target){target.value+=String(a.text||'');target.dispatchEvent(new Event('input',{bubbles:true}))}return{ok:true,type}} - const evName=type==='keyUp'?'keyup':'keydown';target.dispatchEvent(new KeyboardEvent(evName,{bubbles:true,cancelable:true,key:String(a.key||''),code:a.code||'',ctrlKey:!!a.ctrlKey,shiftKey:!!a.shiftKey,altKey:!!a.altKey,metaKey:!!a.metaKey}));if(type==='press')target.dispatchEvent(new KeyboardEvent('keyup',{bubbles:true,key:String(a.key||''),code:a.code||''}));return{ok:true,type,key:a.key}; + setRemoteFrame(jpeg); + $("#liveVisionRemote").setAttribute("data-size",width+"x"+height); + } + + function onMessage(event){ + if(typeof event.data!=="string"){onFramePacket(event.data);return} + let msg;try{msg=JSON.parse(event.data)}catch{return} + if(msg.type==="vision-status"||msg.type==="vision-state"){ + const s=msg.state||msg; + const running=!!s.running; + state.running=running;state.mode=running?"server":"idle"; + state.frameSeq=Math.max(state.frameSeq,Number(s.frameSeq)||0); + state.lastFrameAt=Number(s.lastFrameAt)||state.lastFrameAt; + if(s.fps!=null)$("#liveVisionFps").textContent=Number(s.fps).toFixed(1)+" fps"; + stateText(running?"Live Chromium stream":"Idle",running); + setTransport(running?"WebSocket / Playwright":"—"); } - if(type==='touchStart'||type==='touchMove'||type==='touchEnd'){ - const p=normalize(a.x,a.y,a.screenWidth,a.screenHeight),el=document.elementFromPoint(p.x,p.y)||document.body; - const eventType=type==='touchStart'?'pointerdown':type==='touchMove'?'pointermove':'pointerup'; - return dispatchPointer(eventType,p.x,p.y,{buttons:type==='touchEnd'?0:1,pointerType:'touch'}); + if(msg.type==="session-result"&&msg.ok){ + state.running=true;state.mode="server";stateText("Live Chromium stream",true);setTransport("WebSocket / Playwright"); + log("Vision session started","ok"); } - throw new Error('Unsupported local control action: '+type); + if(msg.type==="action-result"&&!msg.ok)log(msg.error||"Vision action failed","error"); } - async function runManual(action){try{const result=await control(action);log(JSON.stringify(result),'ok')}catch(e){log(e.message||String(e),'error')}} - async function control(action){ - if(state.mode==='cdp'&&state.ws?.readyState===1){ - const r=await fetch('./api/vision/control',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({action})}); - const j=await r.json();if(!j.ok)throw new Error(j.error||'CDP control failed');return j.result; - } - return localControl(action); + + function connectWs(){ + if(!serverCapable()||typeof WebSocket==="undefined")return; + if(state.ws&&(state.ws.readyState===0||state.ws.readyState===1))return; + try{ + const ws=new WebSocket(wsUrl());ws.binaryType="arraybuffer";state.ws=ws; + ws.onopen=()=>{ws.send(JSON.stringify({type:"hello",role:"vision-ui",protocol:5,viewport:{width:innerWidth,height:innerHeight,dpr:devicePixelRatio||1}}));log("Live vision channel connected","ok")}; + ws.onmessage=onMessage; + ws.onerror=()=>log("Vision WebSocket error","error"); + ws.onclose=()=>{state.ws=null;if(state.running){state.running=false;stateText("Disconnected",false)}}; + }catch(e){log(e.message||String(e),"error")} + } + + async function post(path,body){ + const r=await fetch(path,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(body||{})}); + let j={};try{j=await r.json()}catch{} + if(!r.ok||j.ok===false)throw new Error(j.error||"Vision request failed"); + return j; } - async function connectPro(){ - mount();connectVisionWs(); - if(!serverCapable())return log('Pro CDP is only available from the local/server Forge build','warn'); + + async function start(){ + mount();style();connectWs(); + if(!serverCapable())return log("Live server mode requires the local Forge server","warn"); + if(state.running)return log("Live vision is already running","ok"); try{ - const targets=await fetch('./api/vision/targets').then(r=>r.json()); - const list=targets.targets||[]; - const target=list.find(t=>t.type==='page'&&/^https?:/i.test(t.url||'')); - if(!target)throw new Error('No Chromium CDP target found on 127.0.0.1:9222'); - const result=await fetch('./api/vision/connect',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({targetId:target.id,targetUrl:target.url,quality:55,maxWidth:1280,maxHeight:720,everyNthFrame:1})}).then(r=>r.json()); - if(!result.ok)throw new Error(result.error||'CDP connect failed'); - state.mode='cdp';state.running=true;stateText('Direct Chromium live',true);setTransport('CDP');log('CDP live stream started for '+target.url,'ok'); - }catch(e){state.mode='idle';state.running=false;log(e.message||String(e),'error')} + await post("./api/vision/session/start",{target:"/",viewport:{width:Math.max(320,innerWidth),height:Math.max(240,innerHeight)},touch:matchMedia("(pointer:coarse)").matches,mobile:innerWidth<700,streamFps:30}); + state.running=true;state.mode="server";stateText("Live Chromium stream",true);setTransport("WebSocket / Playwright"); + }catch(e){state.running=false;state.mode="idle";stateText("Idle",false);log(e.message||String(e),"error")} } - function visualHealth(){ - const canvas=document.querySelector('#viewport');if(!canvas)return{ok:false,reason:'viewport-missing'}; - const rect=canvas.getBoundingClientRect();return{ok:rect.width>300&&rect.height>300,renderer:window.Forge?.diagnostics?.().renderer||'unknown',viewport:{width:rect.width,height:rect.height},overflow:Math.max(0,document.documentElement.scrollWidth-innerWidth)}; + + async function restart(){ + try{await stop();await sleep(150);await start()}catch(e){log(e.message||String(e),"error")} + } + + async function stop(){ + try{if(serverCapable())await post("./api/vision/session/stop",{})}catch(e){log(e.message||String(e),"error")} + state.running=false;state.mode="idle";stateText("Idle",false);setTransport("—"); + const img=$("#liveVisionRemote"); + if(img)img.removeAttribute("src"); + if(state.remoteUrl){URL.revokeObjectURL(state.remoteUrl);state.remoteUrl=null} + $("#liveVisionPreview")?.classList.remove("remote","local"); + } + + async function control(action,options={}){ + if(!serverCapable())throw new Error("Server vision is unavailable"); + return (await post("./api/vision/action",{action,options})).result; + } + + async function runManual(action){ + try{const result=await control(action);log(JSON.stringify(result),"ok")} + catch(e){log(e.message||String(e),"error")} + } + + async function getState(){const r=await fetch("./api/vision/state",{cache:"no-store"});if(!r.ok)throw new Error("Vision state unavailable");return r.json()} + async function getElements(){const r=await fetch("./api/vision/elements",{cache:"no-store"});if(!r.ok)throw new Error("Vision elements unavailable");return r.json()} + + async function clickSelector(selector){ + const data=await getElements(); + const e=data.elements.find(x=>x.selector===selector&&x.visible); + if(!e)throw new Error("Visible element not found: "+selector); + return control({type:"mouse.click",x:e.x+e.width/2,y:e.y+e.height/2}); } + async function executeAction(a){ - if(a.type==='vision'||a.type==='scene_scan')return window.Forge?.spatial?.sceneVision?.()||window.ForgeSpatial?.sceneVision?.()||{}; - if(a.type==='visual_health'||a.type==='visual_qa')return visualHealth(); - if(a.type==='select_entity'){const F=window.Forge,r=a.id?F.select(a.id):[...F.entities.values()].find(x=>x.name.toLowerCase()===String(a.name||'').toLowerCase());if(!r)throw new Error('Entity not found');return F.spatial?.inspect?.(r)||window.ForgeSpatial?.inspect?.(r)||{id:r.id,name:r.name,kind:r.kind}} - if(a.type==='focus'){window.Forge?.focus?.();return true} - if(a.type==='frame'){window.Forge?.frame?.();return true} - if(a.type==='transform'){if(a.id)window.Forge?.select?.(a.id);window.Forge?.transform?.(a);return window.ForgeSpatial?.inspect?.(window.Forge.selected())||null} + if(a.type==="vision"||a.type==="scene_scan")return getState(); + if(a.type==="visual_health")return window.Forge?.diagnostics?.()?window.Forge.diagnostics():{ok:true}; return control(a); } - async function localAgent(task){ - const F=window.Forge,s=String(task||'').trim(),lower=s.toLowerCase(); - if(/scan|inspect|vision|understand/.test(lower))return executeAction({type:'vision'}); - const names=[...F.entities.values()].map(r=>r.name).filter(Boolean),target=names.find(n=>lower.includes(n.toLowerCase())); - if(target){ - await executeAction({type:'select_entity',name:target}); - if(/delete|remove|erase/.test(lower)){F.delete();return{ok:true,action:'delete',entity:target}} - if(/duplicate|copy|clone/.test(lower)){const d=F.duplicate();return{ok:true,action:'duplicate',entity:d?.name||target}} - if(/center|middle/.test(lower)){F.spatial?.moveToScreen?.(F.selectedId,innerWidth/2,innerHeight/2,.5,0);return{ok:true,action:'move-center',entity:target}} - if(/focus/.test(lower)){F.focus();return{ok:true,action:'focus',entity:target}} - } - if(/play/.test(lower)){F.running=true;F.timelinePlaying=true;return{ok:true,action:'play'}} - if(/stop|pause/.test(lower)){F.running=false;F.timelinePlaying=false;return{ok:true,action:'stop'}} - if(/frame all|frame/.test(lower)){F.frame();return{ok:true,action:'frame'}} - if(/focus/.test(lower)){F.focus();return{ok:true,action:'focus'}} - throw new Error('Task not mapped. Use a deterministic command or a pixel action such as click/drag/wheel/press.'); + + async function executeTask(task){ + const s=String(task||"").trim().toLowerCase(); + if(!s)throw new Error("Task is empty"); + if(s.includes("scan")||s.includes("vision")||s.includes("inspect"))return getState(); + if(s.includes("add box"))return clickSelector('[data-add="box"]'); + if(s.includes("add sphere"))return clickSelector('[data-add="sphere"]'); + if(s.includes("play"))return clickSelector("#play"); + if(s.includes("stop")||s.includes("pause"))return clickSelector("#play"); + if(s.includes("rotate"))return control({type:"keyboard.press",key:"2"}); + if(s.includes("scale"))return control({type:"keyboard.press",key:"3"}); + if(s.includes("move"))return control({type:"keyboard.press",key:"1"}); + if(s.includes("frame"))return clickSelector("#frame"); + if(s.includes("focus"))return clickSelector("#focus"); + throw new Error("Task not mapped; use concrete vision actions for exact control"); } + async function runAgent(){ mount(); - const task=$('#liveVisionTask')?.value?.trim();if(!task)return log('Enter a task first','error');if(state.busy)return; - state.busy=true;$('#liveVisionStart').disabled=true; - try{ - let result; - if(serverCapable()&&state.mode==='cdp')result=await fetch('./api/forge/command',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({command:{op:'agent-task',task}})}).then(r=>r.json()); - else result=await localAgent(task); - log(JSON.stringify(result),'ok'); - }catch(e){log(e.message||String(e),'error')}finally{state.busy=false;$('#liveVisionStart').disabled=false} + const task=$("#liveVisionTask")?.value?.trim(); + if(!task||state.busy)return; + state.busy=true;$("#liveVisionAgent").disabled=true; + try{const result=await executeTask(task);log(JSON.stringify(result),"ok")} + catch(e){log(e.message||String(e),"error")} + finally{state.busy=false;$("#liveVisionAgent").disabled=false} } - async function start(){ - mount();style();connectVisionWs(); - if(state.mode==='cdp'){return log('CDP live is already active','ok')} - if(!navigator.mediaDevices?.getDisplayMedia){stateText('Spatial fallback',true);setTransport('Local DOM');log('Screen capture unavailable; local vision/control remains active','warn');return} - try{ - stop(); - const stream=await navigator.mediaDevices.getDisplayMedia({video:{frameRate:{ideal:30,max:30},width:{ideal:1920,max:1920},height:{ideal:1080,max:1080},displaySurface:'browser'},audio:false,preferCurrentTab:true,selfBrowserSurface:'include',surfaceSwitching:'include'}); - state.stream=stream;state.running=true;state.mode='local-media';stateText('Live MediaStream',true);setTransport('MediaStream'); - const v=$('#liveVisionVideo');v.srcObject=stream;await v.play();$('#liveVisionPreview').classList.add('local');$('#liveVisionPreview').classList.remove('remote'); - stream.getVideoTracks()[0].addEventListener('ended',stop); - sampleLiveFrames(); - }catch(e){state.running=false;state.mode='idle';state.stream=null;stateText('Spatial fallback',true);setTransport('Local DOM');log(e.message||String(e),'warn')} - } - function sampleLiveFrames(){ - if(!state.running||state.mode!=='local-media')return; - const v=$('#liveVisionVideo'); - const tick=()=>{if(!state.running||state.mode!=='local-media')return;state.frameSeq++;state.lastFrameAt=Date.now();$('#liveVisionFps').textContent='live';if('requestVideoFrameCallback' in v)v.requestVideoFrameCallback(()=>setTimeout(sampleLiveFrames,120));else state.captureTimer=setTimeout(sampleLiveFrames,120)}; - tick(); + + function visualHealth(){ + const canvas=document.querySelector("#viewport"); + if(!canvas)return{ok:false,reason:"viewport-missing"}; + const rect=canvas.getBoundingClientRect(); + return{ok:rect.width>300&&rect.height>300,viewport:{width:rect.width,height:rect.height},overflow:Math.max(0,document.documentElement.scrollWidth-innerWidth)}; } - async function stop(){ - try{if(serverCapable())await fetch('./api/vision/disconnect',{method:'POST'}).catch(()=>{})}catch{} - state.running=false;state.mode='idle';if(state.stream){state.stream.getTracks().forEach(t=>t.stop());state.stream=null} - clearTimeout(state.captureTimer);const v=$('#liveVisionVideo');if(v)v.srcObject=null; - const img=$('#liveVisionRemote');if(img)img.removeAttribute('src');if(state.remoteUrl){URL.revokeObjectURL(state.remoteUrl);state.remoteUrl=null} - $('#liveVisionPreview')?.classList.remove('remote','local');stateText('Idle',false);setTransport('—'); - } - window.AssetForgeLiveVision={ - open,start,stop,runAgent,executeAction,visualHealth,connectPro,control, - status:()=>({mode:state.mode,running:state.running,connected:state.ws?.readyState===1,frames:state.frameSeq,lastFrameAt:state.lastFrameAt,transport:state.mode==='cdp'?'chromium-cdp':state.mode==='local-media'?'media-stream':'local-spatial'}) - }; - if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',()=>{mount();style()},{once:true});else{mount();style()} + + const api={open,start,stop,restart,runAgent,executeAction,executeTask,visualHealth,control,status:()=>({mode:state.mode,running:state.running,connected:state.ws?.readyState===1,frames:state.frameSeq,lastFrameAt:state.lastFrameAt,transport:state.mode==="server"?"playwright-screencast-websocket":"idle"})}; + window.AssetForgeLiveVision=api; + if(document.readyState==="loading")document.addEventListener("DOMContentLoaded",()=>{mount();style()},{once:true}); + else{mount();style()} })(); \ No newline at end of file diff --git a/package.json b/package.json index ad17125..7c05256 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "test:core": "playwright test tests/core.smoke.spec.mjs tests/asset-export.spec.mjs tests/visual.smoke.spec.mjs tests/professional.qa.spec.mjs --project=desktop", "test:professional": "playwright test tests/professional.qa.spec.mjs --project=desktop", "validate:gltf": "node scripts/validate-gltf-test-assets.mjs", - "test:vision": "playwright test tests/vision-live.spec.mjs --project=desktop" + "test:vision": "playwright test tests/vision.live.spec.mjs --project=desktop", + "test:vision:mobile": "playwright test tests/vision.live.spec.mjs --project=mobile" }, "devDependencies": { "@playwright/test": "^1.63.0", diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 0000000..378b5bf --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,9 @@ +# Forge Platform Packaging + +This directory is the native-output layer for Forge projects. + +The web packager creates a self-contained Forge runtime directory from a .forge.json project plus optional assets. Capacitor wraps that runtime into Android, while Electron Forge wraps the same runtime into a Windows desktop application. + +The platform workflow builds a real Android debug APK and a Windows package from the committed sample project. It is separate from the main Forge proof workflow so native toolchains cannot regress editor, Vision, asset or gameplay verification. + +The current runtime still loads PlayCanvas/Rapier modules from their CDN URLs. That keeps native builds runtime-equivalent to the proven web build, but it is not an offline engine distribution. Offline/vendor packaging is the next hardening stage. diff --git a/packaging/android/capacitor.config.ts b/packaging/android/capacitor.config.ts new file mode 100644 index 0000000..01323f5 --- /dev/null +++ b/packaging/android/capacitor.config.ts @@ -0,0 +1,7 @@ +const config = { + appId: "com.forge.studio", + appName: "Forge Game", + webDir: "../../dist/forge-web", + bundledWebRuntime: false +}; +export default config; diff --git a/packaging/android/package.json b/packaging/android/package.json new file mode 100644 index 0000000..e2292f3 --- /dev/null +++ b/packaging/android/package.json @@ -0,0 +1,19 @@ +{ + "name": "forge-android-shell", + "private": true, + "version": "3.7.0", + "type": "module", + "scripts": { + "add": "cap add android", + "sync": "cap sync android", + "build": "npx cap sync android && cd android && ./gradlew assembleDebug" + }, + "dependencies": { + "@capacitor/core": "8.5.2", + "@capacitor/android": "8.5.2" + }, + "devDependencies": { + "@capacitor/cli": "8.5.2", + "typescript": "5.9.2" + } +} diff --git a/packaging/sample-project.forge.json b/packaging/sample-project.forge.json new file mode 100644 index 0000000..372fcd3 --- /dev/null +++ b/packaging/sample-project.forge.json @@ -0,0 +1,166 @@ +{ + "format": "forge-project", + "version": 5, + "meta": { + "name": "Forge Packaging Proof", + "engine": "Forge Studio 3.7", + "createdAt": "2026-09-30T00:00:00.000Z", + "units": "meters", + "coordinateSystem": "right-handed +Y up +Z forward" + }, + "scene": { + "format": "forge-scene", + "version": 3, + "meta": { + "engine": "Forge Studio 3.7", + "time": "2026-09-30T00:00:00.000Z", + "units": "meters" + }, + "entities": [ + { + "id": "pkg-camera", + "name": "Main Camera", + "kind": "camera", + "transform": { + "p": [ + 7, + 5, + 9 + ], + "r": [ + -20, + 35, + 0 + ], + "s": [ + 1, + 1, + 1 + ] + }, + "components": { + "camera": { + "active": true + } + }, + "keyframes": [], + "script": null + }, + { + "id": "pkg-sun", + "name": "Sun", + "kind": "light", + "transform": { + "p": [ + 0, + 4, + 0 + ], + "r": [ + 48, + -32, + 0 + ], + "s": [ + 1, + 1, + 1 + ] + }, + "components": { + "light": { + "type": "directional" + } + }, + "keyframes": [], + "script": null + }, + { + "id": "pkg-ground", + "name": "Ground", + "kind": "plane", + "transform": { + "p": [ + 0, + 0, + 0 + ], + "r": [ + 0, + 0, + 0 + ], + "s": [ + 30, + 1, + 30 + ] + }, + "components": { + "physics": { + "mode": "fixed", + "shape": "box" + } + }, + "keyframes": [], + "script": null + }, + { + "id": "pkg-box", + "name": "Packaging Proof Box", + "kind": "box", + "transform": { + "p": [ + 0, + 1, + 0 + ], + "r": [ + 0, + 25, + 0 + ], + "s": [ + 2, + 2, + 2 + ] + }, + "components": { + "physics": { + "mode": "dynamic", + "shape": "box" + } + }, + "keyframes": [], + "script": null + } + ] + }, + "production": { + "assets": [], + "graph": null, + "world": null, + "profiler": null + }, + "runtime": { + "input": null, + "prefabs": [], + "slots": [], + "audio": null + }, + "gameplay": null, + "session": null, + "gameplayData": null, + "twoD": null, + "rendering": null, + "cinematics": null, + "ui": null, + "vfx": null, + "terrain": null, + "shader": null, + "animation": null, + "network": null, + "replay": null, + "spatial": null +} diff --git a/packaging/windows/forge.config.cjs b/packaging/windows/forge.config.cjs new file mode 100644 index 0000000..d0ab570 --- /dev/null +++ b/packaging/windows/forge.config.cjs @@ -0,0 +1,23 @@ +const path = require("node:path"); +module.exports = { + packagerConfig: { + asar: true, + name: "ForgeGame", + executableName: "ForgeGame", + extraResource: [path.resolve(__dirname, "../../dist/forge-web")] + }, + rebuildConfig: {}, + makers: [ + { + name: "@electron-forge/maker-squirrel", + config: { + name: "ForgeGame", + setupExe: "ForgeGameSetup.exe", + shortcutName: "Forge Game", + description: "Forge Game Runtime", + authors: "Forge Studio" + } + }, + { name: "@electron-forge/maker-zip", platforms: ["win32"] } + ] +}; diff --git a/packaging/windows/main.cjs b/packaging/windows/main.cjs new file mode 100644 index 0000000..0c2ce53 --- /dev/null +++ b/packaging/windows/main.cjs @@ -0,0 +1,21 @@ +const { app, BrowserWindow } = require("electron"); +const path = require("node:path"); +function createWindow() { + const win = new BrowserWindow({ + width: 1440, + height: 900, + show: false, + backgroundColor: "#06101d", + webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } + }); + const base = app.isPackaged + ? path.join(process.resourcesPath, "forge-web") + : path.resolve(__dirname, "../../dist/forge-web"); + win.loadFile(path.join(base, "index.html")); + win.once("ready-to-show", () => win.show()); +} +app.whenReady().then(() => { + createWindow(); + app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow(); }); +}); +app.on("window-all-closed", () => { if (process.platform !== "darwin") app.quit(); }); diff --git a/packaging/windows/package.json b/packaging/windows/package.json new file mode 100644 index 0000000..c0fa9c8 --- /dev/null +++ b/packaging/windows/package.json @@ -0,0 +1,17 @@ +{ + "name": "forge-windows-shell", + "productName": "Forge Game", + "version": "3.7.0", + "private": true, + "description": "Forge Studio game runtime", + "main": "main.cjs", + "scripts": { + "make": "electron-forge make" + }, + "devDependencies": { + "electron": "44.0.0", + "@electron-forge/cli": "latest", + "@electron-forge/maker-squirrel": "latest", + "@electron-forge/maker-zip": "latest" + } +} diff --git a/playwright.config.mjs b/playwright.config.mjs index 6029212..f4cf7d7 100644 --- a/playwright.config.mjs +++ b/playwright.config.mjs @@ -5,7 +5,7 @@ export default defineConfig({ reporter:[["list"],["html",{outputFolder:"playwright-report",open:"never"}]], testDir:"./tests", timeout:60000, - use:{baseURL:"http://127.0.0.1:4173",trace:"retain-on-failure",screenshot:"only-on-failure"}, + use:{baseURL:"http://127.0.0.1:4173",channel:process.env.PLAYWRIGHT_CHANNEL||undefined,trace:"retain-on-failure",screenshot:"only-on-failure"}, webServer:{command:"node server.mjs",url:"http://127.0.0.1:4173",reuseExistingServer:true,timeout:30000}, projects:[{name:"desktop",use:{...devices["Desktop Chrome"],viewport:{width:1440,height:900}}},{name:"mobile",use:{...devices["Pixel 7"]}}] }); \ No newline at end of file diff --git a/scripts/fixtures/minimal-triangle.glb b/scripts/fixtures/minimal-triangle.glb new file mode 100644 index 0000000..d9fcaa5 Binary files /dev/null and b/scripts/fixtures/minimal-triangle.glb differ diff --git a/scripts/package-web-project.mjs b/scripts/package-web-project.mjs new file mode 100644 index 0000000..71d51b5 --- /dev/null +++ b/scripts/package-web-project.mjs @@ -0,0 +1,43 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const argv = process.argv.slice(2); +function arg(name, fallback) { const i = argv.indexOf(name); return i >= 0 && argv[i + 1] ? argv[i + 1] : fallback; } +const projectFile = path.resolve(root, arg("--project", "packaging/sample-project.forge.json")); +const out = path.resolve(root, arg("--out", "dist/forge-web")); +const runtimeFiles = [ + "forge-engine.js","forge-runtime.js","forge-project.js","forge-gameplay.js","forge-render.js", + "forge-ui-system.js","forge-vfx.js","forge-2d.js","forge-ai.js","forge-gameplay-data.js", + "forge-network.js","forge-replay.js","forge-session.js","forge-shader.js","forge-terrain.js", + "forge-qa.js","forge-animation.js" +]; +await fs.rm(out, { recursive: true, force: true }); +await fs.mkdir(path.join(out, "assets"), { recursive: true }); +await fs.copyFile(projectFile, path.join(out, "project.forge.json")); +for (const file of runtimeFiles) await fs.copyFile(path.join(root, file), path.join(out, file)); +const assetDirArg = arg("--assets", ""); +if (assetDirArg) { try { await fs.cp(path.resolve(root, assetDirArg), path.join(out, "assets"), { recursive: true }); } catch {} } +const runtimeHtml = [ + "", + "", + "Forge Game Runtime", + "", + "
Forge Runtime loading...
" +].join(""); +await fs.writeFile(path.join(out, "index.html"), runtimeHtml); +await fs.writeFile(path.join(out, "manifest.webmanifest"), JSON.stringify({name:"Forge Game Runtime",short_name:"ForgeGame",start_url:"./",display:"fullscreen",background_color:"#06101d",theme_color:"#06101d"}, null, 2)); +await fs.writeFile(path.join(out, "sw.js"), "self.addEventListener(\"install\",e=>e.waitUntil(caches.open(\"forge-runtime-v1\").then(c=>c.addAll([\"./\",\"./index.html\",\"./project.forge.json\",\"./manifest.webmanifest\"]).then(()=>self.skipWaiting()))));self.addEventListener(\"activate\",e=>e.waitUntil(self.clients.claim()));self.addEventListener(\"fetch\",e=>e.respondWith(caches.match(e.request).then(x=>x||fetch(e.request))));"); +await fs.writeFile(path.join(out, "build-info.json"), JSON.stringify({product:"Forge Game Runtime",schema:1,generatedAt:new Date().toISOString(),project:path.basename(projectFile),runtimeFiles:runtimeFiles.length,assetRoot:"./"}, null, 2)); +console.log(JSON.stringify({ok:true,out,project:projectFile,runtimeFiles:runtimeFiles.length}, null, 2)); \ No newline at end of file diff --git a/scripts/validate-gltf-test-assets.mjs b/scripts/validate-gltf-test-assets.mjs index 95068ab..4aa8f02 100644 --- a/scripts/validate-gltf-test-assets.mjs +++ b/scripts/validate-gltf-test-assets.mjs @@ -3,7 +3,7 @@ import{join,relative,resolve}from"node:path"; import * as ValidatorNS from"gltf-validator"; const validator=ValidatorNS.default||ValidatorNS; -const root=resolve("test-results"); +const roots=[resolve("scripts/fixtures"),resolve("test-results")]; const reportPath="gltf-validation-report.json"; async function walk(dir){ @@ -18,7 +18,7 @@ async function walk(dir){ return out; } -const files=await walk(root); +const files=[...new Set((await Promise.all(roots.map(walk))).flat())]; const results=[]; for(const file of files){ try{ diff --git a/server.mjs b/server.mjs index 6eb9f07..370f9ac 100644 --- a/server.mjs +++ b/server.mjs @@ -5,45 +5,27 @@ import path from 'node:path'; import {fileURLToPath} from 'node:url'; import {WebSocketServer} from 'ws'; import {cookGLB} from './forge-pipeline.mjs'; -import {VisionController} from './vision-controller.mjs'; import {VisionController,installVisionShutdown} from './vision-control.mjs'; const root=path.dirname(fileURLToPath(import.meta.url)); const port=Number(process.env.PORT||4173); -const visionRemote=process.env.FORGE_VISION_REMOTE==='1'; const host=process.env.HOST||'127.0.0.1'; const vision=new VisionController({baseUrl:`http://127.0.0.1:${port}`}); const removeVisionShutdown=installVisionShutdown(vision); const mime={'.html':'text/html; charset=utf-8','.js':'text/javascript; charset=utf-8','.mjs':'text/javascript; charset=utf-8','.css':'text/css; charset=utf-8','.json':'application/json; charset=utf-8','.webmanifest':'application/manifest+json; charset=utf-8','.png':'image/png','.jpg':'image/jpeg','.jpeg':'image/jpeg','.webp':'image/webp','.svg':'image/svg+xml','.ico':'image/x-icon','.wasm':'application/wasm'}; -const clients=new Set(),visionClients=new Set(),rooms=new Map(); -const vision=new VisionController({ - port:Number(process.env.FORGE_CDP_PORT||9222), - host:process.env.FORGE_CDP_HOST||'127.0.0.1', - broadcast:message=>{ - if(message?.type==='vision-frame'&&message.data){ - const jpeg=Buffer.from(message.data,'base64'); - const header=Buffer.alloc(16); - header.writeUInt32BE(0x46563337,0); - header.writeUInt32BE(Number(message.seq)||0,4); - header.writeUInt16BE(Number(message.width)||0,8); - header.writeUInt16BE(Number(message.height)||0,10); - header.writeUInt32BE(Math.floor((Number(message.at)||Date.now())/1000),12); - const packet=Buffer.concat([header,jpeg]); - for(const peer of visionClients)if(peer.readyState===1)peer.send(packet); - return; - } - const raw=JSON.stringify(message); - for(const peer of visionClients)if(peer.readyState===1)peer.send(raw) - } -}); -function trustedVisionRequest(req){ - if(visionRemote)return true; - const ip=String(req.socket?.remoteAddress||'').replace(/^::ffff:/,''); - return ip==='127.0.0.1'||ip==='::1'||ip==='localhost'; -} +const clients=new Set(),rooms=new Map(); function json(res,o,status=200){res.writeHead(status,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','Access-Control-Allow-Origin':'*'});res.end(JSON.stringify(o))} function body(req,max=64*1024){return new Promise((resolve,reject)=>{let raw='';req.on('data',chunk=>{raw+=chunk;if(raw.length>max){reject(new Error('Request body too large'));req.destroy()}});req.on('end',()=>{try{resolve(raw?JSON.parse(raw):{})}catch(e){reject(new Error('Invalid JSON'))}});req.on('error',reject)})} -async function routeVisionAction(req,res){try{const q=await body(req);if(!q.action)return json(res,{ok:false,error:'Missing action'},400);return json(res,await vision.action(q.action,q.options||{}))}catch(e){return json(res,{ok:false,error:e.message||String(e)},500)}} +async function routeVisionAction(req,res){ + try{ + const q=await body(req); + if(!q.action)return json(res,{ok:false,error:'Missing action'},400); + const result=await vision.action(q.action,q.options||{}); + return json(res,{ok:true,result}); + }catch(e){ + return json(res,{ok:false,error:e.message||String(e),vision:vision.status()},500); + } +} function safeFile(urlPath){let p=decodeURIComponent(urlPath||'/');if(p==='/'||p==='')p='/index.html';if(p.includes('..'))return null;const full=path.resolve(root,'.'+p);return full.startsWith(root+path.sep)?full:null} function serve(req,res){const u=new URL(req.url,'http://localhost');const file=safeFile(u.pathname);fs.stat(file||'',(err,st)=>{if(!err&&st.isFile()){const ext=path.extname(file).toLowerCase();res.writeHead(200,{'Content-Type':mime[ext]||'application/octet-stream','Cache-Control':ext==='.html'||ext==='.js'||ext==='.css'?'no-cache':'public, max-age=31536000, immutable','X-Content-Type-Options':'nosniff'});return fs.createReadStream(file).pipe(res)}if(u.pathname!=='/'&&!path.extname(u.pathname)){const fallback=path.join(root,'index.html');res.writeHead(200,{'Content-Type':'text/html; charset=utf-8','Cache-Control':'no-cache'});return fs.createReadStream(fallback).on('error',()=>{res.writeHead(404);res.end('Not found')}).pipe(res)}res.writeHead(404);res.end('Not found')})} const srv=http.createServer(async (req,res)=>{const u=new URL(req.url,'http://localhost');if(req.method==='OPTIONS'){res.writeHead(204,{'Access-Control-Allow-Origin':'*','Access-Control-Allow-Methods':'GET,POST,OPTIONS','Access-Control-Allow-Headers':'Content-Type'});return res.end()} @@ -51,12 +33,22 @@ const srv=http.createServer(async (req,res)=>{const u=new URL(req.url,'http://lo if(u.pathname==='/api/pipeline'&&req.method==='POST'){let body='';req.on('data',c=>body+=c);req.on('end',async()=>{try{const q=JSON.parse(body);if(q.operation!=='cook-glb')return json(res,{ok:false,error:'Unsupported pipeline operation'},400);const bytes=Buffer.from(q.base64||'','base64');if(!bytes.length)return json(res,{ok:false,error:'Empty asset'},400);if(bytes.length>80*1024*1024)return json(res,{ok:false,error:'Asset exceeds 80MB pipeline limit'},413);const cooked=await cookGLB(bytes,q.options||{});return json(res,{ok:true,...cooked})}catch(e){return json(res,{ok:false,error:e.message},500)}});return} if(u.pathname==='/api/live/status')return json(res,{ok:true,agentConfigured:true,transport:'websocket',model:'forge-control-plane-v4',protocol:4,vision:vision.status()}); if(u.pathname==='/api/vision/status')return json(res,{ok:true,protocol:4,vision:vision.status(),capabilities:{liveScreencast:true,mouse:true,keyboard:true,touch:true,wheel:true,drag:true}}); + if(u.pathname==='/api/vision/frame'){ + if(!vision.latestFrame)return json(res,{ok:false,error:'Live Vision frame is not available'},503); + res.writeHead(200,{'Content-Type':'image/jpeg','Cache-Control':'no-store','X-Content-Type-Options':'nosniff'}); + return res.end(vision.latestFrame); + } if(u.pathname==='/api/vision/stream')return vision.stream(req,res); if(u.pathname==='/api/vision/elements')return vision.elements().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503)); if(u.pathname==='/api/vision/state')return vision.state().then(data=>json(res,{ok:true,...data})).catch(e=>json(res,{ok:false,error:e.message||String(e)},503)); if(u.pathname==='/api/vision/session/start'&&req.method==='POST'){ - try{const q=await body(req);const s=await vision.start({url:q.url||q.target||undefined,viewport:q.viewport,touch:q.touch,mobile:q.mobile});return json(res,{ok:true,session:s,stream:'/api/vision/stream',websocket:'/vision'})} - catch(e){return json(res,{ok:false,error:e.message||String(e)},500)} + try{ + const q=await body(req); + const s=await vision.start({url:q.url||q.target||undefined,viewport:q.viewport,touch:q.touch,mobile:q.mobile,streamFps:q.streamFps}); + return json(res,{ok:true,session:s,stream:'/api/vision/stream',websocket:'/vision'}); + }catch(e){ + return json(res,{ok:false,error:e.message||String(e),vision:vision.status()},500); + } } if(u.pathname==='/api/vision/session/stop'&&req.method==='POST'){ try{await vision.stop();return json(res,{ok:true,session:vision.status()})} @@ -100,12 +92,10 @@ visionWss.on('connection',ws=>{ } }); }); -const visionWss=new WebSocketServer({noServer:true}); -visionWss.on('connection',(ws,req)=>{if(!visionRemote&&!trustedVisionRequest(req)){ws.close(1008,'local-only');return}visionClients.add(ws);ws.send(JSON.stringify({type:'vision-state',state:vision.status()}));ws.on('close',()=>visionClients.delete(ws));}); const netWss=new WebSocketServer({noServer:true}); function joinRoom(room,ws,peerId){if(!rooms.has(room))rooms.set(room,new Set());const set=rooms.get(room);for(const p of set)if(p.readyState===1)p.send(JSON.stringify({type:'peer-join',peerId}));set.add(ws);ws.__room=room;ws.__peerId=peerId} function leaveRoom(ws){const room=ws.__room;if(!room||!rooms.has(room))return;const set=rooms.get(room);set.delete(ws);for(const p of set)if(p.readyState===1)p.send(JSON.stringify({type:'peer-leave',peerId:ws.__peerId}));if(!set.size)rooms.delete(room)} netWss.on('connection',ws=>{ws.on('close',()=>leaveRoom(ws));ws.on('message',message=>{let m;try{m=JSON.parse(message)}catch{return}if(m.type==='hello'){joinRoom(String(m.room||'default'),ws,String(m.peerId||crypto.randomUUID()));return}const room=rooms.get(ws.__room);if(!room)return;for(const p of room)if(p!==ws&&p.readyState===1)p.send(JSON.stringify(m))})}); -srv.on('upgrade',(req,socket,head)=>{const pathn=new URL(req.url,'http://localhost').pathname;if(pathn==='/live')return wss.handleUpgrade(req,socket,head,ws=>wss.emit('connection',ws,req));if(pathn==='/vision')return visionWss.handleUpgrade(req,socket,head,ws=>visionWss.emit('connection',ws,req));if(pathn==='/vision')return visionWss.handleUpgrade(req,socket,head,ws=>visionWss.emit('connection',ws,req));if(pathn==='/net')return netWss.handleUpgrade(req,socket,head,ws=>netWss.emit('connection',ws,req));socket.destroy()}); +srv.on('upgrade',(req,socket,head)=>{const pathn=new URL(req.url,'http://localhost').pathname;if(pathn==='/live')return wss.handleUpgrade(req,socket,head,ws=>wss.emit('connection',ws,req));if(pathn==='/vision')return visionWss.handleUpgrade(req,socket,head,ws=>visionWss.emit('connection',ws,req));if(pathn==='/net')return netWss.handleUpgrade(req,socket,head,ws=>netWss.emit('connection',ws,req));socket.destroy()}); srv.listen(port,host,()=>console.log('Forge Studio 3.7 listening on http://'+host+':'+port)); process.once('exit',removeVisionShutdown); \ No newline at end of file diff --git a/tests/packaging-runtime.spec.mjs b/tests/packaging-runtime.spec.mjs new file mode 100644 index 0000000..8cffd35 --- /dev/null +++ b/tests/packaging-runtime.spec.mjs @@ -0,0 +1,48 @@ +import{test,expect}from"@playwright/test"; +import{createServer}from"node:http"; +import{readFile,stat}from"node:fs/promises"; +import path from"path"; +import{fileURLToPath}from"url"; + +const root=path.resolve(path.dirname(fileURLToPath(import.meta.url)),"..","dist","forge-web"); +let server; +let base; + +test.beforeAll(async()=>{ + server=createServer(async(req,res)=>{ + try{ + const clean=(req.url||"/").split("?")[0]; + const rel=clean==="/"?"index.html":clean.replace(/^\/+/,""); + const file=path.resolve(root,rel); + if(!file.startsWith(root+path.sep)&&file!==root)throw new Error("path escape"); + const s=await stat(file); + if(!s.isFile())throw new Error("not file"); + const data=await readFile(file); + const types={".html":"text/html",".js":"text/javascript",".json":"application/json",".webmanifest":"application/manifest+json",".png":"image/png",".jpg":"image/jpeg",".glb":"model/gltf-binary"}; + res.writeHead(200,{"Content-Type":types[path.extname(file).toLowerCase()]||"application/octet-stream","Cache-Control":"no-store"}); + res.end(data); + }catch{res.writeHead(404);res.end("not found")} + }); + await new Promise(resolve=>server.listen(4174,"127.0.0.1",resolve)); + base="http://127.0.0.1:4174/"; +}); +test.afterAll(async()=>{if(server)await new Promise(r=>server.close(r))}); + +test("packaged Forge runtime boots and renders the packaged project",async({page})=>{ + test.setTimeout(60000); + const errors=[]; + page.on("pageerror",e=>errors.push("pageerror: "+String(e))); + page.on("console",m=>{if(m.type()==="error")errors.push("console: "+m.text())}); + await page.goto(base,{waitUntil:"domcontentloaded"}); + await page.waitForFunction(()=>!!window.Forge&&!!window.Forge.diagnostics,{timeout:30000}); + await page.waitForFunction(()=>{ + const d=window.Forge.diagnostics(); + return d?.entities===4&&d?.renderables>0&&!!document.querySelector("canvas"); + },{timeout:30000}); + const d=await page.evaluate(()=>window.Forge.diagnostics()); + expect(d.entities).toBe(4); + expect(d.renderables).toBeGreaterThan(0); + expect(errors).toEqual([]); + await expect(page.locator("canvas")).toHaveCount(1); + await page.screenshot({path:"test-results/packaged-runtime-proof.png",fullPage:true}); +}); \ No newline at end of file diff --git a/tests/professional.qa.spec.mjs b/tests/professional.qa.spec.mjs index 7602b29..938f853 100644 --- a/tests/professional.qa.spec.mjs +++ b/tests/professional.qa.spec.mjs @@ -1,5 +1,6 @@ import{test,expect}from"@playwright/test"; import{writeFile}from"node:fs/promises"; +import{PNG}from"pngjs"; const FOX="https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Assets/main/Models/Fox/glTF-Binary/Fox.glb"; const TOYCAR="https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Assets/main/Models/ToyCar/glTF-Binary/ToyCar.glb"; @@ -47,6 +48,11 @@ async function importViaInput(page,buffer,name,mimeType,{renderable=false}={}){ } } +function wavSilence(seconds=.25,sampleRate=8000){ + const frames=Math.floor(seconds*sampleRate),bytes=44+frames*2,b=Buffer.alloc(bytes); + b.write("RIFF",0);b.writeUInt32LE(bytes-8,4);b.write("WAVE",8);b.write("fmt ",12);b.writeUInt32LE(16,16);b.writeUInt16LE(1,20);b.writeUInt16LE(1,22);b.writeUInt32LE(sampleRate,24);b.writeUInt32LE(sampleRate*2,28);b.writeUInt16LE(2,32);b.writeUInt16LE(16,34);b.write("data",36);b.writeUInt32LE(frames*2,40); + return b; +} async function sha(blob){return pageHash(await blob.arrayBuffer())} async function pageHash(buf){ const d=await crypto.subtle.digest("SHA-256",buf); @@ -73,6 +79,20 @@ test.describe("Forge professional acceptance",()=>{ await page.screenshot({path:"test-results/pro-startup-desktop.png",fullPage:true}); }); + test("editor chrome stays docked and advanced tools do not float over the viewport",async({page})=>{ + await waitForForge(page); + await expect(page.locator(".forge-prod")).toHaveCount(0); + await expect(page.locator("#openEditors")).toBeVisible(); + await page.click("#openEditors"); + await expect(page.locator(".forge-modal")).toBeVisible(); + await expect(page.locator('[data-editor-tool="Asset Lab"]')).toBeVisible(); + await expect(page.locator('[data-editor-tool="Material Lab"]')).toBeVisible(); + await expect(page.locator('[data-editor-tool="Animation Graph + Rig"]')).toBeVisible(); + await page.locator(".forge-modal [data-x]").click(); + await expect(page.locator(".forge-modal")).toHaveCount(0); + await expect(page.locator("#forgeMobileControls")).toHaveCount(0); + }); + test("desktop editor layout follows professional viewport/hierarchy/inspector separation",async({page})=>{ await waitForForge(page); const layout=await page.evaluate(()=>({ @@ -172,6 +192,37 @@ test.describe("Forge professional acceptance",()=>{ expect(result.redone).toEqual(result.changed); }); + test("real audio WAV imports as an Engine Audio Asset with a Sound Slot and listener",async({page})=>{ + await waitForForge(page); + const wav=wavSilence(); + await importViaInput(page,wav,"forge-proof.wav","audio/wav"); + const info=await page.evaluate(()=>{ + const r=window.Forge.selected(); + const slot=r?.entity?.sound?.slot?.("main"); + const cam=window.Forge.camera(); + return{ + kind:r?.kind, + asset:r?.components?.asset?.type, + assetName:r?.components?.asset?.name, + sound:!!r?.entity?.sound, + slot:!!slot, + slotLoaded:!!slot?.isLoaded, + listener:!!cam?.audiolistener, + registry:window.Forge.assets.has("forge-proof.wav") + }; + }); + expect(info.kind).toBe("audio"); + expect(info.asset).toBe("audio"); + expect(info.assetName).toBe("forge-proof.wav"); + expect(info.sound).toBeTruthy(); + expect(info.slot).toBeTruthy(); + expect(info.listener).toBeTruthy(); + expect(info.registry).toBeTruthy(); + await page.click("#assetPlay"); + await page.waitForTimeout(120); + expect(await page.locator("#assetPlay").isEnabled()).toBeTruthy(); + }); + test("real 2D PNG survives source round-trip, validates dimensions, and generates a real viewport PNG",async({page})=>{ await waitForForge(page); const png=await download(page,EXAMPLE_PNG); @@ -183,7 +234,7 @@ test.describe("Forge professional acceptance",()=>{ const check=await window.ForgeExport.validateSelected(); const source=await window.ForgeExport.exportSource({download:false}); const preview=await window.ForgeExport.exportPreview({download:false,name:"developer-reference-preview.png"}); - const info=window.ForgeSpatial.inspect(imported.record); + const info=window.ForgeSpatial.inspect(imported); return{ name:window.Forge.selected()?.name, check, @@ -203,12 +254,142 @@ test.describe("Forge professional acceptance",()=>{ expect(result.sourceSha).toBe(await sha(new Blob([png]))); expect(result.exportedBytes).toBe(result.sourceBytes); expect(result.previewBytes).toBeGreaterThan(50); + expect(result.info.world.size.y).toBeGreaterThan(0.1); + expect(result.info.world.size.y).toBeLessThan(4); + const screen=await page.evaluate(()=>window.ForgeSpatial.screenRect(window.Forge.selected())); + expect(screen.height).toBeGreaterThan(220); expect(await page.locator(".asset-row .asset-thumb").evaluate(img=>img.complete&&img.naturalWidth>0)).toBeTruthy(); expect(await page.locator("#assetMeta").textContent()).toContain("developer-reference.png"); await writeFile("test-results/pro-2d-source.png",png); await page.screenshot({path:"test-results/pro-2d-import.png",fullPage:true}); }); + test("legacy OBJ imports through the real browser conversion adapter and produces a renderable 3D asset",async({page})=>{ + test.setTimeout(120000); + const obj=Buffer.from([ + "o ForgeProof","v -0.7 0 -0.5","v 0.7 0 -0.5","v 0 1.4 -0.5","v -0.7 0 0.5","v 0.7 0 0.5","v 0 1.4 0.5", + "f 1 2 3","f 4 6 5","f 1 4 5 2","f 2 5 6 3","f 3 6 4 1" + ].join("\n"),"utf8"); + await waitForForge(page); + await importViaInput(page,obj,"forge-proof.obj","model/obj",{renderable:true}); + const result=await page.evaluate(()=>{const r=window.Forge.selected(),spatial=window.ForgeSpatial.inspect(r);const stack=[r?.entity],meshes=[];while(stack.length){const n=stack.pop();for(const mi of n?.render?.meshInstances||[])meshes.push(mi);for(const c of n?.children||[])stack.push(c)}const m=meshes[0]?.material;return{name:r?.components?.asset?.name,converted:r?.components?.asset?.converted===true,converter:r?.components?.asset?.converter,sourcePreserved:r?.components?.asset?.sourcePreserved===true,vertices:spatial?.geometry?.vertices||0,triangles:spatial?.geometry?.triangles||0,renderables:meshes.length,diffuse:m?.diffuse?[m.diffuse.r,m.diffuse.g,m.diffuse.b]:[0,0,0]}}); + expect(result.name).toBe("forge-proof.obj");expect(result.converted).toBeTruthy();expect(result.converter).toBe("AssimpJS");expect(result.sourcePreserved).toBeTruthy();expect(result.vertices).toBeGreaterThan(0);expect(result.triangles).toBeGreaterThan(0);expect(result.renderables).toBeGreaterThan(0); + expect(result.diffuse.reduce((a,b)=>a+b,0)).toBeGreaterThan(.3); + await page.screenshot({path:"test-results/pro-legacy-obj-editor.png",fullPage:true}); + }); + test("universal STL source converts through the same production 3D pipeline",async({page})=>{ + test.setTimeout(120000); + const stl=Buffer.from([ + "solid ForgeSTL", + "facet normal 0 0 1", + " outer loop", + " vertex 0 0 0", + " vertex 1 0 0", + " vertex 0 1 0", + " endloop", + "endfacet", + "facet normal 0 0 -1", + " outer loop", + " vertex 0 0 0", + " vertex 0 1 0", + " vertex 1 0 0", + " endloop", + "endfacet", + "endsolid ForgeSTL" + ].join("\n"),"utf8"); + await waitForForge(page); + await importViaInput(page,stl,"forge-proof.stl","model/stl",{renderable:true}); + const result=await page.evaluate(()=>{ + const r=window.Forge.selected(),s=window.ForgeSpatial.inspect(r); + return{name:r?.components?.asset?.name,converted:r?.components?.asset?.converted===true,converter:r?.components?.asset?.converter,sourcePreserved:r?.components?.asset?.sourcePreserved===true,hash:r?.components?.asset?.sourceSha256||"",vertices:s?.geometry?.vertices||0,triangles:s?.geometry?.triangles||0}; + }); + expect(result.name).toBe("forge-proof.stl"); + expect(result.converted).toBeTruthy(); + expect(result.converter).toBe("AssimpJS"); + expect(result.sourcePreserved).toBeTruthy(); + expect(result.hash).toMatch(/^[a-f0-9]{64}$/); + expect(result.vertices).toBeGreaterThan(0); + expect(result.triangles).toBeGreaterThan(0); + await page.screenshot({path:"test-results/pro-universal-stl-editor.png",fullPage:true}); + }); + + test("universal source fallback preserves arbitrary non-renderable files in the asset vault",async({page})=>{ + await waitForForge(page); + const source=Buffer.from("Forge arbitrary source asset\nversion=1\n","utf8"); + await page.locator("#assetInput").setInputFiles({name:"forge-proof.custom",mimeType:"application/octet-stream",buffer:source}); + await page.waitForFunction(()=>window.Forge?.assets?.get?.("forge-proof.custom")?.sourceOnly===true,{timeout:30000}); + const result=await page.evaluate(()=>{const e=window.Forge.assets.get("forge-proof.custom");return{entry:!!e,sourceOnly:e?.sourceOnly===true,bytes:e?.file?.size||0,extension:e?.extension||"",hash:e?.sourceSha256||""}}); + expect(result.entry).toBeTruthy(); + expect(result.sourceOnly).toBeTruthy(); + expect(result.bytes).toBe(source.length); + expect(result.extension).toBe("custom"); + expect(result.hash).toMatch(/^[a-f0-9]{64}$/); + await expect(page.locator("#assetList .asset-row")).toHaveCount(1); + }); + + test("built-in high-fidelity Car Concept imports as a native 3D asset in the Forge viewport",async({page})=>{ + test.setTimeout(150000); + await waitForForge(page); + await expect(page.locator("#importRealCar")).toBeVisible(); + await page.click("#importRealCar"); + await page.waitForFunction(()=>window.Forge?.selected?.()?.components?.asset?.remoteSource?.includes("sceneview.github.io/models/platforms/CarConcept.glb"),{timeout:90000}); + const result=await page.evaluate(()=>{ + const r=window.Forge.selected(); + const stack=[r?.entity],meshes=[]; + while(stack.length){ + const n=stack.pop(); + for(const mi of n?.render?.meshInstances||[])meshes.push(mi); + for(const c of n?.children||[])stack.push(c); + } + const spatial=window.ForgeSpatial.inspect(r); + const materials=new Set(meshes.map(mi=>mi.material?.name||mi.material).filter(Boolean)); + return{ + name:r?.components?.asset?.name, + remoteSource:r?.components?.asset?.remoteSource, + sourceSha256:r?.components?.asset?.sourceSha256, + license:r?.components?.asset?.license, + attribution:r?.components?.asset?.attribution, + catalogSource:r?.components?.asset?.catalogSource, + viewportSanitized:r?.components?.asset?.viewportSanitized===true, + sanitizedExtensions:r?.components?.asset?.sanitizedExtensions||[], + vertices:spatial?.geometry?.vertices||0, + triangles:spatial?.geometry?.triangles||0, + spatialMaterials:spatial?.geometry?.materials||0, + materials:Math.max(spatial?.geometry?.materials||0,materials.size), + renderables:meshes.length, + textured:meshes.filter(mi=>!!mi.material?.diffuseMap).length, + world:spatial?.world?.size||null, + screen:window.ForgeSpatial.screenRect(r) + }; + }); + expect(result.name).toBe("Forge-CarConcept-Khronos.glb"); + expect(result.remoteSource).toContain("sceneview.github.io/models/platforms/CarConcept.glb"); + expect(result.sourceSha256).toMatch(/^[a-f0-9]{64}$/); + expect(result.license).toBe("CC BY 4.0"); + expect(result.attribution).toContain("Darmstadt Graphics Group GmbH"); + expect(result.catalogSource).toContain("KhronosGroup/glTF-Sample-Assets"); + expect(result.vertices).toBeGreaterThan(5000); + expect(result.triangles).toBeGreaterThan(5000); + expect(result.materials).toBeGreaterThanOrEqual(3); + expect(result.renderables).toBeGreaterThan(0); + expect(result.textured).toBeGreaterThan(0); + expect(result.world?.y).toBeGreaterThan(0.5); + expect(result.screen?.width).toBeGreaterThan(250); + expect(result.screen?.height).toBeGreaterThan(140); + const shot=await page.locator("#viewport").screenshot(); + expect(shot.length).toBeGreaterThan(12000); + const png=PNG.sync.read(shot); + let active=0,total=0; + for(let y=0;y28&&spread>10)active++; + total++; + } + expect(active/Math.max(1,total)).toBeGreaterThan(0.01); + await page.screenshot({path:"test-results/pro-high-fidelity-car-editor.png",fullPage:true}); + }); + test("real animated Fox GLB imports with animation clips and exact source export",async({page})=>{ test.setTimeout(120000); const glb=await download(page,FOX); diff --git a/tests/vision-live.spec.mjs b/tests/vision-live.spec.mjs deleted file mode 100644 index 7a3c777..0000000 --- a/tests/vision-live.spec.mjs +++ /dev/null @@ -1,115 +0,0 @@ -import{test,expect,chromium}from"@playwright/test"; -import{writeFile}from"node:fs/promises"; -import{WebSocket as NodeWebSocket}from"ws"; - -test.describe("Forge live computer vision bridge",()=>{ - test("continuous CDP vision stream and real input controls change Forge",async({request})=>{ - test.setTimeout(120000); - const browser=await chromium.launch({ - headless:true, - args:["--remote-debugging-port=9222","--user-data-dir=/tmp/forge-vision-cdp"] - }); - const context=await browser.newContext({viewport:{width:1440,height:900}}); - const page=await context.newPage(); - await page.goto("http://127.0.0.1:4173/"); - await page.waitForFunction(()=>window.ForgeReady===true,{timeout:30000}); - await expect(page.locator("#sceneCount")).toContainText("3 entities"); - - const attach=await request.get("/api/vision/targets"); - expect(attach.ok()).toBeTruthy(); - const targets=(await attach.json()).targets||[]; - const target=targets.find(t=>t.type==="page"&&t.url.includes("127.0.0.1:4173")); - expect(target?.id).toBeTruthy(); - - const streamWs=new NodeWebSocket("ws://127.0.0.1:4173/vision"); - const frames=[]; - const states=[]; - streamWs.on("message",data=>{ - if(Buffer.isBuffer(data)){ - if(data.length>16&&data.readUInt32BE(0)===0x46563337)frames.push({ - seq:data.readUInt32BE(4),width:data.readUInt16BE(8),height:data.readUInt16BE(10),data:data.subarray(16) - }); - return; - } - try{const m=JSON.parse(data.toString());if(m.type==="vision-state")states.push(m.state)}catch{} - }); - const connect=await request.post("/api/vision/connect",{data:{ - targetId:target.id,targetUrl:target.url,quality:55,maxWidth:960,maxHeight:540,everyNthFrame:1 - }}); - expect(connect.ok()).toBeTruthy(); - const connection=await connect.json(); - expect(connection.ok).toBeTruthy(); - expect(connection.state.connected).toBeTruthy(); - expect(connection.state.streaming).toBeTruthy(); - - await expect.poll(()=>frames.length,{timeout:15000}).toBeGreaterThanOrEqual(3); - expect(frames[0].data.length).toBeGreaterThan(1000); - await writeFile("test-results/pro-vision-live-frame.jpg",Buffer.from(frames.at(-1).data)); - - const box=await page.locator('[data-add="box"]').boundingBox(); - expect(box).toBeTruthy(); - const click=await request.post("/api/vision/control",{data:{action:{ - type:"click",x:box.x+box.width/2,y:box.y+box.height/2,screenWidth:1440,screenHeight:900 - }}}); - expect(click.ok()).toBeTruthy(); - await expect(page.locator("#sceneCount")).toContainText("4 entities"); - - const visionMode=await request.post("/api/vision/control",{data:{action:{type:"press",key:"2"}}}); - expect(visionMode.ok()).toBeTruthy(); - await expect.poll(()=>page.evaluate(()=>window.ForgeGizmo?.state?.mode),{timeout:5000}).toBe("rotate"); - - const nameBox=await page.locator("#name").boundingBox(); - expect(nameBox).toBeTruthy(); - await request.post("/api/vision/control",{data:{action:{ - type:"click",x:nameBox.x+nameBox.width/2,y:nameBox.y+nameBox.height/2,screenWidth:1440,screenHeight:900 - }}}); - const typed=await request.post("/api/vision/control",{data:{action:{type:"typeText",text:"_VISION"}}}); - expect(typed.ok()).toBeTruthy(); - expect(await page.locator("#name").inputValue()).toContain("_VISION"); - - const slider=await page.locator("#time").boundingBox(); - expect(slider).toBeTruthy(); - const startTime=await page.locator("#time").inputValue(); - const dragged=await request.post("/api/vision/control",{data:{action:{ - type:"drag", - x:slider.x+6,y:slider.y+slider.height/2, - toX:slider.x+Math.max(50,slider.width*.65),toY:slider.y+slider.height/2, - screenWidth:1440,screenHeight:900,steps:16,stepMs:6 - }}}); - expect(dragged.ok()).toBeTruthy(); - await page.waitForTimeout(80); - const endTime=await page.locator("#time").inputValue(); - expect(endTime).not.toBe(startTime); - - const touchProbe=await page.evaluate(()=>{ - window.__visionTouchCount=0; - const b=document.querySelector('[data-add="sphere"]'); - b?.addEventListener("pointerdown",e=>{if(e.pointerType==="touch")window.__visionTouchCount++},{once:false}); - return b?.getBoundingClientRect().toJSON(); - }); - expect(touchProbe).toBeTruthy(); - const touchStart=await request.post("/api/vision/control",{data:{action:{ - type:"touchStart",x:touchProbe.x+touchProbe.width/2,y:touchProbe.y+touchProbe.height/2,screenWidth:1440,screenHeight:900,id:41 - }}}); - const touchEnd=await request.post("/api/vision/control",{data:{action:{ - type:"touchEnd",x:touchProbe.x+touchProbe.width/2,y:touchProbe.y+touchProbe.height/2,screenWidth:1440,screenHeight:900,id:41 - }}}); - expect(touchStart.ok()).toBeTruthy(); - expect(touchEnd.ok()).toBeTruthy(); - await page.waitForTimeout(50); - expect(await page.evaluate(()=>window.__visionTouchCount)).toBeGreaterThan(0); - - const status=await request.get("/api/vision/status"); - const st=await status.json(); - expect(st.connected).toBeTruthy(); - expect(st.streaming).toBeTruthy(); - expect(st.transport).toBe("chromium-cdp-screencast"); - expect(st.control).toBe("chromium-cdp-input"); - expect(frames.length).toBeGreaterThanOrEqual(3); - - await request.post("/api/vision/disconnect",{data:{}}); - streamWs.close(); - await context.close(); - await browser.close(); - }); -}); diff --git a/tests/vision.live.spec.mjs b/tests/vision.live.spec.mjs index be67fb3..1a0e986 100644 --- a/tests/vision.live.spec.mjs +++ b/tests/vision.live.spec.mjs @@ -1,149 +1,255 @@ import{test,expect}from"@playwright/test"; +import{WebSocket}from"ws"; +import{mkdir,writeFile}from"node:fs/promises"; const BASE="http://127.0.0.1:4173"; +const FOX="https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Assets/main/Models/Fox/glTF-Binary/Fox.glb"; const sleep=ms=>new Promise(r=>setTimeout(r,ms)); -async function stopVision(request){ - try{await request.post("/api/vision/session/stop",{timeout:5000})}catch{} -} - -async function startVision(request,options={}){ - await stopVision(request); - const r=await request.post("/api/vision/session/start",{ - data:{target:"/",viewport:{width:1440,height:900},touch:false,mobile:false,...options}, - timeout:45000 +test.describe.serial("Forge Live Vision — continuous stream + real browser input",()=>{ + test.afterEach(async({request})=>{ + try{await request.post("/api/vision/session/stop",{timeout:5000})}catch{} }); - expect(r.ok()).toBeTruthy(); - const body=await r.json(); - expect(body.ok).toBeTruthy(); - expect(body.session.running).toBeTruthy(); - await expect.poll(async()=>((await request.get("/api/vision/status")).json()).then(j=>j.vision.frameSeq),{timeout:20000}).toBeGreaterThan(3); - return body.session; -} - -async function getJson(request,path){ - const r=await request.get(path,{cache:"no-store"}); - expect(r.ok()).toBeTruthy(); - return r.json(); -} - -test.describe.serial("Forge Live Vision — real continuous vision + input",()=>{ - test.afterEach(async({request})=>{await stopVision(request)}); - - test("publishes a continuous screencast stream rather than one-shot screenshots",async({page,request})=>{ - test.setTimeout(60000); - const initial=await startVision(request); - await sleep(1200); - const later=await getJson(request,"/api/vision/status"); - expect(later.vision.frameSeq).toBeGreaterThan(initial.frameSeq); - expect(later.vision.frame?.bytes).toBeGreaterThan(1000); - expect(later.vision.lastFrameAt).toBeGreaterThan(0); - expect(later.vision.transport).toContain("Playwright Screencast"); - - const controller=new AbortController(); - const res=await fetch(BASE+"/api/vision/stream?proof="+Date.now(),{signal:controller.signal}); - expect(res.ok).toBeTruthy(); - expect(res.headers.get("content-type")).toContain("multipart/x-mixed-replace"); - const reader=res.body.getReader(); - let payload=""; - for(let i=0;i<4&&payload.length<4000;i++){ - const part=await reader.read(); - if(part.done)break; - payload+=new TextDecoder().decode(part.value); - } - controller.abort(); - expect(payload).toContain("--forge-frame"); - expect(payload).toContain("Content-Type: image/jpeg"); + test("continuous frames are pushed over one live WebSocket channel",async({request})=>{ + test.setTimeout(90000); + await mkdir("vision-proof",{recursive:true}); + const socket=new WebSocket(BASE.replace("http","ws")+"/vision"); + const frames=[]; + const open=new Promise((resolve,reject)=>{ + socket.once("open",resolve);socket.once("error",reject); + }); + await open; + socket.on("message",data=>{ + if(!Buffer.isBuffer(data)||data.length<16)return; + if(data.readUInt32BE(0)!==0x46563337)return; + frames.push({seq:data.readUInt32BE(4),width:data.readUInt16BE(8),height:data.readUInt16BE(10),bytes:data.length-16}); + }); + + const start=await request.post("/api/vision/session/start",{data:{ + target:"/",viewport:{width:1280,height:720},touch:false,mobile:false,streamFps:30 + },timeout:45000}); + expect(start.ok()).toBeTruthy(); + const body=await start.json(); + expect(body.session.running).toBeTruthy(); + + await expect.poll(()=>frames.length,{timeout:20000}).toBeGreaterThanOrEqual(5); + const proof=await request.get("/api/vision/frame"); + expect(proof.ok()).toBeTruthy(); + const proofBytes=await proof.body(); + expect(proofBytes.slice(0,2).toString("hex")).toBe("ffd8"); + await writeFile("vision-proof/vision-live-proof.jpg",proofBytes); + expect(frames.at(-1).seq).toBeGreaterThan(frames[0].seq); + expect(frames[0].width).toBeGreaterThan(300); + expect(frames[0].height).toBeGreaterThan(200); + expect(frames[0].bytes).toBeGreaterThan(1000); + + const status=await request.get("/api/vision/status"); + const st=await status.json(); + expect(st.vision.transport).toContain("WebSocket control"); + expect(st.vision.streamFps).toBe(30); + expect(st.vision.streamSubscribers).toBeGreaterThanOrEqual(1); + + socket.close(); }); - test("drives Forge through real pointer coordinates and verifies the resulting engine state",async({request})=>{ - test.setTimeout(90000); - await startVision(request); - const elements=await getJson(request,"/api/vision/elements"); - const box=elements.elements.find(e=>e.selector==='[data-add="box"]'&&e.visible); + test("mouse, keyboard, drag and touch modify real Forge state",async({request})=>{ + test.setTimeout(120000); + const start=await request.post("/api/vision/session/start",{data:{ + target:"/",viewport:{width:1440,height:900},touch:true,mobile:true,streamFps:24 + },timeout:45000}); + expect(start.ok()).toBeTruthy(); + + const elements=(await (await request.get("/api/vision/elements")).json()).elements; + const box=elements.find(e=>e.selector==='[data-add="box"]'&&e.visible); expect(box).toBeTruthy(); - const x=box.x+box.width/2,y=box.y+box.height/2; - const down=await request.post("/api/vision/action",{data:{action:{type:"mouse.move",x,y,steps:4}}}); - expect((await down.json()).ok).toBeTruthy(); - const press=await request.post("/api/vision/action",{data:{action:{type:"mouse.down",button:"left"}}}); - expect((await press.json()).ok).toBeTruthy(); - const release=await request.post("/api/vision/action",{data:{action:{type:"mouse.up",button:"left"}}}); - const released=await release.json(); - expect(released.ok).toBeTruthy(); - expect(released.action.dispatched).toBeTruthy(); - - const state=await getJson(request,"/api/vision/state"); - expect(state.forgeReady).toBeTruthy(); - expect(state.diagnostics.entities).toBeGreaterThanOrEqual(4); - expect(state.selected?.name||"").toMatch(/box/i); - expect(state.selectedScreenRect).toBeTruthy(); - expect(state.canvasRect).toBeTruthy(); - - const moved=await request.post("/api/vision/action",{data:{action:{ - type:"mouse.drag", - from:{x:box.x+box.width/2,y:box.y+box.height/2}, - to:{x:box.x+box.width/2+30,y:box.y+box.height/2+10}, - steps:8 + const click=await request.post("/api/vision/action",{data:{action:{ + type:"mouse.click",x:box.x+box.width/2,y:box.y+box.height/2 + },options:{includeState:true}}}); + const clickBody=await click.json(); + expect(clickBody.ok).toBeTruthy(); + expect(clickBody.result?.action?.dispatched).toBeTruthy(); + expect(clickBody.result?.state?.diagnostics?.entities||0).toBeGreaterThanOrEqual(4); + + const rotate=await request.post("/api/vision/action",{data:{action:{type:"keyboard.press",key:"2"},options:{includeState:true}}}); + expect((await rotate.json()).result.state.gizmoMode).toBe("rotate"); + + const slider=(await (await request.get("/api/vision/elements")).json()).elements.find(e=>e.selector==='#time'&&e.visible); + expect(slider).toBeTruthy(); + const drag=await request.post("/api/vision/action",{data:{action:{ + type:"mouse.drag",from:{x:slider.x+5,y:slider.y+slider.height/2},to:{x:slider.x+Math.max(55,slider.width*0.7),y:slider.y+slider.height/2},steps:16 + }}}); + expect((await drag.json()).result.action.dispatched).toBeTruthy(); + + const touchTarget=(await (await request.get("/api/vision/elements")).json()).elements.find(e=>e.selector==='[data-add="sphere"]'&&e.visible); + expect(touchTarget).toBeTruthy(); + const touched=await request.post("/api/vision/action",{data:{action:{ + type:"touch.swipe", + from:{x:touchTarget.x+touchTarget.width/2,y:touchTarget.y+touchTarget.height/2}, + to:{x:touchTarget.x+touchTarget.width/2+8,y:touchTarget.y+touchTarget.height/2+8},id:7,steps:6 }}}); - expect((await moved.json()).action.dispatched).toBeTruthy(); - const wheel=await request.post("/api/vision/action",{data:{action:{type:"mouse.wheel",deltaX:0,deltaY:180}}}); - expect((await wheel.json()).action.dispatched).toBeTruthy(); + expect((await touched.json()).result.action.dispatched).toBeTruthy(); + + const release=await request.post("/api/vision/action",{data:{action:{type:"releaseAll"}}}); + expect((await release.json()).result.action.released).toBeTruthy(); + + const finalState=await (await request.get("/api/vision/state")).json(); + expect(finalState.forgeReady).toBeTruthy(); + expect(finalState.canvasRect).toBeTruthy(); + + const open=await request.get("/api/vision/elements"); + const openElements=(await open.json()).elements; + let drawOpen=openElements.find(e=>e.selector==="#draw2dOpen"&&e.visible); + expect(drawOpen).toBeTruthy(); + for(let i=0;i<3&&drawOpen.y>850;i++){ + const scroll=await request.post("/api/vision/action",{data:{action:{type:"mouse.wheel",x:110,y:450,deltaY:700}}}); + expect((await scroll.json()).result.action.dispatched).toBeTruthy(); + const refreshed=(await (await request.get("/api/vision/elements")).json()).elements; + drawOpen=refreshed.find(e=>e.selector==="#draw2dOpen"&&e.visible); + expect(drawOpen).toBeTruthy(); + } + expect(drawOpen.y).toBeLessThan(900); + const clickDraw=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:drawOpen.x+drawOpen.width/2,y:drawOpen.y+drawOpen.height/2}}}); + const clickDrawBody=await clickDraw.json(); + expect(clickDrawBody.result.action.target).toMatchObject({id:"draw2dOpen"}); + expect(clickDrawBody.result.action.target.width).toBeGreaterThan(0); + const drawStatus=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"JSON.stringify({api:!!window.ForgeDraw2D,ready:window.ForgeDraw2D?.status?.().ready||false,open:document.querySelector('#draw2dDialog')?.open||false})"}}}); + const drawStatusBody=await drawStatus.json(); + expect(JSON.parse(drawStatusBody.result.action.value).api).toBeTruthy(); + expect(JSON.parse(drawStatusBody.result.action.value).ready).toBeTruthy(); + expect(JSON.parse(drawStatusBody.result.action.value).open).toBeTruthy(); + expect(clickDrawBody.result.action.dispatched).toBeTruthy(); + + await expect.poll(async()=>((await (await request.get("/api/vision/elements")).json()).elements.some(e=>e.selector==="#drawCanvas"&&e.visible)),{timeout:5000}).toBeTruthy(); + const drawingEls=(await (await request.get("/api/vision/elements")).json()).elements; + const canvas=drawingEls.find(e=>e.selector==="#drawCanvas"&&e.visible); + const add=drawingEls.find(e=>e.selector==="#drawAddToScene"&&e.visible); + expect(canvas).toBeTruthy(); expect(add).toBeTruthy(); + + const lines=[ + {from:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.25},to:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.25}}, + {from:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.75},to:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.75}}, + {from:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.25},to:{x:canvas.x+canvas.width*.20,y:canvas.y+canvas.height*.75}}, + {from:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.25},to:{x:canvas.x+canvas.width*.80,y:canvas.y+canvas.height*.75}} + ]; + for(const line of lines){ + const rr=await request.post("/api/vision/action",{data:{action:{type:"mouse.drag",from:line.from,to:line.to,steps:18}}}); + expect((await rr.json()).result.action.dispatched).toBeTruthy(); + } + const canvasProof=await request.get("/api/vision/frame"); + expect(canvasProof.ok()).toBeTruthy(); + const canvasProofBytes=await canvasProof.body(); + expect(canvasProofBytes.slice(0,2).toString("hex")).toBe("ffd8"); + await writeFile("vision-proof/vision-drawing-canvas-proof.jpg",canvasProofBytes); + const addResult=await request.post("/api/vision/action",{data:{action:{type:"mouse.click",x:add.x+add.width/2,y:add.y+add.height/2},options:{includeState:true}}}); + const addBody=await addResult.json(); + expect(addBody.ok).toBeTruthy(); + expect(addBody.result.action.target).toMatchObject({id:"drawAddToScene"}); + await expect.poll(async()=>{ + const st=await (await request.get("/api/vision/state")).json(); + return st.selected?.name||""; + },{timeout:12000}).toBe("forge-drawing.png"); + const importedState=await (await request.get("/api/vision/state")).json(); + expect(importedState.selected?.name).toBe("forge-drawing.png"); + expect(importedState.selected?.kind).toBeTruthy(); + const importedProof=await request.get("/api/vision/frame"); + expect(importedProof.ok()).toBeTruthy(); + await writeFile("vision-proof/vision-imported-asset-proof.jpg",await importedProof.body()); + + const data=await request.post("/api/vision/action",{data:{action:{type:"evaluate",expression:"window.ForgeDraw2D?.lastPngDataUrl || null"}}}); + const dataBody=await data.json(); + expect(typeof dataBody.result.action.value).toBe("string"); + const m=/^data:image\/png;base64,(.+)$/.exec(dataBody.result.action.value); + expect(m).toBeTruthy(); + const pngBytes=Buffer.from(m[1],"base64"); + expect(pngBytes.slice(0,8).toString("hex")).toBe("89504e470d0a1a0a"); + expect(pngBytes.length).toBeGreaterThan(1000); + await writeFile("vision-proof/vision-drawn-asset.png",pngBytes); }); - test("drives the real browser keyboard and proves a visible editor mode change",async({request})=>{ - test.setTimeout(90000); - await startVision(request); - const before=await getJson(request,"/api/vision/state"); - expect(before.forgeReady).toBeTruthy(); - expect(before.diagnostics.entities).toBeGreaterThanOrEqual(3); - - const keyRotate=await request.post("/api/vision/action",{data:{action:{type:"keyboard.press",key:"2"}}}); - const kr=await keyRotate.json(); - expect(kr.ok).toBeTruthy(); - const rotate=await getJson(request,"/api/vision/state"); - expect(rotate.gizmoMode).toBe("rotate"); - - const keyTranslate=await request.post("/api/vision/action",{data:{action:{type:"keyboard.press",key:"1"}}}); - const kt=await keyTranslate.json(); - expect(kt.ok).toBeTruthy(); - const translate=await getJson(request,"/api/vision/state"); - expect(translate.gizmoMode).toBe("translate"); + test("real 3D model is imported and visibly rendered under live Vision control",async({request})=>{ + test.setTimeout(120000); + await mkdir("vision-proof",{recursive:true}); + const fox=await request.get(FOX,{timeout:30000}); + expect(fox.ok()).toBeTruthy(); + const foxBytes=await fox.body(); + expect(foxBytes.slice(0,4).toString()).toBe("glTF"); + await writeFile("vision-proof/Fox.glb",foxBytes); + + const start=await request.post("/api/vision/session/start",{data:{ + target:"/",viewport:{width:1440,height:900},touch:true,mobile:true,streamFps:30 + },timeout:45000}); + expect(start.ok()).toBeTruthy(); + + const controls=await request.post("/api/vision/action",{data:{action:{ + type:"evaluate", + expression:"JSON.stringify({visible:(()=>{const e=document.querySelector('#forgeMobileControls');return !!e&&getComputedStyle(e).display!=='none'&&getComputedStyle(e).visibility!=='hidden'})(),buttons:[...document.querySelectorAll('#forgeMobileControls [data-mobile]')].map(x=>x.textContent)})" + }}}); + const controlsState=JSON.parse((await controls.json()).result.action.value); + expect(controlsState.visible).toBeFalsy(); + const editorChrome=await request.post("/api/vision/action",{data:{action:{ + type:"evaluate", + expression:"JSON.stringify({floatingTools:!!document.querySelector('.forge-prod'),editorButton:!!document.querySelector('#openEditors')})" + }}}); + const editorChromeState=JSON.parse((await editorChrome.json()).result.action.value); + expect(editorChromeState.floatingTools).toBeFalsy(); + expect(editorChromeState.editorButton).toBeTruthy(); + + const input=await request.post("/api/vision/action",{data:{action:{ + type:"file.setInputFiles",selector:"#assetInput",paths:["vision-proof/Fox.glb"] + },options:{includeState:true}}}); + const inputBody=await input.json(); + expect(inputBody.ok).toBeTruthy(); + + await expect.poll(async()=>{ + const st=await (await request.get("/api/vision/state")).json(); + return st.selected?.kind||""; + },{timeout:30000}).toBe("model"); + + const imported=await request.post("/api/vision/action",{data:{action:{ + type:"evaluate", + expression:"JSON.stringify((()=>{const r=window.Forge?.selected?.();const stack=r?.entity?[r.entity]:[];let meshes=0,textured=0;while(stack.length){const n=stack.pop();for(const mi of n?.render?.meshInstances||[]){meshes++;if(mi?.material?.diffuseMap)textured++;}for(const c of n?.children||[])stack.push(c)}return{name:r?.components?.asset?.name,kind:r?.kind,assetType:r?.components?.asset?.type,meshInstances:meshes,texturedMaterials:textured,preview:r?.entity?.__forgeViewportPreview||null,entities:window.Forge?.diagnostics?.()?.entities||0}})())" + },options:{includeState:true}}}); + const importedBody=await imported.json(); + const importedState=JSON.parse(importedBody.result.action.value); + expect(importedState.name).toBe("Fox.glb"); + expect(importedState.kind).toBe("model"); + expect(importedState.assetType).toBe("model"); + expect(importedState.meshInstances).toBeGreaterThan(0); + expect(importedState.texturedMaterials).toBeGreaterThan(0); + expect(importedState.preview?.textured).toBeGreaterThan(0); + expect(importedState.preview?.normalized?.ok).toBeTruthy(); + expect(importedState.preview?.normalized?.scale).toBeGreaterThan(0); + expect(importedState.entities).toBeGreaterThanOrEqual(4); + + const materialDiag=await request.post("/api/vision/action",{data:{action:{ + type:"evaluate", + expression:"JSON.stringify((()=>{const r=window.Forge?.selected?.();const root=r?.entity;let out=[];if(root)root.forEach?.(n=>{for(const mi of n?.render?.meshInstances||[]){const m=mi.material,map=m?.diffuseMap,src=map?.getSource?.();const els=mi?.mesh?.vertexBuffer?.getFormat?.()?.elements||[];out.push({diffuse:m?.diffuse?[m.diffuse.r,m.diffuse.g,m.diffuse.b]:null,useLighting:m?.useLighting,diffuseMap:!!map,textureSize:map?[map.width,map.height]:null,textureSource:!!src,meshSemantics:els.map(e=>e.semantic)})}});return out})())" + }}}); + const materialDiagBody=await materialDiag.json(); + const materialDiagValue=JSON.parse(materialDiagBody.result.action.value); + expect(materialDiagValue.length).toBeGreaterThan(0); + await writeFile("vision-proof/vision-model-material.json",JSON.stringify(materialDiagValue,null,2)); const frame=(await (await request.get("/api/vision/elements")).json()).elements.find(e=>e.selector==="#frame"&&e.visible); + expect(frame).toBeTruthy(); + const frameClick=await request.post("/api/vision/action",{data:{action:{ + type:"mouse.click",x:frame.x+frame.width/2,y:frame.y+frame.height/2 + }}}); + expect((await frameClick.json()).result.action.target.id).toBe("frame"); + + await sleep(700); + const proof=await request.get("/api/vision/frame"); + expect(proof.ok()).toBeTruthy(); + const bytes=await proof.body(); + expect(bytes.slice(0,2).toString("hex")).toBe("ffd8"); + await writeFile("vision-proof/vision-real-3d-asset-proof.jpg",bytes); }); - test("executes real touch start/move/end through CDP on a touch-capable Forge session",async({request})=>{ + test("file-input control is available for real asset import workflows",async({request})=>{ test.setTimeout(90000); - const session=await startVision(request,{viewport:{width:390,height:844},touch:true,mobile:true}); - expect(session.viewport.width).toBe(390); - const elements=await getJson(request,"/api/vision/elements"); - const box=elements.elements.find(e=>e.selector==='[data-add="box"]'&&e.visible); - expect(box).toBeTruthy(); - const bx=box.x+box.width/2,by=box.y+box.height/2; - const start=await request.post("/api/vision/action",{data:{action:{type:"touch.start",points:[{id:1,x:bx,y:by,force:1}]}}}); - const sr=await start.json(); - expect(sr.ok).toBeTruthy(); - expect(sr.action.dispatched).toBeTruthy(); - - const move=await request.post("/api/vision/action",{data:{action:{type:"touch.move",points:[{id:1,x:bx+2,y:by+2,force:1}]}}}); - const mr=await move.json(); - expect(mr.ok).toBeTruthy(); - - const end=await request.post("/api/vision/action",{data:{action:{type:"touch.end"}}}); - const er=await end.json(); - expect(er.ok).toBeTruthy(); - expect(er.action.dispatched).toBeTruthy(); - - const tap=await request.post("/api/vision/action",{data:{action:{type:"touch.tapSelector",selector:'[data-gizmo="rotate"]'}}}); - const tr=await tap.json(); - expect(tr.ok).toBeTruthy(); - expect(tr.action.dispatched).toBeTruthy(); - - const afterTouch=await getJson(request,"/api/vision/state"); - expect(afterTouch.gizmoMode).toBe("rotate"); - - const status=await getJson(request,"/api/vision/status"); - expect(status.vision.frameSeq).toBeGreaterThan(3); - expect(status.capabilities.touch).toBeTruthy(); + const start=await request.post("/api/vision/session/start",{data:{target:"/",viewport:{width:1280,height:720},touch:false,mobile:false},timeout:45000}); + expect(start.ok()).toBeTruthy(); + const input=await (await request.get("/api/vision/elements")).json(); + expect(input.elements.some(e=>e.selector==="#assetInput"&&e.visible)).toBeTruthy(); }); -}); +}); \ No newline at end of file diff --git a/vision-control.mjs b/vision-control.mjs index 2eb6453..c29e33c 100644 --- a/vision-control.mjs +++ b/vision-control.mjs @@ -1,6 +1,10 @@ import { EventEmitter } from 'node:events'; +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const FRAME_MAGIC=0x46563337; +const assetRoot=()=>path.resolve(process.env.VISION_ASSET_ROOT||process.cwd()); function clamp(n,min,max){ const v=Number(n); @@ -52,21 +56,29 @@ export class VisionController extends EventEmitter{ this.lastActionAt=0; this.errors=[]; this.frameTimer=null; + this.streamFps=30; + this.lastStreamFrameAt=0; + this.frameDrops=0; + this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; + this.starting=null; } async start(options={}){ - const viewport={ - width:clamp(options.viewport?.width??1440,320,2560), - height:clamp(options.viewport?.height??900,240,1800) - }; - const touch=!!options.touch; - const mobile=!!options.mobile; - const target=normalizeUrl(options.url||options.target||this.baseUrl+'/',this.baseUrl); + if(this.starting)return this.starting; + const run=async()=>{ + const viewport={ + width:clamp(options.viewport?.width??1440,320,2560), + height:clamp(options.viewport?.height??900,240,1800) + }; + const touch=!!options.touch; + const mobile=!!options.mobile; + const target=normalizeUrl(options.url||options.target||this.baseUrl+'/',this.baseUrl); + this.streamFps=clamp(options.streamFps??30,5,60); - if(this.page && this.targetUrl===target && - this.page.isClosed?.()===false) return this.status(); + if(this.page && this.targetUrl===target && + this.page.isClosed?.()===false) return this.status(); - await this.stop(); + await this.stop(); let chromium; try{ @@ -75,14 +87,17 @@ export class VisionController extends EventEmitter{ throw new Error('Playwright runtime is unavailable; run npm install before starting Live Vision: '+error.message); } - this.sessionId=crypto.randomUUID(); + this.sessionId=randomUUID(); this.targetUrl=target; this.startedAt=Date.now(); this.errors=[]; this.browser=await chromium.launch({ headless:process.env.VISION_HEADLESS!=='false', - args: process.env.VISION_CHROMIUM_ARGS ? String(process.env.VISION_CHROMIUM_ARGS).split(/\s+/).filter(Boolean) : [] + args:[ + '--disable-dev-shm-usage', + ...(process.env.VISION_CHROMIUM_ARGS ? String(process.env.VISION_CHROMIUM_ARGS).split(/\s+/).filter(Boolean) : []) + ] }); this.context=await this.browser.newContext({ @@ -102,22 +117,26 @@ export class VisionController extends EventEmitter{ if(message.type()==='error') this.recordError('console: '+message.text()); }); + await this.page.goto(target,{waitUntil:'domcontentloaded',timeout:30000}); + await this.page.waitForTimeout(50); await this.page.screencast.start({ quality:72, size:viewport, onFrame:frame=>this.acceptFrame(frame) }); - - await this.page.goto(target,{waitUntil:'domcontentloaded',timeout:30000}); try{ await this.page.waitForFunction(()=>window.ForgeReady===true,{timeout:15000}); }catch{ // A live visual session is useful even while an application is still booting. } await this.page.waitForTimeout(250); - this.emit('started',this.status()); - this.emitTelemetry(); - return this.status(); + this.emit('started',this.status()); + this.emitTelemetry(); + return this.status(); + }; + this.starting=run(); + try{return await this.starting} + finally{this.starting=null} } recordError(message){ @@ -128,6 +147,7 @@ export class VisionController extends EventEmitter{ acceptFrame(frame){ const data=Buffer.from(frame.data); const now=Date.now(); + if(!this.page || this.page.isClosed?.()) return; this.frameSeq+=1; this.latestFrame=data; this.latestMeta={ @@ -157,10 +177,32 @@ export class VisionController extends EventEmitter{ this.writeFrame(res,data,this.frameSeq,this.latestMeta); }catch{this.streams.delete(res)} } + const interval=1000/this.streamFps; + if(now-this.lastStreamFrameAt>=interval){ + this.lastStreamFrameAt=now; + this.broadcastFrame(data,this.frameSeq,this.latestMeta); + } + } + + broadcastFrame(data,seq,meta){ + const header=Buffer.alloc(16); + header.writeUInt32BE(FRAME_MAGIC,0); + header.writeUInt32BE(Number(seq)||0,4); + header.writeUInt16BE(Number(meta.width)||0,8); + header.writeUInt16BE(Number(meta.height)||0,10); + header.writeUInt32BE(Math.floor((Number(meta.serverTime)||Date.now())/1000),12); + const packet=Buffer.concat([header,data]); + for(const ws of [...this.telemetry]){ + try{ + if(ws.readyState!==1){this.telemetry.delete(ws);continue;} + if(Number(ws.bufferedAmount||0)>2*1024*1024){this.frameDrops+=1;continue;} + ws.send(packet); + }catch{this.telemetry.delete(ws)} + } } emitTelemetry(){ - const payload=JSON.stringify({type:'vision-frame',...this.status()}); + const payload=JSON.stringify({type:'vision-status',...this.status()}); for(const ws of [...this.telemetry]){ try{ if(ws.readyState===1) ws.send(payload); @@ -212,9 +254,9 @@ export class VisionController extends EventEmitter{ } async stop(){ + try{await this.releaseAll()}catch{} for(const res of this.streams){try{res.end()}catch{}} this.streams.clear(); - this.telemetry.clear(); if(this.frameTimer){clearTimeout(this.frameTimer);this.frameTimer=null} if(this.page){ try{await this.page.screencast.stop()}catch{} @@ -232,6 +274,8 @@ export class VisionController extends EventEmitter{ this.frameSeq=0; this.fps=0; this.framesInWindow=0; + this.lastStreamFrameAt=0; + this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; } status(){ @@ -250,8 +294,17 @@ export class VisionController extends EventEmitter{ viewport, frame:this.latestMeta?{...this.latestMeta}:null, errors:[...this.errors], - controls:['mouse','keyboard','touch','wheel','drag','focus'], - transport:'Playwright Screencast + WebSocket control' + controls:['mouse','keyboard','touch','wheel','drag','focus','file.setInputFiles','releaseAll'], + transport:'Playwright Screencast + WebSocket control', + control:'real-browser-input', + streamFps:this.streamFps, + streamSubscribers:this.telemetry.size, + frameDrops:this.frameDrops, + input:{ + mouseButtons:[...this.inputState.mouseButtons], + keys:[...this.inputState.keys], + touchIds:[...this.inputState.touchIds] + } }; } @@ -294,14 +347,18 @@ export class VisionController extends EventEmitter{ }); } - async elements(){ + async elements(requestedSelectors=[]){ if(!this.page||this.page.isClosed())throw new Error('Vision session is not running'); - const selectors=[ + const defaults=[ '[data-add="box"]','[data-add="sphere"]','[data-add="cylinder"]', '#assetInput','#importAssets','#play','#build','#openProject','#saveProject', '#focus','#frame','#duplicate','#delete','#viewport','.viewport', - '#visionOpen','#selected','#sceneCount','.topbar','.leftpanel','.rightpanel' + '#visionOpen','#selected','#sceneCount','#time','#draw2dOpen','#drawCanvas','#drawAddToScene','#drawClear','#drawUndo','#drawRedo','.topbar','.leftpanel','.rightpanel' ]; + const selectors=[...new Set( + [...(Array.isArray(requestedSelectors)?requestedSelectors:[]),...defaults] + .map(s=>String(s||'').trim()).filter(Boolean).slice(0,64) + )]; return this.page.evaluate(selectors=>({ viewport:{width:innerWidth,height:innerHeight,dpr:devicePixelRatio||1}, elements:selectors.map(selector=>{ @@ -322,24 +379,39 @@ export class VisionController extends EventEmitter{ }),selectors); } - async touchEvent(type,points=[]){ + async touchEvent(type,points=[],modifiers=0){ if(!this.cdp)throw new Error('Touch CDP session unavailable'); const touchPoints=points.map(p=>({ x:Number(p.x)||0, y:Number(p.y)||0, radiusX:Number(p.radiusX)||1, radiusY:Number(p.radiusY)||1, + rotationAngle:Number(p.rotationAngle)||0, force:Number(p.force??1), id:Number(p.id??0) })); await this.cdp.send('Input.dispatchTouchEvent',{ type, touchPoints:type==='touchEnd'||type==='touchCancel'?[]:touchPoints, - modifiers:Number(arguments[1]?.modifiers||0) + modifiers:Number(modifiers)||0 }); + if(type==='touchStart'||type==='touchMove')for(const p of touchPoints)this.inputState.touchIds.add(p.id); + if(type==='touchEnd'||type==='touchCancel')this.inputState.touchIds.clear(); return{dispatched:true,type,touchPoints}; } + async releaseAll(){ + if(!this.page||this.page.isClosed?.())return{released:true,mouseButtons:0,keys:0,touches:0}; + const mouseButtons=[...this.inputState.mouseButtons]; + const keys=[...this.inputState.keys]; + const touchCount=this.inputState.touchIds.size; + for(const button of mouseButtons){try{await this.page.mouse.up({button})}catch{}} + for(const key of keys){try{await this.page.keyboard.up(key)}catch{}} + if(this.cdp&&touchCount){try{await this.touchEvent('touchCancel',[],0)}catch{}} + this.inputState={mouseButtons:new Set(),keys:new Set(),touchIds:new Set()}; + return{released:true,mouseButtons:mouseButtons.length,keys:keys.length,touches:touchCount}; + } + async perform(action){ if(!this.page||this.page.isClosed())throw new Error('Vision session is not running'); const type=String(action?.type||''); @@ -347,43 +419,82 @@ export class VisionController extends EventEmitter{ switch(type){ case 'mouse.move': - await this.page.mouse.move(Number(action.x)||0,Number(action.y)||0,{steps:Math.max(1,Number(action.steps)||1)}); - return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; - case 'mouse.down': - await this.page.mouse.down({button:buttonValue(action.button)}); - return{dispatched:true,type,button:buttonValue(action.button)}; - case 'mouse.up': - await this.page.mouse.up({button:buttonValue(action.button)}); - return{dispatched:true,type,button:buttonValue(action.button)}; - case 'mouse.click': - await this.page.mouse.click(Number(action.x)||0,Number(action.y)||0,{button:buttonValue(action.button),clickCount:Math.max(1,Math.min(3,Number(action.clickCount)||1)),delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; - case 'mouse.dblclick': - await this.page.mouse.dblclick(Number(action.x)||0,Number(action.y)||0,{button:buttonValue(action.button),delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; - case 'mouse.wheel': + case 'mouse.hover':{ + const x=Number(action.x)||0,y=Number(action.y)||0; + await this.page.mouse.move(x,y,{steps:Math.max(1,Number(action.steps)||1)}); + return{dispatched:true,type,x,y}; + } + case 'mouse.down':{ + const button=buttonValue(action.button); + await this.page.mouse.down({button}); + this.inputState.mouseButtons.add(button); + return{dispatched:true,type,button}; + } + case 'mouse.up':{ + const button=buttonValue(action.button); + await this.page.mouse.up({button}); + this.inputState.mouseButtons.delete(button); + return{dispatched:true,type,button}; + } + case 'mouse.click':{ + const x=Number(action.x)||0,y=Number(action.y)||0,button=buttonValue(action.button); + const target=await this.page.evaluate(({x,y})=>{ + const el=document.elementFromPoint(x,y); + if(!el)return null; + const r=el.getBoundingClientRect(); + return{tag:el.tagName||'',id:el.id||'',className:typeof el.className==='string'?el.className:'',text:(el.textContent||'').trim().slice(0,120),x:r.x,y:r.y,width:r.width,height:r.height}; + },{x,y}); + await this.page.mouse.click(x,y,{button,clickCount:Math.max(1,Math.min(3,Number(action.clickCount)||1)),delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,x,y,button,target}; + } + case 'mouse.dblclick':{ + const x=Number(action.x)||0,y=Number(action.y)||0,button=buttonValue(action.button); + await this.page.mouse.dblclick(x,y,{button,delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,x,y,button}; + } + case 'mouse.wheel':{ + const x=Number(action.x),y=Number(action.y); + if(Number.isFinite(x)&&Number.isFinite(y))await this.page.mouse.move(x,y,{steps:Math.max(1,Number(action.steps)||1)}); await this.page.mouse.wheel(Number(action.deltaX)||0,Number(action.deltaY)||0); - return{dispatched:true,type,deltaX:Number(action.deltaX)||0,deltaY:Number(action.deltaY)||0}; + return{dispatched:true,type,x:Number.isFinite(x)?x:null,y:Number.isFinite(y)?y:null,deltaX:Number(action.deltaX)||0,deltaY:Number(action.deltaY)||0}; + } case 'mouse.drag':{ - const from=action.from||{},to=action.to||{}; + const from=action.from||{},to=action.to||{},button=buttonValue(action.button); await this.page.mouse.move(Number(from.x)||0,Number(from.y)||0); - await this.page.mouse.down({button:buttonValue(action.button)}); - await this.page.mouse.move(Number(to.x)||0,Number(to.y)||0,{steps:Math.max(1,Number(action.steps)||12)}); - await this.page.mouse.up({button:buttonValue(action.button)}); - return{dispatched:true,type,from,to}; + await this.page.mouse.down({button}); + this.inputState.mouseButtons.add(button); + await this.page.mouse.move(Number(to.x)||0,Number(to.y)||0,{steps:Math.max(2,Math.min(120,Number(action.steps)||12))}); + await this.page.mouse.up({button}); + this.inputState.mouseButtons.delete(button); + return{dispatched:true,type,from,to,steps:Math.max(2,Math.min(120,Number(action.steps)||12))}; + } + case 'keyboard.down':{ + const key=keyValue(action.key); + await this.page.keyboard.down(key); + this.inputState.keys.add(key); + return{dispatched:true,type,key}; + } + case 'keyboard.up':{ + const key=keyValue(action.key); + await this.page.keyboard.up(key); + this.inputState.keys.delete(key); + return{dispatched:true,type,key}; + } + case 'keyboard.press':{ + const key=keyValue(action.key); + await this.page.keyboard.press(key,{delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,key}; + } + case 'keyboard.type':{ + const text=String(action.text??''); + await this.page.keyboard.type(text,{delay:Math.max(0,Number(action.delay)||0)}); + return{dispatched:true,type,length:text.length}; + } + case 'keyboard.insertText':{ + const text=String(action.text??''); + await this.page.keyboard.insertText(text); + return{dispatched:true,type,length:text.length}; } - case 'keyboard.down': - await this.page.keyboard.down(keyValue(action.key)); - return{dispatched:true,type,key:String(action.key)}; - case 'keyboard.up': - await this.page.keyboard.up(keyValue(action.key)); - return{dispatched:true,type,key:String(action.key)}; - case 'keyboard.press': - await this.page.keyboard.press(keyValue(action.key),{delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,key:String(action.key)}; - case 'keyboard.type': - await this.page.keyboard.type(String(action.text??''),{delay:Math.max(0,Number(action.delay)||0)}); - return{dispatched:true,type,length:String(action.text??'').length}; case 'touch.tap': await this.page.touchscreen.tap(Number(action.x)||0,Number(action.y)||0); return{dispatched:true,type,x:Number(action.x)||0,y:Number(action.y)||0}; @@ -392,35 +503,56 @@ export class VisionController extends EventEmitter{ await this.page.locator(String(action.selector)).tap({timeout:10000}); return{dispatched:true,type,selector:String(action.selector)}; case 'touch.start': - return this.touchEvent('touchStart',action.points||[]); + return this.touchEvent('touchStart',action.points||[],action.modifiers||0); case 'touch.move': - return this.touchEvent('touchMove',action.points||[]); + return this.touchEvent('touchMove',action.points||[],action.modifiers||0); case 'touch.end': - return this.touchEvent('touchEnd',[]); + return this.touchEvent('touchEnd',[],action.modifiers||0); case 'touch.cancel': - return this.touchEvent('touchCancel',[]); + return this.touchEvent('touchCancel',[],action.modifiers||0); case 'touch.swipe':{ const from=action.from||{},to=action.to||{}; const id=Number(action.id??0); - const steps=Math.max(2,Math.min(60,Number(action.steps)||12)); - await this.touchEvent('touchStart',[{id,x:Number(from.x)||0,y:Number(from.y)||0,force:1}]); + const steps=Math.max(2,Math.min(120,Number(action.steps)||12)); + await this.touchEvent('touchStart',[{id,x:Number(from.x)||0,y:Number(from.y)||0,force:1}],action.modifiers||0); for(let i=1;i<=steps;i++){ const t=i/steps; - await this.touchEvent('touchMove',[{id,x:(Number(from.x)||0)+(Number(to.x||0)-Number(from.x||0))*t,y:(Number(from.y)||0)+(Number(to.y||0)-Number(from.y||0))*t,force:1}]); + await this.touchEvent('touchMove',[{id,x:(Number(from.x)||0)+(Number(to.x||0)-Number(from.x||0))*t,y:(Number(from.y)||0)+(Number(to.y||0)-Number(from.y||0))*t,force:1}],action.modifiers||0); await sleep(Math.max(0,Number(action.stepDelay)||8)); } - await this.touchEvent('touchEnd',[]); + await this.touchEvent('touchEnd',[],action.modifiers||0); return{dispatched:true,type,from,to,steps}; } + case 'file.setInputFiles':{ + const selector=String(action.selector||''); + if(!selector)throw new Error('file.setInputFiles requires selector'); + const raw=action.clear?[]:(Array.isArray(action.paths)?action.paths:[action.path]).filter(Boolean); + const root=assetRoot(); + const resolved=raw.map(item=>{ + const full=path.resolve(root,String(item)); + if(full!==root&&!full.startsWith(root+path.sep))throw new Error('Asset path escapes VISION_ASSET_ROOT'); + return full; + }); + await this.page.locator(selector).setInputFiles(resolved); + return{dispatched:true,type,selector,files:resolved.length}; + } + case 'releaseAll': + return this.releaseAll(); case 'focus': await this.page.bringToFront(); return{dispatched:true,type}; + case 'evaluate':{ + const expression=String(action.expression||''); + if(!expression)throw new Error('evaluate requires expression'); + const value=await this.page.evaluate(expression); + return{dispatched:true,type,value}; + } default: throw new Error('Unsupported vision action type: '+type); } } - async action(action,{waitForFrameMs=1500}={}){ + async action(action,{waitForFrameMs=1500,includeState=false}={}){ const before=this.frameSeq; const result=await this.perform(action); const after=await this.waitForFrame(before,waitForFrameMs); @@ -432,7 +564,8 @@ export class VisionController extends EventEmitter{ frameAdvanced:this.frameSeq>before, frame:after?{...after}:null, ageMs:this.lastFrameAt?Date.now()-this.lastFrameAt:null, - inputAcceptedAt:Date.now() + inputAcceptedAt:Date.now(), + state:includeState?await this.state():undefined }; } }