Add image light visual tests

Author: davepagurek

src/core/p5.Renderer3D.js

@@ -1644,6 +1644,9 @@ export class Renderer3D extends Renderer {
       let specularLight = this.getSpecularTexture(this.states.activeImageLight);
 
       shader.setUniform("environmentMapSpecular", specularLight);
+    } else {
+      shader.setUniform("environmentMapDiffused", this._getEmptyTexture());
+      shader.setUniform("environmentMapSpecular", this._getEmptyTexture());
     }
   }
 
@@ -1830,7 +1833,7 @@ export class Renderer3D extends Renderer {
    * an angle to each pixel. This creates and caches textures for reuse, since
    * creating this texture is somewhat expensive.
    */
-  getDiffusedTexture(input) {
+  makeDiffusedTexture(input) {
     // if one already exists for a given input image
     if (this.diffusedTextures.get(input) != null) {
       return this.diffusedTextures.get(input);
@@ -1861,6 +1864,9 @@ export class Renderer3D extends Renderer {
     this.diffusedTextures.set(input, newFramebuffer);
     return newFramebuffer;
   }
+  getDiffusedTexture(input) {
+    return this.diffusedTextures.get(input);
+  }
 
   /*
    * used in imageLight,
@@ -1871,7 +1877,7 @@ export class Renderer3D extends Renderer {
    * Storing the texture for input image in map called `specularTextures`
    * maps the input Image to a p5.MipmapTexture
    */
-  getSpecularTexture(input) {
+  makeSpecularTexture(input) {
     // check if already exits (there are tex of diff resolution so which one to check)
     // currently doing the whole array
     if (this.specularTextures.get(input) != null) {
@@ -1931,6 +1937,9 @@ export class Renderer3D extends Renderer {
     this.specularTextures.set(input, tex);
     return tex;
   }
+  getSpecularTexture(input) {
+    return this.specularTextures.get(input);
+  }
 
   _getSphereMapping(img) {
     if (!this.sphereMapping) {
@@ -1949,6 +1958,7 @@ export class Renderer3D extends Renderer {
           const angleX = p5.mix(uFovX/2.0, -uFovX/2.0, inputs.texCoord.x);
           let rotatedNormal = p5.normalize([angleX, angleY, 1]);
           rotatedNormal = [
+            // Don't mind me, just doing matrix vector multiplication...
             p5.dot(rotatedNormal, uN1),
             p5.dot(rotatedNormal, uN2),
             p5.dot(rotatedNormal, uN3),
@@ -1957,8 +1967,8 @@ export class Renderer3D extends Renderer {
           rotatedNormal.z = rotatedNormal.x;
           rotatedNormal.x = -temp;
           const suv = [
-            0.5 + 0.5 * (-rotatedNormal.y),
-            p5.atan(rotatedNormal.z, rotatedNormal.x) / (2.0 * p5.PI) + 0.5
+            p5.atan(rotatedNormal.z, rotatedNormal.x) / (2.0 * p5.PI) + 0.5,
+            0.5 + 0.5 * (-rotatedNormal.y)
           ];
           return p5.getTexture(uEnvMap, suv);
         })
@@ -1968,10 +1978,12 @@ export class Renderer3D extends Renderer {
     this.scratchMat3.invert(this.scratchMat3); // uNMMatrix is 3x3
     this.sphereMapping.setUniform("uFovY", this.states.curCamera.cameraFOV);
     this.sphereMapping.setUniform("uAspect", this.states.curCamera.aspectRatio);
-    // this.sphereMapping.setUniform("uNewNormalMatrix", this.scratchMat3.mat3);
-    this.sphereMapping.setUniform("uN1", this.scratchMat3.mat3.slice(0, 3));
-    this.sphereMapping.setUniform("uN2", this.scratchMat3.mat3.slice(3, 6));
-    this.sphereMapping.setUniform("uN3", this.scratchMat3.mat3.slice(6));
+    // Pass in the normal matrix as three vectors. TODO replace this with
+    // an actual matrix uniform once we have those again.
+    const m = this.scratchMat3.mat3;
+    this.sphereMapping.setUniform("uN1", [m[0], m[3], m[6]]);
+    this.sphereMapping.setUniform("uN2", [m[1], m[4], m[7]]);
+    this.sphereMapping.setUniform("uN3", [m[2], m[5], m[8]]);
     this.sphereMapping.setUniform("uEnvMap", img);
     return this.sphereMapping;
   }

src/strands/p5.strands.js

@@ -25,7 +25,7 @@ function strands(p5, fn) {
   //////////////////////////////////////////////
   // Global Runtime
   //////////////////////////////////////////////
-  function initStrandsContext(ctx, backend, { active = false } = {}) {
+  function initStrandsContext(ctx, backend, { active = false, renderer = null, baseShader = null } = {}) {
     ctx.dag = createDirectedAcyclicGraph();
     ctx.cfg = createControlFlowGraph();
     ctx.uniforms = [];
@@ -35,6 +35,8 @@ function strands(p5, fn) {
     ctx.globalAssignments = [];
     ctx.backend = backend;
     ctx.active = active;
+    ctx.renderer = renderer;
+    ctx.baseShader = baseShader;
     ctx.previousFES = p5.disableFriendlyErrors;
     ctx.windowOverrides = {};
     ctx.fnOverrides = {};
@@ -77,6 +79,8 @@ function strands(p5, fn) {
       // const backend = glslBackend;
       initStrandsContext(strandsContext, this._renderer.strandsBackend, {
         active: true,
+        renderer: this._renderer,
+        baseShader: this,
       });
       createShaderHooksFunctions(strandsContext, fn, this);
       // TODO: expose this, is internal for debugging for now.

src/strands/strands_codegen.js

@@ -16,7 +16,14 @@ export function generateShaderCode(strandsContext) {
 
   for (const {name, typeInfo, defaultValue} of strandsContext.uniforms) {
     const key = backend.generateHookUniformKey(name, typeInfo);
-    hooksObj.uniforms[key] = defaultValue;
+    if (key !== null) {
+      hooksObj.uniforms[key] = defaultValue;
+    }
+  }
+
+  // Add texture bindings to declarations for WebGPU backend
+  if (backend.addTextureBindingsToDeclarations) {
+    backend.addTextureBindingsToDeclarations(strandsContext);
   }
 
   for (const { hookType, rootNodeID, entryBlockID, shaderContext } of strandsContext.hooks) {

src/webgl/light.js

@@ -1558,6 +1558,9 @@ function light(p5, fn){
     // for sending uniforms to the fillshader
     this.states.setValue('activeImageLight', img);
     this.states.setValue('enableLighting', true);
+    // Make sure textures are cached
+    this.makeDiffusedTexture(img);
+    this.makeSpecularTexture(img);
   };
 
   Renderer3D.prototype.lights = function() {

src/webgl/p5.RendererGL.js

@@ -26,7 +26,6 @@ import webgl2CompatibilityShader from "./shaders/webgl2Compatibility.glsl";
 import normalVert from "./shaders/normal.vert";
 import normalFrag from "./shaders/normal.frag";
 import basicFrag from "./shaders/basic.frag";
-import sphereMappingFrag from "./shaders/sphereMapping.frag";
 import lightVert from "./shaders/light.vert";
 import lightTextureFrag from "./shaders/light_texture.frag";
 import phongVert from "./shaders/phong.vert";
@@ -46,7 +45,6 @@ const defaultShaders = {
   normalVert,
   normalFrag,
   basicFrag,
-  sphereMappingFrag,
   lightVert: lightingShader + lightVert,
   lightTextureFrag,
   phongVert,
@@ -61,7 +59,6 @@ const defaultShaders = {
   filterBaseVert,
   filterBaseFrag,
 };
-let sphereMapping = defaultShaders.sphereMappingFrag;
 for (const key in defaultShaders) {
   defaultShaders[key] = webgl2CompatibilityShader + defaultShaders[key];
 }

src/webgl/shaders/sphereMapping.frag

@@ -997,6 +997,8 @@ class RendererWebGPU extends Renderer3D {
 
     // this._pInst.background('red');
     this._pInst.push();
+    this.states.setValue('enableLighting', false);
+    this.states.setValue('activeImageLight', null);
     this._pInst.setCamera(this.finalCamera);
     this._pInst.resetShader();
     this._pInst.imageMode(this._pInst.CENTER);
@@ -1344,7 +1346,7 @@ class RendererWebGPU extends Renderer3D {
 
     // Extract the struct name from the uniform variable declaration
     const uniformVarRegex = /@group\(0\)\s+@binding\(0\)\s+var<uniform>\s+(\w+)\s*:\s*(\w+);/;
-    const uniformVarMatch = uniformVarRegex.exec(shader._vertSrc);
+    const uniformVarMatch = uniformVarRegex.exec(shader.vertSrc());
     if (!uniformVarMatch) {
       throw new Error('Expected a uniform struct bound to @group(0) @binding(0)');
     }
@@ -1355,8 +1357,8 @@ class RendererWebGPU extends Renderer3D {
     // TODO: support other texture types
     const samplerRegex = /@group\((\d+)\)\s*@binding\((\d+)\)\s*var\s+(\w+)\s*:\s*(texture_2d<f32>|sampler);/g;
     for (const [src, visibility] of [
-      [shader._vertSrc, GPUShaderStage.VERTEX],
-      [shader._fragSrc, GPUShaderStage.FRAGMENT]
+      [shader.vertSrc(), GPUShaderStage.VERTEX],
+      [shader.fragSrc(), GPUShaderStage.FRAGMENT]
     ]) {
       let match;
       while ((match = samplerRegex.exec(src)) !== null) {
@@ -1398,6 +1400,27 @@ class RendererWebGPU extends Renderer3D {
     return [...Object.values(uniforms).sort((a, b) => a.index - b.index), ...Object.values(samplers)];
   }
 
+  getNextBindingIndex(shader, group = 0) {
+    // Get the highest binding index in the specified group and return the next available
+    const samplerRegex = /@group\((\d+)\)\s*@binding\((\d+)\)\s*var\s+(\w+)\s*:\s*(texture_2d<f32>|sampler|uniform)/g;
+    let maxBindingIndex = -1;
+
+    for (const [src, visibility] of [
+      [shader.vertSrc(), GPUShaderStage.VERTEX],
+      [shader.fragSrc(), GPUShaderStage.FRAGMENT]
+    ]) {
+      let match;
+      while ((match = samplerRegex.exec(src)) !== null) {
+        const [_, groupIndex, bindingIndex] = match;
+        if (parseInt(groupIndex) === group) {
+          maxBindingIndex = Math.max(maxBindingIndex, parseInt(bindingIndex));
+        }
+      }
+    }
+
+    return maxBindingIndex + 1;
+  }
+
   updateUniformValue(_shader, uniform, data) {
     if (uniform.isSampler) {
       uniform.texture =

src/webgpu/p5.RendererWebGPU.js

@@ -191,6 +191,27 @@ export const wgslBackend = {
 
     return mutableCopies ? firstLine + '\n' + mutableCopies : firstLine;
   },
+  addTextureBindingsToDeclarations(strandsContext) {
+    // Add texture and sampler bindings for sampler2D uniforms to both vertex and fragment declarations
+    if (!strandsContext.renderer || !strandsContext.baseShader) return;
+
+    // Get the next available binding index from the renderer
+    let bindingIndex = strandsContext.renderer.getNextBindingIndex(strandsContext.baseShader);
+
+    for (const {name, typeInfo} of strandsContext.uniforms) {
+      if (typeInfo.baseType === 'sampler2D') {
+        const textureBinding = `@group(0) @binding(${bindingIndex}) var ${name}: texture_2d<f32>;`;
+        const samplerBinding = `@group(0) @binding(${bindingIndex + 1}) var ${name}_sampler: sampler;`;
+
+        strandsContext.vertexDeclarations.add(textureBinding);
+        strandsContext.vertexDeclarations.add(samplerBinding);
+        strandsContext.fragmentDeclarations.add(textureBinding);
+        strandsContext.fragmentDeclarations.add(samplerBinding);
+
+        bindingIndex += 2;
+      }
+    }
+  },
   getTypeName(baseType, dimension) {
     const primitiveTypeName = TypeNames[baseType + dimension]
     if (!primitiveTypeName) {
@@ -199,6 +220,11 @@ export const wgslBackend = {
     return primitiveTypeName;
   },
   generateHookUniformKey(name, typeInfo) {
+    // For sampler2D types, we don't add them to the uniform struct
+    // Instead, they become separate texture and sampler bindings
+    if (typeInfo.baseType === 'sampler2D') {
+      return null; // Signal that this should not be added to uniform struct
+    }
     return `${name}: ${this.getTypeName(typeInfo.baseType, typeInfo.dimension)}`;
   },
   generateVaryingVariable(varName, typeInfo) {
@@ -332,9 +358,9 @@ export const wgslBackend = {
         }
       }
 
-      // Check if this is a uniform variable
-      const isUniform = generationContext.strandsContext?.uniforms?.some(uniform => uniform.name === node.identifier);
-      if (isUniform) {
+      // Check if this is a uniform variable (but not a texture)
+      const uniform = generationContext.strandsContext?.uniforms?.find(uniform => uniform.name === node.identifier);
+      if (uniform && uniform.typeInfo.baseType !== 'sampler2D') {
         return `uniforms.${node.identifier}`;
       }
 
@@ -366,7 +392,13 @@ export const wgslBackend = {
             return `${left} % ${right}`;
           }
         }
-        
+
+        // Convert atan(y, x) to atan2(y, x) in WGSL
+        if (node.identifier === 'atan' && node.dependsOn.length === 2) {
+          const functionArgs = node.dependsOn.map(arg => this.generateExpression(generationContext, dag, arg));
+          return `atan2(${functionArgs.join(', ')})`;
+        }
+
         const functionArgs = node.dependsOn.map(arg =>this.generateExpression(generationContext, dag, arg));
         return `${node.identifier}(${functionArgs.join(', ')})`;
       }

src/webgpu/strands_wgslBackend.js

@@ -790,4 +790,42 @@ visualSuite('WebGL', function() {
       screenshot();
     });
   });
+
+  visualSuite("Image Based Lighting", function () {
+    const shinesses = [50, 150];
+    for (const shininess of shinesses) {
+      visualTest(
+        `imageLight with panorama and shininess ${shininess}`,
+        async function (p5, screenshot) {
+          p5.createCanvas(100, 100, p5.WEBGL);
+
+          // Load the environment map
+          const env = await p5.loadImage('/unit/assets/spheremap.jpg');
+          p5.clear();
+
+          // Set up panorama background
+          p5.panorama(env);
+
+          // Set up image-based lighting
+          p5.push();
+          p5.imageLight(env);
+          p5.ambientLight(10);
+
+          // Configure materials
+          p5.specularMaterial(255);
+          p5.shininess(shininess);
+          p5.metalness(100);
+          p5.noStroke();
+
+          // Draw a sphere in the center
+          p5.fill('white');
+          p5.sphere(25);
+
+          p5.pop();
+
+          screenshot();
+        },
+      );
+    }
+  });
 });