Add support for right-clicking on images

This patch adds right-click support for images in the PDF, allowing
users to download them. To minimize memory consumption, we:
- Do not store the images separately, and instead crop them out of the
  PDF page canvas
- Only extract the images when needed (i.e. when the user right-clicks
  on them), rather than eagery having all of them available.

To do so, we layer one empty 0x0 canvas per image, stretched to cover
the whole image, and only populate its contents on right click.
These images need to be inside the text layer: they cannot be _behind_
it, otherwise they would be covered by the text layer's container and
not be clickable, and they cannot be in front of it, otherwise they
would make the text spans unselectable.

This feature is managed by a new preference, `imagesRightClickMinSize`:
- when it's set to `-1`, right-click support is disabled
- when set to `0`, all images are available for right click
- when set to a positive integer, only images whose width and height are
  greater than or equal to that value (in the PDF page frame of
  reference) are available for right click.

This features is disabled by default outside of MOZCENTRAL, as it
significantly degrades the text selection experience in non-Firefox
browsers.

Author: nicolo-ribaudo

src/display/api.js

@@ -38,6 +38,10 @@ import {
   PrintAnnotationStorage,
   SerializableEmpty,
 } from "./annotation_storage.js";
+import {
+  CanvasDependencyTracker,
+  CanvasImagesTracker,
+} from "./canvas_dependency_tracker.js";
 import {
   deprecated,
   isDataScheme,
@@ -68,7 +72,6 @@ import {
   NodeStandardFontDataFactory,
   NodeWasmFactory,
 } from "display-node_utils";
-import { CanvasDependencyTracker } from "./canvas_dependency_tracker.js";
 import { CanvasGraphics } from "./canvas.js";
 import { DOMCanvasFactory } from "./canvas_factory.js";
 import { DOMCMapReaderFactory } from "display-cmap_reader_factory";
@@ -1269,6 +1272,7 @@ class PDFDocumentProxy {
  *   annotation ids with canvases used to render them.
  * @property {PrintAnnotationStorage} [printAnnotationStorage]
  * @property {boolean} [isEditing] - Render the page in editing mode.
+ * @property {boolean} [recordImages] - Record the location of images in the PDF
  * @property {boolean} [recordOperations] - Record the dependencies and bounding
  *   boxes of all PDF operations that render onto the canvas.
  * @property {OperationsFilter} [operationsFilter] - If provided, only
@@ -1353,6 +1357,7 @@ class PDFPageProxy {
     this.destroyed = false;
     this.recordedBBoxes = null;
     this.#pagesMapper = pagesMapper;
+    this.imageCoordinates = null;
   }
 
   /**
@@ -1484,6 +1489,7 @@ class PDFPageProxy {
     pageColors = null,
     printAnnotationStorage = null,
     isEditing = false,
+    recordImages = false,
     recordOperations = false,
     operationsFilter = null,
   }) {
@@ -1536,6 +1542,7 @@ class PDFPageProxy {
 
     const shouldRecordOperations =
       !this.recordedBBoxes && (recordOperations || recordForDebugger);
+    const shouldRecordImages = !this.imageCoordinates && recordImages;
 
     const complete = error => {
       intentState.renderTasks.delete(internalRenderTask);
@@ -1555,6 +1562,10 @@ class PDFPageProxy {
         }
       }
 
+      if (shouldRecordImages && !error) {
+        this.imageCoordinates = internalRenderTask.gfx?.imagesTracker.take();
+      }
+
       // Attempt to reduce memory usage during *printing*, by always running
       // cleanup immediately once rendering has finished.
       if (intentPrint) {
@@ -1589,12 +1600,16 @@ class PDFPageProxy {
       params: {
         canvas,
         canvasContext,
-        dependencyTracker: shouldRecordOperations
-          ? new CanvasDependencyTracker(
-              canvas,
-              intentState.operatorList.length,
-              recordForDebugger
-            )
+        dependencyTracker:
+          shouldRecordOperations || shouldRecordImages
+            ? new CanvasDependencyTracker(
+                canvas,
+                intentState.operatorList.length,
+                recordForDebugger
+              )
+            : null,
+        imagesTracker: shouldRecordImages
+          ? new CanvasImagesTracker(canvas)
           : null,
         viewport,
         transform,
@@ -3255,6 +3270,10 @@ class RenderTask {
       (separateAnnots.canvas && annotationCanvasMap?.size > 0)
     );
   }
+
+  get imageCoordinates() {
+    return this._internalRenderTask.imageCoordinates || null;
+  }
 }
 
 /**
@@ -3312,6 +3331,7 @@ class InternalRenderTask {
     this._canvasContext = params.canvas ? null : params.canvasContext;
     this._enableHWA = enableHWA;
     this._dependencyTracker = params.dependencyTracker;
+    this._imagesTracker = params.imagesTracker;
     this._operationsFilter = operationsFilter;
   }
 
@@ -3342,7 +3362,13 @@ class InternalRenderTask {
       this.stepper.init(this.operatorList);
       this.stepper.nextBreakPoint = this.stepper.getNextBreakPoint();
     }
-    const { viewport, transform, background, dependencyTracker } = this.params;
+    const {
+      viewport,
+      transform,
+      background,
+      dependencyTracker,
+      imagesTracker,
+    } = this.params;
 
     // When printing in Firefox, we get a specific context in mozPrintCallback
     // which cannot be created from the canvas itself.
@@ -3362,7 +3388,8 @@ class InternalRenderTask {
       { optionalContentConfig },
       this.annotationCanvasMap,
       this.pageColors,
-      dependencyTracker
+      dependencyTracker,
+      imagesTracker
     );
     this.gfx.beginDrawing({
       transform,

src/display/canvas.js

@@ -658,7 +658,8 @@ class CanvasGraphics {
     { optionalContentConfig, markedContentStack = null },
     annotationCanvasMap,
     pageColors,
-    dependencyTracker
+    dependencyTracker,
+    imagesTracker
   ) {
     this.ctx = canvasCtx;
     this.current = new CanvasExtraState(
@@ -698,6 +699,7 @@ class CanvasGraphics {
     this._cachedBitmapsMap = new Map();
 
     this.dependencyTracker = dependencyTracker ?? null;
+    this.imagesTracker = imagesTracker ?? null;
   }
 
   getObject(opIdx, data, fallback = null) {
@@ -3064,11 +3066,19 @@ class CanvasGraphics {
       imgData.interpolate
     );
 
-    this.dependencyTracker
-      ?.resetBBox(opIdx)
-      .recordBBox(opIdx, ctx, 0, width, -height, 0)
-      .recordDependencies(opIdx, Dependencies.imageXObject)
-      .recordOperation(opIdx);
+    if (this.dependencyTracker) {
+      this.dependencyTracker
+        .resetBBox(opIdx)
+        .recordBBox(opIdx, ctx, 0, width, -height, 0)
+        .recordDependencies(opIdx, Dependencies.imageXObject)
+        .recordOperation(opIdx);
+      this.imagesTracker?.record(
+        ctx,
+        width,
+        height,
+        this.dependencyTracker.clipBox
+      );
+    }
 
     drawImageAtIntegerCoords(
       ctx,

src/display/canvas_dependency_tracker.js

@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-import { Util } from "../shared/util.js";
+import { FeatureTest, Util } from "../shared/util.js";
 
 const FORCED_DEPENDENCY_LABEL = "__forcedDependency";
 
@@ -139,6 +139,10 @@ class CanvasDependencyTracker {
     }
   }
 
+  get clipBox() {
+    return this.#clipBox;
+  }
+
   growOperationsCount(operationsCount) {
     if (operationsCount >= this.#bboxes.length) {
       this.#initializeBBoxes(operationsCount, this.#bboxes);
@@ -644,6 +648,10 @@ class CanvasNestedDependencyTracker {
     this.#ignoreBBoxes = !!ignoreBBoxes;
   }
 
+  get clipBox() {
+    return this.#dependencyTracker.clipBox;
+  }
+
   growOperationsCount() {
     throw new Error("Unreachable");
   }
@@ -918,4 +926,155 @@ const Dependencies = {
   transformAndFill: ["transform", "fillColor"],
 };
 
-export { CanvasDependencyTracker, CanvasNestedDependencyTracker, Dependencies };
+/**
+ * Track the locations of images in the canvas. For each image it computes
+ * a bounding box as a potentially rotated rectangle, matching the rotation of
+ * the current canvas transform.
+ */
+class CanvasImagesTracker {
+  #canvasWidth;
+
+  #canvasHeight;
+
+  #capacity = 4;
+
+  #count = 0;
+
+  // Array of [x1, y1, x2, y2, x3, y3] coordinates.
+  // We need three points to be able to represent a rectangle with a transform
+  // applied.
+  #coords = new CanvasImagesTracker.#CoordsArray(this.#capacity * 6);
+
+  static #CoordsArray =
+    (typeof PDFJSDev !== "undefined" && PDFJSDev.test("MOZCENTRAL")) ||
+    FeatureTest.isFloat16ArraySupported
+      ? Float16Array
+      : Float32Array;
+
+  constructor(canvas) {
+    this.#canvasWidth = canvas.width;
+    this.#canvasHeight = canvas.height;
+  }
+
+  record(ctx, width, height, clipBox) {
+    if (this.#count === this.#capacity) {
+      this.#capacity *= 2;
+      const newCoords = new CanvasImagesTracker.#CoordsArray(
+        this.#capacity * 6
+      );
+      newCoords.set(this.#coords);
+      this.#coords = newCoords;
+    }
+
+    const transform = Util.domMatrixToTransform(ctx.getTransform());
+
+    // We want top left, bottom left, top right.
+    // (0, 0) is the bottom left corner.
+    let coords;
+
+    if (clipBox[0] !== Infinity) {
+      const bbox = [Infinity, Infinity, -Infinity, -Infinity];
+      Util.axialAlignedBoundingBox([0, -height, width, 0], transform, bbox);
+
+      const finalBBox = Util.intersect(clipBox, bbox);
+      if (!finalBBox) {
+        // The image is fully clipped out.
+        return;
+      }
+
+      const [minX, minY, maxX, maxY] = finalBBox;
+
+      if (
+        minX !== bbox[0] ||
+        minY !== bbox[1] ||
+        maxX !== bbox[2] ||
+        maxY !== bbox[3]
+      ) {
+        // The clip box affects the image drawing. We need to compute a
+        // transform that takes the image bbox and fits it into the final bbox,
+        // so that we can then apply it to the original image shape (the
+        // non-axially-aligned rectangle).
+        const rotationAngle = Math.atan2(transform[1], transform[0]);
+
+        // Normalize the angle to be between 0 and 90 degrees.
+        const sin = Math.abs(Math.sin(rotationAngle));
+        const cos = Math.abs(Math.cos(rotationAngle));
+
+        if (
+          sin < 1e-6 ||
+          cos < 1e-6 ||
+          // The logic in the `else` case gives more accurate bounding boxes for
+          // rotated images, but the equation it uses does not give a result
+          // when the rotation is exactly 45 degrees, because there are infinite
+          // possible rectangles that can fit into the same bbox with that same
+          // 45deg rotation. Fallback to returning the whole bbox.
+          Math.abs(sin - cos) < 1e-6
+        ) {
+          coords = [minX, minY, minX, maxY, maxX, minY];
+        } else {
+          // We cannot just scale the bbox into the original bbox, because that
+          // would not preserve the 90deg corners if they have been rotated.
+          // We instead need to find the transform that maps the original
+          // rectangle into the only rectangle that is rotated by the expected
+          // angle and fits into the final bbox.
+          //
+          // This represents the final bbox, with the top-left corner having
+          // coordinates (minX, minY) and the bottom-right corner having
+          // coordinates (maxX, maxY). Alpha is the rotation angle, and a and b
+          // are helper variables used to compute the effective transform.
+          //
+          //               ------------b----------
+          //              +-----------------------*----+
+          //            | |                _ -‾    \   |
+          //            a |           _ -‾          \  |
+          //            | |alpha _ -‾                \ |
+          //            | | _ -‾                      \|
+          //              |\                       _ -‾|
+          //              | \                 _ -‾     |
+          //              |  \           _ -‾          |
+          //              |   \     _ -‾               |
+          //              +----*-----------------------+
+
+          const finalBBoxWidth = maxX - minX;
+          const finalBBoxHeight = maxY - minY;
+
+          const sin2 = sin * sin;
+          const cos2 = cos * cos;
+          const cosSin = cos * sin;
+          const denom = cos2 - sin2;
+
+          const a = (finalBBoxHeight * cos2 - finalBBoxWidth * cosSin) / denom;
+          const b = (finalBBoxHeight * cosSin - finalBBoxWidth * sin2) / denom;
+
+          coords = [minX + b, minY, minX, minY + a, maxX, maxY - a];
+        }
+      }
+    }
+
+    if (!coords) {
+      coords = [0, -height, 0, 0, width, -height];
+      Util.applyTransform(coords, transform, 0);
+      Util.applyTransform(coords, transform, 2);
+      Util.applyTransform(coords, transform, 4);
+    }
+    coords[0] /= this.#canvasWidth;
+    coords[1] /= this.#canvasHeight;
+    coords[2] /= this.#canvasWidth;
+    coords[3] /= this.#canvasHeight;
+    coords[4] /= this.#canvasWidth;
+    coords[5] /= this.#canvasHeight;
+    this.#coords.set(coords, this.#count * 6);
+    this.#count++;
+  }
+
+  take() {
+    return this.#coords.subarray(0, this.#count * 6);
+  }
+}
+
+export {
+  CanvasDependencyTracker,
+  CanvasImagesTracker,
+  CanvasNestedDependencyTracker,
+  Dependencies,
+};