Bug 1397390 - Support better thumbnails for image urls r=k88hudson,Mardak

MozReview-Commit-ID: Ksxo6Gj2rIO

--HG--
extra : rebase_source : e46bbbdbd0ba87eb7475c6c49b46104ae77d9c40

toolkit/components/thumbnails/BackgroundPageThumbs.jsm

@@ -57,6 +57,9 @@ const BackgroundPageThumbs = {
    * @opt timeout    The capture will time out after this many milliseconds have
    *                 elapsed after the capture has progressed to the head of
    *                 the queue and started.  Defaults to 30000 (30 seconds).
+   * @opt isImage    If true, backgroundPageThumbsContent will attempt to render
+   *                 the url directly to canvas. Note that images will mostly get
+   *                 detected and rendered as such anyway, but this will ensure it.
    */
   capture(url, options = {}) {
     if (!PageThumbs._prefEnabled()) {
@@ -404,7 +407,7 @@ Capture.prototype = {
     // didCapture registration
     this._msgMan = messageManager;
     this._msgMan.sendAsyncMessage("BackgroundPageThumbs:capture",
-                                  { id: this.id, url: this.url });
+                                  { id: this.id, url: this.url, isImage: this.options.isImage });
     this._msgMan.addMessageListener("BackgroundPageThumbs:didCapture", this);
   },
 

toolkit/components/thumbnails/PageThumbUtils.jsm

@@ -114,6 +114,47 @@ this.PageThumbUtils = {
     return [width, height];
   },
 
+  /**
+   * Renders an image onto a new canvas of a given width and proportional
+   * height. Uses an image that exists in the window and is loaded, or falls
+   * back to loading the url into a new image element.
+   */
+  async createImageThumbnailCanvas(window, url, targetWidth = 448) {
+    // 224px is the width of cards in ActivityStream; capture thumbnails at 2x
+    const doc = (window || Services.appShell.hiddenDOMWindow).document;
+
+    let image = doc.querySelector("img");
+    if (!image || image.src !== url) {
+      image = doc.createElementNS(this.HTML_NAMESPACE, "img");
+    }
+    if (!image.complete) {
+      await new Promise(resolve => {
+        image.onload = () => resolve();
+        image.onerror = () => { throw new Error("Image failed to load"); }
+        image.src = url;
+      });
+    }
+
+    // <img src="*.svg"> has width/height but not naturalWidth/naturalHeight
+    const imageWidth = image.naturalWidth || image.width;
+    const imageHeight = image.naturalHeight || image.height;
+    if (imageWidth === 0 || imageHeight === 0) {
+      throw new Error("Image has zero dimension");
+    }
+    const width = Math.min(targetWidth, imageWidth);
+    const height = imageHeight * width / imageWidth;
+
+    // As we're setting the width and maintaining the aspect ratio, if an image
+    // is very tall we might get a very large thumbnail. Restricting the canvas
+    // size to {width}x{width} solves this problem. Here we choose to clip the
+    // image at the bottom rather than centre it vertically, based on an
+    // estimate that the focus of a tall image is most likely to be near the top
+    // (e.g., the face of a person).
+    const canvas = this.createCanvas(window, width, Math.min(height, width));
+    canvas.getContext("2d").drawImage(image, 0, 0, width, height);
+    return canvas;
+  },
+
   /** *
    * Given a browser window, this creates a snapshot of the content
    * and returns a canvas with the resulting snapshot of the content

toolkit/components/thumbnails/content/backgroundPageThumbsContent.js

@@ -78,6 +78,7 @@ const backgroundPageThumbsContent = {
     this._nextCapture = {
       id: msg.data.id,
       url: msg.data.url,
+      isImage: msg.data.isImage
     };
     if (this._currentCapture) {
       if (this._state == STATE_LOADING) {
@@ -163,7 +164,7 @@ const backgroundPageThumbsContent = {
   _captureCurrentPage() {
     let win = docShell.QueryInterface(Ci.nsIInterfaceRequestor)
                       .getInterface(Ci.nsIDOMWindow);
-    win.requestIdleCallback(() => {
+    win.requestIdleCallback(async () => {
       let capture = this._currentCapture;
       capture.finalURL = this._webNav.currentURI.spec;
       capture.pageLoadTime = new Date() - capture.pageLoadStartDate;
@@ -171,7 +172,14 @@ const backgroundPageThumbsContent = {
       let canvasDrawDate = new Date();
 
       docShell.isActive = true;
-      let finalCanvas = PageThumbUtils.createSnapshotThumbnail(content, null);
+
+      let finalCanvas;
+      if (capture.isImage || content.document instanceof content.ImageDocument) {
+        finalCanvas = await PageThumbUtils.createImageThumbnailCanvas(content, capture.url);
+      } else {
+        finalCanvas = PageThumbUtils.createSnapshotThumbnail(content, null);
+      }
+
       docShell.isActive = false;
       capture.canvasDrawTime = new Date() - canvasDrawDate;
 

toolkit/components/thumbnails/test/browser.ini

@@ -4,6 +4,9 @@ support-files =
   background_red.html
   background_red_redirect.sjs
   background_red_scroll.html
+  sample_image_red_1920x1080.jpg
+  sample_image_green_1024x1024.jpg
+  sample_image_blue_300x600.jpg
   head.js
   privacy_cache_control.sjs
   thumbnails_background.sjs
@@ -25,6 +28,7 @@ skip-if = !crashreporter
 [browser_thumbnails_bg_no_alert.js]
 [browser_thumbnails_bg_no_duplicates.js]
 [browser_thumbnails_bg_captureIfMissing.js]
+[browser_thumbnails_bg_image_capture.js]
 [browser_thumbnails_bug726727.js]
 [browser_thumbnails_bug727765.js]
 [browser_thumbnails_bug818225.js]

toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js

@@ -0,0 +1,72 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const BASE_URL = "http://mochi.test:8888/browser/toolkit/components/thumbnails/";
+
+/**
+ * These tests ensure that when trying to capture a url that is an image file,
+ * the image itself is captured instead of the the browser window displaying the
+ * image, and that the thumbnail maintains the image aspect ratio.
+ */
+function* runTests() {
+  for (const {url, color, width, height} of [{
+    url: BASE_URL + "test/sample_image_red_1920x1080.jpg",
+    color: [255, 0, 0],
+    width: 1920,
+    height: 1080
+  }, {
+    url: BASE_URL + "test/sample_image_green_1024x1024.jpg",
+    color: [0, 255, 0],
+    width: 1024,
+    height: 1024
+  }, {
+    url: BASE_URL + "test/sample_image_blue_300x600.jpg",
+    color: [0, 0, 255],
+    width: 300,
+    height: 600
+  }]) {
+    dontExpireThumbnailURLs([url]);
+    const capturedPromise = new Promise(resolve => {
+      bgAddPageThumbObserver(url).then(() => {
+        ok(true, `page-thumbnail created for ${url}`);
+        resolve();
+      });
+    });
+    yield bgCapture(url);
+    yield capturedPromise;
+    ok(thumbnailExists(url), "The image thumbnail should exist after capture");
+
+    const thumb = PageThumbs.getThumbnailURL(url);
+    const htmlns = "http://www.w3.org/1999/xhtml";
+    const img = document.createElementNS(htmlns, "img");
+    yield new Promise(resolve => {
+      img.onload = () => resolve();
+      img.src = thumb;
+    });
+
+    // 448px is the default max-width of an image thumbnail
+    const expectedWidth = Math.min(448, width);
+    // Tall images are clipped to {width}x{width}
+    const expectedHeight = Math.min(expectedWidth * height / width, expectedWidth);
+    // Fuzzy equality to account for rounding
+    ok(Math.abs(img.naturalWidth - expectedWidth) <= 1,
+      "The thumbnail should have the right width");
+    ok(Math.abs(img.naturalHeight - expectedHeight) <= 1,
+      "The thumbnail should have the right height");
+
+    // Draw the image to a canvas and compare the pixel color values.
+    const canvas = document.createElementNS(htmlns, "canvas");
+    canvas.width = expectedWidth;
+    canvas.height = expectedHeight;
+    const ctx = canvas.getContext("2d");
+    ctx.drawImage(img, 0, 0, expectedWidth, expectedHeight);
+    const [r, g, b] = ctx.getImageData(0, 0, expectedWidth, expectedHeight).data;
+    // Fuzzy equality to account for image encoding
+    ok((Math.abs(r - color[0]) <= 2 &&
+        Math.abs(g - color[1]) <= 2 &&
+        Math.abs(b - color[2]) <= 2),
+        "The thumbnail should have the right color");
+
+    removeThumbnail(url);
+  }
+}