From afd4276b359c131f652802f1bbe308eb9cc64e84 Mon Sep 17 00:00:00 2001 From: ahillier Date: Thu, 7 Sep 2017 21:18:45 -0400 Subject: [PATCH] Bug 1397390 - Support better thumbnails for image urls r=k88hudson,Mardak MozReview-Commit-ID: Ksxo6Gj2rIO --HG-- extra : rebase_source : e46bbbdbd0ba87eb7475c6c49b46104ae77d9c40 --- .../thumbnails/BackgroundPageThumbs.jsm | 5 +- .../components/thumbnails/PageThumbUtils.jsm | 41 ++++++++++ .../content/backgroundPageThumbsContent.js | 12 ++- .../components/thumbnails/test/browser.ini | 4 + .../browser_thumbnails_bg_image_capture.js | 72 ++++++++++++++++++ .../test/sample_image_blue_300x600.jpg | Bin 0 -> 3581 bytes .../test/sample_image_green_1024x1024.jpg | Bin 0 -> 17077 bytes .../test/sample_image_red_1920x1080.jpg | Bin 0 -> 57332 bytes 8 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js create mode 100644 toolkit/components/thumbnails/test/sample_image_blue_300x600.jpg create mode 100644 toolkit/components/thumbnails/test/sample_image_green_1024x1024.jpg create mode 100644 toolkit/components/thumbnails/test/sample_image_red_1920x1080.jpg diff --git a/toolkit/components/thumbnails/BackgroundPageThumbs.jsm b/toolkit/components/thumbnails/BackgroundPageThumbs.jsm index 917899a15bff..10d4d3cae02b 100644 --- a/toolkit/components/thumbnails/BackgroundPageThumbs.jsm +++ b/toolkit/components/thumbnails/BackgroundPageThumbs.jsm @@ -57,6 +57,9 @@ const BackgroundPageThumbs = { * @opt timeout The capture will time out after this many milliseconds have * elapsed after the capture has progressed to the head of * the queue and started. Defaults to 30000 (30 seconds). + * @opt isImage If true, backgroundPageThumbsContent will attempt to render + * the url directly to canvas. Note that images will mostly get + * detected and rendered as such anyway, but this will ensure it. */ capture(url, options = {}) { if (!PageThumbs._prefEnabled()) { @@ -404,7 +407,7 @@ Capture.prototype = { // didCapture registration this._msgMan = messageManager; this._msgMan.sendAsyncMessage("BackgroundPageThumbs:capture", - { id: this.id, url: this.url }); + { id: this.id, url: this.url, isImage: this.options.isImage }); this._msgMan.addMessageListener("BackgroundPageThumbs:didCapture", this); }, diff --git a/toolkit/components/thumbnails/PageThumbUtils.jsm b/toolkit/components/thumbnails/PageThumbUtils.jsm index ce021aba4b65..83c2d7e27f81 100644 --- a/toolkit/components/thumbnails/PageThumbUtils.jsm +++ b/toolkit/components/thumbnails/PageThumbUtils.jsm @@ -114,6 +114,47 @@ this.PageThumbUtils = { return [width, height]; }, + /** + * Renders an image onto a new canvas of a given width and proportional + * height. Uses an image that exists in the window and is loaded, or falls + * back to loading the url into a new image element. + */ + async createImageThumbnailCanvas(window, url, targetWidth = 448) { + // 224px is the width of cards in ActivityStream; capture thumbnails at 2x + const doc = (window || Services.appShell.hiddenDOMWindow).document; + + let image = doc.querySelector("img"); + if (!image || image.src !== url) { + image = doc.createElementNS(this.HTML_NAMESPACE, "img"); + } + if (!image.complete) { + await new Promise(resolve => { + image.onload = () => resolve(); + image.onerror = () => { throw new Error("Image failed to load"); } + image.src = url; + }); + } + + // has width/height but not naturalWidth/naturalHeight + const imageWidth = image.naturalWidth || image.width; + const imageHeight = image.naturalHeight || image.height; + if (imageWidth === 0 || imageHeight === 0) { + throw new Error("Image has zero dimension"); + } + const width = Math.min(targetWidth, imageWidth); + const height = imageHeight * width / imageWidth; + + // As we're setting the width and maintaining the aspect ratio, if an image + // is very tall we might get a very large thumbnail. Restricting the canvas + // size to {width}x{width} solves this problem. Here we choose to clip the + // image at the bottom rather than centre it vertically, based on an + // estimate that the focus of a tall image is most likely to be near the top + // (e.g., the face of a person). + const canvas = this.createCanvas(window, width, Math.min(height, width)); + canvas.getContext("2d").drawImage(image, 0, 0, width, height); + return canvas; + }, + /** * * Given a browser window, this creates a snapshot of the content * and returns a canvas with the resulting snapshot of the content diff --git a/toolkit/components/thumbnails/content/backgroundPageThumbsContent.js b/toolkit/components/thumbnails/content/backgroundPageThumbsContent.js index 28eae003878c..ae48d401511a 100644 --- a/toolkit/components/thumbnails/content/backgroundPageThumbsContent.js +++ b/toolkit/components/thumbnails/content/backgroundPageThumbsContent.js @@ -78,6 +78,7 @@ const backgroundPageThumbsContent = { this._nextCapture = { id: msg.data.id, url: msg.data.url, + isImage: msg.data.isImage }; if (this._currentCapture) { if (this._state == STATE_LOADING) { @@ -163,7 +164,7 @@ const backgroundPageThumbsContent = { _captureCurrentPage() { let win = docShell.QueryInterface(Ci.nsIInterfaceRequestor) .getInterface(Ci.nsIDOMWindow); - win.requestIdleCallback(() => { + win.requestIdleCallback(async () => { let capture = this._currentCapture; capture.finalURL = this._webNav.currentURI.spec; capture.pageLoadTime = new Date() - capture.pageLoadStartDate; @@ -171,7 +172,14 @@ const backgroundPageThumbsContent = { let canvasDrawDate = new Date(); docShell.isActive = true; - let finalCanvas = PageThumbUtils.createSnapshotThumbnail(content, null); + + let finalCanvas; + if (capture.isImage || content.document instanceof content.ImageDocument) { + finalCanvas = await PageThumbUtils.createImageThumbnailCanvas(content, capture.url); + } else { + finalCanvas = PageThumbUtils.createSnapshotThumbnail(content, null); + } + docShell.isActive = false; capture.canvasDrawTime = new Date() - canvasDrawDate; diff --git a/toolkit/components/thumbnails/test/browser.ini b/toolkit/components/thumbnails/test/browser.ini index 90b89fa690df..d678208507b9 100644 --- a/toolkit/components/thumbnails/test/browser.ini +++ b/toolkit/components/thumbnails/test/browser.ini @@ -4,6 +4,9 @@ support-files = background_red.html background_red_redirect.sjs background_red_scroll.html + sample_image_red_1920x1080.jpg + sample_image_green_1024x1024.jpg + sample_image_blue_300x600.jpg head.js privacy_cache_control.sjs thumbnails_background.sjs @@ -25,6 +28,7 @@ skip-if = !crashreporter [browser_thumbnails_bg_no_alert.js] [browser_thumbnails_bg_no_duplicates.js] [browser_thumbnails_bg_captureIfMissing.js] +[browser_thumbnails_bg_image_capture.js] [browser_thumbnails_bug726727.js] [browser_thumbnails_bug727765.js] [browser_thumbnails_bug818225.js] diff --git a/toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js b/toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js new file mode 100644 index 000000000000..d2e10c013ee7 --- /dev/null +++ b/toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js @@ -0,0 +1,72 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const BASE_URL = "http://mochi.test:8888/browser/toolkit/components/thumbnails/"; + +/** + * These tests ensure that when trying to capture a url that is an image file, + * the image itself is captured instead of the the browser window displaying the + * image, and that the thumbnail maintains the image aspect ratio. + */ +function* runTests() { + for (const {url, color, width, height} of [{ + url: BASE_URL + "test/sample_image_red_1920x1080.jpg", + color: [255, 0, 0], + width: 1920, + height: 1080 + }, { + url: BASE_URL + "test/sample_image_green_1024x1024.jpg", + color: [0, 255, 0], + width: 1024, + height: 1024 + }, { + url: BASE_URL + "test/sample_image_blue_300x600.jpg", + color: [0, 0, 255], + width: 300, + height: 600 + }]) { + dontExpireThumbnailURLs([url]); + const capturedPromise = new Promise(resolve => { + bgAddPageThumbObserver(url).then(() => { + ok(true, `page-thumbnail created for ${url}`); + resolve(); + }); + }); + yield bgCapture(url); + yield capturedPromise; + ok(thumbnailExists(url), "The image thumbnail should exist after capture"); + + const thumb = PageThumbs.getThumbnailURL(url); + const htmlns = "http://www.w3.org/1999/xhtml"; + const img = document.createElementNS(htmlns, "img"); + yield new Promise(resolve => { + img.onload = () => resolve(); + img.src = thumb; + }); + + // 448px is the default max-width of an image thumbnail + const expectedWidth = Math.min(448, width); + // Tall images are clipped to {width}x{width} + const expectedHeight = Math.min(expectedWidth * height / width, expectedWidth); + // Fuzzy equality to account for rounding + ok(Math.abs(img.naturalWidth - expectedWidth) <= 1, + "The thumbnail should have the right width"); + ok(Math.abs(img.naturalHeight - expectedHeight) <= 1, + "The thumbnail should have the right height"); + + // Draw the image to a canvas and compare the pixel color values. + const canvas = document.createElementNS(htmlns, "canvas"); + canvas.width = expectedWidth; + canvas.height = expectedHeight; + const ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0, expectedWidth, expectedHeight); + const [r, g, b] = ctx.getImageData(0, 0, expectedWidth, expectedHeight).data; + // Fuzzy equality to account for image encoding + ok((Math.abs(r - color[0]) <= 2 && + Math.abs(g - color[1]) <= 2 && + Math.abs(b - color[2]) <= 2), + "The thumbnail should have the right color"); + + removeThumbnail(url); + } +} diff --git a/toolkit/components/thumbnails/test/sample_image_blue_300x600.jpg b/toolkit/components/thumbnails/test/sample_image_blue_300x600.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ad5b44ecbc721ef793d81716e9e519aaf2c56d7b GIT binary patch literal 3581 zcmex=_1P|rX?qqI0P zFI~aY%U!`Mz|~!$%)&rZMU_b4?usLlYAdd38%$3nLpnV-q8g zA&i`yoIKn-61=<;Mv5|uMkIs(2N(o7m?9W;m>HEAm;@P_1sVSzVUTBFU}OdQ7UW?l zU}R!uVP#|I;N;>4D%dK(z{JSR%*4XX%F4n5R9y>{XJ8Rz6;d>GWD^cdWLGK_F>0K+ zkVDyN<3Z7&iyu^slZu)+xx~aJB&Af<)HO7&0Dr^+rDGx zu0w~996fgY#K}{aE?>EN?fQ+Iw;n!v{N(Ag=PzEq`uOSdm#^Qx|M>X}>z(JGL-`{vmgtrq9L1*V<3BCp|FxsBZr97#DyCVaw;1KeGpA5 zy2vG_V)9V+BgkuDpAqM=CbE16_ZY%ow-|Vs8G(_*oqEuW@7QqT3#sUQ~@c{1J zaMv5Sdqc(TB#P?|7X-YByPjCw4~|pU#FKjQ^83HM)4aUo`{tt#s*gxqz+dJklv1)v zTSWa#=K6vqMHLH!g-W<;M(z6Qu+rpqIMx&G*pD@Xsnqr%hnqsvo>8?!0k)39> zI`gu!bKG99*Opl@C*L#MmFxAyE}^0*N=C_IGFd!!tKIY0qh2Pmz!H`))IoGIWn`-M z5gQ?t(`sYE--qfLmeccsXfSFAdg6$VGK`L8IF4nt)0?#Kgq68?ySqrAP#)qP5woYY zqfwtU6P z)!Ji)*VfcFG)9}6H?(Zryk+aQ?K^h1@7ld*@4o#94j$?{eB|h{<0np@I(_DB&$;s# zE?&BFweQ;X8#iy=zH|59{ecIAL&GDFpFDl`{Kd;xugAtG-n@PH{=>&lu`a50{+MrJ zzjeu4myTr_mXCE&U9+|sndR(meSA?lABrS6JgtJcxTCYXS4_(djwV*t_ZyPZy(1Z8 zv9zyce-_sEPh~&Desv8KiJ{uhV`MUq40N?mpYV;7u68Fj-T($*00v+H24DaNU;qYS z00v+H24DaNU;qYS00v+H24DaNU;qYS00v+H24DaNU;qYS00v+H24DaNU;qYS00v+H j24DaNU;qYS00v+H24DaNU;qYS00v+H2L7J`MjiSB*UBWl literal 0 HcmV?d00001 diff --git a/toolkit/components/thumbnails/test/sample_image_red_1920x1080.jpg b/toolkit/components/thumbnails/test/sample_image_red_1920x1080.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a2bec4d5e8d2f7169858ea8a5c06173831d5948 GIT binary patch literal 57332 zcmeIzOKcTY7zgk(cjorCg+*_9%7(PG0`4dvl?~xlDLbm7upxkmx@!RoCM0d~0qm** zLRjzy-I&TlU<2h1JA7cN8!Hcy6%+&^CfBKunz%A{Me;k7$z(ETzRdrdne!`j6@Ct7 zbL;BsLKKA%rCTWc6lSJp0~>us238@xyKpk(i<8=1OC04xCLhK5sBk8XPMM(U5tIbCr!lYg*fP7LS?H z(K)Q)@|~)QOPbnBN{5da`P``L=f{p4|H7onFHWg_Y3j^bv+G`-Q$P2WS6`d|`Wp)t zE`4*^Tg%^m=iL=6SH0J~dd>UmKKSsXkJoS5_{pcAZQ8u$^R3%V|IJ$L@X#Y^9Q_x%r7e(bv1-Sf+}>o;!xdh55_cYE*M|NX(kKOQ~q z%N2#VzhzH?{VP}6gea3FagyuH6=j;!J;vi@NI