Bug 744388 - [Page Thumbnails] implement a custom storage, don't use the file cache; r=dietrich

browser/components/nsBrowserGlue.js

@@ -68,6 +68,9 @@ XPCOMUtils.defineLazyModuleGetter(this, "BookmarkHTMLUtils",
 XPCOMUtils.defineLazyModuleGetter(this, "webappsUI", 
                                   "resource:///modules/webappsUI.jsm");
 
+XPCOMUtils.defineLazyModuleGetter(this, "PageThumbs",
+                                  "resource:///modules/PageThumbs.jsm");
+
 const PREF_PLUGINS_NOTIFYUSER = "plugins.update.notifyUser";
 const PREF_PLUGINS_UPDATEURL  = "plugins.update.url";
 
@@ -358,6 +361,8 @@ BrowserGlue.prototype = {
     // Initialize webapps UI
     webappsUI.init();
 
+    PageThumbs.init();
+
     Services.obs.notifyObservers(null, "browser-ui-startup-complete", "");
   },
 
@@ -379,6 +384,7 @@ BrowserGlue.prototype = {
   _onProfileShutdown: function BG__onProfileShutdown() {
     this._shutdownPlaces();
     this._sanitizer.onShutdown();
+    PageThumbs.uninit();
   },
 
   // All initial windows have opened.

browser/components/thumbnails/PageThumbs.jsm

@@ -4,7 +4,7 @@
 
 "use strict";
 
-let EXPORTED_SYMBOLS = ["PageThumbs", "PageThumbsCache"];
+let EXPORTED_SYMBOLS = ["PageThumbs", "PageThumbsStorage", "PageThumbsCache"];
 
 const Cu = Components.utils;
 const Cc = Components.classes;
@@ -12,6 +12,11 @@ const Ci = Components.interfaces;
 
 const HTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
 
+/**
+ * Name of the directory in the profile that contains the thumbnails.
+ */
+const THUMBNAIL_DIRECTORY = "thumbnails";
+
 /**
  * The default background color for page thumbnails.
  */
@@ -25,11 +30,29 @@ XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
 XPCOMUtils.defineLazyModuleGetter(this, "Services",
   "resource://gre/modules/Services.jsm");
 
+XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
+  "resource://gre/modules/FileUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
+  "resource://gre/modules/PlacesUtils.jsm");
+
+XPCOMUtils.defineLazyGetter(this, "gCryptoHash", function () {
+  return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
+});
+
+XPCOMUtils.defineLazyGetter(this, "gUnicodeConverter", function () {
+  let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                    .createInstance(Ci.nsIScriptableUnicodeConverter);
+  converter.charset = 'utf8';
+  return converter;
+});
+
 /**
  * Singleton providing functionality for capturing web page thumbnails and for
  * accessing them if already cached.
  */
 let PageThumbs = {
+  _initialized: false,
 
   /**
    * The calculated width and height of the thumbnails.
@@ -52,6 +75,20 @@ let PageThumbs = {
    */
   get contentType() "image/png",
 
+  init: function PageThumbs_init() {
+    if (!this._initialized) {
+      this._initialized = true;
+      PlacesUtils.history.addObserver(PageThumbsHistoryObserver, false);
+    }
+  },
+
+  uninit: function PageThumbs_uninit() {
+    if (this._initialized) {
+      this._initialized = false;
+      PlacesUtils.history.removeObserver(PageThumbsHistoryObserver);
+    }
+  },
+
   /**
    * Gets the thumbnail image's url for a given web page's url.
    * @param aUrl The web page's url that is depicted in the thumbnail.
@@ -124,32 +161,14 @@ let PageThumbs = {
           //    Sync and therefore also redirect sources appear on the newtab
           //    page. We also want thumbnails for those.
           if (url != originalURL)
-            PageThumbsCache._copy(url, originalURL);
+            PageThumbsStorage.copy(url, originalURL);
         }
 
         if (aCallback)
           aCallback(aSuccessful);
       }
 
-      // Get a writeable cache entry.
-      PageThumbsCache.getWriteEntry(url, function (aEntry) {
-        if (!aEntry) {
-          finish(false);
-          return;
-        }
-
-        let outputStream = aEntry.openOutputStream(0);
-
-        // Write the image data to the cache entry.
-        NetUtil.asyncCopy(aInputStream, outputStream, function (aResult) {
-          let success = Components.isSuccessCode(aResult);
-          if (success)
-            aEntry.markValid();
-
-          aEntry.close();
-          finish(success);
-        });
-      });
+      PageThumbsStorage.write(url, aInputStream, finish);
     });
   },
 
@@ -208,6 +227,88 @@ let PageThumbs = {
   }
 };
 
+let PageThumbsStorage = {
+  getFileForURL: function Storage_getFileForURL(aURL) {
+    let hash = this._calculateMD5Hash(aURL);
+    let parts = [THUMBNAIL_DIRECTORY, hash[0], hash[1], hash.slice(2) + ".png"];
+    return FileUtils.getFile("ProfD", parts);
+  },
+
+  write: function Storage_write(aURL, aDataStream, aCallback) {
+    let file = this.getFileForURL(aURL);
+    let fos = FileUtils.openSafeFileOutputStream(file);
+
+    NetUtil.asyncCopy(aDataStream, fos, function (aResult) {
+      FileUtils.closeSafeFileOutputStream(fos);
+      aCallback(Components.isSuccessCode(aResult));
+    });
+  },
+
+  copy: function Storage_copy(aSourceURL, aTargetURL) {
+    let sourceFile = this.getFileForURL(aSourceURL);
+    let targetFile = this.getFileForURL(aTargetURL);
+
+    try {
+      sourceFile.copyTo(targetFile.parent, targetFile.leafName);
+    } catch (e) {
+      /* We might not be permitted to write to the file. */
+    }
+  },
+
+  remove: function Storage_remove(aURL) {
+    try {
+      this.getFileForURL(aURL).remove(false);
+    } catch (e) {
+      /* The file might not exist or we're not permitted to remove it. */
+    }
+  },
+
+  wipe: function Storage_wipe() {
+    try {
+      FileUtils.getDir("ProfD", [THUMBNAIL_DIRECTORY]).remove(true);
+    } catch (e) {
+      /* The file might not exist or we're not permitted to remove it. */
+    }
+  },
+
+  _calculateMD5Hash: function Storage_calculateMD5Hash(aValue) {
+    let hash = gCryptoHash;
+    let value = gUnicodeConverter.convertToByteArray(aValue);
+
+    hash.init(hash.MD5);
+    hash.update(value, value.length);
+    return this._convertToHexString(hash.finish(false));
+  },
+
+  _convertToHexString: function Storage_convertToHexString(aData) {
+    let hex = "";
+    for (let i = 0; i < aData.length; i++)
+      hex += ("0" + aData.charCodeAt(i).toString(16)).slice(-2);
+    return hex;
+  },
+
+};
+
+let PageThumbsHistoryObserver = {
+  onDeleteURI: function Thumbnails_onDeleteURI(aURI, aGUID) {
+    PageThumbsStorage.remove(aURI.spec);
+  },
+
+  onClearHistory: function Thumbnails_onClearHistory() {
+    PageThumbsStorage.wipe();
+  },
+
+  onTitleChanged: function () {},
+  onBeginUpdateBatch: function () {},
+  onEndUpdateBatch: function () {},
+  onVisit: function () {},
+  onBeforeDeleteURI: function () {},
+  onPageChanged: function () {},
+  onDeleteVisits: function () {},
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsINavHistoryObserver])
+};
+
 /**
  * A singleton handling the storage of page thumbnails.
  */
@@ -222,64 +323,6 @@ let PageThumbsCache = {
     this._openCacheEntry(aKey, Ci.nsICache.ACCESS_READ, aCallback);
   },
 
-  /**
-   * Calls the given callback with a cache entry opened for writing.
-   * @param aKey The key identifying the desired cache entry.
-   * @param aCallback The callback that is called when the cache entry is ready.
-   */
-  getWriteEntry: function Cache_getWriteEntry(aKey, aCallback) {
-    // Try to open the desired cache entry.
-    this._openCacheEntry(aKey, Ci.nsICache.ACCESS_WRITE, aCallback);
-  },
-
-  /**
-   * Copies an existing cache entry's data to a new cache entry.
-   * @param aSourceKey The key that contains the data to copy.
-   * @param aTargetKey The key that will be the copy of aSourceKey's data.
-   */
-  _copy: function Cache_copy(aSourceKey, aTargetKey) {
-    let sourceEntry, targetEntry, waitingCount = 2;
-
-    function finish() {
-      if (sourceEntry)
-        sourceEntry.close();
-
-      if (targetEntry)
-        targetEntry.close();
-    }
-
-    function copyDataWhenReady() {
-      if (--waitingCount > 0)
-        return;
-
-      if (!sourceEntry || !targetEntry) {
-        finish();
-        return;
-      }
-
-      let inputStream = sourceEntry.openInputStream(0);
-      let outputStream = targetEntry.openOutputStream(0);
-
-      // Copy the image data to a new entry.
-      NetUtil.asyncCopy(inputStream, outputStream, function (aResult) {
-        if (Components.isSuccessCode(aResult))
-          targetEntry.markValid();
-
-        finish();
-      });
-    }
-
-    this.getReadEntry(aSourceKey, function (aSourceEntry) {
-      sourceEntry = aSourceEntry;
-      copyDataWhenReady();
-    });
-
-    this.getWriteEntry(aTargetKey, function (aTargetEntry) {
-      targetEntry = aTargetEntry;
-      copyDataWhenReady();
-    });
-  },
-
   /**
    * Opens the cache entry identified by the given key.
    * @param aKey The key identifying the desired cache entry.

browser/components/thumbnails/PageThumbsProtocol.js

@@ -72,6 +72,14 @@ Protocol.prototype = {
    * @return The newly created channel.
    */
   newChannel: function Proto_newChannel(aURI) {
+    let {url} = parseURI(aURI);
+    let file = PageThumbsStorage.getFileForURL(url);
+
+    if (file.exists()) {
+      let fileuri = Services.io.newFileURI(file);
+      return Services.io.newChannelFromURI(fileuri);
+    }
+
     return new Channel(aURI);
   },
 

browser/components/thumbnails/test/Makefile.in

@@ -14,6 +14,7 @@ include $(topsrcdir)/config/rules.mk
 _BROWSER_FILES = \
 	browser_thumbnails_capture.js \
 	browser_thumbnails_redirect.js \
+	browser_thumbnails_storage.js \
 	browser_thumbnails_bug726727.js \
 	head.js \
 	background_red.html \

browser/components/thumbnails/test/browser_thumbnails_redirect.js

@@ -4,9 +4,6 @@
 const URL = "http://mochi.test:8888/browser/browser/components/thumbnails/" +
             "test/background_red_redirect.sjs";
 
-let cacheService = Cc["@mozilla.org/network/cache-service;1"]
-                   .getService(Ci.nsICacheService);
-
 /**
  * These tests ensure that we save and provide thumbnails for redirecting sites.
  */
@@ -19,33 +16,17 @@ function runTests() {
   yield addTab(URL);
   yield captureAndCheckColor(255, 0, 0, "we have a red thumbnail");
 
-  // Wait until the referrer's thumbnail's cache entry has been written.
-  yield whenCacheEntryExists(URL);
+  // Wait until the referrer's thumbnail's file has been written.
+  yield whenFileExists(URL);
   yield checkThumbnailColor(URL, 255, 0, 0, "referrer has a red thumbnail");
 }
 
-function whenCacheEntryExists(aKey) {
+function whenFileExists(aURL) {
   let callback = next;
 
-  checkCacheEntryExists(aKey, function (aExists) {
-    if (!aExists)
-      callback = function () whenCacheEntryExists(aKey);
+  let file = PageThumbsStorage.getFileForURL(aURL);
+  if (!file.exists())
+    callback = function () whenFileExists(aURL);
 
-    executeSoon(callback);
-  });
-}
-
-function checkCacheEntryExists(aKey, aCallback) {
-  PageThumbsCache.getReadEntry(aKey, function (aEntry) {
-    let inputStream = aEntry && aEntry.openInputStream(0);
-    let exists = inputStream && inputStream.available();
-
-    if (inputStream)
-      inputStream.close();
-
-    if (aEntry)
-      aEntry.close();
-
-    aCallback(exists);
-  });
+  executeSoon(callback);
 }

browser/components/thumbnails/test/browser_thumbnails_storage.js

@@ -0,0 +1,89 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const URL = "http://mochi.test:8888/";
+const URL_COPY = URL + "#copy";
+
+XPCOMUtils.defineLazyGetter(this, "Sanitizer", function () {
+  let tmp = {};
+  Cc["@mozilla.org/moz/jssubscript-loader;1"]
+    .getService(Ci.mozIJSSubScriptLoader)
+    .loadSubScript("chrome://browser/content/sanitize.js", tmp);
+  return tmp.Sanitizer;
+});
+
+/**
+ * These tests ensure that the thumbnail storage is working as intended.
+ * Newly captured thumbnails should be saved as files and they should as well
+ * be removed when the user sanitizes their history.
+ */
+function runTests() {
+  clearHistory();
+
+  // create a thumbnail
+  yield addTab(URL);
+  yield whenFileExists();
+  gBrowser.removeTab(gBrowser.selectedTab);
+
+  // clear all browser history
+  yield clearHistory();
+
+  // create a thumbnail
+  yield addTab(URL);
+  yield whenFileExists();
+  gBrowser.removeTab(gBrowser.selectedTab);
+
+  // make sure copy() updates an existing file
+  PageThumbsStorage.copy(URL, URL_COPY);
+  let copy = PageThumbsStorage.getFileForURL(URL_COPY);
+  let mtime = copy.lastModifiedTime -= 60;
+
+  PageThumbsStorage.copy(URL, URL_COPY);
+  isnot(PageThumbsStorage.getFileForURL(URL_COPY).lastModifiedTime, mtime,
+        "thumbnail file was updated");
+
+  // clear last 10 mins of history
+  yield clearHistory(true);
+  ok(!copy.exists(), "copy of thumbnail has been removed");
+}
+
+function clearHistory(aUseRange) {
+  let s = new Sanitizer();
+  s.prefDomain = "privacy.cpd.";
+
+  let prefs = gPrefService.getBranch(s.prefDomain);
+  prefs.setBoolPref("history", true);
+  prefs.setBoolPref("downloads", false);
+  prefs.setBoolPref("cache", false);
+  prefs.setBoolPref("cookies", false);
+  prefs.setBoolPref("formdata", false);
+  prefs.setBoolPref("offlineApps", false);
+  prefs.setBoolPref("passwords", false);
+  prefs.setBoolPref("sessions", false);
+  prefs.setBoolPref("siteSettings", false);
+
+  if (aUseRange) {
+    let usec = Date.now() * 1000;
+    s.range = [usec - 10 * 60 * 1000 * 1000, usec];
+  }
+
+  s.sanitize();
+  s.range = null;
+
+  executeSoon(function () {
+    if (PageThumbsStorage.getFileForURL(URL).exists())
+      clearHistory(aFile, aUseRange);
+    else
+      next();
+  });
+}
+
+function whenFileExists() {
+  let callback = whenFileExists;
+
+  let file = PageThumbsStorage.getFileForURL(URL);
+  if (file.exists() && file.fileSize)
+    callback = next;
+
+  executeSoon(callback);
+}

browser/components/thumbnails/test/head.js

@@ -4,7 +4,7 @@
 let tmp = {};
 Cu.import("resource:///modules/PageThumbs.jsm", tmp);
 let PageThumbs = tmp.PageThumbs;
-let PageThumbsCache = tmp.PageThumbsCache;
+let PageThumbsStorage = tmp.PageThumbsStorage;
 
 registerCleanupFunction(function () {
   while (gBrowser.tabs.length > 1)