Merge inbound to mozilla-central. a=merge

dom/localstorage/ActorsParent.cpp

@@ -38,6 +38,7 @@
 #include "mozilla/ipc/PBackgroundParent.h"
 #include "mozilla/ipc/PBackgroundSharedTypes.h"
 #include "mozilla/Logging.h"
+#include "mozilla/storage/Variant.h"
 #include "nsClassHashtable.h"
 #include "nsDataHashtable.h"
 #include "nsExceptionHandler.h"
@@ -2344,6 +2345,9 @@ class PrepareDatastoreOp
       public SupportsCheckedUnsafePtr<CheckIf<DiagnosticAssertEnabled>> {
   class LoadDataOp;
 
+  class CompressFunction;
+  class CompressibleFunction;
+
   enum class NestedState {
     // The nesting has not yet taken place. Next step is
     // CheckExistingOperations.
@@ -2537,6 +2541,23 @@ class PrepareDatastoreOp::LoadDataOp final
   void Cleanup() override;
 };
 
+class PrepareDatastoreOp::CompressFunction final : public mozIStorageFunction {
+ private:
+  ~CompressFunction() = default;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+};
+
+class PrepareDatastoreOp::CompressibleFunction final
+    : public mozIStorageFunction {
+ private:
+  ~CompressibleFunction() = default;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+};
+
 class PrepareObserverOp : public LSRequestBase {
   const LSRequestPrepareObserverParams mParams;
   nsCString mOrigin;
@@ -7034,15 +7055,33 @@ nsresult PrepareDatastoreOp::DatabaseWork() {
     mozStorageTransaction transaction(
         connection, false, mozIStorageConnection::TRANSACTION_IMMEDIATE);
 
+    nsCOMPtr<mozIStorageFunction> function = new CompressFunction();
+
+    rv =
+        connection->CreateFunction(NS_LITERAL_CSTRING("compress"), 1, function);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    function = new CompressibleFunction();
+
+    rv = connection->CreateFunction(NS_LITERAL_CSTRING("compressible"), 1,
+                                    function);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
     nsCOMPtr<mozIStorageStatement> stmt;
     rv = connection->CreateStatement(
-        NS_LITERAL_CSTRING("INSERT INTO data (key, value, utf16Length) "
-                           "SELECT key, value, utf16Length(value) "
-                           "FROM webappsstore2 "
-                           "WHERE originKey = :originKey "
-                           "AND originAttributes = :originAttributes;"
+        NS_LITERAL_CSTRING(
+            "INSERT INTO data (key, value, utf16Length, compressed) "
+            "SELECT key, compress(value), utf16Length(value), "
+            "compressible(value) "
+            "FROM webappsstore2 "
+            "WHERE originKey = :originKey "
+            "AND originAttributes = :originAttributes;"
 
-                           ),
+            ),
         getter_AddRefs(stmt));
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
@@ -7058,6 +7097,16 @@ nsresult PrepareDatastoreOp::DatabaseWork() {
       return rv;
     }
 
+    rv = connection->RemoveFunction(NS_LITERAL_CSTRING("compress"));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = connection->RemoveFunction(NS_LITERAL_CSTRING("compressible"));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
     rv = connection->CreateStatement(
         NS_LITERAL_CSTRING("UPDATE database SET usage = :usage;"),
         getter_AddRefs(stmt));
@@ -7766,6 +7815,80 @@ void PrepareDatastoreOp::LoadDataOp::Cleanup() {
   ConnectionDatastoreOperationBase::Cleanup();
 }
 
+NS_IMPL_ISUPPORTS(PrepareDatastoreOp::CompressFunction, mozIStorageFunction)
+
+NS_IMETHODIMP
+PrepareDatastoreOp::CompressFunction::OnFunctionCall(
+    mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aFunctionArguments);
+  MOZ_ASSERT(aResult);
+
+#ifdef DEBUG
+  {
+    uint32_t argCount;
+    MOZ_ALWAYS_SUCCEEDS(aFunctionArguments->GetNumEntries(&argCount));
+    MOZ_ASSERT(argCount == 1);
+
+    int32_t type;
+    MOZ_ALWAYS_SUCCEEDS(aFunctionArguments->GetTypeOfIndex(0, &type));
+    MOZ_ASSERT(type == mozIStorageValueArray::VALUE_TYPE_TEXT);
+  }
+#endif
+
+  nsCString value;
+  nsresult rv = aFunctionArguments->GetUTF8String(0, value);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCString compressed;
+  if (!SnappyCompress(value, compressed)) {
+    compressed = value;
+  }
+
+  nsCOMPtr<nsIVariant> result = new storage::UTF8TextVariant(compressed);
+
+  result.forget(aResult);
+  return NS_OK;
+}
+
+NS_IMPL_ISUPPORTS(PrepareDatastoreOp::CompressibleFunction, mozIStorageFunction)
+
+NS_IMETHODIMP
+PrepareDatastoreOp::CompressibleFunction::OnFunctionCall(
+    mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aFunctionArguments);
+  MOZ_ASSERT(aResult);
+
+#ifdef DEBUG
+  {
+    uint32_t argCount;
+    MOZ_ALWAYS_SUCCEEDS(aFunctionArguments->GetNumEntries(&argCount));
+    MOZ_ASSERT(argCount == 1);
+
+    int32_t type;
+    MOZ_ALWAYS_SUCCEEDS(aFunctionArguments->GetTypeOfIndex(0, &type));
+    MOZ_ASSERT(type == mozIStorageValueArray::VALUE_TYPE_TEXT);
+  }
+#endif
+
+  nsCString value;
+  nsresult rv = aFunctionArguments->GetUTF8String(0, value);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCString compressed;
+  bool compressible = SnappyCompress(value, compressed);
+
+  nsCOMPtr<nsIVariant> result = new storage::IntegerVariant(compressible);
+
+  result.forget(aResult);
+  return NS_OK;
+}
+
 /*******************************************************************************
  * PrepareObserverOp
  ******************************************************************************/

dom/quota/ActorsParent.cpp

@@ -218,7 +218,7 @@ const char kResourceOriginPrefix[] = "resource://";
 #define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
 #define LS_ARCHIVE_TMP_FILE_NAME "ls-archive-tmp.sqlite"
 
-const uint32_t kLocalStorageArchiveVersion = 3;
+const uint32_t kLocalStorageArchiveVersion = 4;
 
 const char kProfileDoChangeTopic[] = "profile-do-change";
 
@@ -446,6 +446,7 @@ nsresult LoadLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
   return NS_OK;
 }
 
+/*
 nsresult SaveLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
                                         uint32_t aVersion) {
   AssertIsOnIOThread();
@@ -471,6 +472,7 @@ nsresult SaveLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
 
   return NS_OK;
 }
+*/
 
 /******************************************************************************
  * Quota manager class declarations
@@ -5331,7 +5333,7 @@ nsresult QuotaManager::DowngradeLocalStorageArchive(
   return NS_OK;
 }
 
-nsresult QuotaManager::UpgradeLocalStorageArchiveFrom0To1(
+nsresult QuotaManager::UpgradeLocalStorageArchiveFromLessThan4To4(
     nsCOMPtr<mozIStorageConnection>& aConnection) {
   AssertIsOnIOThread();
   MOZ_ASSERT(CachedNextGenLocalStorageEnabled());
@@ -5341,7 +5343,7 @@ nsresult QuotaManager::UpgradeLocalStorageArchiveFrom0To1(
     return rv;
   }
 
-  rv = InitializeLocalStorageArchive(aConnection, 1);
+  rv = InitializeLocalStorageArchive(aConnection, 4);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -5349,25 +5351,20 @@ nsresult QuotaManager::UpgradeLocalStorageArchiveFrom0To1(
   return NS_OK;
 }
 
-nsresult QuotaManager::UpgradeLocalStorageArchiveFrom1To2(
+/*
+nsresult QuotaManager::UpgradeLocalStorageArchiveFrom4To5(
     nsCOMPtr<mozIStorageConnection>& aConnection) {
-  nsresult rv = SaveLocalStorageArchiveVersion(aConnection, 2);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
-nsresult QuotaManager::UpgradeLocalStorageArchiveFrom2To3(
-    nsCOMPtr<mozIStorageConnection>& aConnection) {
-  nsresult rv = SaveLocalStorageArchiveVersion(aConnection, 3);
+  AssertIsOnIOThread();
+  MOZ_ASSERT(CachedNextGenLocalStorageEnabled());
+
+  nsresult rv = SaveLocalStorageArchiveVersion(aConnection, 5);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
+*/
 
 #ifdef DEBUG
 
@@ -5591,18 +5588,17 @@ nsresult QuotaManager::EnsureStorageIsInitialized() {
           return rv;
         }
       } else {
-        static_assert(kLocalStorageArchiveVersion == 3,
+        static_assert(kLocalStorageArchiveVersion == 4,
                       "Upgrade function needed due to LocalStorage archive "
                       "version increase.");
 
         while (version != kLocalStorageArchiveVersion) {
-          if (version == 0) {
-            rv = UpgradeLocalStorageArchiveFrom0To1(connection);
-          } else if (version == 1) {
-            rv = UpgradeLocalStorageArchiveFrom1To2(connection);
-          } else if (version == 2) {
-            rv = UpgradeLocalStorageArchiveFrom2To3(connection);
-          } else {
+          if (version < 4) {
+            rv = UpgradeLocalStorageArchiveFromLessThan4To4(connection);
+          } /* else if (version == 4) {
+            rv = UpgradeLocalStorageArchiveFrom4To5(connection);
+          } */
+          else {
             QM_WARNING(
                 "Unable to initialize LocalStorage archive, no upgrade path is "
                 "available!");

dom/quota/QuotaManager.h

@@ -483,14 +483,12 @@ class QuotaManager final : public BackgroundThreadObject {
   nsresult DowngradeLocalStorageArchive(
       nsCOMPtr<mozIStorageConnection>& aConnection);
 
-  nsresult UpgradeLocalStorageArchiveFrom0To1(
+  nsresult UpgradeLocalStorageArchiveFromLessThan4To4(
       nsCOMPtr<mozIStorageConnection>& aConnection);
 
-  nsresult UpgradeLocalStorageArchiveFrom1To2(
-      nsCOMPtr<mozIStorageConnection>& aConnection);
-
-  nsresult UpgradeLocalStorageArchiveFrom2To3(
-      nsCOMPtr<mozIStorageConnection>& aConnection);
+  /*
+  nsresult UpgradeLocalStorageArchiveFrom4To5();
+  */
 
   nsresult InitializeRepository(PersistenceType aPersistenceType);
 

dom/quota/test/unit/test_localStorageArchive3upgrade.js

@@ -1,65 +0,0 @@
-/**
- * Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-/**
- * This test is mainly to verify that local storage directories are not removed
- * during local storage archive upgrade from version 2 to version 3.
- * See bug 1513937.
- */
-
-async function testSteps() {
-  const lsDirs = [
-    "storage/default/http+++example.com/ls",
-    "storage/default/http+++localhost/ls",
-    "storage/default/http+++www.mozilla.org/ls",
-  ];
-
-  info("Clearing");
-
-  let request = clear();
-  await requestFinished(request);
-
-  info("Installing package");
-
-  // The profile contains three initialized origin directories with local
-  // storage data, local storage archive, a script for origin initialization,
-  // the storage database and the web apps store database:
-  // - storage/default/https+++example.com
-  // - storage/default/https+++localhost
-  // - storage/default/https+++www.mozilla.org
-  // - storage/ls-archive.sqlite
-  // - create_db.js
-  // - storage.sqlite
-  // - webappsstore.sqlite
-  // The file create_db.js in the package was run locally (with a build with
-  // local storage archive version 2), specifically it was temporarily added to
-  // xpcshell.ini and then executed:
-  //   mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
-  // Note: to make it become the profile in the test, additional manual steps
-  // are needed.
-  // 1. Remove the folder "storage/temporary".
-  installPackage("localStorageArchive3upgrade_profile");
-
-  info("Checking ls dirs");
-
-  for (let lsDir of lsDirs) {
-    let dir = getRelativeFile(lsDir);
-
-    exists = dir.exists();
-    ok(exists, "ls directory does exist");
-  }
-
-  request = init();
-  request = await requestFinished(request);
-
-  info("Checking ls dirs");
-
-  for (let lsDir of lsDirs) {
-    let dir = getRelativeFile(lsDir);
-
-    exists = dir.exists();
-    ok(exists, "ls directory does exist");
-  }
-}

dom/quota/test/unit/test_localStorageArchive4upgrade.js

@@ -4,18 +4,34 @@
  */
 
 /**
- * This test is mainly to verify that local storage directories are not removed
- * during local storage archive upgrade from version 1 to version 2.
- * See bug 1546310.
+ * This test is mainly to verify that local storage directories are removed
+ * during local storage archive upgrade from version 3 to version 4.
+ * See bug 1549654.
  */
 
 async function testSteps() {
   const lsDirs = [
-    "storage/default/http+++example.com/ls",
     "storage/default/http+++localhost/ls",
     "storage/default/http+++www.mozilla.org/ls",
+    "storage/default/http+++example.com/ls",
   ];
 
+  const principalInfos = [
+    "http://localhost",
+    "http://www.mozilla.org",
+    "http://example.com",
+  ];
+
+  const data = [
+    { key: "foo0", value: "bar" },
+    { key: "foo1", value: "A" },
+    { key: "foo2", value: "A".repeat(100) },
+  ];
+
+  function getLocalStorage(principal) {
+    return Services.domStorageManager.createStorage(null, principal, "");
+  }
+
   info("Clearing");
 
   let request = clear();
@@ -34,13 +50,13 @@ async function testSteps() {
   // - storage.sqlite
   // - webappsstore.sqlite
   // The file create_db.js in the package was run locally (with a build with
-  // local storage archive version 1), specifically it was temporarily added to
+  // local storage archive version 3), specifically it was temporarily added to
   // xpcshell.ini and then executed:
   //   mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
   // Note: to make it become the profile in the test, additional manual steps
   // are needed.
   // 1. Remove the folder "storage/temporary".
-  installPackage("localStorageArchive2upgrade_profile");
+  installPackage("localStorageArchive4upgrade_profile");
 
   info("Checking ls dirs");
 
@@ -60,6 +76,20 @@ async function testSteps() {
     let dir = getRelativeFile(lsDir);
 
     exists = dir.exists();
-    ok(exists, "ls directory does exist");
+    ok(!exists, "ls directory doesn't exist");
+  }
+
+  info("Getting storages");
+
+  let storages = [];
+  for (let i = 0; i < principalInfos.length; i++) {
+    let storage = getLocalStorage(getPrincipal(principalInfos[i]));
+    storages.push(storage);
+  }
+
+  info("Verifying data");
+
+  for (let i = 0; i < storages.length; i++) {
+    is(storages[i].getItem(data[i].key), data[i].value, "Correct value");
   }
 }

dom/quota/test/unit/xpcshell.ini

@@ -14,8 +14,7 @@ support-files =
   idbSubdirUpgrade1_profile.zip
   idbSubdirUpgrade2_profile.zip
   localStorageArchive1upgrade_profile.zip
-  localStorageArchive2upgrade_profile.zip
-  localStorageArchive3upgrade_profile.zip
+  localStorageArchive4upgrade_profile.zip
   localStorageArchiveDowngrade_profile.zip
   morgueCleanup_profile.zip
   obsoleteOriginAttributes_profile.zip
@@ -38,8 +37,7 @@ support-files =
 [test_initTemporaryStorage.js]
 [test_listInitializedOrigins.js]
 [test_localStorageArchive1upgrade.js]
-[test_localStorageArchive2upgrade.js]
-[test_localStorageArchive3upgrade.js]
+[test_localStorageArchive4upgrade.js]
 [test_localStorageArchiveDowngrade.js]
 [test_morgueCleanup.js]
 [test_obsoleteOriginAttributesUpgrade.js]

modules/libpref/init/all.js

@@ -1624,6 +1624,7 @@ pref("network.protocol-handler.external.vbscript", false);
 pref("network.protocol-handler.external.javascript", false);
 pref("network.protocol-handler.external.data", false);
 pref("network.protocol-handler.external.ms-help", false);
+pref("network.protocol-handler.external.res", false);
 pref("network.protocol-handler.external.shell", false);
 pref("network.protocol-handler.external.vnd.ms.radio", false);
 #ifdef XP_MACOSX