Bug 1731739 part 3 - Migrate permissions for 3rdPartyStorage to site keys, r=pbz,timhuang

- Increment permissions database schema version
- Apply GetSite to origin keys with type prefixed by "3rdPartyStorage^"
  - Done in a transaction
- Add unit test for migration

Differential Revision: https://phabricator.services.mozilla.com/D130676

extensions/permissions/PermissionManager.cpp

@@ -39,6 +39,7 @@
 #include "nsIURIMutator.h"
 #include "nsIWritablePropertyBag2.h"
 #include "nsReadLine.h"
+#include "nsTHashSet.h"
 #include "nsToolkitCompsCID.h"
 
 using namespace mozilla::dom;
@@ -46,7 +47,7 @@ using namespace mozilla::dom;
 namespace mozilla {
 
 #define PERMISSIONS_FILE_NAME "permissions.sqlite"
-#define HOSTS_SCHEMA_VERSION 11
+#define HOSTS_SCHEMA_VERSION 12
 
 // Default permissions are read from a URL - this is the preference we read
 // to find that URL. If not set, don't use any default permissions.
@@ -1361,6 +1362,84 @@ nsresult PermissionManager::TryInitDB(bool aRemoveFile,
             "SUBSTR(type, 0, 18) == \"storageAccessAPI^\";"));
         NS_ENSURE_SUCCESS(rv, rv);
 
+        rv = data->mDBConn->SetSchemaVersion(11);
+        NS_ENSURE_SUCCESS(rv, rv);
+      }
+
+        // fall through to the next upgrade
+        [[fallthrough]];
+
+      case 11: {
+        // Migrate 3rdPartyStorage keys to a site scope
+        rv = data->mDBConn->BeginTransaction();
+        NS_ENSURE_SUCCESS(rv, rv);
+        nsCOMPtr<mozIStorageStatement> updateStmt;
+        rv = data->mDBConn->CreateStatement(
+            nsLiteralCString("UPDATE moz_perms SET origin = ?2 WHERE id = ?1"),
+            getter_AddRefs(updateStmt));
+        NS_ENSURE_SUCCESS(rv, rv);
+
+        nsCOMPtr<mozIStorageStatement> deleteStmt;
+        rv = data->mDBConn->CreateStatement(
+            nsLiteralCString("DELETE FROM moz_perms WHERE id = ?1"),
+            getter_AddRefs(deleteStmt));
+        NS_ENSURE_SUCCESS(rv, rv);
+
+        nsCOMPtr<mozIStorageStatement> selectStmt;
+        rv = data->mDBConn->CreateStatement(
+            nsLiteralCString("SELECT id, origin, type FROM moz_perms WHERE "
+                             " SUBSTR(type, 0, 17) == \"3rdPartyStorage^\""),
+            getter_AddRefs(selectStmt));
+        NS_ENSURE_SUCCESS(rv, rv);
+
+        nsTHashSet<nsCStringHashKey> deduplicationSet;
+        bool hasResult;
+        while (NS_SUCCEEDED(selectStmt->ExecuteStep(&hasResult)) && hasResult) {
+          int64_t id;
+          rv = selectStmt->GetInt64(0, &id);
+          NS_ENSURE_SUCCESS(rv, rv);
+
+          nsCString origin;
+          rv = selectStmt->GetUTF8String(1, origin);
+          NS_ENSURE_SUCCESS(rv, rv);
+
+          nsCString type;
+          rv = selectStmt->GetUTF8String(2, type);
+          NS_ENSURE_SUCCESS(rv, rv);
+
+          nsCOMPtr<nsIURI> uri;
+          rv = NS_NewURI(getter_AddRefs(uri), origin);
+          if (NS_FAILED(rv)) {
+            continue;
+          }
+          nsCString site;
+          rv = nsEffectiveTLDService::GetInstance()->GetSite(uri, site);
+          if (NS_WARN_IF(NS_FAILED(rv))) {
+            continue;
+          }
+
+          nsCString deduplicationKey =
+              nsPrintfCString("%s,%s", site.get(), type.get());
+          if (deduplicationSet.Contains(deduplicationKey)) {
+            rv = deleteStmt->BindInt64ByIndex(0, id);
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            rv = deleteStmt->Execute();
+            NS_ENSURE_SUCCESS(rv, rv);
+          } else {
+            deduplicationSet.Insert(deduplicationKey);
+            rv = updateStmt->BindInt64ByIndex(0, id);
+            NS_ENSURE_SUCCESS(rv, rv);
+            rv = updateStmt->BindUTF8StringByIndex(1, site);
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            rv = updateStmt->Execute();
+            NS_ENSURE_SUCCESS(rv, rv);
+          }
+        }
+        rv = data->mDBConn->CommitTransaction();
+        NS_ENSURE_SUCCESS(rv, rv);
+
         rv = data->mDBConn->SetSchemaVersion(HOSTS_SCHEMA_VERSION);
         NS_ENSURE_SUCCESS(rv, rv);
       }

extensions/permissions/test/unit/test_permmanager_load_invalid_entries.js

@@ -232,7 +232,7 @@ function run_test() {
 
   // The schema should be upgraded to 11, and a 'modificationTime' column should
   // exist with all records having a value of 0.
-  Assert.equal(connection.schemaVersion, 11);
+  Assert.equal(connection.schemaVersion, 12);
 
   let select = connection.createStatement(
     "SELECT modificationTime FROM moz_perms"

extensions/permissions/test/unit/test_permmanager_migrate_11-12.js

@@ -0,0 +1,232 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+ChromeUtils.defineModuleGetter(
+  this,
+  "PlacesTestUtils",
+  "resource://testing-common/PlacesTestUtils.jsm"
+);
+
+var PERMISSIONS_FILE_NAME = "permissions.sqlite";
+
+function GetPermissionsFile(profile) {
+  let file = profile.clone();
+  file.append(PERMISSIONS_FILE_NAME);
+  return file;
+}
+
+add_task(async function test() {
+  // Create and set up the permissions database.
+  Services.prefs.setCharPref("permissions.manager.defaultsUrl", "");
+  let profile = do_get_profile();
+
+  // We need to execute a pm method to be sure that the DB is fully
+  // initialized.
+  var pm = Services.perms;
+  Assert.equal(pm.all.length, 0, "No cookies");
+
+  let db = Services.storage.openDatabase(GetPermissionsFile(profile));
+  db.schemaVersion = 11;
+
+  let stmt6Insert = db.createStatement(
+    "INSERT INTO moz_perms (" +
+      "id, origin, type, permission, expireType, expireTime, modificationTime" +
+      ") VALUES (" +
+      ":id, :origin, :type, :permission, :expireType, :expireTime, :modificationTime" +
+      ")"
+  );
+
+  let id = 0;
+
+  function insertOrigin(
+    origin,
+    type,
+    permission,
+    expireType,
+    expireTime,
+    modificationTime
+  ) {
+    let thisId = id++;
+
+    stmt6Insert.bindByName("id", thisId);
+    stmt6Insert.bindByName("origin", origin);
+    stmt6Insert.bindByName("type", type);
+    stmt6Insert.bindByName("permission", permission);
+    stmt6Insert.bindByName("expireType", expireType);
+    stmt6Insert.bindByName("expireTime", expireTime);
+    stmt6Insert.bindByName("modificationTime", modificationTime);
+
+    try {
+      stmt6Insert.execute();
+    } finally {
+      stmt6Insert.reset();
+    }
+
+    return {
+      id: thisId,
+      origin,
+      type,
+      permission,
+      expireType,
+      expireTime,
+      modificationTime,
+    };
+  }
+
+  insertOrigin("https://a.com", "3rdPartyStorage^https://b.com", 2, 0, 0, 0);
+  insertOrigin(
+    "https://www.a.com",
+    "3rdPartyStorage^https://www.c.com",
+    2,
+    0,
+    0,
+    0
+  );
+  insertOrigin(
+    "https://localhost",
+    "3rdPartyStorage^http://www.c.com",
+    2,
+    0,
+    0,
+    0
+  );
+
+  insertOrigin(
+    "https://www.b.co.uk",
+    "3rdPartyStorage^https://www.a.co.uk",
+    2,
+    0,
+    0,
+    0
+  );
+
+  insertOrigin(
+    "https://sub.www.b.co.uk",
+    "3rdPartyStorage^https://sub.www.a.co.uk",
+    2,
+    0,
+    0,
+    0
+  );
+
+  insertOrigin(
+    "https://example.b.co.uk",
+    "3rdPartyStorage^https://www.a.co.uk",
+    2,
+    0,
+    0,
+    0
+  );
+
+  insertOrigin(
+    "https://[::1]",
+    "3rdPartyStorage^https://www.a.co.uk",
+    2,
+    0,
+    0,
+    0
+  );
+  // Close the db connection
+  stmt6Insert.finalize();
+  db.close();
+  db = null;
+  info(Services.perms.all);
+
+  let expected = [
+    ["https://a.com", "3rdPartyStorage^https://b.com", 2, 0, 0, 0],
+    ["https://a.com", "3rdPartyStorage^https://www.c.com", 2, 0, 0, 0],
+    ["https://localhost", "3rdPartyStorage^http://www.c.com", 2, 0, 0, 0],
+    ["https://b.co.uk", "3rdPartyStorage^https://www.a.co.uk", 2, 0, 0, 0],
+    ["https://b.co.uk", "3rdPartyStorage^https://sub.www.a.co.uk", 2, 0, 0, 0],
+    ["https://[::1]", "3rdPartyStorage^https://www.a.co.uk", 2, 0, 0, 0],
+  ];
+
+  let found = expected.map(it => 0);
+
+  // Add some places to the places database
+  await PlacesTestUtils.addVisits(
+    Services.io.newURI("https://foo.com/some/other/subdirectory")
+  );
+  await PlacesTestUtils.addVisits(
+    Services.io.newURI("ftp://some.subdomain.of.foo.com:8000/some/subdirectory")
+  );
+  await PlacesTestUtils.addVisits(Services.io.newURI("ftp://127.0.0.1:8080"));
+  await PlacesTestUtils.addVisits(Services.io.newURI("https://localhost:8080"));
+
+  // This will force the permission-manager to reload the data.
+  Services.obs.notifyObservers(null, "testonly-reload-permissions-from-disk");
+
+  info(Services.perms.all);
+
+  // Force initialization of the PermissionManager
+  for (let permission of Services.perms.all) {
+    let isExpected = false;
+
+    expected.forEach((it, i) => {
+      if (
+        permission.principal.origin == it[0] &&
+        permission.type == it[1] &&
+        permission.capability == it[2] &&
+        permission.expireType == it[3] &&
+        permission.expireTime == it[4]
+      ) {
+        isExpected = true;
+        found[i]++;
+      }
+    });
+
+    Assert.ok(
+      isExpected,
+      "Permission " +
+        (isExpected ? "should" : "shouldn't") +
+        " be in permission database: " +
+        permission.principal.origin +
+        ", " +
+        permission.type +
+        ", " +
+        permission.capability +
+        ", " +
+        permission.expireType +
+        ", " +
+        permission.expireTime
+    );
+  }
+  info(expected);
+  info(found);
+
+  found.forEach((count, i) => {
+    Assert.ok(
+      count == 1,
+      "Expected count = 1, got count = " +
+        count +
+        " for permission " +
+        expected[i]
+    );
+  });
+
+  // Check to make sure that all of the tables which we care about are present
+  {
+    db = Services.storage.openDatabase(GetPermissionsFile(profile));
+    Assert.ok(db.tableExists("moz_perms"));
+    Assert.ok(db.tableExists("moz_hosts"));
+    Assert.ok(!db.tableExists("moz_perms_v6"));
+
+    let mozHostsCount = db.createStatement("SELECT count(*) FROM moz_hosts");
+    try {
+      mozHostsCount.executeStep();
+      Assert.equal(mozHostsCount.getInt64(0), 0);
+    } finally {
+      mozHostsCount.finalize();
+    }
+
+    let mozPermsCount = db.createStatement("SELECT count(*) FROM moz_perms");
+    try {
+      mozPermsCount.executeStep();
+      Assert.equal(mozPermsCount.getInt64(0), expected.length);
+    } finally {
+      mozPermsCount.finalize();
+    }
+
+    db.close();
+  }
+});

extensions/permissions/test/unit/xpcshell.ini

@@ -43,6 +43,8 @@ skip-if = toolkit == 'android' # Android doesn't use places
 skip-if = toolkit == 'android' # Android doesn't use places
 [test_permmanager_migrate_10-11.js]
 skip-if = toolkit == 'android' # Android doesn't use places
+[test_permmanager_migrate_11-12.js]
+skip-if = toolkit == 'android' # Android doesn't use places
 [test_permmanager_oa_strip.js]
 [test_permmanager_site_scope.js]
 [test_permmanager_remove_add_update.js]