Bug 1923663, part 1 - Cookie DB migration to remove first-party partitioned cookies. r=cookie-reviewers,edgul

Depends on D226127

Differential Revision: https://phabricator.services.mozilla.com/D225692

modules/libpref/init/StaticPrefList.yaml

@@ -12383,6 +12383,20 @@
   value: @IS_NIGHTLY_BUILD@
   mirror: always
 
+# Updated to match the target count when we migrate the unpartitioned CHIPS
+# cookies to their first-party partition.
+- name: network.cookie.CHIPS.lastMigrateDatabase
+  type: RelaxedAtomicUint32
+  value: 0
+  mirror: always
+
+# Used to increase the number of times we want to have migrated the database.
+# This lets us remotely perform a database migration with Nimbus.
+- name: network.cookie.CHIPS.migrateDatabaseTarget
+  type: RelaxedAtomicUint32
+  value: 0
+  mirror: always
+
 # Stale threshold for cookies in seconds.
 - name: network.cookie.staleThreshold
   type: uint32_t

netwerk/cookie/CookiePersistentStorage.cpp

@@ -9,6 +9,7 @@
 #include "CookiePersistentStorage.h"
 
 #include "mozilla/FileUtils.h"
+#include "mozilla/StaticPrefs_network.h"
 #include "mozilla/glean/GleanMetrics.h"
 #include "mozilla/ScopeExit.h"
 #include "mozilla/Telemetry.h"
@@ -205,6 +206,149 @@ SetInBrowserFromOriginAttributesSQLFunction::OnFunctionCall(
   return NS_OK;
 }
 
+class CalculatePartitionKeyFromHostSQLFunction final
+    : public mozIStorageFunction {
+  ~CalculatePartitionKeyFromHostSQLFunction() = default;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+};
+
+NS_IMPL_ISUPPORTS(CalculatePartitionKeyFromHostSQLFunction,
+                  mozIStorageFunction);
+
+NS_IMETHODIMP
+CalculatePartitionKeyFromHostSQLFunction::OnFunctionCall(
+    mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
+  nsresult rv;
+
+  nsAutoCString host;
+  rv = aFunctionArguments->GetUTF8String(0, host);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // This is a bit hacky. However, CHIPS cookies can only be set in secure
+  // contexts. So, the scheme has to be https.
+  nsAutoCString schemeHost;
+  schemeHost.AssignLiteral("https://");
+
+  if (*host.get() == '.') {
+    schemeHost.Append(nsDependentCSubstring(host, 1));
+  } else {
+    schemeHost.Append(host);
+  }
+
+  nsCOMPtr<nsIURI> uri;
+  rv = NS_NewURI(getter_AddRefs(uri), schemeHost);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  OriginAttributes attrsFromHost;
+  attrsFromHost.SetPartitionKey(uri, false);
+
+  RefPtr<nsVariant> outVar(new nsVariant());
+  rv = outVar->SetAsAString(attrsFromHost.mPartitionKey);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  outVar.forget(aResult);
+
+  return NS_OK;
+}
+
+class FetchPartitionKeyFromOAsSQLFunction final : public mozIStorageFunction {
+  ~FetchPartitionKeyFromOAsSQLFunction() = default;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+};
+
+NS_IMPL_ISUPPORTS(FetchPartitionKeyFromOAsSQLFunction, mozIStorageFunction);
+
+NS_IMETHODIMP
+FetchPartitionKeyFromOAsSQLFunction::OnFunctionCall(
+    mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
+  nsresult rv;
+
+  nsAutoCString suffix;
+  rv = aFunctionArguments->GetUTF8String(0, suffix);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  OriginAttributes attrsFromSuffix;
+  bool success = attrsFromSuffix.PopulateFromSuffix(suffix);
+  NS_ENSURE_TRUE(success, NS_ERROR_FAILURE);
+
+  RefPtr<nsVariant> outVar(new nsVariant());
+  rv = outVar->SetAsAString(attrsFromSuffix.mPartitionKey);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  outVar.forget(aResult);
+
+  return NS_OK;
+}
+
+class UpdateOAsWithPartitionHostSQLFunction final : public mozIStorageFunction {
+  ~UpdateOAsWithPartitionHostSQLFunction() = default;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+};
+
+NS_IMPL_ISUPPORTS(UpdateOAsWithPartitionHostSQLFunction, mozIStorageFunction);
+
+NS_IMETHODIMP
+UpdateOAsWithPartitionHostSQLFunction::OnFunctionCall(
+    mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
+  nsresult rv;
+
+  nsAutoCString formattedOriginAttributes;
+  rv = aFunctionArguments->GetUTF8String(0, formattedOriginAttributes);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoCString partitionKeyHost;
+  rv = aFunctionArguments->GetUTF8String(1, partitionKeyHost);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  OriginAttributes attrsFromSuffix;
+  bool success = attrsFromSuffix.PopulateFromSuffix(formattedOriginAttributes);
+  // On failure, do not alter the OA.
+  if (!success) {
+    RefPtr<nsVariant> outVar(new nsVariant());
+    rv = outVar->SetAsACString(formattedOriginAttributes);
+    NS_ENSURE_SUCCESS(rv, rv);
+    outVar.forget(aResult);
+    return NS_OK;
+  }
+
+  // This is a bit hacky. However, CHIPS cookies can only be set in secure
+  // contexts. So, the scheme has to be https.
+  nsAutoCString schemeHost;
+  schemeHost.AssignLiteral("https://");
+
+  if (*partitionKeyHost.get() == '.') {
+    schemeHost.Append(nsDependentCSubstring(partitionKeyHost, 1));
+  } else {
+    schemeHost.Append(partitionKeyHost);
+  }
+
+  nsCOMPtr<nsIURI> uri;
+  rv = NS_NewURI(getter_AddRefs(uri), schemeHost);
+  // On failure, do not alter the OA.
+  if (NS_FAILED(rv)) {
+    RefPtr<nsVariant> outVar(new nsVariant());
+    rv = outVar->SetAsACString(formattedOriginAttributes);
+    NS_ENSURE_SUCCESS(rv, rv);
+    outVar.forget(aResult);
+    return NS_OK;
+  }
+
+  attrsFromSuffix.SetPartitionKey(uri, false);
+  attrsFromSuffix.CreateSuffix(formattedOriginAttributes);
+
+  RefPtr<nsVariant> outVar(new nsVariant());
+  rv = outVar->SetAsACString(formattedOriginAttributes);
+  NS_ENSURE_SUCCESS(rv, rv);
+  outVar.forget(aResult);
+  return NS_OK;
+}
+
 /******************************************************************************
  * DBListenerErrorHandler impl:
  * Parent class for our async storage listeners that handles the logging of
@@ -1481,6 +1625,12 @@ CookiePersistentStorage::OpenDBResult CookiePersistentStorage::TryInitDB(
     return RESULT_OK;
   }
 
+  if (StaticPrefs::network_cookie_CHIPS_enabled() &&
+      StaticPrefs::network_cookie_CHIPS_lastMigrateDatabase() <
+          StaticPrefs::network_cookie_CHIPS_migrateDatabaseTarget()) {
+    CookiePersistentStorage::MoveUnpartitionedChipsCookies();
+  }
+
   // check whether to import or just read in the db
   if (tableExists) {
     return Read();
@@ -1489,6 +1639,47 @@ CookiePersistentStorage::OpenDBResult CookiePersistentStorage::TryInitDB(
   return RESULT_OK;
 }
 
+void CookiePersistentStorage::MoveUnpartitionedChipsCookies() {
+  nsCOMPtr<mozIStorageFunction> fetchPartitionKeyFromOAs(
+      new FetchPartitionKeyFromOAsSQLFunction());
+  NS_ENSURE_TRUE_VOID(fetchPartitionKeyFromOAs);
+
+  constexpr auto fetchPartitionKeyFromOAsName =
+      "FETCH_PARTITIONKEY_FROM_OAS"_ns;
+
+  nsresult rv = mSyncConn->CreateFunction(fetchPartitionKeyFromOAsName, 1,
+                                          fetchPartitionKeyFromOAs);
+  NS_ENSURE_SUCCESS_VOID(rv);
+
+  nsCOMPtr<mozIStorageFunction> updateOAsWithPartitionHost(
+      new UpdateOAsWithPartitionHostSQLFunction());
+  NS_ENSURE_TRUE_VOID(updateOAsWithPartitionHost);
+
+  constexpr auto updateOAsWithPartitionHostName =
+      "UPDATE_OAS_WITH_PARTITION_HOST"_ns;
+
+  rv = mSyncConn->CreateFunction(updateOAsWithPartitionHostName, 2,
+                                 updateOAsWithPartitionHost);
+  NS_ENSURE_SUCCESS_VOID(rv);
+
+  // Move all cookies with the Partitioned attribute set into their first-party
+  // partitioned storage by updating the origin attributes. Overwrite any
+  // existing cookies that may already be there.
+  rv = mSyncConn->ExecuteSimpleSQL(nsLiteralCString(
+      "UPDATE OR REPLACE moz_cookies  "
+      "SET originAttributes = UPDATE_OAS_WITH_PARTITION_HOST(originAttributes, "
+      "host) "
+      "WHERE FETCH_PARTITIONKEY_FROM_OAS(originAttributes) = '' "
+      "AND isPartitionedAttributeSet = 1;"));
+  NS_ENSURE_SUCCESS_VOID(rv);
+
+  rv = mSyncConn->RemoveFunction(fetchPartitionKeyFromOAsName);
+  NS_ENSURE_SUCCESS_VOID(rv);
+
+  rv = mSyncConn->RemoveFunction(updateOAsWithPartitionHostName);
+  NS_ENSURE_SUCCESS_VOID(rv);
+}
+
 void CookiePersistentStorage::RebuildCorruptDB() {
   NS_ASSERTION(!mDBConn, "shouldn't have an open db connection");
   NS_ASSERTION(mCorruptFlag == CookiePersistentStorage::CLOSING_FOR_REBUILD,
@@ -1871,6 +2062,15 @@ void CookiePersistentStorage::InitDBConn() {
     RemoveCookieFromDB(*cookie);
   }
 
+  // We will have migrated CHIPS cookies if the pref is set, and .unset it
+  // to prevent dupliacted work. This has to happen in the main thread though,
+  // so we waited to this point.
+  if (StaticPrefs::network_cookie_CHIPS_enabled()) {
+    Preferences::SetUint(
+        "network.cookie.CHIPS.lastMigrateDatabase",
+        StaticPrefs::network_cookie_CHIPS_migrateDatabaseTarget());
+  }
+
   nsCOMPtr<nsIObserverService> os = services::GetObserverService();
   if (os) {
     os->NotifyObservers(nullptr, "cookie-db-read", nullptr);

netwerk/cookie/CookiePersistentStorage.h

@@ -97,6 +97,7 @@ class CookiePersistentStorage final : public CookieStorage {
 
   OpenDBResult TryInitDB(bool aRecreateDB);
   OpenDBResult Read();
+  void MoveUnpartitionedChipsCookies();
 
   nsresult CreateTableWorker(const char* aName);
   nsresult CreateTable();

netwerk/test/unit/test_cookies_partition_migration.js

@@ -0,0 +1,236 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function test_chips_migration() {
+  // Set up a profile.
+  let profile = do_get_profile();
+
+  // Start the cookieservice, to force creation of a database.
+  Services.cookies.sessionCookies;
+
+  // Close the profile.
+  await promise_close_profile();
+
+  // Remove the cookie file in order to create another database file.
+  do_get_cookie_file(profile).remove(false);
+
+  // Create a schema 14 database.
+  let database = new CookieDatabaseConnection(do_get_cookie_file(profile), 14);
+
+  let now = Date.now() * 1000;
+  let expiry = Math.round(now / 1e6 + 1000);
+
+  // Populate db with a first-party unpartitioned cookies
+  let cookie = new Cookie(
+    "test",
+    "Some data",
+    "example.com",
+    "/",
+    expiry,
+    now,
+    now,
+    false,
+    false,
+    false,
+    false,
+    {},
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SCHEME_UNSET,
+    false // isPartitioned
+  );
+  database.insertCookie(cookie);
+
+  // Populate db with a first-party unpartitioned cookies with the partitioned attribute
+  cookie = new Cookie(
+    "test partitioned",
+    "Some data",
+    "example.com",
+    "/",
+    expiry,
+    now,
+    now,
+    false,
+    false,
+    false,
+    false,
+    {},
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SCHEME_UNSET,
+    true // isPartitioned
+  );
+  database.insertCookie(cookie);
+
+  // Populate db with a first-party unpartitioned cookies with the partitioned attribute
+  cookie = new Cookie(
+    "test overwrite",
+    "Overwritten",
+    "example.com",
+    "/",
+    expiry,
+    now,
+    now,
+    false,
+    false,
+    false,
+    false,
+    {},
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SCHEME_UNSET,
+    true // isPartitioned
+  );
+  database.insertCookie(cookie);
+
+  // Populate db with a first-party unpartitioned cookies with the partitioned attribute
+  cookie = new Cookie(
+    "test overwrite",
+    "Did not overwrite",
+    "example.com",
+    "/",
+    expiry,
+    now,
+    now,
+    false,
+    false,
+    false,
+    false,
+    { partitionKey: "(https,example.com)" },
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SAMESITE_NONE,
+    Ci.nsICookie.SCHEME_UNSET,
+    true // isPartitioned
+  );
+  database.insertCookie(cookie);
+
+  database.close();
+  database = null;
+
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref("network.cookie.CHIPS.enabled");
+    Services.prefs.clearUserPref("network.cookie.CHIPS.migrateDatabase");
+  });
+
+  // Reload profile.
+  Services.prefs.setBoolPref("network.cookie.CHIPS.enabled", true);
+  Services.prefs.setIntPref("network.cookie.CHIPS.lastMigrateDatabase", 0);
+  Services.prefs.setIntPref("network.cookie.CHIPS.migrateDatabaseTarget", 0);
+  await promise_load_profile();
+
+  // Make sure there were no changes
+  Assert.equal(
+    Services.cookies.getCookiesFromHost("example.com", {}).length,
+    3
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {})
+      .filter(cookie => cookie.name == "test").length,
+    1
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {})
+      .filter(cookie => cookie.name == "test partitioned").length,
+    1
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {})
+      .filter(cookie => cookie.name == "test overwrite").length,
+    1
+  );
+  Assert.equal(
+    Services.cookies.getCookiesFromHost("example.com", {
+      partitionKey: "(https,example.com)",
+    }).length,
+    1
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {})
+      .filter(cookie => cookie.name == "test overwrite").length,
+    1
+  );
+
+  // Close the profile.
+  await promise_close_profile();
+
+  // Reload profile.
+  await Services.prefs.setBoolPref("network.cookie.CHIPS.enabled", true);
+  await Services.prefs.setIntPref(
+    "network.cookie.CHIPS.migrateDatabaseTarget",
+    1000
+  );
+  await promise_load_profile();
+
+  // Check if the first-party unpartitioned cookie is still there
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {})
+      .filter(cookie => cookie.name == "test").length,
+    1
+  );
+
+  // Check that we no longer have Partitioned cookies in the unpartitioned storage
+  Assert.equal(
+    Services.cookies.getCookiesFromHost("example.com", {}).length,
+    1
+  );
+
+  // Check that we only have our two partitioned cookies
+  Assert.equal(
+    Services.cookies.getCookiesFromHost("example.com", {
+      partitionKey: "(https,example.com)",
+    }).length,
+    2
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {
+        partitionKey: "(https,example.com)",
+      })
+      .filter(cookie => cookie.name == "test").length,
+    0
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {
+        partitionKey: "(https,example.com)",
+      })
+      .filter(cookie => cookie.name == "test partitioned").length,
+    1
+  );
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {
+        partitionKey: "(https,example.com)",
+      })
+      .filter(cookie => cookie.name == "test overwrite").length,
+    1
+  );
+
+  // Test that we overwrote the value of the cookie in the partition with the
+  // value that was not partitioned
+  Assert.equal(
+    Services.cookies
+      .getCookiesFromHost("example.com", {
+        partitionKey: "(https,example.com)",
+      })
+      .filter(cookie => cookie.name == "test overwrite")[0].value,
+    "Overwritten"
+  );
+
+  // Make sure we cleared the migration pref as part of the migration
+  Assert.equal(
+    Services.prefs.getIntPref("network.cookie.CHIPS.lastMigrateDatabase"),
+    1000
+  );
+
+  // Cleanup
+  Services.cookies.removeAll();
+  do_close_profile();
+});

netwerk/test/unit/xpcshell.toml

@@ -505,6 +505,8 @@ skip-if = ["os == 'linux' && bits == 64 && !debug"] #Bug 1553353
 
 ["test_cookies_partition_counting.js"]
 
+["test_cookies_partition_migration.js"]
+
 ["test_cookies_privatebrowsing.js"]
 
 ["test_cookies_profile_close.js"]

toolkit/components/nimbus/FeatureManifest.yaml

@@ -2208,6 +2208,12 @@ networking:
       setPref:
         branch: default
         pref: "network.cookie.CHIPS.enabled"
+    chipsMigrationTarget:
+      description: What CHIPS migration count target the browser should reach.
+      type: int
+      setPref:
+        branch: default
+        pref: "network.cookie.CHIPS.migrateDatabaseTarget"
     chipsPartitionLimitEnabled:
       description: Whether we enforce CHIPS partition limit
       type: boolean