diff --git a/browser/base/content/browser.js b/browser/base/content/browser.js
index 721825e9dfdf..36090b46816d 100644
--- a/browser/base/content/browser.js
+++ b/browser/base/content/browser.js
@@ -3539,7 +3539,7 @@ function BrowserReloadWithFlags(reloadFlags) {
   // This is done here because we only want to reset
   // permissions on user reload.
   for (let tab of unchangedRemoteness) {
-    SitePermissions.clearTemporaryPermissions(tab.linkedBrowser);
+    SitePermissions.clearTemporaryBlockPermissions(tab.linkedBrowser);
     // Also reset DOS mitigations for the basic auth prompt on reload.
     delete tab.linkedBrowser.authPromptAbuseCounter;
   }
diff --git a/browser/base/content/tabbrowser.js b/browser/base/content/tabbrowser.js
index 2dcbfbb41e1d..80ad701431fb 100644
--- a/browser/base/content/tabbrowser.js
+++ b/browser/base/content/tabbrowser.js
@@ -4361,7 +4361,7 @@
       let browser = this.getBrowserForTab(aTab);
       // Reset temporary permissions on the current tab. This is done here
       // because we only want to reset permissions on user reload.
-      SitePermissions.clearTemporaryPermissions(browser);
+      SitePermissions.clearTemporaryBlockPermissions(browser);
       // Also reset DOS mitigations for the basic auth prompt on reload.
       delete browser.authPromptAbuseCounter;
       gIdentityHandler.hidePopup();
diff --git a/browser/components/urlbar/UrlbarInput.jsm b/browser/components/urlbar/UrlbarInput.jsm
index 72d4660b92bf..18fa77c5e990 100644
--- a/browser/components/urlbar/UrlbarInput.jsm
+++ b/browser/components/urlbar/UrlbarInput.jsm
@@ -2391,7 +2391,7 @@ class UrlbarInput {
         browser.currentURI &&
         url === browser.currentURI.spec
       ) {
-        this.window.SitePermissions.clearTemporaryPermissions(browser);
+        this.window.SitePermissions.clearTemporaryBlockPermissions(browser);
       }
     }
 
diff --git a/browser/modules/SitePermissions.jsm b/browser/modules/SitePermissions.jsm
index 4a380d241f4b..17af0fc7195c 100644
--- a/browser/modules/SitePermissions.jsm
+++ b/browser/modules/SitePermissions.jsm
@@ -240,21 +240,54 @@ const TemporaryPermissions = {
   // Clears all permissions for the specified browser.
   // Unlike other methods, this does NOT clear only for
   // the currentURI but the whole browser state.
-  clear(browser) {
-    let entry = this._stateByBrowser.get(browser);
-    this._stateByBrowser.delete(browser);
 
+  /**
+   * Clear temporary permissions for the specified browser. Unlike other
+   * methods, this does NOT clear only for the currentURI but the whole browser
+   * state.
+   * @param {Browser} browser - Browser to clear permissions for.
+   * @param {Number} [filterState] - Only clear permissions with the given state
+   * value. Defaults to all permissions.
+   */
+  clear(browser, filterState = null) {
+    let entry = this._stateByBrowser.get(browser);
     if (!entry?.uriToPerm) {
       return;
     }
-    Object.values(entry.uriToPerm).forEach(permissions => {
-      Object.values(permissions).forEach(({ expireTimeout }) => {
-        if (!expireTimeout) {
-          return;
+
+    let { uriToPerm } = entry;
+    Object.entries(uriToPerm).forEach(([uriKey, permissions]) => {
+      Object.entries(permissions).forEach(
+        ([permId, { state, expireTimeout }]) => {
+          // We need to explicitly check for null or undefined here, because the
+          // permission state may be 0.
+          if (filterState != null) {
+            if (state != filterState) {
+              // Skip permission entry if it doesn't match the filter.
+              return;
+            }
+            delete permissions[permId];
+          }
+          // For the clear-all case we remove the entire browser entry, so we
+          // only need to clear the timeouts.
+          if (!expireTimeout) {
+            return;
+          }
+          clearTimeout(expireTimeout);
         }
-        clearTimeout(expireTimeout);
-      });
+      );
+      // If there are no more permissions, remove the entry from the URI map.
+      if (filterState != null && !Object.keys(permissions).length) {
+        delete uriToPerm[uriKey];
+      }
     });
+
+    // We're either clearing all permissions or only the permissions with state
+    // == filterState. If we have a filter, we can only clean up the browser if
+    // there are no permission entries left in the map.
+    if (filterState == null || !Object.keys(uriToPerm).length) {
+      this._stateByBrowser.delete(browser);
+    }
   },
 
   // Copies the temporary permission state of one browser
@@ -866,13 +899,13 @@ var SitePermissions = {
   },
 
   /**
-   * Clears all permissions that were temporarily saved.
+   * Clears all block permissions that were temporarily saved.
    *
    * @param {Browser} browser
    *        The browser object to clear.
    */
-  clearTemporaryPermissions(browser) {
-    TemporaryPermissions.clear(browser);
+  clearTemporaryBlockPermissions(browser) {
+    TemporaryPermissions.clear(browser, SitePermissions.BLOCK);
   },
 
   /**
diff --git a/browser/modules/test/browser/browser_SitePermissions_tab_urls.js b/browser/modules/test/browser/browser_SitePermissions_tab_urls.js
index b855787b8898..c2f4321f917c 100644
--- a/browser/modules/test/browser/browser_SitePermissions_tab_urls.js
+++ b/browser/modules/test/browser/browser_SitePermissions_tab_urls.js
@@ -77,7 +77,7 @@ add_task(async function testTemporaryPermissionTabURLs() {
         );
       }
 
-      SitePermissions.clearTemporaryPermissions(browser);
+      SitePermissions.clearTemporaryBlockPermissions(browser);
     }
 
     for (let principal of different) {
@@ -126,7 +126,7 @@ add_task(async function testTemporaryPermissionTabURLs() {
         }
       }
 
-      SitePermissions.clearTemporaryPermissions(browser);
+      SitePermissions.clearTemporaryBlockPermissions(browser);
     }
   });
 });