Bug 1625935 - [devtools] Implement ResourceWatcher with legacy listeners for Storage panel r=ochameau

This is D86050, rebased on top of current central. I kept it a separate commit, so the new stuff can be reviewed separately in a different revision.

Differential Revision: https://phabricator.services.mozilla.com/D91122

devtools/client/storage/ui.js

@@ -59,8 +59,6 @@ const REASON = {
   UPDATE: "update",
 };
 
-const SAFE_HOSTS_PREFIXES_REGEX = /^(about:|https?:|file:|moz-extension:)/;
-
 // Maximum length of item name to show in context menu label - will be
 // trimmed with ellipsis if it's longer.
 const ITEM_NAME_MAX_LENGTH = 32;
@@ -167,9 +165,6 @@ class StorageUI {
       this.searchBox.focus();
     });
 
-    this.onEdit = this.onEdit.bind(this);
-    this.onCleared = this.onCleared.bind(this);
-
     this.handleKeypress = this.handleKeypress.bind(this);
     this._panelDoc.addEventListener("keypress", this.handleKeypress);
 
@@ -279,6 +274,34 @@ class StorageUI {
       this._onTargetAvailable,
       this._onTargetDestroyed
     );
+
+    this.storageTypes = {};
+
+    this._onResourceListAvailable = this._onResourceListAvailable.bind(this);
+    this._onResourceUpdated = this._onResourceUpdated.bind(this);
+    this._onResourceListUpdated = this._onResourceListUpdated.bind(this);
+    this._onResourceDestroyed = this._onResourceDestroyed.bind(this);
+    this._onResourceListDestroyed = this._onResourceListDestroyed.bind(this);
+
+    const { resourceWatcher } = this._toolbox;
+
+    await this._toolbox.resourceWatcher.watchResources(
+      [
+        // The first item in this list will be the first selected storage item
+        // Tests assume Cookie -- moving cookie will break tests
+        resourceWatcher.TYPES.COOKIE,
+        resourceWatcher.TYPES.CACHE_STORAGE,
+        resourceWatcher.TYPES.EXTENSION_STORAGE,
+        resourceWatcher.TYPES.INDEXED_DB,
+        resourceWatcher.TYPES.LOCAL_STORAGE,
+        resourceWatcher.TYPES.SESSION_STORAGE,
+      ],
+      {
+        onAvailable: this._onResourceListAvailable,
+        onUpdated: this._onResourceListUpdated,
+        onDestroyed: this._onResourceListDestroyed,
+      }
+    );
   }
 
   async _onTargetAvailable({ targetFront }) {
@@ -289,31 +312,39 @@ class StorageUI {
     }
 
     this.front = await targetFront.getFront("storage");
-    this.front.on("stores-update", this.onEdit);
-    this.front.on("stores-cleared", this.onCleared);
+  }
+
+  async _onResourceListAvailable(resources) {
+    const {
+      COOKIE,
+      LOCAL_STORAGE,
+      SESSION_STORAGE,
+      EXTENSION_STORAGE,
+      CACHE_STORAGE,
+      INDEXED_DB,
+    } = this._toolbox.resourceWatcher.TYPES;
+
+    const storages = {};
+
+    for (const resource of resources) {
+      const { resourceType } = resource;
+      if (resourceType == COOKIE) {
+        storages.cookies = resource;
+      } else if (resourceType == LOCAL_STORAGE) {
+        storages.localStorage = resource;
+      } else if (resourceType == SESSION_STORAGE) {
+        storages.sessionStorage = resource;
+      } else if (resourceType == EXTENSION_STORAGE) {
+        storages.extensionStorage = resource;
+      } else if (resourceType == CACHE_STORAGE) {
+        storages.Cache = resource;
+      } else if (resourceType == INDEXED_DB) {
+        storages.indexedDB = resource;
+      }
+    }
+
     try {
-      const storageTypes = await this.front.listStores();
-      // When we are in the browser console we list indexedDBs internal to
-      // Firefox e.g. defined inside a .jsm. Because there is no way before this
-      // point to know whether or not we are inside the browser toolbox we have
-      // already fetched the hostnames of these databases.
-      //
-      // If we are not inside the browser toolbox we need to delete these
-      // hostnames.
-      if (!targetFront.chrome && storageTypes.indexedDB) {
-        const hosts = storageTypes.indexedDB.hosts;
-        const newHosts = {};
-
-        for (const [host, dbs] of Object.entries(hosts)) {
-          if (SAFE_HOSTS_PREFIXES_REGEX.test(host)) {
-            newHosts[host] = dbs;
-          }
-        }
-
-        storageTypes.indexedDB.hosts = newHosts;
-      }
-
-      await this.populateStorageTree(storageTypes);
+      await this.populateStorageTree(storages);
     } catch (e) {
       if (!this._toolbox || this._toolbox._destroyer) {
         // The toolbox is in the process of being destroyed... in this case throwing here
@@ -336,9 +367,6 @@ class StorageUI {
     this.table.clear();
     this.hideSidebar();
     this.tree.clear();
-
-    this.front.off("stores-update", this.onEdit);
-    this.front.off("stores-cleared", this.onCleared);
   }
 
   set animationsEnabled(value) {
@@ -482,32 +510,38 @@ class StorageUI {
     await this.updateObjectSidebar();
   }
 
+  async _onResourceListDestroyed(clearedList) {
+    for (const cleared of clearedList) {
+      await this._onResourceDestroyed(cleared);
+    }
+  }
+
   /**
    * Event handler for "stores-cleared" event coming from the storage actor.
    *
-   * @param {object} response
-   *        An object containing which storage types were cleared
+   * @param {object}
+   *        An object containing which hosts/paths are cleared from a
+   *        storage
    */
-  onCleared(response) {
+  _onResourceDestroyed({ resourceKey, clearedHostsOrPaths }) {
     function* enumPaths() {
-      for (const type in response) {
-        if (Array.isArray(response[type])) {
+      if (Array.isArray(clearedHostsOrPaths)) {
         // Handle the legacy response with array of hosts
-          for (const host of response[type]) {
-            yield [type, host];
+        for (const host of clearedHostsOrPaths) {
+          yield [host];
         }
       } else {
         // Handle the new format that supports clearing sub-stores in a host
-          for (const host in response[type]) {
-            const paths = response[type][host];
+        for (const host in clearedHostsOrPaths) {
+          const paths = clearedHostsOrPaths[host];
 
           if (!paths.length) {
-              yield [type, host];
+            yield [host];
           } else {
             for (let path of paths) {
               try {
                 path = JSON.parse(path);
-                  yield [type, host, ...path];
+                yield [host, ...path];
               } catch (ex) {
                 // ignore
               }
@@ -516,11 +550,10 @@ class StorageUI {
         }
       }
     }
-    }
 
     for (const path of enumPaths()) {
       // Find if the path is selected (there is max one) and clear it
-      if (this.tree.isSelected(path)) {
+      if (this.tree.isSelected([resourceKey, ...path])) {
         this.table.clear();
         this.hideSidebar();
 
@@ -534,6 +567,12 @@ class StorageUI {
     }
   }
 
+  async _onResourceListUpdated(updates) {
+    for (const update of updates) {
+      await this._onResourceUpdated(update.update);
+    }
+  }
+
   /**
    * Event handler for "stores-update" event coming from the storage actor.
    *
@@ -557,7 +596,7 @@ class StorageUI {
    *        of the changed store objects. This array is empty for deleted object
    *        if the host was completely removed.
    */
-  async onEdit({ changed, added, deleted }) {
+  async _onResourceUpdated({ changed, added, deleted }) {
     if (added) {
       await this.handleAddedItems(added);
     }
@@ -780,7 +819,6 @@ class StorageUI {
         }
 
         await this._readSupportsTraits(type);
-
         await this.resetColumns(type, host, subType);
       }
 
@@ -865,8 +903,6 @@ class StorageUI {
    *        StorageFront.listStores call.
    */
   async populateStorageTree(storageTypes) {
-    this.storageTypes = {};
-
     // When can we expect the "store-objects-updated" event?
     //   -> TreeWidget setter `selectedItem` emits a "select" event
     //   -> on tree "select" event, this module calls `onHostSelect`
@@ -892,6 +928,7 @@ class StorageUI {
       } catch (e) {
         console.error("Unable to localize tree label type:" + type);
       }
+
       this.tree.add([{ id: type, label: typeLabel, type: "store" }]);
       if (!storageTypes[type].hosts) {
         continue;

devtools/shared/resources/legacy-listeners/cache-storage.js

@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const {
+  makeStorageLegacyListener,
+} = require("devtools/shared/resources/legacy-listeners/storage-utils");
+
+module.exports = makeStorageLegacyListener(
+  "Cache",
+  ResourceWatcher.TYPES.CACHE_STORAGE
+);

devtools/shared/resources/legacy-listeners/cookie.js

@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const {
+  makeStorageLegacyListener,
+} = require("devtools/shared/resources/legacy-listeners/storage-utils");
+
+module.exports = makeStorageLegacyListener(
+  "cookies",
+  ResourceWatcher.TYPES.COOKIE
+);

devtools/shared/resources/legacy-listeners/extension-storage.js

@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const {
+  makeStorageLegacyListener,
+} = require("devtools/shared/resources/legacy-listeners/storage-utils");
+
+module.exports = makeStorageLegacyListener(
+  "extensionStorage",
+  ResourceWatcher.TYPES.EXTENSION_STORAGE
+);

devtools/shared/resources/legacy-listeners/indexed-db.js

@@ -0,0 +1,19 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const {
+  // getFilteredStorageEvents,
+  makeStorageLegacyListener,
+} = require("devtools/shared/resources/legacy-listeners/storage-utils");
+
+module.exports = makeStorageLegacyListener(
+  "indexedDB",
+  ResourceWatcher.TYPES.INDEXED_DB
+);

devtools/shared/resources/legacy-listeners/local-storage.js

@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const {
+  makeStorageLegacyListener,
+} = require("devtools/shared/resources/legacy-listeners/storage-utils");
+
+module.exports = makeStorageLegacyListener(
+  "localStorage",
+  ResourceWatcher.TYPES.LOCAL_STORAGE
+);

devtools/shared/resources/legacy-listeners/moz.build

@@ -3,14 +3,21 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 DevToolsModules(
+    'cache-storage.js',
     'console-messages.js',
+    'cookie.js',
     'css-changes.js',
     'css-messages.js',
     'error-messages.js',
+    'extension-storage.js',
+    'indexed-db.js',
+    'local-storage.js',
     'network-event-stacktraces.js',
     'network-events.js',
     'platform-messages.js',
     'root-node.js',
+    'session-storage.js',
+    'storage-utils.js',
     'stylesheet.js',
     'websocket.js',
 )

devtools/shared/resources/legacy-listeners/session-storage.js

@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const {
+  makeStorageLegacyListener,
+} = require("devtools/shared/resources/legacy-listeners/storage-utils");
+
+module.exports = makeStorageLegacyListener(
+  "sessionStorage",
+  ResourceWatcher.TYPES.SESSION_STORAGE
+);

devtools/shared/resources/legacy-listeners/storage-utils.js

@@ -0,0 +1,96 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+// Filters "stores-update" response to only include events for
+// the storage type we desire
+function getFilteredStorageEvents(updates, storageType) {
+  const filteredUpdate = Object.create(null);
+
+  // updateType will be "added", "changed", or "deleted"
+  for (const updateType in updates) {
+    if (updates[updateType][storageType]) {
+      if (!filteredUpdate[updateType]) {
+        filteredUpdate[updateType] = {};
+      }
+      filteredUpdate[updateType][storageType] =
+        updates[updateType][storageType];
+    }
+  }
+
+  return Object.keys(filteredUpdate).length > 0 ? filteredUpdate : null;
+}
+
+// This is a mixin that provides all shared cored between storage legacy
+// listeners
+function makeStorageLegacyListener(storageKey, storageType) {
+  return async function({
+    targetList,
+    targetType,
+    targetFront,
+    isFissionEnabledOnContentToolbox,
+    onAvailable,
+    onUpdated,
+    onDestroyed,
+  }) {
+    if (!targetFront.isTopLevel) {
+      return;
+    }
+
+    const storageFront = await targetFront.getFront("storage");
+    const storageTypes = await storageFront.listStores();
+
+    // Initialization
+    const storage = storageTypes[storageKey];
+
+    // extension storage might not be available
+    if (!storage) {
+      return;
+    }
+
+    storage.resourceType = storageType;
+    // storage resources are singletons, and thus we can set their ID to their
+    // storage type
+    storage.resourceId = storageType;
+    onAvailable([storage]);
+
+    // Any item in the store gets updated
+    storageFront.on("stores-update", response => {
+      response = getFilteredStorageEvents(response, storageKey);
+      if (!response) {
+        return;
+      }
+      onUpdated([
+        {
+          resourceId: storageType,
+          resourceType: storageType,
+          changed: response.changed,
+          added: response.added,
+          deleted: response.deleted,
+        },
+      ]);
+    });
+
+    // When a store gets cleared
+    storageFront.on("stores-cleared", response => {
+      const cleared = response[storageKey];
+
+      if (!cleared) {
+        return;
+      }
+
+      onDestroyed([
+        {
+          resourceId: storageType,
+          resourceType: storageType,
+          resourceKey: storageKey,
+          clearedHostsOrPaths: cleared,
+        },
+      ]);
+    });
+  };
+}
+
+module.exports = { makeStorageLegacyListener };

devtools/shared/resources/resource-watcher.js

@@ -417,6 +417,10 @@ class ResourceWatcher {
         nestedResourceUpdates,
       } = update;
 
+      if (!resourceId) {
+        console.warn(`Expected resource ${resourceType} to have a resourceId`);
+      }
+
       const existingResource = this._cache.find(
         cachedResource =>
           cachedResource.resourceType === resourceType &&
@@ -664,8 +668,9 @@ class ResourceWatcher {
     }
 
     const onAvailable = this._onResourceAvailable.bind(this, { targetFront });
-    const onDestroyed = this._onResourceDestroyed.bind(this, { targetFront });
     const onUpdated = this._onResourceUpdated.bind(this, { targetFront });
+    const onDestroyed = this._onResourceDestroyed.bind(this, { targetFront });
+
     return LegacyListeners[resourceType]({
       targetList: this.targetList,
       targetFront,
@@ -746,6 +751,12 @@ ResourceWatcher.TYPES = ResourceWatcher.prototype.TYPES = {
   STYLESHEET: "stylesheet",
   NETWORK_EVENT: "network-event",
   WEBSOCKET: "websocket",
+  COOKIE: "cookie",
+  LOCAL_STORAGE: "local-storage",
+  SESSION_STORAGE: "session-storage",
+  CACHE_STORAGE: "Cache",
+  EXTENSION_STORAGE: "extension-storage",
+  INDEXED_DB: "indexed-db",
   NETWORK_EVENT_STACKTRACE: "network-event-stacktrace",
 };
 module.exports = { ResourceWatcher };
@@ -789,6 +800,18 @@ const LegacyListeners = {
     .NETWORK_EVENT]: require("devtools/shared/resources/legacy-listeners/network-events"),
   [ResourceWatcher.TYPES
     .WEBSOCKET]: require("devtools/shared/resources/legacy-listeners/websocket"),
+  [ResourceWatcher.TYPES
+    .COOKIE]: require("devtools/shared/resources/legacy-listeners/cookie"),
+  [ResourceWatcher.TYPES
+    .LOCAL_STORAGE]: require("devtools/shared/resources/legacy-listeners/local-storage"),
+  [ResourceWatcher.TYPES
+    .SESSION_STORAGE]: require("devtools/shared/resources/legacy-listeners/session-storage"),
+  [ResourceWatcher.TYPES
+    .CACHE_STORAGE]: require("devtools/shared/resources/legacy-listeners/cache-storage"),
+  [ResourceWatcher.TYPES
+    .EXTENSION_STORAGE]: require("devtools/shared/resources/legacy-listeners/extension-storage"),
+  [ResourceWatcher.TYPES
+    .INDEXED_DB]: require("devtools/shared/resources/legacy-listeners/indexed-db"),
   [ResourceWatcher.TYPES
     .NETWORK_EVENT_STACKTRACE]: require("devtools/shared/resources/legacy-listeners/network-event-stacktraces"),
 };
@@ -802,6 +825,8 @@ const ResourceTransformers = {
     .CONSOLE_MESSAGE]: require("devtools/shared/resources/transformers/console-messages"),
   [ResourceWatcher.TYPES
     .ERROR_MESSAGE]: require("devtools/shared/resources/transformers/error-messages"),
+  [ResourceWatcher.TYPES
+    .INDEXED_DB]: require("devtools/shared/resources/transformers/storage-indexed-db.js"),
   [ResourceWatcher.TYPES
     .ROOT_NODE]: require("devtools/shared/resources/transformers/root-node"),
 };

devtools/shared/resources/transformers/moz.build

@@ -6,4 +6,5 @@ DevToolsModules(
     'console-messages.js',
     'error-messages.js',
     'root-node.js',
+    'storage-indexed-db.js',
 )

devtools/shared/resources/transformers/storage-indexed-db.js

@@ -0,0 +1,32 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+module.exports = function({ resource, targetFront }) {
+  // When we are in the browser console we list indexedDBs internal to
+  // Firefox e.g. defined inside a .jsm. Because there is no way before this
+  // point to know whether or not we are inside the browser toolbox we have
+  // already fetched the hostnames of these databases.
+  //
+  // If we are not inside the browser toolbox we need to delete these
+  // hostnames.
+
+  const SAFE_HOSTS_PREFIXES_REGEX = /^(about:|https?:|file:|moz-extension:)/;
+  const { hosts } = resource;
+
+  if (!targetFront.chrome) {
+    const newHosts = {};
+
+    for (const [host, dbs] of Object.entries(hosts)) {
+      if (SAFE_HOSTS_PREFIXES_REGEX.test(host)) {
+        newHosts[host] = dbs;
+      }
+    }
+
+    resource.hosts = newHosts;
+  }
+
+  return resource;
+};