Bug 1620280 - [devtools] Implement and use a SOURCE resource. r=jdescottes,bomsy,nchevobbe

Differential Revision: https://phabricator.services.mozilla.com/D86916

devtools/client/debugger/panel.js

@@ -51,6 +51,7 @@ class DebuggerPanel {
       client,
     } = await this.panelWin.Debugger.bootstrap({
       targetList: this.toolbox.targetList,
+      resourceWatcher: this.toolbox.resourceWatcher,
       devToolsClient: this.toolbox.target.client,
       workers: {
         sourceMaps: this.toolbox.sourceMapService,

devtools/client/debugger/src/actions/sources/newSources.js

@@ -47,9 +47,7 @@ import { validateNavigateContext, ContextError } from "../../utils/context";
 
 import type {
   Source,
-  SourceActorId,
   SourceId,
-  ThreadId,
   Context,
   OriginalSourceData,
   GeneratedSourceData,
@@ -400,18 +398,3 @@ function checkNewSources(cx: Context, sources: Source[]) {
     return sources;
   };
 }
-
-export function ensureSourceActor(
-  thread: ThreadId,
-  sourceActor: SourceActorId
-) {
-  return async function({ dispatch, getState, client }: ThunkArgs) {
-    await sourceQueue.flush();
-    if (hasSourceActor(getState(), sourceActor)) {
-      return Promise.resolve();
-    }
-
-    const sources = await client.fetchThreadSources(thread);
-    await dispatch(newGeneratedSources(sources));
-  };
-}

devtools/client/debugger/src/actions/threads.js

@@ -7,7 +7,6 @@
 import type { Target } from "../client/firefox/types";
 import type { Action, ThunkArgs } from "./types";
 import { removeSourceActors } from "./source-actors";
-import { newGeneratedSources } from "./sources";
 import { validateContext } from "../utils/context";
 
 import { getContext, getThread, getSourceActorsForThread } from "../selectors";
@@ -20,19 +19,6 @@ export function addTarget(targetFront: Target) {
     validateContext(getState(), cx);
 
     dispatch(({ type: "INSERT_THREAD", cx, newThread: thread }: Action));
-
-    // Fetch the sources and install breakpoints on any new workers.
-    try {
-      const sources = await client.fetchThreadSources(thread.actor);
-      validateContext(getState(), cx);
-
-      await dispatch(newGeneratedSources(sources));
-    } catch (e) {
-      // NOTE: This fails quietly because it is pretty easy for sources to
-      // throw during the fetch if their thread shuts down,
-      // which would cause test failures.
-      console.error(e);
-    }
   };
 }
 

devtools/client/debugger/src/client/firefox.js

@@ -7,20 +7,27 @@
 import { setupCommands, clientCommands } from "./firefox/commands";
 import { setupEvents, clientEvents } from "./firefox/events";
 import { features, prefs } from "../utils/prefs";
+import { prepareSourcePayload } from "./firefox/create";
+import sourceQueue from "../utils/source-queue";
 
 let actions;
 let targetList;
 
 export async function onConnect(
   connection: any,
-  _actions: Object
+  _actions: Object,
+  store: any
 ): Promise<void> {
-  const { devToolsClient, targetList: _targetList } = connection;
+  const {
+    devToolsClient,
+    targetList: _targetList,
+    resourceWatcher,
+  } = connection;
   actions = _actions;
   targetList = _targetList;
 
   setupCommands({ devToolsClient, targetList });
-  setupEvents({ actions, devToolsClient });
+  setupEvents({ actions, devToolsClient, store, resourceWatcher });
   const { targetFront } = targetList;
   if (targetFront.isBrowsingContext || targetFront.isParentProcess) {
     targetList.listenForWorkers = true;
@@ -36,6 +43,16 @@ export async function onConnect(
     onTargetAvailable,
     onTargetDestroyed
   );
+
+  await resourceWatcher.watchResources([resourceWatcher.TYPES.SOURCE], {
+    onAvailable: onSourceAvailable,
+  });
+
+  // Tests like browser_webconsole_eval_sources.js using viewSourceInDebugger
+  // are expecting to find sources in the debugger store immediately for existing sources.
+  // So flush the queue immediately after calling watchResources, which will
+  // process all existing sources.
+  await sourceQueue.flush();
 }
 
 async function onTargetAvailable({
@@ -112,4 +129,23 @@ function onTargetDestroyed({ targetFront }): void {
   actions.removeTarget(targetFront);
 }
 
+async function onSourceAvailable(sources) {
+  for (const source of sources) {
+    const threadFront = await source.targetFront.getFront("thread");
+    const frontendSource = prepareSourcePayload(threadFront, source);
+
+    // Use SourceQueue, which will throttle all incoming sources and only display merged set of
+    // action every 100ms. Unless SourceQueue.flush is manually called when:
+    // - the thread is paused and we have to retrieve the source immediately
+    // - after fetching all existing sources (tests expect sources to be in redux store immediately)
+    //
+    // This throttling code could probably be migrated in the ResourceWatcher API
+    // so that we use a generic throttling algorithm for all resources.
+    sourceQueue.queue({
+      type: "generated",
+      data: frontendSource,
+    });
+  }
+}
+
 export { clientCommands, clientEvents };

devtools/client/debugger/src/client/firefox/commands.js

@@ -4,7 +4,7 @@
 
 // @flow
 
-import { prepareSourcePayload, createThread, createFrame } from "./create";
+import { createThread, createFrame } from "./create";
 import {
   addThreadEventListeners,
   clientEvents,
@@ -22,7 +22,6 @@ import type {
   PendingLocation,
   Frame,
   FrameId,
-  GeneratedSourceData,
   Script,
   SourceId,
   SourceActor,
@@ -39,7 +38,6 @@ import type {
   ThreadFront,
   ObjectFront,
   ExpressionResult,
-  SourcesPacket,
 } from "./types";
 
 import type { EventListenerCategoryList } from "../../actions/types";
@@ -396,14 +394,6 @@ function registerSourceActor(sourceActorId: string, sourceId: SourceId) {
   sourceActors[sourceActorId] = sourceId;
 }
 
-async function getSources(
-  client: ThreadFront
-): Promise<Array<GeneratedSourceData>> {
-  const { sources }: SourcesPacket = await client.getSources();
-
-  return sources.map(source => prepareSourcePayload(client, source));
-}
-
 async function toggleEventLogging(logEventBreakpoints: boolean) {
   return forEachThread(thread =>
     thread.toggleEventLogging(logEventBreakpoints)
@@ -418,21 +408,6 @@ function getAllThreadFronts(): ThreadFront[] {
   return fronts;
 }
 
-// Fetch the sources for all the targets
-async function fetchSources(): Promise<Array<GeneratedSourceData>> {
-  let sources = [];
-  for (const threadFront of getAllThreadFronts()) {
-    sources = sources.concat(await getSources(threadFront));
-  }
-  return sources;
-}
-
-async function fetchThreadSources(
-  thread: string
-): Promise<Array<GeneratedSourceData>> {
-  return getSources(lookupThreadFront(thread));
-}
-
 // Check if any of the targets were paused before we opened
 // the debugger. If one is paused. Fake a `pause` RDP event
 // by directly calling the client event listener.
@@ -555,8 +530,6 @@ const clientCommands = {
   getFrames,
   pauseOnExceptions,
   toggleEventLogging,
-  fetchSources,
-  fetchThreadSources,
   checkIfAlreadyPaused,
   registerSourceActor,
   addThread,

devtools/client/debugger/src/client/firefox/events.js

@@ -5,7 +5,6 @@
 // @flow
 
 import type {
-  SourcePacket,
   PausedPacket,
   ThreadFront,
   Target,
@@ -14,19 +13,23 @@ import type {
 
 import Actions from "../../actions";
 
-import { createPause, prepareSourcePayload } from "./create";
+import { createPause } from "./create";
 import sourceQueue from "../../utils/source-queue";
 import { recordEvent } from "../../utils/telemetry";
 import { prefs } from "../../utils/prefs";
+import { hasSourceActor } from "../../selectors";
+import { stringToSourceActorId } from "../../reducers/source-actors";
 
 type Dependencies = {
   actions: typeof Actions,
   devToolsClient: DevToolsClient,
+  store: any,
 };
 
 let actions: typeof Actions;
 let isInterrupted: boolean;
 let threadFrontListeners: WeakMap<ThreadFront, Array<Function>>;
+let store: any;
 
 function addThreadEventListeners(thread: ThreadFront): void {
   const removeListeners = [];
@@ -53,6 +56,7 @@ function attachAllTargets(currentTarget: Target): boolean {
 function setupEvents(dependencies: Dependencies): void {
   actions = dependencies.actions;
   sourceQueue.initialize(actions);
+  store = dependencies.store;
 
   threadFrontListeners = new WeakMap();
 }
@@ -78,10 +82,7 @@ async function paused(
   if (packet.frame) {
     // When reloading we might receive a pause event before the
     // top frame's source has arrived.
-    await actions.ensureSourceActor(
+    await ensureSourceActor(packet.frame.where.actor);
-      threadFront.actorID,
-      packet.frame.where.actor
-    );
   }
 
   const pause = createPause(threadFront.actor, packet);
@@ -102,17 +103,36 @@ function resumed(threadFront: ThreadFront): void {
   actions.resumed(threadFront.actorID);
 }
 
-function newSource(threadFront: ThreadFront, { source }: SourcePacket): void {
+/**
-  sourceQueue.queue({
+ * This method wait for the given source is registered in Redux store.
-    type: "generated",
+ *
-    data: prepareSourcePayload(threadFront, source),
+ * @param {String} sourceActor
-  });
+ *                 Actor ID of the source to be waiting for.
+ */
+async function ensureSourceActor(sourceActor: string) {
+  const sourceActorId = stringToSourceActorId(sourceActor);
+  if (!hasSourceActor(store.getState(), sourceActorId)) {
+    await new Promise(resolve => {
+      const unsubscribe = store.subscribe(check);
+      let currentState = null;
+      function check() {
+        const previousState = currentState;
+        currentState = store.getState().sourceActors.values;
+        if (previousState == currentState) {
+          return;
+        }
+        if (hasSourceActor(store.getState(), sourceActorId)) {
+          unsubscribe();
+          resolve();
+        }
+      }
+    });
+  }
 }
 
 const clientEvents = {
   paused,
   resumed,
-  newSource,
 };
 
 export {

devtools/client/debugger/src/client/index.js

@@ -87,7 +87,7 @@ export async function onConnect(
     initialState
   );
 
-  const connected = client.onConnect(connection, actions);
+  const connected = client.onConnect(connection, actions, store);
 
   await syncBreakpoints();
   syncXHRBreakpoints();

devtools/client/debugger/src/main.js

@@ -15,11 +15,18 @@ function unmountRoot() {
 }
 
 module.exports = {
-  bootstrap: ({ targetList, devToolsClient, workers, panel }: any) =>
+  bootstrap: ({
+    targetList,
+    resourceWatcher,
+    devToolsClient,
+    workers,
+    panel,
+  }: any) =>
     onConnect(
       {
         tab: { clientType: "firefox" },
         targetList,
+        resourceWatcher,
         devToolsClient,
       },
       workers,

devtools/client/debugger/src/utils/resource/core.js

@@ -47,7 +47,11 @@ export function insertResources<R: ResourceBound>(
   for (const resource of resources) {
     const { id } = resource;
     if (state.identity[id]) {
-      throw new Error(`Resource "${id}" already exists, cannot insert`);
+      throw new Error(
+        `Resource "${id}" already exists, cannot insert ${JSON.stringify(
+          resource
+        )}`
+      );
     }
     if (state.values[id]) {
       throw new Error(

devtools/client/debugger/test/mochitest/browser_dbg-fission-switch-target.js

@@ -15,6 +15,8 @@ add_task(async function() {
 
   // Navigate to a content process URL and check that the sources tree updates
   await navigate(dbg, EXAMPLE_URL + "doc-scripts.html", "simple1.js");
+  info("Wait for all sources to be in the store");
+  await waitFor(() => dbg.selectors.getSourceCount() == 5);
   is(dbg.selectors.getSourceCount(), 5, "5 sources are loaded.");
 
   // Check that you can still break after target switching.

devtools/shared/resources/legacy-listeners/moz.build

@@ -11,6 +11,7 @@ DevToolsModules(
     'network-events.js',
     'platform-messages.js',
     'root-node.js',
+    'source.js',
     'stylesheet.js',
     'websocket.js',
 )

devtools/shared/resources/legacy-listeners/source.js

@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+/**
+ * Emit SOURCE resources, which represents a Javascript source and has the following attributes set on "available":
+ *
+ * - introductionType {null|String}: A string indicating how this source code was introduced into the system.
+ *   This will typically be set to "scriptElement", "eval", ...
+ *   But this may have many other values:
+ *     https://searchfox.org/mozilla-central/rev/ac142717cc067d875e83e4b1316f004f6e063a46/dom/script/ScriptLoader.cpp#2628-2639
+ *     https://searchfox.org/mozilla-central/search?q=symbol:_ZN2JS14CompileOptions19setIntroductionTypeEPKc&redirect=false
+ *     https://searchfox.org/mozilla-central/rev/ac142717cc067d875e83e4b1316f004f6e063a46/devtools/server/actors/source.js#160-169
+ * - sourceMapBaseURL {String}: Base URL where to look for a source map.
+ *   This isn't the source map URL.
+ * - sourceMapURL {null|String}: URL of the source map, if there is one.
+ * - url {null|String}: URL of the source, if it relates to a particular URL.
+ *   Evaled sources won't have any related URL.
+ * - isBlackBoxed {Boolean}: Specifying whether the source actor's 'black-boxed' flag is set.
+ * - extensionName {null|String}: If the source comes from an add-on, the add-on name.
+ */
+module.exports = async function({
+  targetList,
+  targetFront,
+  isFissionEnabledOnContentToolbox,
+  onAvailable,
+}) {
+  const isBrowserToolbox = targetList.targetFront.isParentProcess;
+  const isNonTopLevelFrameTarget =
+    !targetFront.isTopLevel &&
+    targetFront.targetType === targetList.TYPES.FRAME;
+
+  if (isBrowserToolbox && isNonTopLevelFrameTarget) {
+    // In the BrowserToolbox, non-top-level frame targets are already
+    // debugged via content-process targets.
+    return;
+  }
+
+  const threadFront = await targetFront.getFront("thread");
+
+  // Use a list of all notified SourceFront as we don't have a newSource event for all sources
+  // but we sometime get sources notified both via newSource event *and* sources() method...
+  // We store actor ID instead of SourceFront as it appears that multiple SourceFront for the same
+  // actor are created...
+  const sourcesActorIDCache = new Set();
+
+  // Forward new sources (but also existing ones, see next comment)
+  threadFront.on("newSource", ({ source }) => {
+    if (sourcesActorIDCache.has(source.actor)) {
+      return;
+    }
+    sourcesActorIDCache.add(source.actor);
+    // source is a SourceActor's form, add the resourceType attribute on it
+    source.resourceType = ResourceWatcher.TYPES.SOURCE;
+    onAvailable([source]);
+  });
+
+  // Forward already existing sources
+  // Note that calling `sources()` will end up emitting `newSource` event for all existing sources.
+  // But not in some cases, for example, when the thread is already paused.
+  // (And yes, it means that already existing sources can be transfered twice over the wire)
+  let { sources } = await threadFront.sources();
+
+  // Note that `sources()` doesn't encapsulate SourceFront into a `source` attribute
+  // while `newSource` event does.
+  sources = sources.filter(source => {
+    return !sourcesActorIDCache.has(source.actor);
+  });
+  for (const source of sources) {
+    sourcesActorIDCache.add(source.actor);
+    // source is a SourceActor's form, add the resourceType attribute on it
+    source.resourceType = ResourceWatcher.TYPES.SOURCE;
+  }
+  onAvailable(sources);
+};

devtools/shared/resources/resource-watcher.js

@@ -666,6 +666,9 @@ class ResourceWatcher {
     const onAvailable = this._onResourceAvailable.bind(this, { targetFront });
     const onDestroyed = this._onResourceDestroyed.bind(this, { targetFront });
     const onUpdated = this._onResourceUpdated.bind(this, { targetFront });
+    if (!(resourceType in LegacyListeners)) {
+      throw new Error(`Missing legacy listener for ${resourceType}`);
+    }
     return LegacyListeners[resourceType]({
       targetList: this.targetList,
       targetFront,
@@ -747,6 +750,7 @@ ResourceWatcher.TYPES = ResourceWatcher.prototype.TYPES = {
   NETWORK_EVENT: "network-event",
   WEBSOCKET: "websocket",
   NETWORK_EVENT_STACKTRACE: "network-event-stacktrace",
+  SOURCE: "source",
 };
 module.exports = { ResourceWatcher };
 
@@ -791,6 +795,8 @@ const LegacyListeners = {
     .WEBSOCKET]: require("devtools/shared/resources/legacy-listeners/websocket"),
   [ResourceWatcher.TYPES
     .NETWORK_EVENT_STACKTRACE]: require("devtools/shared/resources/legacy-listeners/network-event-stacktraces"),
+  [ResourceWatcher.TYPES
+    .SOURCE]: require("devtools/shared/resources/legacy-listeners/source"),
 };
 
 // Optional transformers for each type of resource.

devtools/shared/resources/tests/browser.ini

@@ -9,6 +9,8 @@ support-files =
   network_document.html
   fission_document.html
   fission_iframe.html
+  sources.html
+  sources.js
   style_document.css
   style_document.html
   style_iframe.css
@@ -19,6 +21,7 @@ support-files =
   test_worker.js
   websocket_backend_wsh.py
   websocket_frontend.html
+  worker-sources.js
 
 [browser_resources_client_caching.js]
 [browser_resources_console_messages.js]
@@ -33,6 +36,7 @@ skip-if = os == "linux" #Bug 1655183
 [browser_resources_platform_messages.js]
 [browser_resources_root_node.js]
 [browser_resources_several_resources.js]
+[browser_resources_sources.js]
 [browser_resources_stylesheets.js]
 [browser_resources_target_destroy.js]
 [browser_resources_target_resources_race.js]

devtools/shared/resources/tests/browser_resources_sources.js

@@ -0,0 +1,176 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test the ResourceWatcher API around SOURCE.
+
+const {
+  ResourceWatcher,
+} = require("devtools/shared/resources/resource-watcher");
+
+const TEST_URL = URL_ROOT + "sources.html";
+
+add_task(async function() {
+  const tab = await addTab(TEST_URL);
+
+  const htmlRequest = await fetch(TEST_URL);
+  const htmlContent = await htmlRequest.text();
+
+  const {
+    client,
+    resourceWatcher,
+    targetList,
+  } = await initResourceWatcherAndTarget(tab);
+
+  // Force the target list to cover workers
+  targetList.listenForWorkers = true;
+  await targetList.startListening();
+
+  const targets = [];
+  await targetList.watchTargets(targetList.ALL_TYPES, async function({
+    targetFront,
+  }) {
+    targets.push(targetFront);
+  });
+  is(targets.length, 2, "Got expected number of targets");
+
+  info("Check already available resources");
+  const availableResources = [];
+  await resourceWatcher.watchResources([ResourceWatcher.TYPES.SOURCE], {
+    onAvailable: resources => availableResources.push(...resources),
+  });
+
+  const expectedExistingResources = [
+    {
+      description: "independent js file",
+      sourceForm: {
+        introductionType: "scriptElement",
+        sourceMapBaseURL:
+          "http://example.com/browser/devtools/shared/resources/tests/sources.js",
+        url:
+          "http://example.com/browser/devtools/shared/resources/tests/sources.js",
+        isBlackBoxed: false,
+        sourceMapURL: null,
+        extensionName: null,
+      },
+      sourceContent: {
+        contentType: "text/javascript",
+        source: "/* eslint-disable */\nfunction scriptSource() {}\n",
+      },
+    },
+    {
+      description: "eval",
+      sourceForm: {
+        introductionType: "eval",
+        sourceMapBaseURL:
+          "http://example.com/browser/devtools/shared/resources/tests/sources.html",
+        url: null,
+        isBlackBoxed: false,
+        sourceMapURL: null,
+        extensionName: null,
+      },
+      sourceContent: {
+        contentType: "text/javascript",
+        source: "this.global = function evalFunction() {}",
+      },
+    },
+    {
+      description: "inline JS",
+      sourceForm: {
+        introductionType: "scriptElement",
+        sourceMapBaseURL:
+          "http://example.com/browser/devtools/shared/resources/tests/sources.html",
+        url:
+          "http://example.com/browser/devtools/shared/resources/tests/sources.html",
+        isBlackBoxed: false,
+        sourceMapURL: null,
+        extensionName: null,
+      },
+      sourceContent: {
+        contentType: "text/html",
+        source: htmlContent,
+      },
+    },
+    {
+      description: "worker script",
+      sourceForm: {
+        introductionType: undefined,
+        sourceMapBaseURL:
+          "http://example.com/browser/devtools/shared/resources/tests/worker-sources.js",
+        url:
+          "http://example.com/browser/devtools/shared/resources/tests/worker-sources.js",
+        isBlackBoxed: false,
+        sourceMapURL: null,
+        extensionName: null,
+      },
+      sourceContent: {
+        contentType: "text/javascript",
+        source: "/* eslint-disable */\nfunction workerSource() {}\n",
+      },
+    },
+  ];
+  await assertResources(availableResources, expectedExistingResources);
+
+  await targetList.stopListening();
+  await client.close();
+});
+
+async function assertResources(resources, expected) {
+  is(
+    resources.length,
+    expected.length,
+    "Length of existing resources is correct at initial"
+  );
+  for (let i = 0; i < resources.length; i++) {
+    await assertResource(resources[i], expected);
+  }
+}
+
+async function assertResource(source, expected) {
+  info(`Checking resource "#${expected.description}"`);
+
+  is(
+    source.resourceType,
+    ResourceWatcher.TYPES.SOURCE,
+    "Resource type is correct"
+  );
+
+  const threadFront = await source.targetFront.getFront("thread");
+  // `source` is SourceActor's form()
+  // so try to instantiate the related SourceFront:
+  const sourceFront = threadFront.source(source);
+  // then fetch source content
+  const sourceContent = await sourceFront.source();
+
+  // Order of sources is random, so we have to find the best expected resource.
+  // The only unique attribute is the JS Source text content.
+  const matchingExpected = expected.find(res => {
+    return res.sourceContent.source == sourceContent.source;
+  });
+  ok(
+    matchingExpected,
+    `This source was expected with source content being "${sourceContent.source}"`
+  );
+  assertObject(
+    sourceContent,
+    matchingExpected.sourceContent,
+    matchingExpected.description
+  );
+
+  assertObject(
+    source,
+    matchingExpected.sourceForm,
+    matchingExpected.description
+  );
+}
+
+function assertObject(object, expected, description) {
+  for (const field in expected) {
+    is(
+      object[field],
+      expected[field],
+      `The value of ${field} is correct for "#${description}"`
+    );
+  }
+}

devtools/shared/resources/tests/sources.html

@@ -0,0 +1,21 @@
+<!-- Any copyright is dedicated to the Public Domain.
+     http://creativecommons.org/publicdomain/zero/1.0/ -->
+<!doctype HTML>
+<html>
+  <head>
+    <meta charset="utf-8"/>
+  </head>
+  <body>
+    <script type="text/javascript">
+      "use strict";
+      /* eslint-disable */
+      function inlineSource() {}
+      // Assign it to a global in order to avoid it being GCed
+      eval("this.global = function evalFunction() {}");
+      // Assign the worker to a global variable in order to avoid
+      // having it be GCed.
+      this.worker = new Worker("worker-sources.js");
+    </script>
+     <script src="sources.js"></script>
+  </body>
+</html>

devtools/shared/resources/tests/sources.js

@@ -0,0 +1,2 @@
+/* eslint-disable */
+function scriptSource() {}

devtools/shared/resources/tests/worker-sources.js

@@ -0,0 +1,2 @@
+/* eslint-disable */
+function workerSource() {}