gecko-dev/testing/marionette/reftest.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

"use strict";

ChromeUtils.import("resource://gre/modules/Log.jsm");
ChromeUtils.import("resource://gre/modules/Preferences.jsm");

ChromeUtils.import("chrome://marionette/content/assert.js");
ChromeUtils.import("chrome://marionette/content/capture.js");
const {InvalidArgumentError} =
    ChromeUtils.import("chrome://marionette/content/error.js", {});

this.EXPORTED_SYMBOLS = ["reftest"];

const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
const PREF_E10S = "browser.tabs.remote.autostart";

const logger = Log.repository.getLogger("Marionette");

const SCREENSHOT_MODE = {
  unexpected: 0,
  fail: 1,
  always: 2,
};

const STATUS = {
  PASS: "PASS",
  FAIL: "FAIL",
  ERROR: "ERROR",
  TIMEOUT: "TIMEOUT",
};

/**
 * Implements an fast runner for web-platform-tests format reftests
 * c.f. http://web-platform-tests.org/writing-tests/reftests.html.
 *
 * @namespace
 */
this.reftest = {};

/**
 * @memberof reftest
 * @class Runner
 */
reftest.Runner = class {
  constructor(driver) {
    this.driver = driver;
    this.canvasCache = new Map([[null, []]]);
    this.windowUtils = null;
    this.lastURL = null;
    this.remote = Preferences.get(PREF_E10S);
  }

  /**
   * Setup the required environment for running reftests.
   *
   * This will open a non-browser window in which the tests will
   * be loaded, and set up various caches for the reftest run.
   *
   * @param {Object.<Number>} urlCount
   *     Object holding a map of URL: number of times the URL
   *     will be opened during the reftest run, where that's
   *     greater than 1.
   * @param {string} screenshotMode
   *     String enum representing when screenshots should be taken
   */
  async setup(urlCount, screenshotMode) {
    this.parentWindow =  assert.open(this.driver.getCurrentWindow());

    this.screenshotMode = SCREENSHOT_MODE[screenshotMode] ||
        SCREENSHOT_MODE.unexpected;

    this.urlCount = Object.keys(urlCount || {})
        .reduce((map, key) => map.set(key, urlCount[key]), new Map());

    await this.ensureWindow();
  }

  async ensureWindow() {
    if (this.reftestWin && !this.reftestWin.closed) {
      return this.reftestWin;
    }

    let reftestWin = await this.openWindow();

    let found = this.driver.findWindow([reftestWin], () => true);
    await this.driver.setWindowHandle(found, true);

    this.windowUtils = reftestWin.QueryInterface(Ci.nsIInterfaceRequestor)
      .getInterface(Ci.nsIDOMWindowUtils);
    this.reftestWin = reftestWin;
    return reftestWin;
  }

  async openWindow() {
    let reftestWin = this.parentWindow.open(
        "chrome://marionette/content/reftest.xul",
        "reftest",
        "chrome,dialog,height=600,width=600");

    await new Promise(resolve => {
      reftestWin.addEventListener("load", resolve, {once: true});
    });

    let browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
    browser.permanentKey = {};
    browser.setAttribute("id", "browser");
    browser.setAttribute("anonid", "initialBrowser");
    browser.setAttribute("type", "content");
    browser.setAttribute("primary", "true");

    if (this.remote) {
      browser.setAttribute("remote", "true");
      browser.setAttribute("remoteType", "web");
    }
    // Make sure the browser element is exactly 600x600, no matter
    // what size our window is
    const windowStyle = `padding: 0px; margin: 0px; border:none;
min-width: 600px; min-height: 600px; max-width: 600px; max-height: 600px`;
    browser.setAttribute("style", windowStyle);

    let doc = reftestWin.document.documentElement;
    while (doc.firstChild) {
      doc.firstChild.remove();
    }
    doc.appendChild(browser);
    reftestWin.gBrowser = browser;

    return reftestWin;
  }

  abort() {
    if (this.reftestWin) {
      this.driver.closeChromeWindow();
    }
    this.reftestWin = null;
  }

  /**
   * Run a specific reftest.
   *
   * The assumed semantics are those of web-platform-tests where
   * references form a tree and each test must meet all the conditions
   * to reach one leaf node of the tree in order for the overall test
   * to pass.
   *
   * @param {string} testUrl
   *     URL of the test itself.
   * @param {Array.<Array>} references
   *     Array representing a tree of references to try.
   *
   *     Each item in the array represents a single reference node and
   *     has the form <code>[referenceUrl, references, relation]</code>,
   *     where <var>referenceUrl</var> is a string to the URL, relation
   *     is either <code>==</code> or <code>!=</code> depending on the
   *     type of reftest, and references is another array containing
   *     items of the same form, representing further comparisons treated
   *     as AND with the current item. Sibling entries are treated as OR.
   *
   *     For example with testUrl of T:
   *
   *     <pre><code>
   *       references = [[A, [[B, [], ==]], ==]]
   *       Must have T == A AND A == B to pass
   *
   *       references = [[A, [], ==], [B, [], !=]
   *       Must have T == A OR T != B
   *
   *       references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]]
   *       Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D)
   *     </code></pre>
   *
   * @param {string} expected
   *     Expected test outcome (e.g. <tt>PASS</tt>, <tt>FAIL</tt>).
   * @param {number} timeout
   *     Test timeout in milliseconds.
   *
   * @return {Object}
   *     Result object with fields status, message and extra.
   */
  async run(testUrl, references, expected, timeout) {

    let timeoutHandle;

    let timeoutPromise = new Promise(resolve => {
      timeoutHandle = this.parentWindow.setTimeout(() => {
        resolve({status: STATUS.TIMEOUT, message: null, extra: {}});
      }, timeout);
    });

    let testRunner = (async () => {
      let result;
      try {
        result = await this.runTest(testUrl, references, expected, timeout);
      } catch (e) {
        result = {
          status: STATUS.ERROR,
          message: String(e),
          stack: e.stack,
          extra: {},
        };
      }
      return result;
    })();

    let result = await Promise.race([testRunner, timeoutPromise]);
    this.parentWindow.clearTimeout(timeoutHandle);
    if (result.status === STATUS.TIMEOUT) {
      this.abort();
    }

    return result;
  }

  async runTest(testUrl, references, expected, timeout) {
    let win = await this.ensureWindow();

    function toBase64(screenshot) {
      let dataURL = screenshot.canvas.toDataURL();
      return dataURL.split(",")[1];
    }

    win.innerWidth = 600;
    win.innerHeight = 600;

    let message = "";

    let screenshotData = [];

    let stack = [];
    for (let i = references.length - 1; i >= 0; i--) {
      let item = references[i];
      stack.push([testUrl, item[0], item[1], item[2]]);
    }

    let status = STATUS.FAIL;

    while (stack.length) {
      let [lhsUrl, rhsUrl, references, relation] = stack.pop();
      message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`;

      let comparison = await this.compareUrls(
          win, lhsUrl, rhsUrl, relation, timeout);

      function recordScreenshot() {
        let encodedLHS = toBase64(comparison.lhs);
        let encodedRHS = toBase64(comparison.rhs);
        screenshotData.push([{url: lhsUrl, screenshot: encodedLHS},
          relation,
          {url: rhsUrl, screenshot: encodedRHS}]);
      }

      if (this.screenshotMode === SCREENSHOT_MODE.always) {
        recordScreenshot();
      }

      if (comparison.passed) {
        if (references.length) {
          for (let i = references.length - 1; i >= 0; i--) {
            let item = references[i];
            stack.push([testUrl, item[0], item[1], item[2]]);
          }
        } else {
          // Reached a leaf node so all of one reference chain passed
          status = STATUS.PASS;
          if (this.screenshotMode <= SCREENSHOT_MODE.fail &&
              expected != status) {
            recordScreenshot();
          }
          break;
        }
      } else if (!stack.length) {
        // If we don't have any alternatives to try then this will be
        // the last iteration, so save the failing screenshots if required.
        let isFail = this.screenshotMode === SCREENSHOT_MODE.fail;
        let isUnexpected = this.screenshotMode === SCREENSHOT_MODE.unexpected;
        if (isFail || (isUnexpected && expected != status)) {
          recordScreenshot();
        }
      }

      // Return any reusable canvases to the pool
      let canvasPool = this.canvasCache.get(null);
      [comparison.lhs, comparison.rhs].map(screenshot => {
        if (screenshot.reuseCanvas) {
          canvasPool.push(screenshot.canvas);
        }
      });
      logger.debug(`Canvas pool is of length ${canvasPool.length}`);
    }

    let result = {status, message, extra: {}};
    if (screenshotData.length) {
      // For now the tbpl formatter only accepts one screenshot, so just
      // return the last one we took.
      let lastScreenshot = screenshotData[screenshotData.length - 1];
      // eslint-disable-next-line camelcase
      result.extra.reftest_screenshots = lastScreenshot;
    }

    return result;
  }

  async compareUrls(win, lhsUrl, rhsUrl, relation, timeout) {
    logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`);

    // Take the reference screenshot first so that if we pause
    // we see the test rendering
    let rhs = await this.screenshot(win, rhsUrl, timeout);
    let lhs = await this.screenshot(win, lhsUrl, timeout);

    let maxDifferences = {};

    let differences = this.windowUtils.compareCanvases(
        lhs.canvas, rhs.canvas, maxDifferences);

    let passed;
    switch (relation) {
      case "==":
        passed = differences === 0;
        if (!passed) {
          logger.info(`Found ${differences} pixels different, ` +
              `maximum difference per channel ${maxDifferences.value}`);
        }
        break;

      case "!=":
        passed = differences !== 0;
        break;

      default:
        throw new InvalidArgumentError("Reftest operator should be '==' or '!='");
    }

    return {lhs, rhs, passed};
  }

  async screenshot(win, url, timeout) {
    let canvas = null;
    let remainingCount = this.urlCount.get(url) || 1;
    let cache = remainingCount > 1;
    logger.debug(`screenshot ${url} remainingCount: ` +
        `${remainingCount} cache: ${cache}`);
    let reuseCanvas = false;
    if (this.canvasCache.has(url)) {
      logger.debug(`screenshot ${url} taken from cache`);
      canvas = this.canvasCache.get(url);
      if (!cache) {
        this.canvasCache.delete(url);
      }
    } else {
      let canvases = this.canvasCache.get(null);
      if (canvases.length) {
        canvas = canvases.pop();
      } else {
        canvas = null;
      }
      reuseCanvas = !cache;

      let ctxInterface = win.CanvasRenderingContext2D;
      let flags = ctxInterface.DRAWWINDOW_DRAW_CARET |
          ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS |
          ctxInterface.DRAWWINDOW_DRAW_VIEW;

      logger.debug(`Starting load of ${url}`);
      let navigateOpts = {
        commandId: this.driver.listener.activeMessageId,
        pageTimeout: timeout,
      };
      if (this.lastURL === url) {
        logger.debug(`Refreshing page`);
        await this.driver.listener.refresh(navigateOpts);
      } else {
        navigateOpts.url = url;
        navigateOpts.loadEventExpected = false;
        await this.driver.listener.get(navigateOpts);
        this.lastURL = url;
      }

      this.driver.curBrowser.contentBrowser.focus();
      await this.driver.listener.reftestWait(url, this.remote);

      canvas = capture.canvas(
          win,
          0, // left
          0, // top
          win.innerWidth,
          win.innerHeight,
          {canvas, flags});
    }
    if (cache) {
      this.canvasCache.set(url, canvas);
    }
    this.urlCount.set(url, remainingCount - 1);
    return {canvas, reuseCanvas};
  }
};