/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ "use strict"; const { Preferences } = ChromeUtils.import( "resource://gre/modules/Preferences.jsm" ); const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); const { XPCOMUtils } = ChromeUtils.import( "resource://gre/modules/XPCOMUtils.jsm" ); const { assert } = ChromeUtils.import("chrome://marionette/content/assert.js"); const { capture } = ChromeUtils.import( "chrome://marionette/content/capture.js" ); const { InvalidArgumentError } = ChromeUtils.import( "chrome://marionette/content/error.js" ); const { Log } = ChromeUtils.import("chrome://marionette/content/log.js"); XPCOMUtils.defineLazyGetter(this, "logger", Log.get); this.EXPORTED_SYMBOLS = ["reftest"]; const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"; const PREF_E10S = "browser.tabs.remote.autostart"; const SCREENSHOT_MODE = { unexpected: 0, fail: 1, always: 2, }; const STATUS = { PASS: "PASS", FAIL: "FAIL", ERROR: "ERROR", TIMEOUT: "TIMEOUT", }; const DEFAULT_REFTEST_WIDTH = 600; const DEFAULT_REFTEST_HEIGHT = 600; /** * Implements an fast runner for web-platform-tests format reftests * c.f. http://web-platform-tests.org/writing-tests/reftests.html. * * @namespace */ this.reftest = {}; /** * @memberof reftest * @class Runner */ reftest.Runner = class { constructor(driver) { this.driver = driver; this.canvasCache = new DefaultMap(undefined, () => new Map([[null, []]])); this.windowUtils = null; this.lastURL = null; this.remote = Preferences.get(PREF_E10S); } /** * Setup the required environment for running reftests. * * This will open a non-browser window in which the tests will * be loaded, and set up various caches for the reftest run. * * @param {Object.} urlCount * Object holding a map of URL: number of times the URL * will be opened during the reftest run, where that's * greater than 1. * @param {string} screenshotMode * String enum representing when screenshots should be taken */ setup(urlCount, screenshotMode) { this.parentWindow = assert.open(this.driver.getCurrentWindow()); this.screenshotMode = SCREENSHOT_MODE[screenshotMode] || SCREENSHOT_MODE.unexpected; this.urlCount = Object.keys(urlCount || {}).reduce( (map, key) => map.set(key, urlCount[key]), new Map() ); } async ensureWindow(timeout, width, height) { logger.debug(`ensuring we have a window ${width}x${height}`); if (this.reftestWin && !this.reftestWin.closed) { let browserRect = this.reftestWin.gBrowser.getBoundingClientRect(); if (browserRect.width === width && browserRect.height === height) { return this.reftestWin; } logger.debug(`current: ${browserRect.width}x${browserRect.height}`); } let reftestWin; if (Services.appinfo.OS == "Android") { logger.debug("Using current window"); reftestWin = this.parentWindow; await this.driver.listener.get({ commandID: this.driver.listener.activeMessageId, pageTimeout: timeout, url: "about:blank", loadEventExpected: false, }); } else { logger.debug("Using separate window"); if (this.reftestWin && !this.reftestWin.closed) { this.reftestWin.close(); } reftestWin = await this.openWindow(width, height); } this.setupWindow(reftestWin, width, height); this.windowUtils = reftestWin.windowUtils; this.reftestWin = reftestWin; let found = this.driver.findWindow([reftestWin], () => true); await this.driver.setWindowHandle(found, true); let browserRect = reftestWin.gBrowser.getBoundingClientRect(); logger.debug(`new: ${browserRect.width}x${browserRect.height}`); return reftestWin; } async openWindow(width, height) { assert.positiveInteger(width); assert.positiveInteger(height); let reftestWin = this.parentWindow.open( "chrome://marionette/content/reftest.xhtml", "reftest", `chrome,height=${height},width=${width}` ); await new Promise(resolve => { reftestWin.addEventListener("load", resolve, { once: true }); }); return reftestWin; } setupWindow(reftestWin, width, height) { let browser; if (Services.appinfo.OS === "Android") { browser = reftestWin.document.getElementsByTagName("browser")[0]; browser.setAttribute("remote", "false"); } else { browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser"); browser.permanentKey = {}; browser.setAttribute("id", "browser"); browser.setAttribute("type", "content"); browser.setAttribute("primary", "true"); if (this.remote) { browser.setAttribute("remote", "true"); browser.setAttribute("remoteType", "web"); } else { browser.setAttribute("remote", "false"); } } // Make sure the browser element is exactly the right size, no matter // what size our window is const windowStyle = `padding: 0px; margin: 0px; border:none; min-width: ${width}px; min-height: ${height}px; max-width: ${width}px; max-height: ${height}px`; browser.setAttribute("style", windowStyle); if (Services.appinfo.OS !== "Android") { let doc = reftestWin.document.documentElement; while (doc.firstChild) { doc.firstChild.remove(); } doc.appendChild(browser); } if (reftestWin.BrowserApp) { reftestWin.BrowserApp = browser; } reftestWin.gBrowser = browser; return reftestWin; } async abort() { if (this.reftestWin && this.reftestWin != this.parentWindow) { this.driver.closeChromeWindow(); let parentHandle = this.driver.findWindow( [this.parentWindow], () => true ); await this.driver.setWindowHandle(parentHandle); } this.reftestWin = null; } /** * Run a specific reftest. * * The assumed semantics are those of web-platform-tests where * references form a tree and each test must meet all the conditions * to reach one leaf node of the tree in order for the overall test * to pass. * * @param {string} testUrl * URL of the test itself. * @param {Array.} references * Array representing a tree of references to try. * * Each item in the array represents a single reference node and * has the form [referenceUrl, references, relation], * where referenceUrl is a string to the URL, relation * is either == or != depending on the * type of reftest, and references is another array containing * items of the same form, representing further comparisons treated * as AND with the current item. Sibling entries are treated as OR. * * For example with testUrl of T: * *

   *       references = [[A, [[B, [], ==]], ==]]
   *       Must have T == A AND A == B to pass
   *
   *       references = [[A, [], ==], [B, [], !=]
   *       Must have T == A OR T != B
   *
   *       references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]]
   *       Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D)
   *     
* * @param {string} expected * Expected test outcome (e.g. PASS, FAIL). * @param {number} timeout * Test timeout in milliseconds. * * @return {Object} * Result object with fields status, message and extra. */ async run( testUrl, references, expected, timeout, width = DEFAULT_REFTEST_WIDTH, height = DEFAULT_REFTEST_HEIGHT ) { let timeoutHandle; let timeoutPromise = new Promise(resolve => { timeoutHandle = this.parentWindow.setTimeout(() => { resolve({ status: STATUS.TIMEOUT, message: null, extra: {} }); }, timeout); }); let testRunner = (async () => { let result; try { result = await this.runTest( testUrl, references, expected, timeout, width, height ); } catch (e) { result = { status: STATUS.ERROR, message: String(e), stack: e.stack, extra: {}, }; } return result; })(); let result = await Promise.race([testRunner, timeoutPromise]); this.parentWindow.clearTimeout(timeoutHandle); if (result.status === STATUS.TIMEOUT) { await this.abort(); } return result; } async runTest(testUrl, references, expected, timeout, width, height) { let win = await this.ensureWindow(timeout, width, height); function toBase64(screenshot) { let dataURL = screenshot.canvas.toDataURL(); return dataURL.split(",")[1]; } let result = { status: STATUS.FAIL, message: "", stack: null, extra: {}, }; let screenshotData = []; let stack = []; for (let i = references.length - 1; i >= 0; i--) { let item = references[i]; stack.push([testUrl, ...item]); } let done = false; while (stack.length && !done) { let [lhsUrl, rhsUrl, references, relation, extras = {}] = stack.pop(); result.message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`; let comparison; try { comparison = await this.compareUrls( win, lhsUrl, rhsUrl, relation, timeout, extras ); } catch (e) { comparison = { lhs: null, rhs: null, passed: false, error: e, msg: null, }; } if (comparison.msg) { result.message += `${comparison.msg}\n`; } if (comparison.error !== null) { result.status = STATUS.ERROR; result.message += String(comparison.error); result.stack = comparison.error.stack; } function recordScreenshot() { let encodedLHS = comparison.lhs ? toBase64(comparison.lhs) : ""; let encodedRHS = comparison.rhs ? toBase64(comparison.rhs) : ""; screenshotData.push([ { url: lhsUrl, screenshot: encodedLHS }, relation, { url: rhsUrl, screenshot: encodedRHS }, ]); } if (this.screenshotMode === SCREENSHOT_MODE.always) { recordScreenshot(); } if (comparison.passed) { if (references.length) { for (let i = references.length - 1; i >= 0; i--) { let item = references[i]; stack.push([rhsUrl, ...item]); } } else { // Reached a leaf node so all of one reference chain passed result.status = STATUS.PASS; if ( this.screenshotMode <= SCREENSHOT_MODE.fail && expected != result.status ) { recordScreenshot(); } done = true; } } else if (!stack.length || result.status == STATUS.ERROR) { // If we don't have any alternatives to try then this will be // the last iteration, so save the failing screenshots if required. let isFail = this.screenshotMode === SCREENSHOT_MODE.fail; let isUnexpected = this.screenshotMode === SCREENSHOT_MODE.unexpected; if (isFail || (isUnexpected && expected != result.status)) { recordScreenshot(); } } // Return any reusable canvases to the pool let cacheKey = width + "x" + height; let canvasPool = this.canvasCache.get(cacheKey).get(null); [comparison.lhs, comparison.rhs].map(screenshot => { if (screenshot !== null && screenshot.reuseCanvas) { canvasPool.push(screenshot.canvas); } }); logger.debug( `Canvas pool (${cacheKey}) is of length ${canvasPool.length}` ); } if (screenshotData.length) { // For now the tbpl formatter only accepts one screenshot, so just // return the last one we took. let lastScreenshot = screenshotData[screenshotData.length - 1]; // eslint-disable-next-line camelcase result.extra.reftest_screenshots = lastScreenshot; } return result; } async compareUrls(win, lhsUrl, rhsUrl, relation, timeout, extras) { logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`); // Take the reference screenshot first so that if we pause // we see the test rendering let rhs = await this.screenshot(win, rhsUrl, timeout); let lhs = await this.screenshot(win, lhsUrl, timeout); logger.debug(`lhs canvas size ${lhs.canvas.width}x${lhs.canvas.height}`); logger.debug(`rhs canvas size ${rhs.canvas.width}x${rhs.canvas.height}`); let passed; let error = null; let pixelsDifferent = null; let maxDifferences = {}; let msg = null; try { pixelsDifferent = this.windowUtils.compareCanvases( lhs.canvas, rhs.canvas, maxDifferences ); } catch (e) { passed = false; error = e; } if (error === null) { passed = this.isAcceptableDifference( maxDifferences.value, pixelsDifferent, extras.fuzzy ); switch (relation) { case "==": if (!passed) { msg = `Found ${pixelsDifferent} pixels different, ` + `maximum difference per channel ${maxDifferences.value}`; } break; case "!=": passed = !passed; break; default: throw new InvalidArgumentError( "Reftest operator should be '==' or '!='" ); } } return { lhs, rhs, passed, error, msg }; } isAcceptableDifference(maxDifference, pixelsDifferent, allowed) { if (!allowed) { logger.info(`No differences allowed`); return pixelsDifferent === 0; } let [allowedDiff, allowedPixels] = allowed; logger.info( `Allowed ${allowedPixels.join("-")} pixels different, ` + `maximum difference per channel ${allowedDiff.join("-")}` ); return ( (pixelsDifferent === 0 && allowedPixels[0] == 0) || (maxDifference === 0 && allowedDiff[0] == 0) || (maxDifference >= allowedDiff[0] && maxDifference <= allowedDiff[1] && (pixelsDifferent >= allowedPixels[0] || pixelsDifferent <= allowedPixels[1])) ); } ensureFocus(win) { const focusManager = Services.focus; if (focusManager.activeWindow != win) { focusManager.activeWindow = win; } this.driver.curBrowser.contentBrowser.focus(); } async screenshot(win, url, timeout) { // On windows the above doesn't *actually* set the window to be the // reftest size; but *does* set the content area to be the right size; // the window is given some extra borders that aren't explicable from CSS let browserRect = win.gBrowser.getBoundingClientRect(); let canvas = null; let remainingCount = this.urlCount.get(url) || 1; let cache = remainingCount > 1; let cacheKey = browserRect.width + "x" + browserRect.height; logger.debug( `screenshot ${url} remainingCount: ` + `${remainingCount} cache: ${cache} cacheKey: ${cacheKey}` ); let reuseCanvas = false; let sizedCache = this.canvasCache.get(cacheKey); if (sizedCache.has(url)) { logger.debug(`screenshot ${url} taken from cache`); canvas = sizedCache.get(url); if (!cache) { sizedCache.delete(url); } } else { let canvasPool = sizedCache.get(null); if (canvasPool.length) { logger.debug("reusing canvas from canvas pool"); canvas = canvasPool.pop(); } else { logger.debug("using new canvas"); canvas = null; } reuseCanvas = !cache; let ctxInterface = win.CanvasRenderingContext2D; let flags = ctxInterface.DRAWWINDOW_DRAW_CARET | ctxInterface.DRAWWINDOW_DRAW_VIEW | ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS; if ( !( 0 <= browserRect.left && 0 <= browserRect.top && win.innerWidth >= browserRect.width && win.innerHeight >= browserRect.height ) ) { logger.error(`Invalid window dimensions: browserRect.left: ${browserRect.left} browserRect.top: ${browserRect.top} win.innerWidth: ${win.innerWidth} browserRect.width: ${browserRect.width} win.innerHeight: ${win.innerHeight} browserRect.height: ${browserRect.height}`); throw new Error("Window has incorrect dimensions"); } url = new URL(url).href; // normalize the URL logger.debug(`Starting load of ${url}`); let navigateOpts = { commandId: this.driver.listener.activeMessageId, pageTimeout: timeout, }; if (this.lastURL === url) { logger.debug(`Refreshing page`); await this.driver.listener.refresh(navigateOpts); } else { navigateOpts.url = url; navigateOpts.loadEventExpected = false; await this.driver.listener.get(navigateOpts); this.lastURL = url; } this.ensureFocus(win); await this.driver.listener.reftestWait(url, this.remote); canvas = await capture.canvas( win, win.docShell.browsingContext, 0, // left 0, // top browserRect.width, browserRect.height, { canvas, flags, readback: true } ); } if ( canvas.width !== browserRect.width || canvas.height !== browserRect.height ) { logger.warn( `Canvas dimensions changed to ${canvas.width}x${canvas.height}` ); reuseCanvas = false; cache = false; } if (cache) { sizedCache.set(url, canvas); } this.urlCount.set(url, remainingCount - 1); return { canvas, reuseCanvas }; } }; class DefaultMap extends Map { constructor(iterable, defaultFactory) { super(iterable); this.defaultFactory = defaultFactory; } get(key) { if (this.has(key)) { return super.get(key); } let v = this.defaultFactory(); this.set(key, v); return v; } }