Adds failed test tracking

2018-06-11 15:08:15 -07:00 · 2018-06-11 15:08:15 -07:00 · 0944c29a47
--- a/.gitignore
+++ b/.gitignore
@ -73,4 +73,5 @@ tests/cases/user/*/**/*.d.ts
 !tests/cases/user/zone.js/
 !tests/cases/user/bignumber.js/
 !tests/cases/user/discord.js/
-tests/baselines/reference/dt
+tests/baselines/reference/dt
+.failed-tests
--- a/Gulpfile.js
+++ b/Gulpfile.js
@ -38,7 +38,7 @@ const constEnumCaptureRegexp = /^(\s*)(export )?const enum (\S+) {(\s*)$/gm;
 const constEnumReplacement = "$1$2enum $3 {$4";

 const cmdLineOptions = minimist(process.argv.slice(2), {
-    boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"],
+    boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"],
    string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
    alias: {
        "b": "browser",
@ -598,7 +598,6 @@ gulp.task("LKG", "Makes a new LKG out of the built js files", ["clean", "dontUse
    return seq;
 });

-
 // Task to build the tests infrastructure using the built compiler
 const run = path.join(builtLocalDirectory, "run.js");
 gulp.task(run, /*help*/ false, [servicesFile, tsserverLibraryFile], () => {
@ -653,10 +652,12 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
        let testTimeout = cmdLineOptions.timeout;
        const debug = cmdLineOptions.debug;
        const inspect = cmdLineOptions.inspect;
-        const tests = cmdLineOptions.tests;
+        let tests = cmdLineOptions.tests;
        const runners = cmdLineOptions.runners;
        const light = cmdLineOptions.light;
        const stackTraceLimit = cmdLineOptions.stackTraceLimit;
+        const failed = cmdLineOptions.failed;
+        const keepFailed = cmdLineOptions.keepFailed || failed;
        const testConfigFile = "test.config";
        if (fs.existsSync(testConfigFile)) {
            fs.unlinkSync(testConfigFile);
@ -679,8 +680,8 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
            testTimeout = 400000;
        }

-        if (tests || runners || light || testTimeout || taskConfigsFolder) {
-            writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout);
+        if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
+            writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
        }

        const colors = cmdLineOptions.colors;
@ -690,7 +691,8 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
        // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
        if (!runInParallel) {
            const args = [];
-            args.push("-R", reporter);
+            args.push("-R", "scripts/failed-tests");
+            args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
            if (tests) {
                args.push("-g", `"${tests}"`);
            }
@ -711,8 +713,12 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
            }
            args.push(run);
            setNodeEnvToDevelopment();
-            exec(mocha, args, lintThenFinish, finish);
-
+            if (failed) {
+                exec(host, ["scripts/run-failed-tests.js"].concat(args), lintThenFinish, finish);
+            }
+            else {
+                exec(mocha, args, lintThenFinish, finish);
+            }
        }
        else {
            // run task to load all tests and partition them between workers
@ -888,8 +894,9 @@ function cleanTestDirs(done) {
 * @param {number=} workerCount
 * @param {string=} stackTraceLimit
 * @param {number=} timeout
+ * @param {boolean=} keepFailed
 */
-function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) {
+function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
    const testConfigContents = JSON.stringify({
        test: tests ? [tests] : undefined,
        runner: runners ? runners.split(",") : undefined,
@ -899,6 +906,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
        taskConfigsFolder,
        noColor: !cmdLineOptions.colors,
        timeout,
+        keepFailed
    });
    console.log("Running tests with config: " + testConfigContents);
    fs.writeFileSync("test.config", testConfigContents);
--- a/Jakefile.js
+++ b/Jakefile.js
@ -388,6 +388,8 @@ function runConsoleTests(defaultReporter, runInParallel) {
    const runners = process.env.runners || process.env.runner || process.env.ru;
    const tests = process.env.test || process.env.tests || process.env.t;
    const light = process.env.light === undefined || process.env.light !== "false";
+    const failed = process.env.failed;
+    const keepFailed = process.env.keepFailed || failed;
    const stackTraceLimit = process.env.stackTraceLimit;
    const colorsFlag = process.env.color || process.env.colors;
    const colors = colorsFlag !== "false" && colorsFlag !== "0";
@ -418,8 +420,8 @@ function runConsoleTests(defaultReporter, runInParallel) {
        testTimeout = 800000;
    }

-    if (tests || runners || light || testTimeout || taskConfigsFolder) {
-        writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout);
+    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
+        writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed);
    }

    // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
@ -427,7 +429,8 @@ function runConsoleTests(defaultReporter, runInParallel) {
    if (!runInParallel) {
        var startTime = Travis.mark();
        var args = [];
-        args.push("-R", reporter);
+        args.push("-R", "scripts/failed-tests");
+        args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
        if (tests) args.push("-g", `"${tests}"`);
        args.push(colors ? "--colors" : "--no-colors");
        if (bail) args.push("--bail");
@ -438,7 +441,14 @@ function runConsoleTests(defaultReporter, runInParallel) {
        }
        args.push(Paths.builtLocalRun);

-        var cmd = "mocha " + args.join(" ");
+        var cmd;
+        if (failed) {
+            args.unshift("scripts/run-failed-tests.js");
+            cmd = host + " " + args.join(" ");
+        }
+        else {
+            cmd = "mocha " + args.join(" ");
+        }
        var savedNodeEnv = process.env.NODE_ENV;
        process.env.NODE_ENV = "development";
        exec(cmd, function () {
@ -499,7 +509,7 @@ function runConsoleTests(defaultReporter, runInParallel) {
 }

 // used to pass data from jake command line directly to run.js
-function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) {
+function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed) {
    var testConfigContents = JSON.stringify({
        runners: runners ? runners.split(",") : undefined,
        test: tests ? [tests] : undefined,
@ -508,7 +518,8 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
        taskConfigsFolder: taskConfigsFolder,
        stackTraceLimit: stackTraceLimit,
        noColor: !colors,
-        timeout: testTimeout
+        timeout: testTimeout,
+        keepFailed: keepFailed
    });
    fs.writeFileSync('test.config', testConfigContents, { encoding: "utf-8" });
 }
--- a/scripts/failed-tests.d.ts
+++ b/scripts/failed-tests.d.ts
@ -0,0 +1,22 @@
+import Mocha = require("mocha");
+
+export = FailedTestsReporter;
+
+declare class FailedTestsReporter extends Mocha.reporters.Base {
+    passes: Mocha.Test[];
+    failures: Mocha.Test[];
+    reporterOptions: FailedTestsReporter.ReporterOptions;
+    reporter?: Mocha.reporters.Base;
+    constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions });
+    static writeFailures(file: string, passes: ReadonlyArray<Mocha.Test>, failures: ReadonlyArray<Mocha.Test>, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void;
+    done(failures: number, fn?: (failures: number) => void): void;
+}
+
+declare namespace FailedTestsReporter {
+    interface ReporterOptions {
+        file?: string;
+        keepFailed?: boolean;
+        reporter?: string | Mocha.ReporterConstructor;
+        reporterOptions?: any;
+    }
+}
--- a/scripts/failed-tests.js
+++ b/scripts/failed-tests.js
@ -0,0 +1,117 @@
+// @ts-check
+const Mocha = require("mocha");
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+
+/**
+ * .failed-tests reporter
+ * 
+ * @typedef {Object} ReporterOptions
+ * @property {string} [file]
+ * @property {boolean} [keepFailed]
+ * @property {string|Mocha.ReporterConstructor} [reporter]
+ * @property {*} [reporterOptions]
+ */
+class FailedTestsReporter extends Mocha.reporters.Base {
+    /**
+     * @param {Mocha.Runner} runner 
+     * @param {{ reporterOptions?: ReporterOptions }} [options]
+     */
+    constructor(runner, options) {
+        super(runner, options);
+        if (!runner) return;
+
+        const reporterOptions = this.reporterOptions = options.reporterOptions || {};
+        if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests";
+        if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false;
+        if (reporterOptions.reporter) {
+            /** @type {Mocha.ReporterConstructor} */
+            let reporter;
+            if (typeof reporterOptions.reporter === "function") {
+                reporter = reporterOptions.reporter;
+            }
+            else if (Mocha.reporters[reporterOptions.reporter]) {
+                reporter = Mocha.reporters[reporterOptions.reporter];
+            }
+            else {
+                try {
+                    reporter = require(reporterOptions.reporter);
+                }
+                catch (_) {
+                    reporter = require(path.resolve(process.cwd(), reporterOptions.reporter));
+                }
+            }
+
+            const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} });
+            this.reporter = new reporter(runner, newOptions);
+        }
+
+        /** @type {Mocha.Test[]} */
+        this.passes = [];
+        
+        /** @type {Mocha.Test[]} */
+        this.failures = [];
+        
+        runner.on("pass", test => this.passes.push(test));
+        runner.on("fail", test => this.failures.push(test));
+    }
+
+    /**
+     * @param {string} file 
+     * @param {ReadonlyArray<Mocha.Test>} passes 
+     * @param {ReadonlyArray<Mocha.Test>} failures 
+     * @param {boolean} keepFailed 
+     * @param {(err?: NodeJS.ErrnoException) => void} done 
+     */
+    static writeFailures(file, passes, failures, keepFailed, done) {
+        const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined);
+        if (failingTests.size > 0) {
+            for (const test of passes) {
+                const title = test.fullTitle().trim();
+                if (title) failingTests.delete(title);
+            }
+        }
+        for (const test of failures) {
+            const title = test.fullTitle().trim();
+            if (title) failingTests.add(title);
+        }
+        if (failingTests.size > 0) {
+            const failed = Array.from(failingTests).join(os.EOL);
+            fs.writeFile(file, failed, "utf8", done);
+        }
+        else if (!keepFailed) {
+            fs.unlink(file, done);
+        }
+        else {
+            done();
+        }
+
+        function readTests() {
+            return fs.readFileSync(file, "utf8")
+                .split(/\r?\n/g)
+                .map(line => line.trim())
+                .filter(line => line.length > 0);
+        }
+    }
+
+    /**
+     * @param {number} failures 
+     * @param {(failures: number) => void} [fn]
+     */
+    done(failures, fn) {
+        FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => {
+            const reporter = this.reporter;
+            if (reporter && reporter.done) {
+                reporter.done(failures, fn);
+            }
+            else if (fn) {
+                fn(failures);
+            }
+
+            if (err) console.error(err);
+        });
+    }
+}
+
+module.exports = FailedTestsReporter;
--- a/scripts/run-failed-tests.js
+++ b/scripts/run-failed-tests.js
@ -0,0 +1,93 @@
+const spawn = require('child_process').spawn;
+const os = require("os");
+const fs = require("fs");
+const path = require("path");
+
+let grep;
+try {
+    const failedTests = fs.readFileSync(".failed-tests", "utf8");
+    grep = failedTests
+        .split(/\r?\n/g)
+        .map(test => test.trim())
+        .filter(test => test.length > 0)
+        .map(escapeRegExp);
+}
+catch (e) {
+    grep = [];
+}
+
+let args = [];
+let waitForGrepValue = false;
+let grepIndex = -1;
+process.argv.slice(2).forEach((arg, index) => {
+    const [flag, value] = arg.split('=');
+    if (flag === "g" || flag === "grep") {
+        grepIndex = index - 1;
+        waitForGrepValue = arg !== flag;
+        if (!waitForGrepValue) grep.push(value.replace(/^"|"$/g, ""));
+        return;
+    }
+    if (waitForGrepValue) {
+        grep.push(arg.replace(/^"|"$/g, ""));
+        waitForGrepValue = false;
+        return;
+    }
+    args.push(arg);
+});
+
+let mocha = "./node_modules/mocha/bin/mocha";
+let grepOption;
+let grepOptionValue;
+let grepFile;
+if (grep.length) {
+    grepOption = "--grep";
+    grepOptionValue = grep.join("|");
+    if (grepOptionValue.length > 20) {
+        grepFile = path.resolve(os.tmpdir(), ".failed-tests.opts");
+        fs.writeFileSync(grepFile, `--grep ${grepOptionValue}`, "utf8");
+        grepOption = "--opts";
+        grepOptionValue = grepFile;
+        mocha = "./node_modules/mocha/bin/_mocha";
+    }
+}
+
+if (grepOption) {
+    if (grepIndex >= 0) {
+        args.splice(grepIndex, 0, grepOption, grepOptionValue);
+    }
+    else {
+        args.push(grepOption, grepOptionValue);
+    }
+}
+
+args.unshift(path.resolve(mocha));
+
+console.log(args.join(" "));
+const proc = spawn(process.execPath, args, {
+    stdio: 'inherit'
+});
+proc.on('exit', (code, signal) => {
+    process.on('exit', () => {
+        if (grepFile) {
+            fs.unlinkSync(grepFile);
+        }
+
+        if (signal) {
+            process.kill(process.pid, signal);
+        } else {
+            process.exit(code);
+        }
+    });
+});
+
+// terminate children.
+process.on('SIGINT', () => {
+    proc.kill('SIGINT'); // calls runner.abort()
+    proc.kill('SIGTERM'); // if that didn't work, we're probably in an infinite loop, so make it die.
+});
+
+function escapeRegExp(pattern) {
+    return pattern
+        .replace(/[^-\w\d\s]/g, match => "\\" + match)
+        .replace(/\s/g, "\\s");
+}
--- a/src/testRunner/parallel/host.ts
+++ b/src/testRunner/parallel/host.ts
@ -16,6 +16,10 @@ namespace Harness.Parallel.Host {
        const { fork } = require("child_process") as typeof import("child_process");
        const { statSync } = require("fs") as typeof import("fs");

+        // NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js
+        // tslint:disable-next-line:variable-name
+        const FailedTestReporter = require(path.resolve(__dirname, "../../scripts/failed-tests")) as typeof import("../../../scripts/failed-tests");
+
        const perfData = readSavedPerfData(configOption);
        const newTasks: Task[] = [];
        let tasks: Task[] = [];
@ -54,7 +58,7 @@ namespace Harness.Parallel.Host {
        interface Worker {
            process: import("child_process").ChildProcess;
            accumulatedOutput: string;
-            currentTasks?: {file: string}[];
+            currentTasks?: { file: string }[];
            timer?: any;
        }

@ -115,7 +119,7 @@ namespace Harness.Parallel.Host {
            update(index: number, percentComplete: number, color: string, title: string | undefined, titleColor?: string) {
                percentComplete = minMax(percentComplete, 0, 1);

-                const progressBar = this._progressBars[index] || (this._progressBars[index] = { });
+                const progressBar = this._progressBars[index] || (this._progressBars[index] = {});
                const width = this._options.width;
                const n = Math.floor(width * percentComplete);
                const i = width - n;
@ -177,7 +181,7 @@ namespace Harness.Parallel.Host {
            return `${perfdataFileNameFragment}${target ? `.${target}` : ""}.json`;
        }

-        function readSavedPerfData(target?: string): {[testHash: string]: number} | undefined {
+        function readSavedPerfData(target?: string): { [testHash: string]: number } | undefined {
            const perfDataContents = IO.readFile(perfdataFileName(target));
            if (perfDataContents) {
                return JSON.parse(perfDataContents);
@ -189,7 +193,7 @@ namespace Harness.Parallel.Host {
            return `tsrunner-${runner}://${test}`;
        }

-        function startDelayed(perfData: {[testHash: string]: number} | undefined, totalCost: number) {
+        function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) {
            console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : "."));
            console.log("Discovering runner-based tests...");
            const discoverStart = +(new Date());
@ -247,7 +251,7 @@ namespace Harness.Parallel.Host {
            const progressUpdateInterval = 1 / progressBars._options.width;
            let nextProgress = progressUpdateInterval;

-            const newPerfData: {[testHash: string]: number} = {};
+            const newPerfData: { [testHash: string]: number } = {};

            const workers: Worker[] = [];
            let closedWorkers = 0;
@ -531,10 +535,26 @@ namespace Harness.Parallel.Host {
                patchStats(consoleReporter.stats);

                let xunitReporter: import("mocha").reporters.XUnit | undefined;
-                if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser && process.env.CI === "true") {
-                    xunitReporter = new Mocha.reporters.XUnit(replayRunner, { reporterOptions: { suiteName: "Tests", output: "./TEST-results.xml" } });
-                    patchStats(xunitReporter.stats);
-                    xunitReporter.write(`<?xml version="1.0" encoding="UTF-8"?>\n`);
+                let failedTestReporter: import("../../../scripts/failed-tests") | undefined;
+                if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
+                    if (process.env.CI === "true") {
+                        xunitReporter = new Mocha.reporters.XUnit(replayRunner, {
+                            reporterOptions: {
+                                suiteName: "Tests",
+                                output: "./TEST-results.xml"
+                            }
+                        });
+                        patchStats(xunitReporter.stats);
+                        xunitReporter.write(`<?xml version="1.0" encoding="UTF-8"?>\n`);
+                    }
+                    else {
+                        failedTestReporter = new FailedTestReporter(replayRunner, {
+                            reporterOptions: {
+                                file: path.resolve(".failed-tests"),
+                                keepFailed
+                            }
+                        });
+                    }
                }

                const savedUseColors = Base.useColors;
@ -551,6 +571,9 @@ namespace Harness.Parallel.Host {
                if (xunitReporter) {
                    xunitReporter.done(errorResults.length, failures => process.exit(failures));
                }
+                else if (failedTestReporter) {
+                    failedTestReporter.done(errorResults.length, failures => process.exit(failures));
+                }
                else {
                    process.exit(errorResults.length);
                }
--- a/src/testRunner/runner.ts
+++ b/src/testRunner/runner.ts
@ -62,6 +62,7 @@ let workerCount: number;
 let runUnitTests: boolean | undefined;
 let stackTraceLimit: number | "full" | undefined;
 let noColors = false;
+let keepFailed = false;

 interface TestConfig {
    light?: boolean;
@ -74,6 +75,7 @@ interface TestConfig {
    runUnitTests?: boolean;
    noColors?: boolean;
    timeout?: number;
+    keepFailed?: boolean;
 }

 interface TaskSet {
@ -102,6 +104,9 @@ function handleTestConfig() {
        if (testConfig.noColors !== undefined) {
            noColors = testConfig.noColors;
        }
+        if (testConfig.keepFailed) {
+            keepFailed = true;
+        }

        if (testConfig.stackTraceLimit === "full") {
            (<any>Error).stackTraceLimit = Infinity;