Introduce Code Coverage Metrics Reporting (#37)

@W-4092804@

.appveyor.yml

@@ -23,6 +23,7 @@ install:
   - 7z x sfdx-windows-amd64.tar
   - SET PATH=%APPVEYOR_BUILD_FOLDER%\sfdx\bin;%PATH%
   - sfdx update
+  - npm install -g codecov 
 
 build_script:
   - npm run compile
@@ -32,3 +33,7 @@ test_script:
   - node --version
   - npm --version
   - npm run test
+
+on_finish:
+  - codecov
+ 

.gitignore

@@ -26,6 +26,10 @@ lib-cov
 # Coverage directory used by tools like istanbul
 coverage
 
+# test results
+test-results.xml
+xunit.xml
+
 # nyc test coverage
 .nyc_output
 

.travis.yml

@@ -52,8 +52,10 @@ before_install:
 
 install:
   - npm install
+  - npm install -g codecov 
 
 script:
   - npm run compile
   - npm run lint
   - npm run test
+  - codecov

packages/salesforcedx-utils-vscode/package.json

@@ -5,20 +5,26 @@
   "version": "40.4.0",
   "publisher": "salesforce",
   "license": "BSD-3-Clause",
-  "categories": [
-    "Other"
-  ],
+  "categories": ["Other"],
   "dependencies": {
     "rxjs": "^5.4.1",
     "tree-kill": "^1.1.0"
   },
   "devDependencies": {
     "@types/chai": "^4.0.0",
-    "@types/mocha": "2.2.38",
+    "@types/glob": "^5.0.30",
+    "@types/mocha": "^2.2.38",
     "@types/node": "^6.0.40",
     "chai": "^4.0.2",
-    "mocha": "3.2.0",
+    "decache": "^4.1.0",
+    "glob": "^7.1.2",
+    "istanbul": "^0.4.5",
+    "mocha": "^3.2.0",
+    "mocha-junit-reporter": "^1.13.0",
+    "mocha-multi-reporters": "^1.1.4",
     "nyc": "^11.0.2",
+    "remap-istanbul": "^0.9.5",
+    "source-map-support": "^0.4.15",
     "typescript": "2.4.0"
   },
   "scripts": {
@@ -26,7 +32,7 @@
     "lint": "tslint --project .",
     "watch": "tsc -watch -p .",
     "clean": "shx rm -rf node_modules && shx rm -rf out",
-    "test": "./node_modules/.bin/_mocha --recursive out/test",
+    "test": "./node_modules/.bin/_mocha --recursive out/test --reporter mocha-multi-reporters",
     "coverage": "./node_modules/.bin/nyc npm test"
   },
   "main": "./out/src/"

packages/salesforcedx-utils-vscode/src/test/testrunner.ts

@@ -0,0 +1,261 @@
+/*
+ * rewritten test runner for integrating code coverage;
+ * https://github.com/codecov/example-typescript-vscode-extension
+ */
+
+'use strict';
+
+import * as fs from 'fs';
+import * as glob from 'glob';
+import * as paths from 'path';
+
+// tslint:disable:no-var-requires
+const istanbul = require('istanbul');
+// tslint:disable-next-line:variable-name
+const Mocha = require('mocha');
+const remapIstanbul = require('remap-istanbul');
+
+// Linux: prevent a weird NPE when mocha on Linux requires the window size from the TTY
+// Since we are not running in a tty environment, we just implementt he method statically
+const tty = require('tty');
+if (!tty.getWindowSize) {
+  tty.getWindowSize = (): number[] => {
+    return [80, 75];
+  };
+}
+
+let mocha = new Mocha({
+  ui: 'tdd',
+  useColors: true,
+  reporter: 'mocha-multi-reporters'
+});
+
+function configure(mochaOpts: any): void {
+  mocha = new Mocha(mochaOpts);
+}
+exports.configure = configure;
+
+function _mkDirIfExists(dir: string): void {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir);
+  }
+}
+
+function _readCoverOptions(testsRoot: string): ITestRunnerOptions | undefined {
+  const coverConfigPath = paths.join(testsRoot, '..', '..', 'coverconfig.json');
+  let coverConfig: ITestRunnerOptions | undefined = undefined;
+  if (fs.existsSync(coverConfigPath)) {
+    const configContent = fs.readFileSync(coverConfigPath, 'utf-8');
+    coverConfig = JSON.parse(configContent);
+  }
+  return coverConfig;
+}
+
+function run(testsRoot: any, clb: any): any {
+  // Enable source map support
+  require('source-map-support').install();
+
+  // Read configuration for the coverage file
+  const coverOptions: ITestRunnerOptions | undefined = _readCoverOptions(
+    testsRoot
+  );
+  if (coverOptions && coverOptions.enabled) {
+    // Setup coverage pre-test, including post-test hook to report
+    const coverageRunner = new CoverageRunner(coverOptions, testsRoot, clb);
+    coverageRunner.setupCoverage();
+  }
+
+  // Glob test files
+  glob('**/**.test.js', { cwd: testsRoot }, (error, files): any => {
+    if (error) {
+      return clb(error);
+    }
+    try {
+      // Fill into Mocha
+      files.forEach((f): Mocha => {
+        return mocha.addFile(paths.join(testsRoot, f));
+      });
+      // Run the tests
+      let failureCount = 0;
+
+      mocha
+        .run()
+        .on('fail', (test: any, err: any): void => {
+          failureCount++;
+        })
+        .on('end', (): void => {
+          clb(undefined, failureCount);
+        });
+    } catch (error) {
+      return clb(error);
+    }
+  });
+}
+exports.run = run;
+
+interface ITestRunnerOptions {
+  enabled?: boolean;
+  relativeCoverageDir: string;
+  relativeSourcePath: string;
+  ignorePatterns: string[];
+  includePid?: boolean;
+  reports?: string[];
+  verbose?: boolean;
+}
+
+declare var global: {
+  [key: string]: any; // missing index defintion
+};
+
+class CoverageRunner {
+  private coverageVar: string = '$$cov_' + new Date().getTime() + '$$';
+  private transformer: any = undefined;
+  private matchFn: any = undefined;
+  private instrumenter: any = undefined;
+
+  constructor(
+    private options: ITestRunnerOptions,
+    private testsRoot: string,
+    errorRunCallback: (error: string) => any
+  ) {
+    if (!options.relativeSourcePath) {
+      return errorRunCallback(
+        'Error - relativeSourcePath must be defined for code coverage to work'
+      );
+    }
+  }
+
+  public setupCoverage(): void {
+    // Set up Code Coverage, hooking require so that instrumented code is returned
+    const self = this;
+    self.instrumenter = new istanbul.Instrumenter({
+      coverageVariable: self.coverageVar
+    });
+    const sourceRoot = paths.join(
+      self.testsRoot,
+      self.options.relativeSourcePath
+    );
+
+    // Glob source files
+    const srcFiles = glob.sync('**/**.js', {
+      cwd: sourceRoot,
+      ignore: self.options.ignorePatterns
+    });
+
+    // Create a match function - taken from the run-with-cover.js in istanbul.
+    const decache = require('decache');
+    interface FileMap {
+      [key: string]: boolean;
+    }
+    const fileMap: FileMap = {};
+    srcFiles.forEach(file => {
+      const fullPath = paths.join(sourceRoot, file);
+      fileMap[fullPath] = true;
+
+      // On Windows, extension is loaded pre-test hooks and this mean we lose
+      // our chance to hook the Require call. In order to instrument the code
+      // we have to decache the JS file so on next load it gets instrumented.
+      // This doesn"t impact tests, but is a concern if we had some integration
+      // tests that relied on VSCode accessing our module since there could be
+      // some shared global state that we lose.
+      decache(fullPath);
+    });
+
+    self.matchFn = (file: string): boolean => {
+      return fileMap[file];
+    };
+    self.matchFn.files = Object.keys(fileMap);
+
+    // Hook up to the Require function so that when this is called, if any of our source files
+    // are required, the instrumented version is pulled in instead. These instrumented versions
+    // write to a global coverage variable with hit counts whenever they are accessed
+    self.transformer = self.instrumenter.instrumentSync.bind(self.instrumenter);
+    const hookOpts = { verbose: false, extensions: ['.js'] };
+    istanbul.hook.hookRequire(self.matchFn, self.transformer, hookOpts);
+
+    // initialize the global variable to stop mocha from complaining about leaks
+    global[self.coverageVar] = {};
+
+    // Hook the process exit event to handle reporting
+    // Only report coverage if the process is exiting successfully
+    process.on('exit', (code: any) => {
+      self.reportCoverage();
+    });
+  }
+
+  /**
+   * Writes a coverage report. Note that as this is called in the process exit callback, all calls must be synchronous.
+   *
+   * @returns {void}
+   *
+   * @memberOf CoverageRunner
+   */
+  public reportCoverage(): void {
+    const self = this;
+    istanbul.hook.unhookRequire();
+    let cov: any;
+    if (
+      typeof global[self.coverageVar] === 'undefined' ||
+      Object.keys(global[self.coverageVar]).length === 0
+    ) {
+      console.error(
+        'No coverage information was collected, exit without writing coverage information'
+      );
+      return;
+    } else {
+      cov = global[self.coverageVar];
+    }
+
+    // TODO consider putting this under a conditional flag
+    // Files that are not touched by code ran by the test runner is manually instrumented, to
+    // illustrate the missing coverage.
+    self.matchFn.files.forEach((file: string) => {
+      if (!cov[file]) {
+        self.transformer(fs.readFileSync(file, 'utf-8'), file);
+
+        // When instrumenting the code, istanbul will give each FunctionDeclaration a value of 1 in coverState.s,
+        // presumably to compensate for function hoisting. We need to reset this, as the function was not hoisted,
+        // as it was never loaded.
+        Object.keys(self.instrumenter.coverState.s).forEach(key => {
+          self.instrumenter.coverState.s[key] = 0;
+        });
+
+        cov[file] = self.instrumenter.coverState;
+      }
+    });
+
+    // TODO Allow config of reporting directory with
+    const reportingDir = paths.join(
+      self.testsRoot,
+      self.options.relativeCoverageDir
+    );
+    const includePid = self.options.includePid;
+    const pidExt = includePid ? '-' + process.pid : '';
+    const coverageFile = paths.resolve(
+      reportingDir,
+      'coverage' + pidExt + '.json'
+    );
+
+    _mkDirIfExists(reportingDir); // yes, do this again since some test runners could clean the dir initially created
+
+    fs.writeFileSync(coverageFile, JSON.stringify(cov), { encoding: 'utf8' });
+
+    const remappedCollector = remapIstanbul.remap(cov, {
+      warn: (warning: string) => {
+        // We expect some warnings as any JS file without a typescript mapping will cause this.
+        // By default, we"ll skip printing these to the console as it clutters it up
+        if (self.options.verbose) {
+          console.warn(warning);
+        }
+      }
+    });
+
+    const reporter = new istanbul.Reporter(undefined, reportingDir);
+    const reportTypes =
+      self.options.reports instanceof Array ? self.options.reports : ['lcov'];
+    reporter.addAll(reportTypes);
+    reporter.write(remappedCollector, true, () => {
+      console.log(`reports written to ${reportingDir}`);
+    });
+  }
+}

packages/salesforcedx-vscode-apex/coverconfig.json

@@ -0,0 +1,9 @@
+{
+  "enabled": true,
+  "relativeSourcePath": "../src",
+  "relativeCoverageDir": "../../coverage",
+  "ignorePatterns": ["**/node_modules/**"],
+  "includePid": false,
+  "reports": ["json", "html", "lcov"],
+  "verbose": false
+}

packages/salesforcedx-vscode-apex/package.json

@@ -21,10 +21,9 @@
   "engines": {
     "vscode": "^1.13.0"
   },
-  "categories": [
-    "Languages"
-  ],
+  "categories": ["Languages"],
   "devDependencies": {
+    "@salesforce/salesforcedx-utils-vscode": "40.4.0",
     "@types/chai": "^4.0.0",
     "@types/mocha": "2.2.38",
     "@types/node": "^6.0.40",
@@ -47,9 +46,7 @@
     "postinstall": "node ./node_modules/vscode/bin/install",
     "test": "node ./node_modules/vscode/bin/test"
   },
-  "activationEvents": [
-    "workspaceContains:sfdx-project.json"
-  ],
+  "activationEvents": ["workspaceContains:sfdx-project.json"],
   "main": "./out/src",
   "contributes": {
     "configuration": {
@@ -57,10 +54,7 @@
       "title": "%configuration_title%",
       "properties": {
         "salesforcedx-vscode-apex.java.home": {
-          "type": [
-            "string",
-            "null"
-          ],
+          "type": ["string", "null"],
           "default": null,
           "description": "%java_home_description%"
         }
@@ -69,14 +63,8 @@
     "languages": [
       {
         "id": "apex",
-        "aliases": [
-          "Apex",
-          "apex"
-        ],
-        "extensions": [
-          ".cls",
-          ".trigger"
-        ],
+        "aliases": ["Apex", "apex"],
+        "extensions": [".cls", ".trigger"],
         "configuration": "./syntaxes/apex.configuration.json"
       }
     ],

packages/salesforcedx-vscode-apex/test/index.ts

@@ -5,20 +5,8 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-//
-// PLEASE DO NOT MODIFY / DELETE UNLESS YOU KNOW WHAT YOU ARE DOING
-//
-// This file is providing the test runner to use when running extension tests.
-// By default the test runner in use is Mocha based.
-//
-// You can provide your own test runner if you want to override it by exporting
-// a function run(testRoot: string, clb: (error:Error) => void) that the extension
-// host can call to run the tests. The test runner is expected to use console.log
-// to report the results back to the caller. When the tests are finished, return
-// a possible error to the callback or null if none.
-
 // tslint:disable-next-line:no-var-requires
-const testRunner = require('vscode/lib/testrunner');
+const testRunner = require('@salesforce/salesforcedx-utils-vscode/out/src/test/testrunner');
 
 // You can directly control Mocha options by uncommenting the following lines
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info

packages/salesforcedx-vscode-core/coverconfig.json

@@ -0,0 +1,9 @@
+{
+  "enabled": true,
+  "relativeSourcePath": "../src",
+  "relativeCoverageDir": "../../coverage",
+  "ignorePatterns": ["**/node_modules/**"],
+  "includePid": false,
+  "reports": ["json", "html", "lcov"],
+  "verbose": false
+}

packages/salesforcedx-vscode-core/test/index.ts

@@ -18,7 +18,7 @@
 // a possible error to the callback or null if none.
 
 // tslint:disable-next-line:no-var-requires
-const testRunner = require('vscode/lib/testrunner');
+const testRunner = require('@salesforce/salesforcedx-utils-vscode/out/src/test/testrunner');
 
 // You can directly control Mocha options by uncommenting the following lines
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info

packages/salesforcedx-vscode-lightning/coverconfig.json

@@ -0,0 +1,9 @@
+{
+  "enabled": true,
+  "relativeSourcePath": "../src",
+  "relativeCoverageDir": "../../coverage",
+  "ignorePatterns": ["**/node_modules/**"],
+  "includePid": false,
+  "reports": ["json", "html", "lcov"],
+  "verbose": false
+}

packages/salesforcedx-vscode-lightning/package.json

@@ -21,10 +21,9 @@
   "engines": {
     "vscode": "^1.13.0"
   },
-  "categories": [
-    "Languages"
-  ],
+  "categories": ["Languages"],
   "devDependencies": {
+    "@salesforce/salesforcedx-utils-vscode": "40.4.0",
     "@types/chai": "^4.0.0",
     "@types/mocha": "2.2.38",
     "@types/node": "^6.0.40",

packages/salesforcedx-vscode-lightning/test/index.ts

@@ -5,20 +5,8 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-//
-// PLEASE DO NOT MODIFY / DELETE UNLESS YOU KNOW WHAT YOU ARE DOING
-//
-// This file is providing the test runner to use when running extension tests.
-// By default the test runner in use is Mocha based.
-//
-// You can provide your own test runner if you want to override it by exporting
-// a function run(testRoot: string, clb: (error:Error) => void) that the extension
-// host can call to run the tests. The test runner is expected to use console.log
-// to report the results back to the caller. When the tests are finished, return
-// a possible error to the callback or null if none.
-
 // tslint:disable-next-line:no-var-requires
-const testRunner = require('vscode/lib/testrunner');
+const testRunner = require('@salesforce/salesforcedx-utils-vscode/out/src/test/testrunner');
 
 // You can directly control Mocha options by uncommenting the following lines
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info

packages/salesforcedx-vscode-visualforce/coverconfig.json

@@ -0,0 +1,9 @@
+{
+  "enabled": true,
+  "relativeSourcePath": "../src",
+  "relativeCoverageDir": "../../coverage",
+  "ignorePatterns": ["**/node_modules/**"],
+  "includePid": false,
+  "reports": ["json", "html", "lcov"],
+  "verbose": false
+}

packages/salesforcedx-vscode-visualforce/package.json

@@ -21,10 +21,9 @@
   "engines": {
     "vscode": "^1.13.0"
   },
-  "categories": [
-    "Languages"
-  ],
+  "categories": ["Languages"],
   "devDependencies": {
+    "@salesforce/salesforcedx-utils-vscode": "40.4.0",
     "@types/chai": "^4.0.0",
     "@types/mocha": "2.2.38",
     "@types/node": "^6.0.40",
@@ -51,10 +50,7 @@
     "languages": [
       {
         "id": "html",
-        "extensions": [
-          ".page",
-          ".component"
-        ]
+        "extensions": [".page", ".component"]
       }
     ]
   }

packages/salesforcedx-vscode-visualforce/test/index.ts

@@ -5,20 +5,8 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-//
-// PLEASE DO NOT MODIFY / DELETE UNLESS YOU KNOW WHAT YOU ARE DOING
-//
-// This file is providing the test runner to use when running extension tests.
-// By default the test runner in use is Mocha based.
-//
-// You can provide your own test runner if you want to override it by exporting
-// a function run(testRoot: string, clb: (error:Error) => void) that the extension
-// host can call to run the tests. The test runner is expected to use console.log
-// to report the results back to the caller. When the tests are finished, return
-// a possible error to the callback or null if none.
-
 // tslint:disable-next-line:no-var-requires
-const testRunner = require('vscode/lib/testrunner');
+const testRunner = require('@salesforce/salesforcedx-utils-vscode/out/src/test/testrunner');
 
 // You can directly control Mocha options by uncommenting the following lines
 // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info