test: new github checks: 'yarn test' + coveralls (#58)

* test: add 'yarn test' GitHub action for PR to main * fix: run 'tsc build' before 'yarn test' * fix: missing import in github-client.ts * chore: in GH Actions, skip tests requiring fiddle * chore: rename task as "unit tests" * test: integration tests use a fake electron-fiddle We don't have electron-fiddle on GitHub Actions, so add a fake that outputs pre-recorded logs from the real electron-fiddle. * fixup! test: integration tests use a fake electron-fiddle * fix: omit platform-specific args from canned logs * fix: still finding portable checksum for test argv * chore: rename workflow: 'unit tests' => 'tests' * fix oops in 1b14ea * chore: experiment with adding coveralls * chore: one coverage file for top-level 'yarn test'
2021-06-14 15:26:19 -05:00 · 2021-06-14 15:26:19 -05:00 · d7fe7f6745
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@ -0,0 +1,29 @@
+name: tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2.1.2
+        with:
+          node-version: '14.x'
+      - name: Install dependencies
+        run: yarn install
+      - name: Build 
+        run: yarn build
+      - name: Run tests
+        run: yarn test:ci
+      - name: Coveralls
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1,12 @@
-node_modules/
 .DS_Store
+.env
+node_modules/
+
+# Coverage
+coverage
+integration-tests/coverage
+modules/bot/coverage
+modules/broker/coverage
+modules/runner/coverage
+modules/shared/coverage
+
--- a/integration-tests/package.json
+++ b/integration-tests/package.json
@ -4,12 +4,14 @@
  "version": "0.0.1",
  "scripts": {
    "build": "# no-op",
-    "test": "jest"
+    "test": "jest",
+    "test:ci": "jest --coverage --runInBand"
  },
  "devDependencies": {
    "@types/jest": "^26.0.23",
    "dayjs": "^1.10.5",
    "jest": "^26.6.3",
+    "object-hash": "^2.2.0",
    "uuid": "^8.3.2"
  }
 }
--- a/integration-tests/test/broker-runner.test.ts
+++ b/integration-tests/test/broker-runner.test.ts
@ -1,5 +1,5 @@
+import * as path from 'path';
 import debug from 'debug';
-import dayjs from 'dayjs';
 import { v4 as mkuuid } from 'uuid';

 import { Broker } from '../../modules/broker/src/broker';
@ -35,6 +35,7 @@ describe('runner', () => {
  function createRunner(opts: Record<string, any> = {}) {
    runner = new Runner({
      brokerUrl: `http://localhost:${brokerServer.port}`,
+      fiddleExecPath: path.resolve(__dirname, 'fixtures', 'electron-fiddle'),
      platform,
      ...opts,
    });
--- a/integration-tests/test/fixtures/electron-fiddle
+++ b/integration-tests/test/fixtures/electron-fiddle
@ -0,0 +1,20 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const hash = require('object-hash');
+const path = require('path');
+const util = require('util');
+
+// skip past the absolute paths to node and electron-fiddle
+const argv = process.argv.slice(process.argv.indexOf(__filename) + 1);
+
+// find the logfile by hashing the remaining args
+const argv_hash = hash(argv);
+const logfile = path.resolve(__dirname, 'electron-fiddle-logs', argv_hash);
+
+// dump the logfile so that Runner thinks it actually called electron-fiddle
+if (fs.existsSync(logfile)) {
+  console.log(fs.readFileSync(logfile, { encoding: 'utf8' }));
+} else {
+  throw new Error(`No logfile found for ${argv_hash} ${util.inspect(argv)}`);
+}
--- a/integration-tests/test/fixtures/electron-fiddle-logs/6f156b6731ee67650ab593b9f1efbe1628dd2b3a
+++ b/integration-tests/test/fixtures/electron-fiddle-logs/6f156b6731ee67650ab593b9f1efbe1628dd2b3a
@ -0,0 +1,94 @@
+Creating main window
+[6:43:13 PM] Console ready 🔬
+[6:43:14 PM] Task: Open fiddle "{"gistId":"8c5fc0c6a5153d49b5a4a56d3ed9da8f"}"
+[6:43:17 PM] Task: Bisect  [10.0.0, 10.0.1, 10.1.0, 10.1.1, 10.1.2, 10.1.3, 10.1.4, 10.1.5, 10.1.6, 10.1.7, 10.2.0, 10.3.0, 10.3.1, 10.3.2, 10.4.0, 10.4.1, 10.4.2, 10.4.3, 10.4.4, 10.4.5, 10.4.6, 10.4.7, 11.0.0-nightly.20200525, 11.0.0-nightly.20200526, 11.0.0-nightly.20200529, 11.0.0-nightly.20200602, 11.0.0-nightly.20200603, 11.0.0-nightly.20200604, 11.0.0-nightly.20200609, 11.0.0-nightly.20200610, 11.0.0-nightly.20200611, 11.0.0-nightly.20200615, 11.0.0-nightly.20200616, 11.0.0-nightly.20200617, 11.0.0-nightly.20200618, 11.0.0-nightly.20200619, 11.0.0-nightly.20200701, 11.0.0-nightly.20200702, 11.0.0-nightly.20200703, 11.0.0-nightly.20200706, 11.0.0-nightly.20200707, 11.0.0-nightly.20200708, 11.0.0-nightly.20200709, 11.0.0-nightly.20200716, 11.0.0-nightly.20200717, 11.0.0-nightly.20200720, 11.0.0-nightly.20200721, 11.0.0-nightly.20200723, 11.0.0-nightly.20200724, 11.0.0-nightly.20200729, 11.0.0-nightly.20200730, 11.0.0-nightly.20200731, 11.0.0-nightly.20200803, 11.0.0-nightly.20200804, 11.0.0-nightly.20200805, 11.0.0-nightly.20200811, 11.0.0-nightly.20200812, 11.0.0-nightly.20200822, 11.0.0-nightly.20200824, 11.0.0-nightly.20200825, 11.0.0-nightly.20200826, 11.0.0-beta.1, 11.0.0-beta.3, 11.0.0-beta.4, 11.0.0-beta.5, 11.0.0-beta.6, 11.0.0-beta.7, 11.0.0-beta.8, 11.0.0-beta.9, 11.0.0-beta.11, 11.0.0-beta.12, 11.0.0-beta.13, 11.0.0-beta.16, 11.0.0-beta.17, 11.0.0-beta.18, 11.0.0-beta.19, 11.0.0-beta.20, 11.0.0-beta.22, 11.0.0-beta.23, 11.0.0, 11.0.1, 11.0.2, 11.0.3, 11.0.4, 11.0.5, 11.1.0, 11.1.1, 11.2.0]
+[6:43:17 PM] Runner: autobisect Electron 11.0.0-nightly.20200716 - setting version
+[6:43:17 PM] Runner: autobisect Electron 11.0.0-nightly.20200716 - starting test
+[6:43:17 PM] Saving files to temp directory...
+[6:43:17 PM] Saved files to /tmp/tmp-304016-SN2JSuBO50p6
+[6:43:17 PM] Electron v11.0.0-nightly.20200716 started.
+[6:43:17 PM] (electron) Sending uncompressed crash reports is deprecated and will be removed in a future version of Electron. Set { compress: true } to opt-in to the new behavior. Crash reports will be uploaded gzipped, which most crash reporting servers support.
+[6:43:17 PM] test passed
+[6:43:17 PM] 
+[6:43:17 PM] Electron exited with code 0.
+[6:43:17 PM] Runner: autobisect Electron 11.0.0-nightly.20200716 - finished test ✅ passed
+[6:43:17 PM] Runner: autobisect Electron 11.0.0-beta.6 - setting version
+[6:43:17 PM] Runner: autobisect Electron 11.0.0-beta.6 - starting test
+[6:43:17 PM] Saving files to temp directory...
+[6:43:17 PM] Saved files to /tmp/tmp-304016-vZECnE31rWwG
+[6:43:17 PM] Electron v11.0.0-beta.6 started.
+[6:43:17 PM] (electron) Sending uncompressed crash reports is deprecated and will be removed in a future version of Electron. Set { compress: true } to opt-in to the new behavior. Crash reports will be uploaded gzipped, which most crash reporting servers support.
+[6:43:17 PM] test failed
+[6:43:17 PM] Error: test failed
+    at Object.done (/tmp/tmp-304016-vZECnE31rWwG/preload.js:11:32)
+    at Object.fail (/tmp/tmp-304016-vZECnE31rWwG/preload.js:9:27)
+    at file:///tmp/tmp-304016-vZECnE31rWwG/renderer.js:17:8
+TypeError: remote.screen.getPrimaryDisplay is not a function
+    at file:///tmp/tmp-304016-vZECnE31rWwG/renderer.js:11:40
+[6:43:18 PM] 
+[6:43:18 PM] Electron exited with code 1.
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-beta.6 - finished test ❌ failed
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200805 - setting version
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200805 - starting test
+[6:43:18 PM] Saving files to temp directory...
+[6:43:18 PM] Saved files to /tmp/tmp-304016-l4hes2UOLw3q
+[6:43:18 PM] Electron v11.0.0-nightly.20200805 started.
+[6:43:18 PM] (electron) Sending uncompressed crash reports is deprecated and will be removed in a future version of Electron. Set { compress: true } to opt-in to the new behavior. Crash reports will be uploaded gzipped, which most crash reporting servers support.
+[6:43:18 PM] test failed
+[6:43:18 PM] Error: test failed
+    at Object.done (/tmp/tmp-304016-l4hes2UOLw3q/preload.js:11:32)
+    at Object.fail (/tmp/tmp-304016-l4hes2UOLw3q/preload.js:9:27)
+    at file:///tmp/tmp-304016-l4hes2UOLw3q/renderer.js:17:8
+TypeError: remote.screen.getPrimaryDisplay is not a function
+    at file:///tmp/tmp-304016-l4hes2UOLw3q/renderer.js:11:40
+[6:43:18 PM] 
+[6:43:18 PM] Electron exited with code 1.
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200805 - finished test ❌ failed
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200724 - setting version
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200724 - starting test
+[6:43:18 PM] Saving files to temp directory...
+[6:43:18 PM] Saved files to /tmp/tmp-304016-9l1WS3ejKtJ9
+[6:43:18 PM] Electron v11.0.0-nightly.20200724 started.
+[6:43:18 PM] (electron) Sending uncompressed crash reports is deprecated and will be removed in a future version of Electron. Set { compress: true } to opt-in to the new behavior. Crash reports will be uploaded gzipped, which most crash reporting servers support.
+[6:43:18 PM] test passed
+[6:43:18 PM] 
+[6:43:18 PM] Electron exited with code 0.
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200724 - finished test ✅ passed
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200731 - setting version
+[6:43:18 PM] Runner: autobisect Electron 11.0.0-nightly.20200731 - starting test
+[6:43:18 PM] Saving files to temp directory...
+[6:43:18 PM] Saved files to /tmp/tmp-304016-cm8EohaSIbK2
+[6:43:18 PM] Electron v11.0.0-nightly.20200731 started.
+[6:43:19 PM] (electron) Sending uncompressed crash reports is deprecated and will be removed in a future version of Electron. Set { compress: true } to opt-in to the new behavior. Crash reports will be uploaded gzipped, which most crash reporting servers support.
+[6:43:19 PM] test failed
+[6:43:19 PM] Error: test failed
+    at Object.done (/tmp/tmp-304016-cm8EohaSIbK2/preload.js:11:32)
+    at Object.fail (/tmp/tmp-304016-cm8EohaSIbK2/preload.js:9:27)
+    at file:///tmp/tmp-304016-cm8EohaSIbK2/renderer.js:17:8
+TypeError: remote.screen.getPrimaryDisplay is not a function
+    at file:///tmp/tmp-304016-cm8EohaSIbK2/renderer.js:11:40
+[6:43:19 PM] 
+[6:43:19 PM] Electron exited with code 1.
+[6:43:19 PM] Runner: autobisect Electron 11.0.0-nightly.20200731 - finished test ❌ failed
+[6:43:19 PM] Runner: autobisect Electron 11.0.0-nightly.20200729 - setting version
+[6:43:19 PM] Runner: autobisect Electron 11.0.0-nightly.20200729 - starting test
+[6:43:19 PM] Saving files to temp directory...
+[6:43:19 PM] Saved files to /tmp/tmp-304016-rFfmvTFNIOdj
+[6:43:19 PM] Electron v11.0.0-nightly.20200729 started.
+[6:43:19 PM] (electron) Sending uncompressed crash reports is deprecated and will be removed in a future version of Electron. Set { compress: true } to opt-in to the new behavior. Crash reports will be uploaded gzipped, which most crash reporting servers support.
+[6:43:19 PM] test failed
+[6:43:19 PM] Error: test failed
+    at Object.done (/tmp/tmp-304016-rFfmvTFNIOdj/preload.js:11:32)
+    at Object.fail (/tmp/tmp-304016-rFfmvTFNIOdj/preload.js:9:27)
+    at file:///tmp/tmp-304016-rFfmvTFNIOdj/renderer.js:17:8
+TypeError: remote.screen.getPrimaryDisplay is not a function
+    at file:///tmp/tmp-304016-rFfmvTFNIOdj/renderer.js:11:40
+[6:43:19 PM] 
+[6:43:19 PM] Electron exited with code 1.
+[6:43:19 PM] Runner: autobisect Electron 11.0.0-nightly.20200729 - finished test ❌ failed
+[6:43:19 PM] Runner: autobisect complete
+[6:43:19 PM] Runner: autobisect ✅ passed 11.0.0-nightly.20200724
+[6:43:19 PM] Runner: autobisect ❌ failed 11.0.0-nightly.20200729
+[6:43:19 PM] Runner: autobisect Commits between versions:
+[6:43:19 PM] https://github.com/electron/electron/compare/v11.0.0-nightly.20200724...v11.0.0-nightly.20200729
+[6:43:19 PM] Task: Bisect  success
--- a/jest.config.js
+++ b/jest.config.js
@ -0,0 +1,15 @@
+process.env.SPEC_RUNNING = '1';
+
+module.exports = {
+  clearMocks: true,
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  preset: 'ts-jest',
+  roots: [
+    '<rootDir>/integration-tests',
+    '<rootDir>/modules/bot',
+    '<rootDir>/modules/broker',
+    '<rootDir>/modules/runner',
+    '<rootDir>/modules/shared',
+  ],
+  testRegex: '(/spec/.*|(\\.|/)(test|spec))\\.tsx?$',
+};
--- a/modules/bot/package.json
+++ b/modules/bot/package.json
@ -7,7 +7,8 @@
    "build": "tsc -b",
    "heroku-postbuild": "yarn build",
    "start": "probot run ./dist/github-client.js",
-    "test": "jest --coverage"
+    "test": "jest",
+    "test:ci": "jest --coverage --runInBand"
  },
  "dependencies": {
    "@electron/bugbot-shared": "*",
--- a/modules/bot/src/github-client.ts
+++ b/modules/bot/src/github-client.ts
@ -1,5 +1,6 @@
 import debug from 'debug';
 import { Context, Probot } from 'probot';
+import { URL } from 'url';
 import { inspect } from 'util';

 import { env } from '@electron/bugbot-shared/lib/env-vars';
--- a/modules/broker/package.json
+++ b/modules/broker/package.json
@ -6,7 +6,8 @@
  "scripts": {
    "build": "tsc -b",
    "start": "node ./dist/main.js",
-    "test": "jest --coverage"
+    "test": "jest",
+    "test:ci": "jest --coverage --runInBand"
  },
  "dependencies": {
    "@electron/bugbot-shared": "*",
--- a/modules/runner/package.json
+++ b/modules/runner/package.json
@ -6,7 +6,8 @@
  "scripts": {
    "build": "tsc -b",
    "start": "node ./dist/main.js",
-    "test": "jest --coverage"
+    "test": "jest",
+    "test:ci": "jest --coverage --runInBand"
  },
  "dependencies": {
    "@electron/bugbot-shared": "*",
--- a/modules/shared/package.json
+++ b/modules/shared/package.json
@ -5,7 +5,8 @@
  "main": "lib/index.js",
  "scripts": {
    "build": "tsc -b",
-    "test": "jest --coverage"
+    "test": "jest",
+    "test:ci": "jest --coverage --runInBand"
  },
  "dependencies": {
    "mdast-util-from-markdown": "^0.8.5",
--- a/package.json
+++ b/package.json
@ -10,7 +10,8 @@
    "start:bot": "yarn workspace @electron/bugbot-bot start",
    "start:broker": "yarn workspace @electron/bugbot-broker start",
    "start:runner": "yarn workspace @electron/bugbot-runner start",
-    "test": "yarn workspaces run test",
+    "test": "jest",
+    "test:ci": "jest --coverage --runInBand",
    "lint": "run-p lint:eslint lint:prettier",
    "lint:fix": "run-p lint:eslint:fix lint:prettier:fix",
    "lint:eslint": "eslint modules/*/src/**/*.ts",
--- a/tsconfig.json
+++ b/tsconfig.json
@ -1,9 +1,6 @@
-/**
- * This file contains the root TypeScript project which ties together the modules.
- *
- * For more info see: https://www.typescriptlang.org/docs/handbook/project-references.html
- */
 {
+  "extends": "@electron/bugbot-shared/tsconfig.base.json",
+
  // NOTE: this list should be synced with the "workspaces" field in the root `package.json` file.
  "references": [
    { "path": "./modules/shared" },
--- a/yarn.lock
+++ b/yarn.lock
@ -4789,6 +4789,11 @@ object-copy@^0.1.0:
    define-property "^0.2.5"
    kind-of "^3.0.3"

+object-hash@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5"
+  integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==
+
 object-inspect@^1.9.0:
  version "1.10.3"
  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.10.3.tgz#c2aa7d2d09f50c99375704f7a0adf24c5782d369"