Merge pull request #550 from quicktype/buildkite

CI with Buildkite

test/.prettierrc

@@ -0,0 +1 @@
+{ "printWidth": 110 }

test/buildkite.ts

@@ -0,0 +1,50 @@
+import * as _ from "lodash";
+import { exec } from "shelljs";
+
+import { WorkItem } from "./test";
+import { allFixtures, Fixture } from "./fixtures";
+
+function getChangedFiles(base: string, commit: string): string[] {
+  let diff = exec(`git fetch -v origin ${base} && git diff --name-only origin/${base}..${commit}`).stdout;
+  return diff.trim().split("\n");
+}
+
+export function affectedFixtures(changedFiles: string[] | undefined = undefined): Fixture[] {
+  if (changedFiles === undefined) {
+    const { BUILDKITE_PULL_REQUEST_BASE_BRANCH: base, BUILDKITE_COMMIT: commit } = process.env;
+    return commit === undefined ? allFixtures : affectedFixtures(getChangedFiles(base || "master", commit));
+  }
+
+  // We can ignore changes in Markdown files
+  changedFiles = _.reject(changedFiles, file => _.endsWith(file, ".md"));
+
+  // All fixtures are dirty if any changed file is not included as a sourceFile of some fixture.
+  const fileDependencies = _.flatMap(allFixtures, f => f.language.sourceFiles || []);
+  const allFixturesDirty = _.some(changedFiles, f => !_.includes(fileDependencies, f));
+
+  if (allFixturesDirty) return allFixtures;
+
+  const dirtyFixtures = allFixtures.filter(
+    fixture =>
+      // Fixtures that don't specify dependencies are always dirty
+      fixture.language.sourceFiles === undefined ||
+      // Fixtures that have a changed file are dirty
+      _.some(changedFiles, f => _.includes(fixture.language.sourceFiles, f))
+  );
+
+  return dirtyFixtures;
+}
+
+export function divideParallelJobs(workItems: WorkItem[]): WorkItem[] {
+  const { BUILDKITE_PARALLEL_JOB: pjob, BUILDKITE_PARALLEL_JOB_COUNT: pcount } = process.env;
+
+  if (pjob === undefined || pcount === undefined) return workItems;
+
+  try {
+    const segment = Math.ceil(workItems.length / parseFloat(pcount));
+    const start = parseInt(pjob, 10) * segment;
+    return workItems.slice(start, start + segment);
+  } catch {
+    return workItems;
+  }
+}

test/fixtures.ts

@@ -69,6 +69,8 @@ function jsonTestFiles(base: string): string[] {
 export abstract class Fixture {
   abstract name: string;
 
+  constructor(public language: languages.Language) {}
+
   runForName(name: string): boolean {
     return this.name === name;
   }
@@ -91,33 +93,36 @@ export abstract class Fixture {
     return `test/runs/${this.name}-${randomBytes(3).toString("hex")}`;
   }
 
-  printRunMessage(
+  runMessageStart(
     sample: Sample,
     index: number,
     total: number,
     cwd: string,
     shouldSkip: boolean
-  ): void {
+  ): string {
     const rendererOptions = _.map(
       sample.additionalRendererOptions,
       (v, k) => `${k}: ${v}`
     ).join(", ");
-    console.error(
+    const message = [
       `*`,
       chalk.dim(`[${index + 1}/${total}]`),
       chalk.magenta(this.name) + chalk.dim(`(${rendererOptions})`),
       path.join(cwd, chalk.cyan(path.basename(sample.path))),
       shouldSkip ? chalk.red("SKIP") : ""
-    );
+    ].join(" ");
+    console.time(message);
+    return message;
+}
+
+  runMessageEnd(message: string) {
+    console.timeEnd(message);
   }
 }
 
 abstract class LanguageFixture extends Fixture {
-  protected language: languages.Language;
-
   constructor(language: languages.Language) {
-    super();
-    this.language = language;
+    super(language);
   }
 
   async setup() {
@@ -154,7 +159,7 @@ abstract class LanguageFixture extends Fixture {
     let shouldSkip = this.shouldSkipTest(sample);
     const additionalFiles = this.additionalFiles(sample);
 
-    this.printRunMessage(sample, index, total, cwd, shouldSkip);
+    const message = this.runMessageStart(sample, index, total, cwd, shouldSkip);
 
     if (shouldSkip) {
       return;
@@ -181,6 +186,8 @@ abstract class LanguageFixture extends Fixture {
     });
 
     shell.rm("-rf", cwd);
+
+    this.runMessageEnd(message);
   }
 }
 
@@ -296,16 +303,12 @@ class JSONFixture extends LanguageFixture {
       priority = quickTestSamples.concat(priority);
     }
 
-    if (IS_CI && !IS_PR && !IS_BLESSED) {
-      // Run only priority sources on low-priority CI branches
-      others = [];
-    } else if (IS_CI) {
+    if (IS_CI) {
       // On CI, we run a maximum number of test samples. First we test
       // the priority samples to fail faster, then we continue testing
       // until testMax with random sources.
       const testMax = 100;
       others = _.chain(samplesFromPaths(miscSamples))
-        .shuffle()
         .take(testMax - prioritySamples.length)
         .value();
     }
@@ -342,7 +345,8 @@ class JSONSchemaJSONFixture extends JSONFixture {
       skipMiscJSON: false,
       skipSchema: [],
       rendererOptions: {},
-      quickTestRendererOptions: []
+      quickTestRendererOptions: [],
+      sourceFiles: language.sourceFiles
     };
     super(schemaLanguage);
     this.runLanguage = language;

test/languages.ts

@@ -17,6 +17,7 @@ export interface Language {
   skipSchema: string[];
   rendererOptions: RendererOptions;
   quickTestRendererOptions: RendererOptions[];
+  sourceFiles?: string[];
 }
 
 export const CSharpLanguage: Language = {
@@ -41,7 +42,8 @@ export const CSharpLanguage: Language = {
     { "array-type": "list" },
     { "csharp-version": "5" },
     { density: "dense" }
-  ]
+  ],
+  sourceFiles: ["src/Language/CSharp.ts"]
 };
 
 export const JavaLanguage: Language = {
@@ -64,7 +66,8 @@ export const JavaLanguage: Language = {
   skipMiscJSON: false,
   skipSchema: ["keyword-unions.schema"], // generates classes with names that are case-insensitively equal
   rendererOptions: {},
-  quickTestRendererOptions: []
+  quickTestRendererOptions: [],
+  sourceFiles: ["src/Language/Java.ts"]
 };
 
 export const RustLanguage: Language = {
@@ -83,7 +86,8 @@ export const RustLanguage: Language = {
   skipSchema: [],
   skipMiscJSON: true,
   rendererOptions: {},
-  quickTestRendererOptions: [{ density: "dense" }]
+  quickTestRendererOptions: [{ density: "dense" }],
+  sourceFiles: ["src/Language/Rust.ts"]
 };
 
 export const GoLanguage: Language = {
@@ -105,7 +109,8 @@ export const GoLanguage: Language = {
   skipMiscJSON: false,
   skipSchema: [],
   rendererOptions: {},
-  quickTestRendererOptions: []
+  quickTestRendererOptions: [],
+  sourceFiles: ["src/Language/Golang.ts"]
 };
 
 export const CPlusPlusLanguage: Language = {
@@ -130,7 +135,8 @@ export const CPlusPlusLanguage: Language = {
   skipMiscJSON: false,
   skipSchema: [],
   rendererOptions: {},
-  quickTestRendererOptions: [{ unions: "indirection" }]
+  quickTestRendererOptions: [{ unions: "indirection" }],
+  sourceFiles: ["src/Language/CPlusPlus.ts"]
 };
 
 export const ElmLanguage: Language = {
@@ -164,7 +170,8 @@ export const ElmLanguage: Language = {
     "keyword-unions.schema" // can't handle "hasOwnProperty" for some reason
   ],
   rendererOptions: {},
-  quickTestRendererOptions: [{ "array-type": "list" }]
+  quickTestRendererOptions: [{ "array-type": "list" }],
+  sourceFiles: ["src/Language/Elm.ts"]
 };
 
 export const SwiftLanguage: Language = {
@@ -193,7 +200,8 @@ export const SwiftLanguage: Language = {
     { "struct-or-class": "class" },
     { density: "dense" },
     { density: "normal" }
-  ]
+  ],
+  sourceFiles: ["src/Language/Swift.ts"]
 };
 
 export const ObjectiveCLanguage: Language = {
@@ -223,7 +231,8 @@ export const ObjectiveCLanguage: Language = {
   skipMiscJSON: false,
   skipSchema: [],
   rendererOptions: { functions: "true" },
-  quickTestRendererOptions: []
+  quickTestRendererOptions: [],
+  sourceFiles: ["src/Language/Objective-C.ts"]
 };
 
 export const TypeScriptLanguage: Language = {
@@ -243,5 +252,6 @@ export const TypeScriptLanguage: Language = {
   skipMiscJSON: false,
   skipSchema: ["keyword-unions.schema"], // can't handle "constructor" property
   rendererOptions: { "explicit-unions": "yes" },
-  quickTestRendererOptions: []
+  quickTestRendererOptions: [],
+  sourceFiles: ["src/Language/TypeScript.ts"]
 };

test/test.ts

@@ -6,29 +6,28 @@ import * as _ from "lodash";
 import { inParallel } from "./lib/multicore";
 import { exec, execAsync, Sample } from "./utils";
 import { Fixture, allFixtures } from "./fixtures";
+import { affectedFixtures, divideParallelJobs } from "./buildkite";
 
 const exit = require("exit");
-
-//////////////////////////////////////
-// Constants
-/////////////////////////////////////
-
 const CPUs = parseInt(process.env.CPUs || "0", 10) || os.cpus().length;
 
 //////////////////////////////////////
 // Test driver
 /////////////////////////////////////
 
-type WorkItem = { sample: Sample; fixtureName: string };
+export type WorkItem = { sample: Sample; fixtureName: string };
 
 async function main(sources: string[]) {
   let fixtures = allFixtures;
   const fixturesFromCmdline = process.env.FIXTURE;
   if (fixturesFromCmdline) {
     const fixtureNames = fixturesFromCmdline.split(",");
-    fixtures = _.filter(fixtures, fixture =>
-      _.some(fixtureNames, name => fixture.runForName(name))
-    );
+    fixtures = _.filter(fixtures, fixture => _.some(fixtureNames, name => fixture.runForName(name)));
+  } else {
+    fixtures = affectedFixtures();
+    if (allFixtures.length !== fixtures.length) {
+      console.error(`* Running a subset of fixtures: ${fixtures.map(f => f.name).join(", ")}`);
+    }
   }
   // Get an array of all { sample, fixtureName } objects we'll run.
   // We can't just put the fixture in there because these WorkItems
@@ -44,7 +43,7 @@ async function main(sources: string[]) {
     _.map(x.samples.others, s => ({ fixtureName: x.fixtureName, sample: s }))
   );
 
-  const tests = _.concat(_.shuffle(priority), _.shuffle(others));
+  const tests = divideParallelJobs(_.concat(priority, others));
 
   await inParallel({
     queue: tests,