diff --git a/.buildkite/build-pr.sh b/.buildkite/build-pr.sh
index bf47dfe6..066fc52e 100755
--- a/.buildkite/build-pr.sh
+++ b/.buildkite/build-pr.sh
@@ -32,6 +32,9 @@ if [ -d quicktype-outputs ] ; then
 fi
 GIT_SSH_COMMAND='ssh -i id_rsa' git clone git@github.com:quicktype/quicktype-outputs.git
 cd ./quicktype-outputs
+if [ ! -d outputs ] ; then
+    mkdir outputs
+fi
 COMMIT_DIR="`pwd`/outputs/$BUILDKITE_COMMIT"
 if [ ! -d "$COMMIT_DIR" ] ; then
     mkdir "$COMMIT_DIR"
diff --git a/test/fixtures.ts b/test/fixtures.ts
index 63869a1b..f1caf143 100644
--- a/test/fixtures.ts
+++ b/test/fixtures.ts
@@ -155,7 +155,7 @@ abstract class LanguageFixture extends Fixture {
 
     // FIXME: This is an ugly hack to exclude Java, which has multiple
     // output files.  We have to support that eventually.
-    if (OUTPUT_DIR !== undefined && this.language.output.indexOf("/") < 0) {
+    if (sample.saveOutput && OUTPUT_DIR !== undefined && this.language.output.indexOf("/") < 0) {
       const outputDir = path.join(
         OUTPUT_DIR,
         this.language.name,
@@ -245,7 +245,8 @@ class JSONFixture extends LanguageFixture {
     if (sources.length === 0) {
       const quickTestSamples = _.map(this.language.quickTestRendererOptions, ro => ({
         path: combinationsInput,
-        additionalRendererOptions: ro
+        additionalRendererOptions: ro,
+        saveOutput: false
       }));
       priority = quickTestSamples.concat(priority);
     }
diff --git a/test/utils.ts b/test/utils.ts
index f9e0a365..edcbefe7 100644
--- a/test/utils.ts
+++ b/test/utils.ts
@@ -126,10 +126,11 @@ export function testsInDir(dir: string, extension: string): string[] {
 export interface Sample {
   path: string;
   additionalRendererOptions: RendererOptions;
+  saveOutput: boolean;
 }
 
 export function samplesFromPaths(paths: string[]): Sample[] {
-  return paths.map(p => ({ path: p, additionalRendererOptions: {} }));
+  return paths.map(p => ({ path: p, additionalRendererOptions: {}, saveOutput: true }));
 }
 
 export function samplesFromSources(
@@ -178,11 +179,11 @@ export function compareJsonFileToJson(args: ComparisonArgs) {
   const allowMissingNull = !!args.allowMissingNull;
   let jsonAreEqual = strict
     ? callAndReportFailure("Failed to strictly compare objects", () =>
-        strictDeepEquals(givenJSON, expectedJSON)
-      )
+      strictDeepEquals(givenJSON, expectedJSON)
+    )
     : callAndReportFailure("Failed to compare objects.", () =>
-        deepEquals(expectedJSON, givenJSON, allowMissingNull)
-      );
+      deepEquals(expectedJSON, givenJSON, allowMissingNull)
+    );
 
   if (!jsonAreEqual) {
     failWith("Error: Output is not equivalent to input.", {