Samples as tests (#2215)

fix [#3017](https://github.com/Azure/typespec-azure/issues/3017)

Run the samples as test which allows all of them to run without crashing
on the first one, lets us run it in the test explorer.

This also make the sample regeneration much faster(typespec azure repo):
- Before `~90s`
- Now `~12s`


![image](https://github.com/microsoft/typespec/assets/1031227/a3356a90-7847-43cf-a473-4ecda0c53330)

## New `resolveCompilerOptions` utils
This also include the addition of a new util `resolveCompilerOptions`
that resolve the compiler options programtically in the same way that
the cli would have for a given entrypoint.

I can move this into a dedicated PR if people prefer.

---------

Co-authored-by: Mark Cowlishaw <markcowl@microsoft.com>

.prettierignore

@@ -24,7 +24,7 @@ CODE_OF_CONDUCT.md
 packages/compiler/test/formatter/**/*.tsp
 
 # That is an example with error and can't be formatted
-packages/samples/local-typespec/test.tsp
+packages/samples/specs/local-typespec/test.tsp
 packages/website/build/
 packages/website/versioned_sidebars/
 packages/website/versions.json

.vscode/settings.json

@@ -91,6 +91,7 @@
   },
   "typescript.tsdk": "./packages/compiler/node_modules/typescript/lib",
   "git.ignoreLimitWarning": true,
+  "testExplorer.useNativeTesting": true,
   "mochaExplorer.parallel": false,
   "mochaExplorer.files": [
     "./packages/*/dist/test/**/*.test.js",

CONTRIBUTING.md

@@ -209,7 +209,7 @@ debug the last one you chose.
    attach to both the VS Code client process and the language server
    process automatically.
 2. **Compile Scratch**: Use this to debug compiling
-   `packages/typespec-samples/scratch/*.tsp`. The TypeSpec source code in that
+   `packages/samples/scratch/*.tsp`. The TypeSpec source code in that
    folder is excluded from source control by design. Create TypeSpec files
    there to experiment and debug how the compiler reacts.
 3. **Compile Scratch (nostdlib)**: Same as above, but skips parsing

common/changes/@typespec/compiler/feature-samples-snapshot-tests_2023-08-03-15-54.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/compiler",
+      "comment": "Add a new util `resolveCompilerOptions` to resolve compiler options from a given entrypoint. This will resolve the options from the tspconfig.yaml in the same way the cli would.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/compiler"
+}

common/config/rush/pnpm-lock.yaml

@@ -1183,12 +1183,30 @@ importers:
         specifier: workspace:~0.46.0
         version: link:../versioning
     devDependencies:
+      '@types/mocha':
+        specifier: ~10.0.1
+        version: 10.0.1
+      '@types/node':
+        specifier: ~18.11.9
+        version: 18.11.9
+      '@typespec/eslint-config-typespec':
+        specifier: workspace:~0.46.0
+        version: link:../eslint-config-typespec
       '@typespec/internal-build-utils':
         specifier: workspace:~0.46.0
         version: link:../internal-build-utils
       autorest:
         specifier: ~3.3.2
         version: 3.3.2
+      cross-env:
+        specifier: ~7.0.3
+        version: 7.0.3
+      eslint:
+        specifier: ^8.42.0
+        version: 8.42.0
+      mocha:
+        specifier: ~10.2.0
+        version: 10.2.0
       rimraf:
         specifier: ~5.0.1
         version: 5.0.1

eng/pipelines/pull-request-common.yml

@@ -46,9 +46,6 @@ steps:
     displayName: Lint
     condition: ne(variables['Agent.OS'], 'Windows_NT')
 
-  - script: cd packages/samples && npm run regen-samples
-    displayName: Regenerate Samples
-
   - script: node eng/scripts/check-for-changed-files.js
     displayName: Check Git Status For Changed Files
 

packages/compiler/src/config/config-to-options.ts

@@ -0,0 +1,96 @@
+import { createDiagnosticCollector, getDirectoryPath, normalizePath } from "../core/index.js";
+import { CompilerOptions } from "../core/options.js";
+import { CompilerHost, Diagnostic } from "../core/types.js";
+import { deepClone, omitUndefined } from "../core/util.js";
+import { expandConfigVariables } from "./config-interpolation.js";
+import { loadTypeSpecConfigForPath, validateConfigPathsAbsolute } from "./config-loader.js";
+import { EmitterOptions, TypeSpecConfig } from "./types.js";
+
+export interface ResolveCompilerOptionsOptions {
+  /** Absolute entrypoint path */
+  entrypoint: string;
+
+  /** Explicit config path. */
+  configPath?: string;
+
+  /** Current working directory. This will be used to interpolate `{cwd}` in the config.
+   * @default to `process.cwd()`
+   */
+  cwd?: string;
+
+  /**
+   * Environment variables.
+   * @default process.env
+   */
+  env?: Record<string, string | undefined>;
+
+  /**
+   * Any arguments to interpolate the config.
+   */
+  args?: Record<string, string>;
+
+  /** Compiler options to override the config */
+  overrides?: Partial<TypeSpecConfig>;
+}
+
+/**
+ * Resolve the compiler options for the given entrypoint by resolving the tspconfig.yaml.
+ * @param host Compiler host
+ * @param compilerOptions
+ */
+export async function resolveCompilerOptions(
+  host: CompilerHost,
+  options: ResolveCompilerOptionsOptions
+): Promise<[CompilerOptions, readonly Diagnostic[]]> {
+  const cwd = normalizePath(options.cwd ?? process.cwd());
+  const diagnostics = createDiagnosticCollector();
+
+  const entrypointStat = await host.stat(options.entrypoint);
+  const configPath =
+    options.configPath ?? entrypointStat.isDirectory()
+      ? options.entrypoint
+      : getDirectoryPath(options.entrypoint);
+  const config = await loadTypeSpecConfigForPath(host, configPath);
+  const configWithOverrides: TypeSpecConfig = {
+    ...config,
+    ...options.overrides,
+    options: mergeOptions(config.options, options.overrides?.options),
+  };
+  const expandedConfig = diagnostics.pipe(
+    expandConfigVariables(configWithOverrides, {
+      cwd,
+      outputDir: options.overrides?.outputDir,
+      env: options.env ?? process.env,
+      args: options.args,
+    })
+  );
+  validateConfigPathsAbsolute(expandedConfig).forEach((x) => diagnostics.add(x));
+
+  const resolvedOptions: CompilerOptions = omitUndefined({
+    outputDir: expandedConfig.outputDir,
+    config: config.filename,
+    additionalImports: expandedConfig["imports"],
+    warningAsError: expandedConfig.warnAsError,
+    trace: expandedConfig.trace,
+    emit: expandedConfig.emit,
+    options: expandedConfig.options,
+    linterRuleSet: expandedConfig.linter,
+  });
+  return diagnostics.wrap(resolvedOptions);
+}
+
+function mergeOptions(
+  base: Record<string, Record<string, unknown>> | undefined,
+  overrides: Record<string, Record<string, unknown>> | undefined
+): Record<string, EmitterOptions> {
+  const configuredEmitters: Record<string, Record<string, unknown>> = deepClone(base ?? {});
+
+  for (const [emitterName, cliOptionOverride] of Object.entries(overrides ?? {})) {
+    configuredEmitters[emitterName] = {
+      ...(configuredEmitters[emitterName] ?? {}),
+      ...cliOptionOverride,
+    };
+  }
+
+  return configuredEmitters;
+}

packages/compiler/src/config/index.ts

@@ -1,2 +1,3 @@
 export * from "./config-loader.js";
+export { ResolveCompilerOptionsOptions, resolveCompilerOptions } from "./config-to-options.js";
 export * from "./types.js";

packages/compiler/src/core/cli/actions/compile/args.ts

@@ -1,14 +1,9 @@
-import { expandConfigVariables } from "../../../../config/config-interpolation.js";
-import {
-  loadTypeSpecConfigForPath,
-  validateConfigPathsAbsolute,
-} from "../../../../config/config-loader.js";
-import { EmitterOptions, TypeSpecConfig } from "../../../../config/types.js";
-import { createDiagnosticCollector } from "../../../index.js";
+import { resolveCompilerOptions } from "../../../../config/config-to-options.js";
+import { createDiagnosticCollector } from "../../../diagnostics.js";
 import { CompilerOptions } from "../../../options.js";
-import { getDirectoryPath, normalizePath, resolvePath } from "../../../path-utils.js";
+import { resolvePath } from "../../../path-utils.js";
 import { CompilerHost, Diagnostic } from "../../../types.js";
-import { deepClone, omitUndefined } from "../../../util.js";
+import { omitUndefined } from "../../../util.js";
 
 export interface CompileCliArgs {
   "output-dir"?: string;
@@ -34,65 +29,39 @@ export async function getCompilerOptions(
   args: CompileCliArgs,
   env: Record<string, string | undefined>
 ): Promise<[CompilerOptions | undefined, readonly Diagnostic[]]> {
-  cwd = normalizePath(cwd);
-
   const diagnostics = createDiagnosticCollector();
+
   const pathArg = args["output-dir"] ?? args["output-path"];
-  const configPath = args["config"]
-    ? resolvePath(cwd, args["config"])
-    : getDirectoryPath(entrypoint);
-
-  const config = await loadTypeSpecConfigForPath(host, configPath, "config" in args);
-  if (config.diagnostics.length > 0) {
-    if (config.diagnostics.some((d) => d.severity === "error")) {
-      return [undefined, config.diagnostics];
-    }
-    config.diagnostics.forEach((x) => diagnostics.add(x));
-  }
-
-  const cliOptions = resolveCliOptions(args);
-
-  const configWithCliArgs: TypeSpecConfig = {
-    ...config,
-    outputDir: config.outputDir,
-    imports: args["import"] ?? config["imports"],
-    warnAsError: args["warn-as-error"] ?? config.warnAsError,
-    trace: args.trace ?? config.trace,
-    emit: args.emit ?? config.emit,
-    options: resolveEmitterOptions(config, cliOptions),
-  };
   const cliOutputDir = pathArg
     ? pathArg.startsWith("{")
       ? pathArg
       : resolvePath(cwd, pathArg)
     : undefined;
 
-  const expandedConfig = diagnostics.pipe(
-    expandConfigVariables(configWithCliArgs, {
-      cwd: cwd,
-      outputDir: cliOutputDir,
-      env,
+  const cliOptions = resolveCliOptions(args);
+  const resolvedOptions = diagnostics.pipe(
+    await resolveCompilerOptions(host, {
+      entrypoint,
+      configPath: args["config"] && resolvePath(cwd, args["config"]),
+      cwd,
       args: resolveConfigArgs(args),
+      env,
+      overrides: omitUndefined({
+        outputDir: cliOutputDir,
+        imports: args["import"],
+        warnAsError: args["warn-as-error"],
+        trace: args.trace,
+        emit: args.emit,
+        options: cliOptions.options,
+      }),
+    })
+  );
+  return diagnostics.wrap(
+    omitUndefined({
+      ...resolvedOptions,
+      miscOptions: cliOptions.miscOptions,
     })
   );
-  validateConfigPathsAbsolute(expandedConfig).forEach((x) => diagnostics.add(x));
-
-  const options: CompilerOptions = omitUndefined({
-    nostdlib: args["nostdlib"],
-    watchForChanges: args["watch"],
-    noEmit: args["no-emit"],
-    ignoreDeprecated: args["ignore-deprecated"],
-    miscOptions: cliOptions.miscOptions,
-    outputDir: expandedConfig.outputDir,
-    config: config.filename,
-    additionalImports: expandedConfig["imports"],
-    warningAsError: expandedConfig.warnAsError,
-    trace: expandedConfig.trace,
-    emit: expandedConfig.emit,
-    options: expandedConfig.options,
-    linterRuleSet: expandedConfig.linter,
-  });
-  return diagnostics.wrap(options);
 }
 
 function resolveConfigArgs(args: CompileCliArgs): Record<string, string> {
@@ -108,9 +77,11 @@ function resolveConfigArgs(args: CompileCliArgs): Record<string, string> {
 
   return map;
 }
-function resolveCliOptions(
-  args: CompileCliArgs
-): Record<string | "miscOptions", Record<string, unknown>> {
+function resolveCliOptions(args: CompileCliArgs): {
+  options: Record<string, Record<string, unknown>>;
+  miscOptions: Record<string, string> | undefined;
+} {
+  let miscOptions: Record<string, string> | undefined;
   const options: Record<string, Record<string, string>> = {};
   for (const option of args.options ?? []) {
     const optionParts = option.split("=");
@@ -122,10 +93,10 @@ function resolveCliOptions(
     let optionKeyParts = optionParts[0].split(".");
     if (optionKeyParts.length === 1) {
       const key = optionKeyParts[0];
-      if (!("miscOptions" in options)) {
-        options.miscOptions = {};
+      if (miscOptions === undefined) {
+        miscOptions = {};
       }
-      options.miscOptions[key] = optionParts[1];
+      miscOptions[key] = optionParts[1];
       continue;
     } else if (optionKeyParts.length > 2) {
       // support emitter/path/file.js.option=xyz
@@ -141,26 +112,5 @@ function resolveCliOptions(
     }
     options[emitterName][key] = optionParts[1];
   }
-  return options;
-}
-
-function resolveEmitterOptions(
-  config: TypeSpecConfig,
-  cliOptions: Record<string | "miscOptions", Record<string, unknown>>
-): Record<string, EmitterOptions> {
-  const configuredEmitters: Record<string, Record<string, unknown>> = deepClone(
-    config.options ?? {}
-  );
-
-  for (const [emitterName, cliOptionOverride] of Object.entries(cliOptions)) {
-    if (emitterName === "miscOptions") {
-      continue;
-    }
-    configuredEmitters[emitterName] = {
-      ...(configuredEmitters[emitterName] ?? {}),
-      ...cliOptionOverride,
-    };
-  }
-
-  return configuredEmitters;
+  return { options, miscOptions };
 }

packages/compiler/src/index.ts

@@ -1,3 +1,4 @@
+export { ResolveCompilerOptionsOptions, resolveCompilerOptions } from "./config/index.js";
 export * from "./core/index.js";
 export * from "./lib/decorators.js";
 export * as decorators from "./lib/decorators.js";

packages/compiler/test/cli.test.ts

@@ -25,7 +25,7 @@ describe("compiler: cli", () => {
     async function resolveCompilerOptions(args: CompileCliArgs, env: Record<string, string> = {}) {
       const [options, diagnostics] = await getCompilerOptions(
         host.compilerHost,
-        "ws/main.cadl",
+        "ws/main.tsp",
         cwd,
         args,
         env
@@ -139,7 +139,7 @@ describe("compiler: cli", () => {
       it("emit diagnostic if passing unknown parameter", async () => {
         const [_, diagnostics] = await getCompilerOptions(
           host.compilerHost,
-          "ws/main.cadl",
+          "ws/main.tsp",
           cwd,
           {
             args: ["not-defined-arg=my-value"],
@@ -162,7 +162,7 @@ describe("compiler: cli", () => {
         );
         const [_, diagnostics] = await getCompilerOptions(
           host.compilerHost,
-          "ws/main.cadl",
+          "ws/main.tsp",
           cwd,
           {},
           {}

packages/samples/.eslintrc.cjs

@@ -0,0 +1,7 @@
+require("@typespec/eslint-config-typespec/patch/modern-module-resolution");
+
+module.exports = {
+  plugins: ["@typespec/eslint-plugin"],
+  extends: ["@typespec/eslint-config-typespec", "plugin:@typespec/eslint-plugin/recommended"],
+  parserOptions: { tsconfigRootDir: __dirname },
+};

packages/samples/.mocharc.yaml

@@ -0,0 +1,4 @@
+timeout: 5000
+require: source-map-support/register
+spec: "dist/test/**/*.js"
+ignore: "dist/test/manual/**/*.js"

packages/samples/README.md

@@ -3,3 +3,13 @@
 This project has a collection of samples used to demonstrate and test various TypeSpec features.
 
 It is not published as an npm package.
+
+```bash
+npm run test # Check Samples match snapshots
+npm run test-official # run test same as CI
+
+
+npm run test:regen -- -g "<sample-name>" # Regen of this name
+
+npm run regen-samples # Regen all samples.
+```

packages/samples/package.json

@@ -19,6 +19,12 @@
     "cli"
   ],
   "type": "module",
+  "exports": {
+    ".": {
+      "default": "./dist/src/index.js",
+      "types": "./dist/src/index.d.ts"
+    }
+  },
   "engines": {
     "node": ">=16.0.0"
   },
@@ -26,7 +32,10 @@
     "clean": "rimraf ./dist ./temp",
     "build": "tsc -p .",
     "watch": "tsc -p . --watch",
-    "regen-samples": "node scripts/regen-samples.js"
+    "test": "mocha",
+    "test-official": "mocha --forbid-only",
+    "test:regen": "cross-env RECORD=true mocha",
+    "regen-samples": "RECORD=true mocha"
   },
   "files": [
     "lib/*.tsp",
@@ -45,6 +54,12 @@
   },
   "devDependencies": {
     "@typespec/internal-build-utils": "workspace:~0.46.0",
+    "@typespec/eslint-config-typespec": "workspace:~0.46.0",
+    "@types/mocha": "~10.0.1",
+    "@types/node": "~18.11.9",
+    "cross-env": "~7.0.3",
+    "eslint": "^8.42.0",
+    "mocha": "~10.2.0",
     "autorest": "~3.3.2",
     "rimraf": "~5.0.1",
     "typescript": "~5.1.3"

packages/samples/rest-metadata-emitter/tspconfig.yaml

@@ -1,2 +0,0 @@
-emit:
-  - ../dist/rest-emitter/rest-emitter-sample.js

packages/samples/scripts/regen-samples.js

@@ -1,83 +0,0 @@
-#!/usr/bin/env node
-// @ts-check
-import { run } from "@typespec/internal-build-utils";
-import { readdirSync, rmSync } from "fs";
-import { mkdir } from "fs/promises";
-import { dirname, join, normalize, resolve } from "path";
-import { fileURLToPath } from "url";
-
-const excludedSamples = [
-  // fails compilation by design to demo language server
-  "local-typespec",
-
-  // no actual samples in these dirs
-  "node_modules",
-  "dist",
-  "scratch",
-  "scripts",
-  "test",
-  ".rush",
-];
-
-const rootInputPath = resolvePath("../");
-const rootOutputPath = resolvePath("../test/output");
-const restEmitterSamplePath = resolvePath("../rest-metadata-emitter");
-
-main().catch((e) => {
-  console.error(e);
-  process.exit(1);
-});
-
-async function main() {
-  // clear any previous output as otherwise failing to emit anything could
-  // escape PR validation. Also ensures we delete output for samples that
-  // no longer exist.
-  rmSync(rootOutputPath, { recursive: true });
-
-  for (const folderName of getSampleFolders()) {
-    const inputPath = join(rootInputPath, folderName);
-    const outputPath = join(rootOutputPath, folderName);
-    await mkdir(outputPath, { recursive: true });
-
-    let emitter = "@typespec/openapi3";
-    if (inputPath === restEmitterSamplePath) {
-      emitter = resolvePath("../dist/rest-metadata-emitter/rest-metadata-emitter-sample.js");
-    }
-
-    await run(process.execPath, [
-      "../../packages/compiler/entrypoints/cli.js",
-      "compile",
-      inputPath,
-      `--option="${emitter}.emitter-output-dir=${outputPath}"`,
-      `--emit=${emitter}`,
-      `--warn-as-error`,
-      `--debug`,
-    ]);
-  }
-}
-
-function getSampleFolders() {
-  const samples = new Set();
-  const excludes = new Set(excludedSamples.map(normalize));
-  walk("");
-  return samples;
-
-  function walk(relativeDir) {
-    if (samples.has(relativeDir) || excludes.has(relativeDir)) {
-      return;
-    }
-    const fullDir = join(rootInputPath, relativeDir);
-    for (const entry of readdirSync(fullDir, { withFileTypes: true })) {
-      if (entry.isDirectory()) {
-        walk(join(relativeDir, entry.name));
-      } else if (relativeDir && (entry.name === "main.tsp" || entry.name === "package.json")) {
-        samples.add(relativeDir);
-      }
-    }
-  }
-}
-
-function resolvePath(...parts) {
-  const dir = dirname(fileURLToPath(import.meta.url));
-  return resolve(dir, ...parts);
-}

packages/samples/specs/rest-metadata-emitter/tspconfig.yaml

@@ -0,0 +1,2 @@
+emit:
+  - ../../dist/specs/rest-metadata-emitter/rest-metadata-emitter-sample.js

packages/samples/src/index.ts

@@ -0,0 +1 @@
+export * from "./sample-snapshot-testing.js";

packages/samples/src/sample-snapshot-testing.ts

@@ -0,0 +1,229 @@
+import {
+  CompilerHost,
+  NodeHost,
+  ResolveCompilerOptionsOptions,
+  compile,
+  getDirectoryPath,
+  getRelativePathFromDirectory,
+  joinPaths,
+  resolveCompilerOptions,
+  resolvePath,
+} from "@typespec/compiler";
+import { expectDiagnosticEmpty } from "@typespec/compiler/testing";
+import { fail, ok, strictEqual } from "assert";
+import { readdirSync } from "fs";
+import { mkdir, readFile, readdir, rm, writeFile } from "fs/promises";
+
+const shouldUpdateSnapshots = process.env.RECORD === "true";
+
+export interface SampleSnapshotTestOptions {
+  /**  Sample root directory. */
+  sampleDir: string;
+
+  /** Output directory for snapshots. */
+  outputDir: string;
+
+  /** Folders to exclude from testing. */
+  exclude?: string[];
+
+  /** Override the emitters to use. */
+  emit?: string[];
+}
+
+export interface TestContext {
+  runCount: number;
+  registerSnapshot(filename: string): void;
+}
+export function defineSampleSnaphotTests(config: SampleSnapshotTestOptions) {
+  const samples = resolveSamples(config);
+  let existingSnapshots: string[];
+  const writtenSnapshots: string[] = [];
+  const context = {
+    runCount: 0,
+    registerSnapshot(filename: string) {
+      writtenSnapshots.push(filename);
+    },
+  };
+  before(async () => {
+    existingSnapshots = await readFilesInDirRecursively(config.outputDir);
+  });
+
+  after(async function (this: any) {
+    if (context.runCount !== samples.length) {
+      return; // Not running the full test suite, so don't bother checking snapshots.
+    }
+
+    if (this.test.parent.tests.some((x: any) => x.state === "failed")) {
+      return; // Do not check snapshots if the test failed so we don't get a confusing error message about the missing snapshot if there is already a failure.
+    }
+
+    const missingSnapshots = new Set<string>(existingSnapshots);
+    for (const writtenSnapshot of writtenSnapshots) {
+      missingSnapshots.delete(writtenSnapshot);
+    }
+    if (missingSnapshots.size > 0) {
+      if (shouldUpdateSnapshots) {
+        for (const file of [...missingSnapshots].map((x) => joinPaths(config.outputDir, x))) {
+          await rm(file);
+        }
+      } else {
+        const snapshotList = [...missingSnapshots].map((x) => `  ${x}`).join("\n");
+        fail(
+          `The following snapshot are still present in the output dir but were not generated:\n${snapshotList}\n Run with RECORD=true to regenerate them.`
+        );
+      }
+    }
+  });
+  samples.forEach((samples) => defineSampleSnaphotTest(context, config, samples));
+}
+
+function defineSampleSnaphotTest(
+  context: TestContext,
+  config: SampleSnapshotTestOptions,
+  sample: Sample
+) {
+  it(sample.name, async () => {
+    context.runCount++;
+    const host = createSampleSnapshotTestHost(config);
+
+    const outputDir = resolvePath(config.outputDir, sample.name);
+
+    const overrides: Partial<ResolveCompilerOptionsOptions["overrides"]> = {
+      outputDir,
+    };
+    if (config.emit) {
+      overrides.emit = config.emit;
+    }
+    const [options, diagnostics] = await resolveCompilerOptions(host, {
+      entrypoint: sample.fullPath,
+      overrides,
+    });
+    expectDiagnosticEmpty(diagnostics);
+
+    const emit = options.emit;
+    if (emit === undefined || emit.length === 0) {
+      fail(
+        `No emitters configured for sample "${sample.name}". Make sure the  config at: "${options.config}" is correct.`
+      );
+    }
+
+    const program = await compile(host, sample.fullPath, options);
+    expectDiagnosticEmpty(program.diagnostics);
+
+    if (shouldUpdateSnapshots) {
+      try {
+        await host.rm(outputDir, { recursive: true });
+      } catch (e) {}
+      await mkdir(outputDir, { recursive: true });
+
+      for (const [snapshotPath, content] of host.outputs.entries()) {
+        const relativePath = getRelativePathFromDirectory(outputDir, snapshotPath, false);
+
+        try {
+          await mkdir(getDirectoryPath(snapshotPath), { recursive: true });
+          await writeFile(snapshotPath, content);
+          context.registerSnapshot(resolvePath(sample.name, relativePath));
+        } catch (e) {
+          throw new Error(`Failure to write snapshot: "${snapshotPath}"\n Error: ${e}`);
+        }
+      }
+    } else {
+      for (const [snapshotPath, content] of host.outputs.entries()) {
+        const relativePath = getRelativePathFromDirectory(outputDir, snapshotPath, false);
+        let existingContent;
+        try {
+          existingContent = await readFile(snapshotPath);
+        } catch (e: unknown) {
+          if (isEnoentError(e)) {
+            fail(`Snapshot "${snapshotPath}" is missing. Run with RECORD=true to regenerate it.`);
+          }
+          throw e;
+        }
+        context.registerSnapshot(resolvePath(sample.name, relativePath));
+        strictEqual(content, existingContent.toString());
+      }
+
+      for (const filename of await readFilesInDirRecursively(outputDir)) {
+        const snapshotPath = resolvePath(outputDir, filename);
+        ok(
+          host.outputs.has(snapshotPath),
+          `Snapshot for "${snapshotPath}" was not emitted. Run with RECORD=true to remove it.`
+        );
+      }
+    }
+  });
+}
+
+interface SampleSnapshotTestHost extends CompilerHost {
+  outputs: Map<string, string>;
+}
+
+function createSampleSnapshotTestHost(config: SampleSnapshotTestOptions): SampleSnapshotTestHost {
+  const outputs = new Map<string, string>();
+  return {
+    ...NodeHost,
+    outputs,
+    mkdirp: (path: string) => Promise.resolve(path),
+    rm: (path: string) => Promise.resolve(),
+    writeFile: async (path: string, content: string) => {
+      outputs.set(path, content);
+    },
+  };
+}
+async function readFilesInDirRecursively(dir: string): Promise<string[]> {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch (e) {
+    if (isEnoentError(e)) {
+      return [];
+    } else {
+      throw new Error(`Failed to read dir "${dir}"\n Error: ${e}`);
+    }
+  }
+  const files: string[] = [];
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      for (const file of await readFilesInDirRecursively(resolvePath(dir, entry.name))) {
+        files.push(resolvePath(entry.name, file));
+      }
+    } else {
+      files.push(entry.name);
+    }
+  }
+  return files;
+}
+
+interface Sample {
+  name: string;
+  /** Sample folder */
+  fullPath: string;
+}
+
+function resolveSamples(config: SampleSnapshotTestOptions): Sample[] {
+  const samples: Sample[] = [];
+  const excludes = new Set(config.exclude);
+  walk("");
+  return samples;
+
+  async function walk(relativeDir: string) {
+    if (excludes.has(relativeDir)) {
+      return;
+    }
+    const fullDir = joinPaths(config.sampleDir, relativeDir);
+    for (const entry of readdirSync(fullDir, { withFileTypes: true })) {
+      if (entry.isDirectory()) {
+        walk(joinPaths(relativeDir, entry.name));
+      } else if (relativeDir && (entry.name === "main.tsp" || entry.name === "package.json")) {
+        samples.push({
+          name: relativeDir,
+          fullPath: joinPaths(config.sampleDir, relativeDir),
+        });
+      }
+    }
+  }
+}
+
+function isEnoentError(e: unknown): e is { code: "ENOENT" } {
+  return typeof e === "object" && e !== null && "code" in e;
+}