Rename `ref-doc` package to `tspd` and provide a cli that can be used by each package (#2225)

Progress towards providing the ref doc generation as a usable tool by
any typespec library.

Detach the generation from the website, instead each package define
where they want their doc and we generate it.

This adds a new `tspd` package and CLI tool that is meant to be used by
library authors. It will have the following functionalities
- generate docs
- generate TS signatures
- lint library?

---------

Co-authored-by: Mark Cowlishaw <markcowl@microsoft.com>

common/changes/@typespec/http/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/http",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/http"
+}

common/changes/@typespec/json-schema/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/json-schema",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/json-schema"
+}

common/changes/@typespec/openapi/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/openapi",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/openapi"
+}

common/changes/@typespec/openapi3/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/openapi3",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/openapi3"
+}

common/changes/@typespec/protobuf/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/protobuf",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/protobuf"
+}

common/changes/@typespec/rest/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/rest",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/rest"
+}

common/changes/@typespec/versioning/ref-doc-cmd_2023-07-28-18-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/versioning",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/versioning"
+}

common/config/rush/command-line.json

@@ -56,7 +56,7 @@
       "commandKind": "global",
       "name": "regen-docs",
       "summary": "(CUSTOM) Regenerate the reference docs.",
-      "shellCommand": "node eng/scripts/npm-run.js regen-ref-docs"
+      "shellCommand": "node eng/scripts/npm-run.js regen-docs"
     },
     {
       "commandKind": "global",

common/config/rush/pnpm-lock.yaml

@@ -383,6 +383,9 @@ importers:
       '@typespec/library-linter':
         specifier: workspace:~0.46.0
         version: link:../library-linter
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       c8:
         specifier: ~8.0.0
         version: 8.0.0
@@ -484,6 +487,9 @@ importers:
       '@typespec/library-linter':
         specifier: workspace:~0.46.0
         version: link:../library-linter
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       ajv:
         specifier: ~8.12.0
         version: 8.12.0
@@ -695,6 +701,9 @@ importers:
       '@typespec/rest':
         specifier: workspace:~0.46.0
         version: link:../rest
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       c8:
         specifier: ~8.0.0
         version: 8.0.0
@@ -753,6 +762,9 @@ importers:
       '@typespec/rest':
         specifier: workspace:~0.46.0
         version: link:../rest
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       '@typespec/versioning':
         specifier: workspace:~0.46.0
         version: link:../versioning
@@ -1086,6 +1098,9 @@ importers:
       '@typespec/eslint-plugin':
         specifier: workspace:~0.46.0
         version: link:../eslint-plugin-typespec
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       c8:
         specifier: ~8.0.0
         version: 8.0.0
@@ -1105,64 +1120,6 @@ importers:
         specifier: ~5.1.3
         version: 5.1.3
 
-  ../../packages/ref-doc:
-    dependencies:
-      '@typespec/compiler':
-        specifier: workspace:~0.46.0
-        version: link:../compiler
-      js-yaml:
-        specifier: ~4.1.0
-        version: 4.1.0
-      prettier:
-        specifier: ~2.8.7
-        version: 2.8.7
-    devDependencies:
-      '@types/js-yaml':
-        specifier: ~4.0.1
-        version: 4.0.1
-      '@types/mocha':
-        specifier: ~10.0.1
-        version: 10.0.1
-      '@types/node':
-        specifier: ~18.11.9
-        version: 18.11.9
-      '@types/prettier':
-        specifier: 2.6.0
-        version: 2.6.0
-      '@typespec/eslint-config-typespec':
-        specifier: workspace:~0.46.0
-        version: link:../eslint-config-typespec
-      '@typespec/prettier-plugin-typespec':
-        specifier: workspace:~0.46.0
-        version: link:../prettier-plugin-typespec
-      c8:
-        specifier: ~8.0.0
-        version: 8.0.0
-      eslint:
-        specifier: ^8.42.0
-        version: 8.42.0
-      mocha:
-        specifier: ~10.2.0
-        version: 10.2.0
-      mocha-junit-reporter:
-        specifier: ~2.2.0
-        version: 2.2.0(mocha@10.2.0)
-      mocha-multi-reporters:
-        specifier: ~1.5.1
-        version: 1.5.1(mocha@10.2.0)
-      rimraf:
-        specifier: ~5.0.1
-        version: 5.0.1
-      typedoc:
-        specifier: ~0.24.8
-        version: 0.24.8(typescript@5.1.3)
-      typedoc-plugin-markdown:
-        specifier: ~4.0.0-next.17
-        version: 4.0.0-next.17(prettier@2.8.7)(typedoc@0.24.8)
-      typescript:
-        specifier: ~5.1.3
-        version: 5.1.3
-
   ../../packages/rest:
     devDependencies:
       '@types/mocha':
@@ -1186,6 +1143,9 @@ importers:
       '@typespec/library-linter':
         specifier: workspace:~0.46.0
         version: link:../library-linter
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       c8:
         specifier: ~8.0.0
         version: 8.0.0
@@ -1288,6 +1248,76 @@ importers:
         specifier: ~5.1.3
         version: 5.1.3
 
+  ../../packages/tspd:
+    dependencies:
+      '@typespec/compiler':
+        specifier: workspace:~0.46.0
+        version: link:../compiler
+      js-yaml:
+        specifier: ~4.1.0
+        version: 4.1.0
+      picocolors:
+        specifier: ~1.0.0
+        version: 1.0.0
+      prettier:
+        specifier: ~2.8.7
+        version: 2.8.7
+      yargs:
+        specifier: ~17.7.1
+        version: 17.7.1
+    devDependencies:
+      '@types/js-yaml':
+        specifier: ~4.0.1
+        version: 4.0.1
+      '@types/mocha':
+        specifier: ~10.0.1
+        version: 10.0.1
+      '@types/node':
+        specifier: ~18.11.9
+        version: 18.11.9
+      '@types/prettier':
+        specifier: 2.6.0
+        version: 2.6.0
+      '@types/yargs':
+        specifier: ~17.0.24
+        version: 17.0.24
+      '@typespec/eslint-config-typespec':
+        specifier: workspace:~0.46.0
+        version: link:../eslint-config-typespec
+      '@typespec/prettier-plugin-typespec':
+        specifier: workspace:~0.46.0
+        version: link:../prettier-plugin-typespec
+      c8:
+        specifier: ~8.0.0
+        version: 8.0.0
+      eslint:
+        specifier: ^8.42.0
+        version: 8.42.0
+      mocha:
+        specifier: ~10.2.0
+        version: 10.2.0
+      mocha-junit-reporter:
+        specifier: ~2.2.0
+        version: 2.2.0(mocha@10.2.0)
+      mocha-multi-reporters:
+        specifier: ~1.5.1
+        version: 1.5.1(mocha@10.2.0)
+      rimraf:
+        specifier: ~5.0.1
+        version: 5.0.1
+      source-map-support:
+        specifier: ~0.5.19
+        version: 0.5.19
+      typedoc:
+        specifier: ~0.24.8
+        version: 0.24.8(typescript@5.1.3)
+      typedoc-plugin-markdown:
+        specifier: ~4.0.0-next.17
+        version: 4.0.0-next.17(prettier@2.8.7)(typedoc@0.24.8)
+      typescript:
+        specifier: ~5.1.3
+        version: 5.1.3
+
   ../../packages/typespec-vs:
     devDependencies:
       '@typespec/internal-build-utils':
@@ -1374,6 +1404,9 @@ importers:
       '@typespec/library-linter':
         specifier: workspace:~0.46.0
         version: link:../library-linter
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       c8:
         specifier: ~8.0.0
         version: 8.0.0
@@ -1453,15 +1486,15 @@ importers:
       '@typespec/protobuf':
         specifier: workspace:~0.46.0
         version: link:../protobuf
-      '@typespec/ref-doc':
-        specifier: workspace:~0.1.0
-        version: link:../ref-doc
       '@typespec/rest':
         specifier: workspace:~0.46.0
         version: link:../rest
       '@typespec/spec':
         specifier: workspace:*
         version: link:../spec
+      '@typespec/tspd':
+        specifier: workspace:~0.46.0
+        version: link:../tspd
       '@typespec/versioning':
         specifier: workspace:~0.46.0
         version: link:../versioning

cspell.yaml

@@ -59,6 +59,7 @@ words:
   - strs
   - TRYIT
   - TSES
+  - tspd
   - tsvs
   - Uhoh
   - unassignable

docs/standard-library/protobuf/reference/data-types.md

@@ -153,3 +153,45 @@ Uses variable-length encoding. These more efficiently encode negative numbers th
 ```typespec
 scalar TypeSpec.Protobuf.sint64
 ```
+
+## TypeSpec.Protobuf.WellKnown
+
+### `Any` {#TypeSpec.Protobuf.WellKnown.Any}
+
+Any value.
+
+This model references `google.protobuf.Any` from `google/protobuf/any.proto`.
+
+```typespec
+model TypeSpec.Protobuf.WellKnown.Any
+```
+
+### `Empty` {#TypeSpec.Protobuf.WellKnown.Empty}
+
+An empty message.
+
+This model references `google.protobuf.Empty` from `google/protobuf/empty.proto`.
+
+```typespec
+model TypeSpec.Protobuf.WellKnown.Empty
+```
+
+### `LatLng` {#TypeSpec.Protobuf.WellKnown.LatLng}
+
+A latitude and longitude.
+
+This model references `google.type.LatLng` from `google/type/latlng.proto`.
+
+```typespec
+model TypeSpec.Protobuf.WellKnown.LatLng
+```
+
+### `Timestamp` {#TypeSpec.Protobuf.WellKnown.Timestamp}
+
+A timestamp.
+
+This model references `google.protobuf.Timestamp` from `google/protobuf/timestamp.proto`.
+
+```typespec
+model TypeSpec.Protobuf.WellKnown.Timestamp
+```

docs/standard-library/protobuf/reference/index.md

@@ -49,3 +49,12 @@ npm install --save-peer @typespec/protobuf
 - [`Extern`](./data-types.md#TypeSpec.Protobuf.Extern)
 - [`Map`](./data-types.md#TypeSpec.Protobuf.Map)
 - [`PackageDetails`](./data-types.md#TypeSpec.Protobuf.PackageDetails)
+
+## TypeSpec.Protobuf.WellKnown
+
+### Models
+
+- [`Any`](./data-types.md#TypeSpec.Protobuf.WellKnown.Any)
+- [`Empty`](./data-types.md#TypeSpec.Protobuf.WellKnown.Empty)
+- [`LatLng`](./data-types.md#TypeSpec.Protobuf.WellKnown.LatLng)
+- [`Timestamp`](./data-types.md#TypeSpec.Protobuf.WellKnown.Timestamp)

eng/scripts/helpers.js

@@ -19,16 +19,33 @@ export const tsc = resolve(repoRoot, "packages/compiler/node_modules/.bin/tsc");
 
 const rush = read(`${repoRoot}/rush.json`);
 
-export function forEachProject(onEach) {
+export function forEachProject(onEach, filter) {
   // load all the projects
   for (const each of rush.projects) {
     const packageName = each.packageName;
+    if (filter !== undefined && !filter.includes(packageName)) continue;
     const projectFolder = resolve(`${repoRoot}/${each.projectFolder}`);
     const project = JSON.parse(readFileSync(`${projectFolder}/package.json`, "utf-8"));
     onEach(packageName, projectFolder, project);
   }
 }
 
+export function npmForEachDependency(cmd, projectDir, options) {
+  const project = JSON.parse(readFileSync(`${projectDir}/package.json`, "utf-8"));
+  const deps = [
+    Object.keys(project.dependencies || {}),
+    Object.keys(project.devDependencies || {}),
+    Object.keys(project.peerDependencies || {}),
+  ].flat();
+
+  forEachProject((name, location, project) => {
+    if (project.scripts[cmd] || cmd === "pack") {
+      const args = cmd === "pack" ? [cmd] : ["run", cmd];
+      run("npm", args, { cwd: location, ...options });
+    }
+  }, deps);
+}
+
 export function npmForEach(cmd, options) {
   forEachProject((name, location, project) => {
     if (cmd === "test-official" && !project.scripts[cmd] && project.scripts["test"]) {

eng/scripts/npm-run-for-deps.js

@@ -0,0 +1,6 @@
+// @ts-check
+
+// Runs the npm run command on each project that has it.
+import { npmForEachDependency } from "./helpers.js";
+
+npmForEachDependency(process.argv[2], process.cwd());

packages/http/package.json

@@ -44,7 +44,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only --reporter mocha-multi-reporters",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/http/reference"
   },
   "files": [
     "lib/*.tsp",
@@ -61,6 +62,7 @@
     "@typespec/eslint-config-typespec": "workspace:~0.46.0",
     "@typespec/library-linter": "workspace:~0.46.0",
     "@typespec/eslint-plugin": "workspace:~0.46.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",
     "mocha-junit-reporter": "~2.2.0",

packages/json-schema/package.json

@@ -34,7 +34,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only --reporter mocha-multi-reporters",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/json-schema/reference"
   },
   "files": [
     "lib/*.tsp",
@@ -51,6 +52,7 @@
     "@typespec/library-linter": "workspace:~0.46.0",
     "@typespec/eslint-plugin": "workspace:~0.46.0",
     "@typespec/eslint-config-typespec": "workspace:~0.46.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",
     "mocha-junit-reporter": "~2.2.0",

packages/openapi/package.json

@@ -44,7 +44,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only --reporter mocha-multi-reporters",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/openapi/reference"
   },
   "files": [
     "lib/*.tsp",
@@ -65,6 +66,7 @@
     "@typespec/eslint-config-typespec": "workspace:~0.46.0",
     "@typespec/library-linter": "workspace:~0.46.0",
     "@typespec/eslint-plugin": "workspace:~0.46.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",
     "mocha-junit-reporter": "~2.2.0",

packages/openapi3/package.json

@@ -44,7 +44,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only --reporter mocha-multi-reporters",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/openapi3/reference"
   },
   "files": [
     "lib/*.tsp",
@@ -73,6 +74,7 @@
     "@typespec/eslint-config-typespec": "workspace:~0.46.0",
     "@typespec/library-linter": "workspace:~0.46.0",
     "@typespec/eslint-plugin": "workspace:~0.46.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",
     "mocha-junit-reporter": "~2.2.0",

packages/protobuf/package.json

@@ -28,7 +28,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/protobuf/reference"
   },
   "peerDependencies": {
     "@typespec/compiler": "workspace:~0.46.0"
@@ -39,6 +40,7 @@
     "@typespec/eslint-plugin": "workspace:~0.46.0",
     "@types/mocha": "~10.0.1",
     "@types/node": "~18.11.9",
+    "@typespec/tspd": "workspace:~0.46.0",
     "c8": "~8.0.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",

packages/ref-doc/src/cli.ts

@@ -1,32 +0,0 @@
-/* eslint-disable no-console */
-import { normalizePath } from "@typespec/compiler";
-import { resolve } from "path";
-import { generateLibraryDocs } from "./index.js";
-
-main().catch((e) => {
-  console.error(e);
-  process.exit(1);
-});
-
-async function main() {
-  const args = process.argv.slice(2);
-  const entrypoint = args[0];
-  const outputDir = args[1];
-  const namespaces = args.slice(2);
-  if (entrypoint === undefined) {
-    console.error("Specify doc entrypoint as 1st parameter");
-    process.exit(1);
-  }
-  if (outputDir === undefined) {
-    console.error("Specify output-dir as 2nd parameter");
-    process.exit(1);
-  }
-  const main = normalizePath(resolve(entrypoint));
-  const resolvedOutputDir = normalizePath(resolve(outputDir));
-  console.log(`Generate docs`, {
-    main,
-    resolvedOutputDir,
-  });
-
-  await generateLibraryDocs(main, namespaces, resolvedOutputDir);
-}

packages/rest/package.json

@@ -44,7 +44,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only --reporter mocha-multi-reporters",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/rest/reference"
   },
   "files": [
     "lib/*.tsp",
@@ -62,6 +63,7 @@
     "@typespec/eslint-config-typespec": "workspace:~0.46.0",
     "@typespec/library-linter": "workspace:~0.46.0",
     "@typespec/eslint-plugin": "workspace:~0.46.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",
     "mocha-junit-reporter": "~2.2.0",

packages/tspd/cmd/tspd.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "../dist/src/cli.js";

packages/tspd/package.json

@@ -1,10 +1,10 @@
 {
-  "name": "@typespec/ref-doc",
-  "version": "0.1.0",
+  "name": "@typespec/tspd",
+  "version": "0.46.0",
   "author": "Microsoft Corporation",
   "description": "TypeSpec library for generating typespec docs",
   "homepage": "https://microsoft.github.io/typespec",
-  "readme": "https://github.com/microsoft/typespec/blob/master/README.md",
+  "readme": "https://github.com/microsoft/typespec/blob/main/README.md",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -17,22 +17,24 @@
     "typespec"
   ],
   "type": "module",
-  "main": "dist/src/index.js",
-  "exports": {
-    ".": "./dist/src/index.js",
-    "./emitters/docusaurus": "./dist/src/emitters/docusaurus.js"
+  "bin": {
+    "tspd": "./cmd/tspd.js"
   },
-  "typesVersions": {
-    "*": {
-      "*": [
-        "./dist/src/index.d.ts"
-      ],
-      "emitters/docusaurus": [
-        "./dist/src/emitters/docusaurus.d.ts"
-      ]
+  "main": "./dist/src/index.js",
+  "exports": {
+    ".": {
+      "default": "./dist/src/index.js",
+      "types": "./dist/src/index.d.ts"
+    },
+    "./ref-doc": {
+      "default": "./dist/src/ref-doc/index.js",
+      "types": "./dist/src/ref-doc/index.d.ts"
+    },
+    "./ref-doc/emitters/docusaurus": {
+      "default": "./dist/src/ref-doc/emitters/docusaurus.js",
+      "types": "./dist/src/ref-doc/emitters/docusaurus.d.ts"
     }
   },
-  "tspMain": "dist/src/index.js",
   "engines": {
     "node": ">=16.0.0"
   },
@@ -53,7 +55,9 @@
   "dependencies": {
     "@typespec/compiler": "workspace:~0.46.0",
     "js-yaml": "~4.1.0",
-    "prettier": "~2.8.7"
+    "prettier": "~2.8.7",
+    "picocolors": "~1.0.0",
+    "yargs": "~17.7.1"
   },
   "devDependencies": {
     "@typespec/compiler": "workspace:~0.46.0",
@@ -63,11 +67,13 @@
     "@types/node": "~18.11.9",
     "@types/prettier": "2.6.0",
     "@types/js-yaml": "~4.0.1",
+    "@types/yargs": "~17.0.24",
     "c8": "~8.0.0",
     "eslint": "^8.42.0",
     "mocha-junit-reporter": "~2.2.0",
     "mocha-multi-reporters": "~1.5.1",
     "mocha": "~10.2.0",
+    "source-map-support": "~0.5.19",
     "rimraf": "~5.0.1",
     "typedoc-plugin-markdown": "~4.0.0-next.17",
     "typedoc": "~0.24.8",

packages/tspd/src/cli.ts

@@ -0,0 +1,118 @@
+/* eslint-disable no-console */
+import { NodeHost, logDiagnostics, resolvePath, typespecVersion } from "@typespec/compiler";
+import pc from "picocolors";
+import yargs from "yargs";
+import { generateLibraryDocs } from "./ref-doc/experimental.js";
+
+try {
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  await import("source-map-support/register.js");
+} catch {
+  // package only present in dev.
+}
+
+function logExperimentalWarning(type: "log" | "error") {
+  const log =
+    type === "error"
+      ? (message: string) => console.error(pc.red(message))
+      : (message: string) => console.log(pc.yellow(message));
+  log("-".repeat(100));
+  log(
+    `tspd (TypeSpec Library Developer Cli) is experimental and might be ${pc.bold(
+      "BREAKING"
+    )} between versions.`
+  );
+  if (type === "error") {
+    log(`Add "--enable-experimental" flag to acknowledge this and continue.`);
+  }
+  log("-".repeat(100));
+}
+
+async function main() {
+  console.log(`TypeSpec compiler v${typespecVersion}\n`);
+
+  await yargs(process.argv.slice(2))
+    .scriptName("tspd")
+    .help()
+    .strict()
+    .parserConfiguration({
+      "greedy-arrays": false,
+      "boolean-negation": false,
+    })
+    .option("debug", {
+      type: "boolean",
+      description: "Output debug log messages.",
+      default: false,
+    })
+    .option("pretty", {
+      type: "boolean",
+      description:
+        "Enable color and formatting in TypeSpec's output to make compiler errors easier to read.",
+      default: true,
+    })
+    .option("enable-experimental", {
+      type: "boolean",
+      description: "Acknowledge that the tspd command line is experiemental.",
+      default: false,
+    })
+    .check((args) => {
+      if (args["enable-experimental"]) {
+        logExperimentalWarning("log");
+        return true;
+      } else {
+        logExperimentalWarning("error");
+        process.exit(1);
+      }
+    })
+    .command(
+      "doc <entrypoint>",
+      "Generate documentation for a TypeSpec library.",
+      (cmd) => {
+        return cmd
+          .positional("entrypoint", {
+            description: "Path to the library entrypoint.",
+            type: "string",
+            demandOption: true,
+          })
+          .option("output-dir", {
+            type: "string",
+          });
+      },
+      async (args) => {
+        const resolvedRoot = resolvePath(process.cwd(), args.entrypoint);
+        const host = NodeHost;
+        const diagnostics = await generateLibraryDocs(
+          resolvedRoot,
+          args["output-dir"] ?? resolvePath(resolvedRoot, "docs")
+        );
+        // const diagnostics = await generateExternSignatures(host, resolvedRoot);
+        if (diagnostics.length > 0) {
+          logDiagnostics(diagnostics, host.logSink);
+        }
+      }
+    )
+    .version(typespecVersion)
+    .demandCommand(1, "You must use one of the supported commands.").argv;
+}
+
+function internalError(error: unknown): never {
+  // NOTE: An expected error, like one thrown for bad input, shouldn't reach
+  // here, but be handled somewhere else. If we reach here, it should be
+  // considered a bug and therefore we should not suppress the stack trace as
+  // that risks losing it in the case of a bug that does not repro easily.
+
+  console.error("Internal error!");
+  console.error("File issue at https://github.com/microsoft/typespec");
+  console.error();
+  console.error(error);
+
+  process.exit(1);
+}
+
+process.on("unhandledRejection", (error: unknown) => {
+  console.error("Unhandled promise rejection!");
+  internalError(error);
+});
+
+main().catch(internalError);

packages/tspd/src/ref-doc/experimental.ts

@@ -9,7 +9,7 @@ import {
 import { mkdir, readFile, writeFile } from "fs/promises";
 import { generateJsApiDocs } from "./api-docs.js";
 import { renderToDocusaurusMarkdown } from "./emitters/docusaurus.js";
-import { extractLibraryRefDocs, extractRefDocs } from "./extractor.js";
+import { extractLibraryRefDocs, ExtractRefDocOptions, extractRefDocs } from "./extractor.js";
 import { TypeSpecRefDocBase } from "./types.js";
 
 /**
@@ -17,13 +17,12 @@ import { TypeSpecRefDocBase } from "./types.js";
  */
 export async function generateLibraryDocs(
   libraryPath: string,
-  namespaces: string[],
   outputDir: string,
   skipJSApi: boolean = false
 ): Promise<readonly Diagnostic[]> {
   const diagnostics = createDiagnosticCollector();
   const pkgJson = await readPackageJson(libraryPath);
-  const refDoc = diagnostics.pipe(await extractLibraryRefDocs(libraryPath, namespaces));
+  const refDoc = diagnostics.pipe(await extractLibraryRefDocs(libraryPath));
   const files = renderToDocusaurusMarkdown(refDoc);
   await mkdir(outputDir, { recursive: true });
   for (const [name, content] of Object.entries(files)) {
@@ -37,7 +36,7 @@ export async function generateLibraryDocs(
 
 export async function resolveLibraryRefDocsBase(
   libraryPath: string,
-  namespaces: string[]
+  options: ExtractRefDocOptions = {}
 ): Promise<[TypeSpecRefDocBase, readonly Diagnostic[]] | undefined> {
   const diagnostics = createDiagnosticCollector();
   const pkgJson = await readPackageJson(libraryPath);
@@ -46,7 +45,7 @@ export async function resolveLibraryRefDocsBase(
     const program = await compile(NodeHost, main, {
       parseOptions: { comments: true, docs: true },
     });
-    const refDoc = extractRefDocs(program, namespaces);
+    const refDoc = diagnostics.pipe(extractRefDocs(program, options));
     for (const diag of program.diagnostics ?? []) {
       diagnostics.add(diag);
     }

packages/tspd/src/ref-doc/extractor.ts

@@ -8,9 +8,9 @@ import {
   DocUnknownTagNode,
   Enum,
   getDoc,
+  getLocationContext,
   getSourceLocation,
   getTypeName,
-  ignoreDiagnostics,
   Interface,
   isDeclaredType,
   isTemplateDeclaration,
@@ -18,6 +18,7 @@ import {
   JSONSchemaType,
   Model,
   Namespace,
+  navigateProgram,
   navigateTypesInNamespace,
   NodeHost,
   NodePackage,
@@ -53,8 +54,7 @@ import {
 import { getQualifier, getTypeSignature } from "./utils/type-signature.js";
 
 export async function extractLibraryRefDocs(
-  libraryPath: string,
-  namespaces: string[]
+  libraryPath: string
 ): Promise<[TypeSpecLibraryRefDoc, readonly Diagnostic[]]> {
   const diagnostics = createDiagnosticCollector();
   const pkgJson = await readPackageJson(libraryPath);
@@ -68,7 +68,7 @@ export async function extractLibraryRefDocs(
     const program = await compile(NodeHost, main, {
       parseOptions: { comments: true, docs: true },
     });
-    refDoc.namespaces = extractRefDocs(program, namespaces).namespaces;
+    refDoc.namespaces = diagnostics.pipe(extractRefDocs(program)).namespaces;
     for (const diag of program.diagnostics ?? []) {
       diagnostics.add(diag);
     }
@@ -92,13 +92,51 @@ async function readPackageJson(libraryPath: string): Promise<NodePackage> {
   return JSON.parse(buffer.toString());
 }
 
+export interface ExtractRefDocOptions {
+  namespaces?: {
+    include?: string[];
+    exclude?: string[];
+  };
+}
+
+function resolveNamespaces(
+  program: Program,
+  options: ExtractRefDocOptions
+): [Namespace[], readonly Diagnostic[]] {
+  const diagnostics = createDiagnosticCollector();
+  let namespaceTypes: Namespace[] = [];
+
+  const { include, exclude } = options?.namespaces ?? {};
+  if (include) {
+    namespaceTypes = include
+      .map((x) => diagnostics.pipe(program.resolveTypeReference(x)))
+      .filter((x): x is Namespace => x !== undefined);
+  }
+  navigateProgram(program, {
+    namespace(namespace) {
+      if (getLocationContext(program, namespace).type !== "project") {
+        return;
+      }
+      if (namespace.name === "Private") {
+        return;
+      }
+      namespaceTypes.push(namespace);
+    },
+  });
+
+  if (exclude !== undefined) {
+    namespaceTypes = namespaceTypes.filter((x) => {
+      return exclude.includes(getTypeName(x));
+    });
+  }
+  return diagnostics.wrap(namespaceTypes);
+}
 export function extractRefDocs(
   program: Program,
-  filterToNamespace: string[] = []
-): TypeSpecRefDocBase {
-  const namespaceTypes = filterToNamespace
-    .map((x) => ignoreDiagnostics(program.resolveTypeReference(x)))
-    .filter((x): x is Namespace => x !== undefined);
+  options: ExtractRefDocOptions = {}
+): [TypeSpecRefDocBase, readonly Diagnostic[]] {
+  const diagnostics = createDiagnosticCollector();
+  const namespaceTypes = diagnostics.pipe(resolveNamespaces(program, options));
 
   const refDoc: TypeSpecRefDocBase = {
     namespaces: [],
@@ -183,7 +221,7 @@ export function extractRefDocs(
     arr.sort((a, b) => a.id.localeCompare(b.id, "en"));
   }
 
-  return refDoc;
+  return diagnostics.wrap(refDoc);
 }
 
 function extractTemplateParameterDocs(program: Program, type: TemplatedType) {

packages/tspd/src/ref-doc/lib.ts

@@ -1,7 +1,7 @@
 import { createTypeSpecLibrary, paramMessage } from "@typespec/compiler";
 
 export const libDef = {
-  name: "@typespec/ref-doc",
+  name: "@typespec/tspd",
   diagnostics: {
     "documentation-missing": {
       severity: "warning",

packages/tspd/test/ref-doc/markdown.test.ts

@@ -1,5 +1,5 @@
 import { strictEqual } from "assert";
-import { table } from "../src/utils/markdown.js";
+import { table } from "../../src/ref-doc/utils/markdown.js";
 
 describe("ref-doc: markdown", () => {
   it("escapes table content", () => {

packages/tspd/tsconfig.json

@@ -2,8 +2,6 @@
   "extends": "../tsconfig.json",
   "references": [{ "path": "../compiler/tsconfig.json" }, { "path": "../rest/tsconfig.json" }],
   "compilerOptions": {
-    "module": "Node16",
-    "moduleResolution": "Node16",
     "outDir": "dist",
     "rootDir": ".",
     "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo",

packages/versioning/package.json

@@ -44,7 +44,8 @@
     "test": "mocha",
     "test-official": "c8 mocha --forbid-only --reporter mocha-multi-reporters",
     "lint": "eslint . --ext .ts --max-warnings=0",
-    "lint:fix": "eslint . --fix --ext .ts"
+    "lint:fix": "eslint . --fix --ext .ts",
+    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../docs/standard-library/versioning/reference"
   },
   "files": [
     "lib/*.tsp",
@@ -61,6 +62,7 @@
     "@typespec/eslint-config-typespec": "workspace:~0.46.0",
     "@typespec/library-linter": "workspace:~0.46.0",
     "@typespec/eslint-plugin": "workspace:~0.46.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "eslint": "^8.42.0",
     "mocha": "~10.2.0",
     "mocha-junit-reporter": "~2.2.0",

packages/website/.scripts/regen-compiler-docs.mjs

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+// @ts-check
+import { NodeHost, logDiagnostics } from "@typespec/compiler";
+import { generateJsApiDocs, resolveLibraryRefDocsBase } from "@typespec/tspd/ref-doc";
+import { renderDecoratorFile } from "@typespec/tspd/ref-doc/emitters/docusaurus";
+
+import assert from "assert";
+import { writeFile } from "fs/promises";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+
+export const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
+
+const diagnostics = new Map();
+
+// Compiler
+const compilerDiag = await generateCompilerDocs();
+if (compilerDiag.length) {
+  diagnostics.set("@typespec/compiler", compilerDiag);
+}
+
+let exitCode = 0;
+// Log the diagnostics
+for (const pkg of diagnostics.keys()) {
+  console.log(`\nIssues in ${pkg}:`);
+  const diags = diagnostics.get(pkg);
+  logDiagnostics(diags, NodeHost.logSink);
+  exitCode = 1;
+}
+process.exit(exitCode);
+
+async function generateCompilerDocs() {
+  const compilerPath = join(repoRoot, "packages/compiler");
+  const outputDir = join(repoRoot, "docs/standard-library");
+  const results = await resolveLibraryRefDocsBase(compilerPath, {
+    namespaces: { include: ["TypeSpec"] },
+  });
+  assert(results, "Unexpected ref doc should have been resolved for compiler.");
+  const [refDoc, diagnostics] = results;
+  const decoratorContent = renderDecoratorFile(refDoc, { title: "Built-in Decorators" });
+  assert(decoratorContent, "Unexpected decorator file shouldn't be empty for compiler.");
+  await writeFile(join(outputDir, "built-in-decorators.md"), decoratorContent);
+
+  await generateJsApiDocs(compilerPath, join(outputDir, "reference/js-api"));
+  return diagnostics;
+}

packages/website/.scripts/regen-ref-docs.mjs

@@ -1,118 +0,0 @@
-#!/usr/bin/env node
-// @ts-check
-import { NodeHost, logDiagnostics } from "@typespec/compiler";
-import {
-  generateJsApiDocs,
-  generateLibraryDocs,
-  resolveLibraryRefDocsBase,
-} from "@typespec/ref-doc";
-import { renderDecoratorFile } from "@typespec/ref-doc/emitters/docusaurus";
-import assert from "assert";
-import { writeFile } from "fs/promises";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-
-export const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
-
-const diagnostics = new Map();
-
-// Compiler
-const compilerDiag = await generateCompilerDocs();
-if (compilerDiag.length) {
-  diagnostics.set("@typespec/compiler", compilerDiag);
-}
-
-// Http
-const httpDiag = await generateLibraryDocs(
-  join(repoRoot, "packages/http"),
-  ["TypeSpec.Http"],
-  join(repoRoot, "docs/standard-library/http/reference")
-);
-if (httpDiag.length) {
-  diagnostics.set("@typespec/http", httpDiag);
-}
-
-// Rest
-const restDiag = await generateLibraryDocs(
-  join(repoRoot, "packages/rest"),
-  ["TypeSpec.Rest", "TypeSpec.Rest.Resource"],
-  join(repoRoot, "docs/standard-library/rest/reference")
-);
-if (restDiag.length) {
-  diagnostics.set("@typespec/rest", restDiag);
-}
-
-// OpenAPI
-const openapiDiag = await generateLibraryDocs(
-  join(repoRoot, "packages/openapi"),
-  ["OpenAPI"],
-  join(repoRoot, "docs/standard-library/openapi/reference")
-);
-if (openapiDiag.length) {
-  diagnostics.set("@typespec/openapi", openapiDiag);
-}
-
-// OpenAPI3
-const openapi3Diag = await generateLibraryDocs(
-  join(repoRoot, "packages/openapi3"),
-  ["OpenAPI"],
-  join(repoRoot, "docs/standard-library/openapi3/reference"),
-  true
-);
-if (openapi3Diag.length) {
-  diagnostics.set("@typespec/openapi3", openapi3Diag);
-}
-
-// Protobuf
-const protobufDiag = await generateLibraryDocs(
-  join(repoRoot, "packages/protobuf"),
-  ["TypeSpec.Protobuf"],
-  join(repoRoot, "docs/standard-library/protobuf/reference")
-);
-if (protobufDiag.length) {
-  diagnostics.set("@typespec/protobuf", protobufDiag);
-}
-
-// JSON Schema
-const jsonSchema = await generateLibraryDocs(
-  join(repoRoot, "packages/json-schema"),
-  ["TypeSpec.JsonSchema"],
-  join(repoRoot, "docs/standard-library/json-schema/reference")
-);
-if (jsonSchema.length) {
-  diagnostics.set("@typespec/json-schema", jsonSchema);
-}
-
-// Versioning
-const versioningDiag = await generateLibraryDocs(
-  join(repoRoot, "packages/versioning"),
-  ["TypeSpec.Versioning"],
-  join(repoRoot, "docs/standard-library/versioning/reference")
-);
-if (versioningDiag.length) {
-  diagnostics.set("@typespec/versioning", versioningDiag);
-}
-
-let exitCode = 0;
-// Log the diagnostics
-for (const pkg of diagnostics.keys()) {
-  console.log(`\nIssues in ${pkg}:`);
-  const diags = diagnostics.get(pkg);
-  logDiagnostics(diags, NodeHost.logSink);
-  exitCode = 1;
-}
-process.exit(exitCode);
-
-async function generateCompilerDocs() {
-  const compilerPath = join(repoRoot, "packages/compiler");
-  const outputDir = join(repoRoot, "docs/standard-library");
-  const results = await resolveLibraryRefDocsBase(compilerPath, ["TypeSpec"]);
-  assert(results, "Unexpected ref doc should have been resolved for compiler.");
-  const [refDoc, diagnostics] = results;
-  const decoratorContent = renderDecoratorFile(refDoc, { title: "Built-in Decorators" });
-  assert(decoratorContent, "Unexpected decorator file shouldn't be empty for compiler.");
-  await writeFile(join(outputDir, "built-in-decorators.md"), decoratorContent);
-
-  await generateJsApiDocs(compilerPath, join(outputDir, "reference/js-api"));
-  return diagnostics;
-}

packages/website/.scripts/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "allowJs": true
+  },
+  "include": ["**/*.mjs", "**/*.ts"]
+}

packages/website/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
-    "build": "npm run regen-ref-docs && node  .scripts/docusaurus-build.mjs 2>&1",
+    "build": "npm run regen-all-packages-docs && node  .scripts/docusaurus-build.mjs 2>&1",
     "swizzle": "docusaurus swizzle",
     "clear": "docusaurus clear",
     "clean": "docusaurus clear",
@@ -16,7 +16,8 @@
     "update-latest-docs": "rimraf versions.json ./versioned_docs ./versioned_sidebars && docusaurus docs:version latest",
     "lint": "eslint . --ext .ts,.js --max-warnings=0",
     "lint:fix": "eslint . --fix --ext .ts,.js",
-    "regen-ref-docs": "node ./.scripts/regen-ref-docs.mjs"
+    "regen-docs": "node ./.scripts/regen-compiler-docs.mjs",
+    "regen-all-packages-docs": "node ../../eng/scripts/npm-run-for-deps.js regen-docs"
   },
   "keywords": [],
   "author": "",
@@ -32,7 +33,7 @@
   },
   "devDependencies": {
     "@typespec/compiler": "workspace:~0.46.0",
-    "@typespec/ref-doc": "workspace:~0.1.0",
+    "@typespec/tspd": "workspace:~0.46.0",
     "@typespec/spec": "workspace:*",
     "@typespec/http": "workspace:~0.46.0",
     "@typespec/rest": "workspace:~0.46.0",

packages/website/tsconfig.json

@@ -5,5 +5,5 @@
     "jsx": "react-jsx"
   },
   "include": ["**/*.tsx", "**/*.ts", "**/*.js"],
-  "exclude": ["build/**/*.js"]
+  "exclude": ["build/**/*.js", ".scripts/**/*.mjs"]
 }

rush.json

@@ -200,8 +200,8 @@
       "versionPolicyName": "typespec"
     },
     {
-      "packageName": "@typespec/ref-doc",
-      "projectFolder": "packages/ref-doc",
+      "packageName": "@typespec/tspd",
+      "projectFolder": "packages/tspd",
       "reviewCategory": "production",
       "shouldPublish": false
     },

tsconfig.json

@@ -19,7 +19,7 @@
     { "path": "packages/bundler/tsconfig.json" },
     { "path": "packages/playground/tsconfig.json" },
     { "path": "packages/playground-website/tsconfig.json" },
-    { "path": "packages/ref-doc/tsconfig.json" },
+    { "path": "packages/tspd/tsconfig.json" },
     { "path": "packages/samples/tsconfig.json" },
     { "path": "packages/json-schema/tsconfig.json" },
     { "path": "packages/best-practices/tsconfig.json" }