diff --git a/common/changes/@typespec/internal-build-utils/extensions-cleanup_2023-12-05-20-04.json b/common/changes/@typespec/internal-build-utils/extensions-cleanup_2023-12-05-20-04.json
new file mode 100644
index 000000000..a231477ac
--- /dev/null
+++ b/common/changes/@typespec/internal-build-utils/extensions-cleanup_2023-12-05-20-04.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/internal-build-utils",
+      "comment": "Fix third party generation when there is invalid sourcemaps",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/internal-build-utils"
+}
\ No newline at end of file
diff --git a/common/changes/typespec-vscode/extensions-cleanup_2023-12-05-17-22.json b/common/changes/typespec-vscode/extensions-cleanup_2023-12-05-17-22.json
new file mode 100644
index 000000000..4ec67f8df
--- /dev/null
+++ b/common/changes/typespec-vscode/extensions-cleanup_2023-12-05-17-22.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "typespec-vscode",
+      "comment": "",
+      "type": "none"
+    }
+  ],
+  "packageName": "typespec-vscode"
+}
\ No newline at end of file
diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml
index 820862c23..ef5083171 100644
--- a/common/config/rush/pnpm-lock.yaml
+++ b/common/config/rush/pnpm-lock.yaml
@@ -1417,6 +1417,9 @@ importers:
       '@rollup/plugin-node-resolve':
         specifier: ~15.2.1
         version: 15.2.3(rollup@4.9.0)
+      '@rollup/plugin-typescript':
+        specifier: ~11.1.5
+        version: 11.1.5(rollup@4.9.0)(typescript@5.3.3)
       '@types/mocha':
         specifier: ~10.0.6
         version: 10.0.6
@@ -6930,6 +6933,7 @@ packages:
 
   /@swagger-api/apidom-ns-json-schema-draft-4@0.87.0:
     resolution: {integrity: sha512-dea5jN2F9w8wt/YBZLrNdqF6gcxLOUhCEOJ/qsIK9ZT/Hyb/HjEOAqpLQhQMdZgIyefc/Mh0KGKBdQAxegKMqw==}
+    requiresBuild: true
     dependencies:
       '@babel/runtime-corejs3': 7.23.6
       '@swagger-api/apidom-ast': 0.87.0
diff --git a/packages/internal-build-utils/src/generate-third-party-notice.ts b/packages/internal-build-utils/src/generate-third-party-notice.ts
index 0d8ed810f..5a9c139b8 100644
--- a/packages/internal-build-utils/src/generate-third-party-notice.ts
+++ b/packages/internal-build-utils/src/generate-third-party-notice.ts
@@ -51,8 +51,11 @@ async function findThirdPartyPackages() {
     const contents = JSON.parse(await readFile(map, "utf-8"));
     const sources = contents.sources;
     for (const source of sources) {
-      const sourcePath = join(dirname(map), source);
+      const sourcePath = resolve(dirname(map), source);
       const packageRoot = await getPackageRoot(sourcePath);
+      if (packageRoot === undefined) {
+        continue;
+      }
       const pkg = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf-8"));
 
       if (pkg.name === rootName || /microsoft/i.test(JSON.stringify(pkg.author))) {
@@ -80,15 +83,18 @@ async function* projectSourcemaps(rootPath: string): any {
 
       yield* projectSourcemaps(filepath);
     } else {
-      if (file.name.endsWith(".js.map")) {
+      if (file.name.endsWith(".js.map") || file.name.endsWith(".cjs.map")) {
         yield filepath;
       }
     }
   }
 }
 
-async function getPackageRoot(filename: string): Promise<any> {
+async function getPackageRoot(filename: string): Promise<string | undefined> {
   const dir = dirname(filename);
+  if (dir === "/") {
+    return undefined;
+  }
   try {
     const pkgPath = join(dir, "package.json");
     await stat(pkgPath);
diff --git a/packages/typespec-vscode/.vscodeignore b/packages/typespec-vscode/.vscodeignore
index 23e1012e2..63dbd54be 100644
--- a/packages/typespec-vscode/.vscodeignore
+++ b/packages/typespec-vscode/.vscodeignore
@@ -7,9 +7,9 @@
 # Then explicitly include what we want to ship in VSIX
 !dist/**/*.tmLanguage
 !dist/**/*.js
+!dist/**/*.cjs
 !dist/**/*.js.map
 !dist/**/language-configuration.json
-!extension-shim.js
 !markdown-typespec.json
 !README.md
 !ThirdPartyNotices.txt
diff --git a/packages/typespec-vscode/extension-shim.js b/packages/typespec-vscode/extension-shim.js
deleted file mode 100644
index 613d6ac8e..000000000
--- a/packages/typespec-vscode/extension-shim.js
+++ /dev/null
@@ -1,3 +0,0 @@
-module.exports = require(
-  process.env.TYPESPEC_DEVELOPMENT_MODE ? "./dist-dev/src/extension.js" : "./dist/src/extension.js"
-);
diff --git a/packages/typespec-vscode/package.json b/packages/typespec-vscode/package.json
index b0a804b0f..e95c1e48f 100644
--- a/packages/typespec-vscode/package.json
+++ b/packages/typespec-vscode/package.json
@@ -21,8 +21,8 @@
   "categories": [
     "Programming Languages"
   ],
-  "type": "commonjs",
-  "main": "./extension-shim.js",
+  "type": "module",
+  "main": "./dist/src/extension.cjs",
   "engines": {
     "vscode": "^1.85.0"
   },
@@ -106,20 +106,20 @@
     ]
   },
   "scripts": {
-    "clean": "rimraf ./dist ./dist-dev ./temp",
-    "build": "npm run compile && npm run rollup && npm run copy-tmlanguage && npm run generate-language-configuration && npm run generate-third-party-notices && npm run package-vsix",
-    "compile": "tsc -p .",
-    "watch": "tsc -p . --watch",
+    "clean": "rimraf ./dist ./temp",
+    "build": "npm run compile && npm run copy-tmlanguage && npm run generate-language-configuration && npm run generate-third-party-notices && npm run package-vsix",
+    "compile": "rollup --config rollup.config.ts --configPlugin typescript --failAfterWarnings 2>&1",
+    "watch": "rollup --config rollup.config.ts --configPlugin typescript --watch",
     "dogfood": "node scripts/dogfood.js",
     "copy-tmlanguage": "node scripts/copy-tmlanguage.js",
     "generate-language-configuration": "node scripts/generate-language-configuration.js",
     "generate-third-party-notices": "typespec-build-tool generate-third-party-notices",
-    "rollup": "rollup --config --failAfterWarnings 2>&1",
     "package-vsix": "vsce package"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "~25.0.4",
     "@rollup/plugin-node-resolve": "~15.2.1",
+    "@rollup/plugin-typescript": "~11.1.5",
     "@types/mocha": "~10.0.6",
     "@types/node": "~18.11.9",
     "@types/vscode": "~1.85.0",
diff --git a/packages/typespec-vscode/rollup.config.mjs b/packages/typespec-vscode/rollup.config.ts
similarity index 70%
rename from packages/typespec-vscode/rollup.config.mjs
rename to packages/typespec-vscode/rollup.config.ts
index e5ad837e7..d59c960cd 100644
--- a/packages/typespec-vscode/rollup.config.mjs
+++ b/packages/typespec-vscode/rollup.config.ts
@@ -1,19 +1,24 @@
-// @ts-check
 import commonjs from "@rollup/plugin-commonjs";
 import resolve from "@rollup/plugin-node-resolve";
+import typescript from "@rollup/plugin-typescript";
+
 import { defineConfig } from "rollup";
 
 export default defineConfig({
-  input: "dist-dev/src/extension.js",
+  input: "src/extension.ts",
   output: {
-    file: "dist/src/extension.js",
+    file: "dist/src/extension.cjs",
     format: "commonjs",
     sourcemap: true,
     exports: "named",
     inlineDynamicImports: true,
   },
   external: ["fs/promises", "vscode"],
-  plugins: [resolve({ preferBuiltins: true }), commonjs()],
+  plugins: [
+    (resolve as any)({ preferBuiltins: true }),
+    (commonjs as any)(),
+    (typescript as any)({ tsconfig: "./tsconfig.build.json" }),
+  ],
   onwarn: (warning, warn) => {
     if (warning.code === "CIRCULAR_DEPENDENCY") {
       // filter out warnings about circular dependencies out of our control
diff --git a/packages/typespec-vscode/tsconfig.build.json b/packages/typespec-vscode/tsconfig.build.json
new file mode 100644
index 000000000..469b70897
--- /dev/null
+++ b/packages/typespec-vscode/tsconfig.build.json
@@ -0,0 +1,13 @@
+// tsconfig for building the playground
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "sourceMap": true,
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "declaration": false,
+    "declarationMap": false,
+    "sourceRoot": ".."
+  },
+  "include": ["src/**/*.ts"]
+}
diff --git a/packages/typespec-vscode/tsconfig.json b/packages/typespec-vscode/tsconfig.json
index af1aba200..ffedbe4e0 100644
--- a/packages/typespec-vscode/tsconfig.json
+++ b/packages/typespec-vscode/tsconfig.json
@@ -2,13 +2,14 @@
   "extends": "../tsconfig.json",
   "references": [{ "path": "../tmlanguage-generator/tsconfig.json" }],
   "compilerOptions": {
-    "outDir": "dist-dev",
-    "rootDir": ".",
-    "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo",
-    "module": "ES2022",
-    "moduleResolution": "Bundler",
-    "skipLibCheck": true,
-    "types": ["node", "mocha"]
+    "incremental": false,
+    "composite": false,
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "sourceMap": false,
+    "resolveJsonModule": true,
+    "outDir": "dist",
+    "rootDir": "."
   },
-  "include": ["src/**/*.ts", "test/**/*.ts"]
+  "include": ["rollup.config.ts", "src/**/*.ts", "test/**/*.ts"]
 }
diff --git a/tsconfig.json b/tsconfig.json
index 8207ac41b..e29d43467 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -10,7 +10,6 @@
     { "path": "packages/openapi3/tsconfig.json" },
     { "path": "packages/lint/tsconfig.json" },
     { "path": "packages/migrate/tsconfig.json" },
-    { "path": "packages/typespec-vscode/tsconfig.json" },
     { "path": "packages/internal-build-utils/tsconfig.json" },
     { "path": "packages/bundle-uploader/tsconfig.json" },
     { "path": "packages/tmlanguage-generator/tsconfig.json" },