Connect emitter to generator and use that path during generate.ps1 (#3320)

Fixes https://github.com/Azure/autorest.csharp/issues/4464

---------

Co-authored-by: Timothee Guerin <timothee.guerin@outlook.com>

.prettierignore

@@ -61,3 +61,6 @@ eng/feeds/
 
 # Tm Grammars generated files
 grammars/
+
+# auto generated code model files
+packages/http-client-csharp/generator/TestProjects/**/tspCodeModel.json

eng/emitters/pipelines/templates/stages/emitter-stages.yml

@@ -215,6 +215,14 @@ stages:
               BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
               Os: windows
               TestMatrix: ${{ parameters.TestMatrix }}
+          - template: /eng/emitters/pipelines/templates/jobs/test-job.yml
+            parameters:
+              AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
+              PackagePath: ${{ parameters.PackagePath }}
+              LanguageShortName: ${{ parameters.LanguageShortName }}
+              BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
+              Os: linux
+              TestMatrix: ${{ parameters.TestMatrix }}
 
   # Publish stage
   # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`

packages/http-client-csharp/emitter/src/emitter.ts

@@ -6,13 +6,18 @@ import {
   EmitContext,
   Program,
   createTypeSpecLibrary,
+  getDirectoryPath,
+  joinPaths,
   logDiagnostics,
   paramMessage,
   resolvePath,
 } from "@typespec/compiler";
 
-import fs from "fs";
+import { SpawnOptions, spawn } from "child_process";
+import fs, { statSync } from "fs";
 import { PreserveType, stringifyRefs } from "json-serialize-refs";
+import { dirname } from "path";
+import { fileURLToPath } from "url";
 import { configurationFileName, tspOutputFileName } from "./constants.js";
 import { createModel } from "./lib/client-model-builder.js";
 import { LoggerLevel, logger } from "./lib/logger.js";
@@ -51,6 +56,26 @@ export const $lib = createTypeSpecLibrary({
   },
 });
 
+/**
+ * Look for the project root by looking up until a `package.json` is found.
+ * @param path Path to start looking
+ */
+function findProjectRoot(path: string): string | undefined {
+  let current = path;
+  while (true) {
+    const pkgPath = joinPaths(current, "package.json");
+    const stats = checkFile(pkgPath);
+    if (stats?.isFile()) {
+      return current;
+    }
+    const parent = getDirectoryPath(current);
+    if (parent === current) {
+      return undefined;
+    }
+    current = parent;
+  }
+}
+
 export async function $onEmit(context: EmitContext<NetEmitterOptions>) {
   const program: Program = context.program;
   const options = resolveOptions(context);
@@ -75,16 +100,14 @@ export async function $onEmit(context: EmitContext<NetEmitterOptions>) {
     const tspNamespace = root.Name; // this is the top-level namespace defined in the typespec file, which is actually always different from the namespace of the SDK
     // await program.host.writeFile(outPath, prettierOutput(JSON.stringify(root, null, 2)));
     if (root) {
-      const generatedFolder = outputFolder.endsWith("src")
-        ? resolvePath(outputFolder, "Generated")
-        : resolvePath(outputFolder, "src", "Generated");
+      const generatedFolder = resolvePath(outputFolder, "src", "Generated");
 
       if (!fs.existsSync(generatedFolder)) {
         fs.mkdirSync(generatedFolder, { recursive: true });
       }
 
       await program.host.writeFile(
-        resolvePath(generatedFolder, tspOutputFileName),
+        resolvePath(outputFolder, tspOutputFileName),
         prettierOutput(stringifyRefs(root, null, 1, PreserveType.Objects))
       );
 
@@ -128,41 +151,99 @@ export async function $onEmit(context: EmitContext<NetEmitterOptions>) {
       };
 
       await program.host.writeFile(
-        resolvePath(generatedFolder, configurationFileName),
+        resolvePath(outputFolder, configurationFileName),
         prettierOutput(JSON.stringify(configurations, null, 2))
       );
 
       if (options.skipSDKGeneration !== true) {
-        const csProjFile = resolvePath(outputFolder, `${configurations["library-name"]}.csproj`);
+        const csProjFile = resolvePath(
+          outputFolder,
+          "src",
+          `${configurations["library-name"]}.csproj`
+        );
         logger.info(`Checking if ${csProjFile} exists`);
+        const newProjectOption = "";
+        // TODO uncomment when https://github.com/Azure/autorest.csharp/issues/4463 is resolved
+        //  options["new-project"] || !existsSync(csProjFile) ? "--new-project" : "";
+        const existingProjectOption = options["existing-project-folder"]
+          ? `--existing-project-folder ${options["existing-project-folder"]}`
+          : "";
+        const debugFlag = options.debug ?? false ? " --debug" : "";
 
-        logger.info("TODO connect the dotnet generator");
-        //const command = `dotnet --roll-forward Major ${resolvePath(
-        //  options.csharpGeneratorPath
-        //)} --project-path ${outputFolder} ${newProjectOption} ${existingProjectOption} --clear-output-folder ${
-        //  options["clear-output-folder"]
-        //}${debugFlag}`;
-        //logger.info(command);
-        //
-        //try {
-        //  execSync(command, { stdio: "inherit" });
-        //} catch (error: any) {
-        //  if (error.message) logger.info(error.message);
-        //  if (error.stderr) logger.error(error.stderr);
-        //  if (error.stdout) logger.verbose(error.stdout);
-        //  throw error;
-        //}
-      }
+        const projectRoot = findProjectRoot(dirname(fileURLToPath(import.meta.url)));
+        const generatorPath = resolvePath(
+          projectRoot + "/dist/generator/Microsoft.Generator.CSharp.dll"
+        );
 
-      if (!options["save-inputs"]) {
-        // delete
-        deleteFile(resolvePath(generatedFolder, tspOutputFileName));
-        deleteFile(resolvePath(generatedFolder, configurationFileName));
+        const command = `dotnet --roll-forward Major ${generatorPath} ${outputFolder} ${newProjectOption} ${existingProjectOption}${debugFlag}`;
+        logger.info(command);
+
+        await execAsync(
+          "dotnet",
+          [
+            "--roll-forward",
+            "Major",
+            generatorPath,
+            outputFolder,
+            newProjectOption,
+            existingProjectOption,
+            debugFlag,
+          ],
+          { stdio: "inherit" }
+        )
+          .then(() => {
+            if (!options["save-inputs"]) {
+              // delete
+              deleteFile(resolvePath(outputFolder, tspOutputFileName));
+              deleteFile(resolvePath(outputFolder, configurationFileName));
+            }
+          })
+          .catch((error: any) => {
+            if (error.message) logger.info(error.message);
+            if (error.stderr) logger.error(error.stderr);
+            if (error.stdout) logger.verbose(error.stdout);
+            throw error;
+          });
       }
     }
   }
 }
 
+async function execAsync(
+  command: string,
+  args: string[] = [],
+  options: SpawnOptions = {}
+): Promise<{ exitCode: number; stdio: string; stdout: string; stderr: string; proc: any }> {
+  const child = spawn(command, args, options);
+
+  return new Promise((resolve, reject) => {
+    child.on("error", (error) => {
+      reject(error);
+    });
+    const stdio: Buffer[] = [];
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    child.stdout?.on("data", (data) => {
+      stdout.push(data);
+      stdio.push(data);
+    });
+    child.stderr?.on("data", (data) => {
+      stderr.push(data);
+      stdio.push(data);
+    });
+
+    child.on("exit", (exitCode) => {
+      resolve({
+        exitCode: exitCode ?? -1,
+        stdio: Buffer.concat(stdio).toString(),
+        stdout: Buffer.concat(stdout).toString(),
+        stderr: Buffer.concat(stderr).toString(),
+        proc: child,
+      });
+    });
+  });
+}
+
 function deleteFile(filePath: string) {
   fs.unlink(filePath, (err) => {
     if (err) {
@@ -176,3 +257,11 @@ function deleteFile(filePath: string) {
 function prettierOutput(output: string) {
   return output + "\n";
 }
+
+function checkFile(pkgPath: string) {
+  try {
+    return statSync(pkgPath);
+  } catch (error) {
+    return undefined;
+  }
+}

packages/http-client-csharp/emitter/src/options.ts

@@ -12,7 +12,6 @@ export type NetEmitterOptions = {
   skipSDKGeneration?: boolean;
   "unreferenced-types-handling"?: "removeOrInternalize" | "internalize" | "keepAll";
   "new-project"?: boolean;
-  csharpGeneratorPath?: string;
   "clear-output-folder"?: boolean;
   "save-inputs"?: boolean;
   "model-namespace"?: boolean;
@@ -48,11 +47,6 @@ export const NetEmitterOptionsSchema: JSONSchemaType<NetEmitterOptions> = {
       nullable: true,
     },
     "new-project": { type: "boolean", nullable: true },
-    csharpGeneratorPath: {
-      type: "string",
-      default: "fake-location",
-      nullable: true,
-    },
     "clear-output-folder": { type: "boolean", nullable: true },
     "save-inputs": { type: "boolean", nullable: true },
     "model-namespace": { type: "boolean", nullable: true },
@@ -120,7 +114,6 @@ const defaultOptions = {
   logFile: "log.json",
   skipSDKGeneration: false,
   "new-project": false,
-  csharpGeneratorPath: "fake-location",
   "clear-output-folder": false,
   "save-inputs": false,
   "generate-protocol-methods": true,

packages/http-client-csharp/eng/scripts/Generate.ps1

@@ -4,11 +4,12 @@ Import-Module "$PSScriptRoot\Generation.psm1" -DisableNameChecking -Force;
 
 $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..' '..')
 
-Invoke "npm run build"
+Invoke "npm run build:emitter"
+# we don't want to build the entire solution because the test projects might not build until after regeneration
+# generating Microsoft.Generator.CSharp.ClientModel.csproj is enough
+Invoke "dotnet build $repoRoot/generator/Microsoft.Generator.CSharp.ClientModel/src"
 
-$mgcArtifactRoot = Join-Path $repoRoot 'dist' 'generator'
-$clientModelTestProjectsDirectory = Join-Path $repoRoot 'generator' 'Microsoft.Generator.CSharp.ClientModel.TestProjects'
+$testProjectsLocalDir = Join-Path $repoRoot 'generator' 'TestProjects' 'Local'
 
-$mgcPath = Join-Path $mgcArtifactRoot "Microsoft.Generator.CSharp.exe"
-$unbrandedTypespecTestProject = Join-Path $clientModelTestProjectsDirectory "Unbranded-TypeSpec"
-Invoke "$mgcPath $unbrandedTypespecTestProject"
+$unbrandedTypespecTestProject = Join-Path $testProjectsLocalDir "Unbranded-TypeSpec"
+Invoke "npx tsp compile $unbrandedTypespecTestProject/Unbranded-TypeSpec.tsp --emit @typespec/http-client-csharp --option @typespec/http-client-csharp.emitter-output-dir=$unbrandedTypespecTestProject --option @typespec/http-client-csharp.save-inputs=true"

packages/http-client-csharp/generator/Microsoft.Generator.CSharp.ClientModel.TestProjects/Microsoft.Generator.CSharp.ClientModel.TestProjects.csproj

@@ -1,15 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <IsPackable>false</IsPackable>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-  <ItemGroup>
-    <Folder Include="Unbranded-TypeSpec\" />
-  </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="System.Memory.Data" />
-  </ItemGroup>
-
-</Project>

packages/http-client-csharp/generator/Microsoft.Generator.CSharp.ClientModel.TestProjects/Unbranded-TypeSpec/Configuration.json

@@ -1,5 +0,0 @@
-{
-  "output-folder": ".",
-  "namespace": "UnbrandedTypeSpec",
-  "library-name": "UnbrandedTypeSpec"
-}

packages/http-client-csharp/generator/Microsoft.Generator.CSharp.ClientModel.TestProjects/Unbranded-TypeSpec/UnbrandedTypeSpec.sln

@@ -1,50 +0,0 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.29709.97
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnbrandedTypeSpec", "src\UnbrandedTypeSpec.csproj", "{28FF4005-4467-4E36-92E7-DEA27DEB1519}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnbrandedTypeSpec.Tests", "tests\UnbrandedTypeSpec.Tests.csproj", "{1F1CD1D4-9932-4B73-99D8-C252A67D4B46}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Release|Any CPU = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{B0C276D1-2930-4887-B29A-D1A33E7009A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B0C276D1-2930-4887-B29A-D1A33E7009A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B0C276D1-2930-4887-B29A-D1A33E7009A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B0C276D1-2930-4887-B29A-D1A33E7009A2}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8E9A77AC-792A-4432-8320-ACFD46730401}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8E9A77AC-792A-4432-8320-ACFD46730401}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8E9A77AC-792A-4432-8320-ACFD46730401}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8E9A77AC-792A-4432-8320-ACFD46730401}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A4241C1F-A53D-474C-9E4E-075054407E74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A4241C1F-A53D-474C-9E4E-075054407E74}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A4241C1F-A53D-474C-9E4E-075054407E74}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A4241C1F-A53D-474C-9E4E-075054407E74}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FA8BD3F1-8616-47B6-974C-7576CDF4717E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FA8BD3F1-8616-47B6-974C-7576CDF4717E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FA8BD3F1-8616-47B6-974C-7576CDF4717E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FA8BD3F1-8616-47B6-974C-7576CDF4717E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{85677AD3-C214-42FA-AE6E-49B956CAC8DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{85677AD3-C214-42FA-AE6E-49B956CAC8DC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{85677AD3-C214-42FA-AE6E-49B956CAC8DC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{85677AD3-C214-42FA-AE6E-49B956CAC8DC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{28FF4005-4467-4E36-92E7-DEA27DEB1519}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{28FF4005-4467-4E36-92E7-DEA27DEB1519}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{28FF4005-4467-4E36-92E7-DEA27DEB1519}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{28FF4005-4467-4E36-92E7-DEA27DEB1519}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1F1CD1D4-9932-4B73-99D8-C252A67D4B46}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{1F1CD1D4-9932-4B73-99D8-C252A67D4B46}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1F1CD1D4-9932-4B73-99D8-C252A67D4B46}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{1F1CD1D4-9932-4B73-99D8-C252A67D4B46}.Release|Any CPU.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {A97F4B90-2591-4689-B1F8-5F21FE6D6CAE}
-	EndGlobalSection
-EndGlobal

packages/http-client-csharp/generator/Microsoft.Generator.CSharp.sln

@@ -11,8 +11,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Generator.CSharp.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Generator.CSharp.Input", "Microsoft.Generator.CSharp.Input\src\Microsoft.Generator.CSharp.Input.csproj", "{B5484903-F77E-4C50-82C6-240AFECDF309}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Generator.CSharp.ClientModel.TestProjects", "Microsoft.Generator.CSharp.ClientModel.TestProjects\Microsoft.Generator.CSharp.ClientModel.TestProjects.csproj", "{2192EE77-150D-425D-B117-B13DF8794BC6}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Generator.CSharp.Customization", "Microsoft.Generator.CSharp.Customization\src\Microsoft.Generator.CSharp.Customization.csproj", "{82508C25-0623-4EDE-A6FA-3DB395A5BB37}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Generator.CSharp.Input.Tests", "Microsoft.Generator.CSharp.Input\test\Microsoft.Generator.CSharp.Input.Tests.csproj", "{043B51D2-52CB-4521-BF9A-9D3A45E8998D}"
@@ -23,6 +21,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		readme.md = readme.md
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestProjects.Local.Tests", "TestProjects\Local\TestProjects.Local.Tests.csproj", "{BFE39EDA-ADA1-4F60-A47C-ADE88C67B3B7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -45,10 +45,6 @@ Global
 		{B5484903-F77E-4C50-82C6-240AFECDF309}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B5484903-F77E-4C50-82C6-240AFECDF309}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B5484903-F77E-4C50-82C6-240AFECDF309}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2192EE77-150D-425D-B117-B13DF8794BC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2192EE77-150D-425D-B117-B13DF8794BC6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2192EE77-150D-425D-B117-B13DF8794BC6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2192EE77-150D-425D-B117-B13DF8794BC6}.Release|Any CPU.Build.0 = Release|Any CPU
 		{82508C25-0623-4EDE-A6FA-3DB395A5BB37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{82508C25-0623-4EDE-A6FA-3DB395A5BB37}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{82508C25-0623-4EDE-A6FA-3DB395A5BB37}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -57,6 +53,10 @@ Global
 		{043B51D2-52CB-4521-BF9A-9D3A45E8998D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{043B51D2-52CB-4521-BF9A-9D3A45E8998D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{043B51D2-52CB-4521-BF9A-9D3A45E8998D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BFE39EDA-ADA1-4F60-A47C-ADE88C67B3B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BFE39EDA-ADA1-4F60-A47C-ADE88C67B3B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BFE39EDA-ADA1-4F60-A47C-ADE88C67B3B7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BFE39EDA-ADA1-4F60-A47C-ADE88C67B3B7}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

packages/http-client-csharp/generator/Microsoft.Generator.CSharp/src/Properties/launchSettings.json

@@ -3,7 +3,7 @@
     "Unbranded-TypeSpec": {
       "commandName": "Executable",
       "executablePath": "$(SolutionDir)/../dist/generator/Microsoft.Generator.CSharp.exe",
-      "commandLineArgs": "$(SolutionDir)/Microsoft.Generator.CSharp.ClientModel.TestProjects/Unbranded-TypeSpec"
+      "commandLineArgs": "$(SolutionDir)/TestProjects/Local/Unbranded-TypeSpec"
     }
   }
 }

packages/http-client-csharp/generator/TestProjects/Local/TestProjects.Local.Tests.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="System.Memory.Data" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="Moq" />
+    <PackageReference Include="NUnit" />
+    <PackageReference Include="NUnit3TestAdapter" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Remove="./**/*.*" />
+    <Compile Remove="./**/*.*" />
+    <Compile Include="./**/*.cs" Exclude="./**/Properties/**/*.cs" />
+  </ItemGroup>
+  
+</Project>

packages/http-client-csharp/generator/TestProjects/Local/Unbranded-TypeSpec/Configuration.json

@@ -0,0 +1,6 @@
+{
+  "output-folder": ".",
+  "namespace": "UnbrandedTypeSpec",
+  "library-name": "UnbrandedTypeSpec",
+  "use-model-reader-writer": true
+}