Add python client emitter to repo (#1366)

package.json

@@ -39,6 +39,7 @@
     "typescript": "^4.7.4"
   },
   "peerDependencies": {
-    "typescript": "^4.7.4"
+    "typescript": "^4.7.4",
+    "prettier": "^2.7.1"
   }
 }

packages/cadl-python-emitter/.c8rc.json

@@ -0,0 +1,3 @@
+{
+  "reporter": ["cobertura", "json", "text"]
+}

packages/cadl-python-emitter/.eslintrc.cjs

@@ -0,0 +1,6 @@
+require("@cadl-lang/eslint-config-cadl/patch/modern-module-resolution");
+
+module.exports = {
+  extends: "@cadl-lang/eslint-config-cadl",
+  parserOptions: { tsconfigRootDir: __dirname },
+};

packages/cadl-python-emitter/.mocharc.yaml

@@ -0,0 +1,4 @@
+timeout: 5000
+require: source-map-support/register
+spec: "dist/test/**/*.js"
+ignore: "dist/test/manual/**/*.js"

packages/cadl-python-emitter/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

packages/cadl-python-emitter/README.md

@@ -0,0 +1 @@
+# Cadl Python Client Emitter

packages/cadl-python-emitter/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "@azure-tools/cadl-python-emitter",
+  "version": "0.1.0",
+  "author": "Microsoft Corporation",
+  "description": "Cadl library for emitting OpenAPI 3.0 from the Cadl REST protocol binding",
+  "homepage": "https://github.com/Azure/autorest.python",
+  "readme": "https://github.com/Azure/autorest.python/packages/cadl-python-emitter/README.md",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Azure/adl.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Azure/adl/issues"
+  },
+  "keywords": [
+    "cadl"
+  ],
+  "type": "module",
+  "main": "dist/src/index.js",
+  "exports": {
+    ".": "./dist/src/index.js",
+    "./testing": "./dist/src/testing/index.js"
+  },
+  "typesVersions": {
+    "*": {
+      "*": [
+        "./dist/src/index.d.ts"
+      ],
+      "testing": [
+        "./dist/src/testing/index.d.ts"
+      ]
+    }
+  },
+  "cadlMain": "dist/src/index.js",
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "scripts": {
+    "clean": "rimraf ./dist ./temp",
+    "build": "tsc -p .",
+    "watch": "tsc -p . --watch",
+    "test": "mocha",
+    "test-official": "c8 mocha --forbid-only",
+    "lint": "eslint . --ext .ts --max-warnings=0",
+    "lint:fix": "eslint . --fix --ext .ts"
+  },
+  "files": [
+    "lib/*.cadl",
+    "dist/**",
+    "!dist/test/**",
+    "get-autorest-python-path.cjs"
+  ],
+  "peerDependencies": {
+    "@cadl-lang/versioning": "~0.6.1",
+    "@cadl-lang/compiler": "~0.34.0-dev.21",
+    "@cadl-lang/rest": "~0.16.0-dev.7",
+    "@azure-tools/cadl-azure-core": "~0.6.0-dev.3"
+  },
+  "dependencies": {
+    "js-yaml": "~4.1.0",
+    "@autorest/python": "~6.1.1"
+  },
+  "devDependencies": {
+    "@types/mocha": "~9.1.0",
+    "@types/node": "~16.0.3",
+    "@types/js-yaml": "~4.0.1",
+    "@cadl-lang/compiler": "~0.34.0-dev.21",
+    "@cadl-lang/rest": "~0.16.0-dev.7",
+    "@cadl-lang/versioning": "~0.6.1",
+    "@cadl-lang/eslint-config-cadl": "~0.3.0",
+    "eslint": "^8.12.0",
+    "mocha": "~9.2.0",
+    "c8": "~7.11.0",
+    "rimraf": "~3.0.2",
+    "typescript": "~4.7.2",
+    "@azure-tools/cadl-azure-core": "~0.6.0-dev.3"
+  }
+}

packages/cadl-python-emitter/src/emitter.ts

@@ -0,0 +1,479 @@
+import {
+  EnumMemberType,
+  EnumType,
+  getDoc,
+  getIntrinsicModelName,
+  getServiceNamespace,
+  getServiceNamespaceString,
+  getServiceTitle,
+  ignoreDiagnostics,
+  isErrorModel,
+  isNeverType,
+  ModelType,
+  ModelTypeProperty,
+  Program,
+  resolvePath,
+  Type,
+} from "@cadl-lang/compiler";
+import {
+  getAllRoutes,
+  getContentTypes,
+  HttpOperationParameter,
+  HttpOperationParameters,
+  HttpOperationResponse,
+  HttpOperationResponseContent,
+  OperationDetails,
+} from "@cadl-lang/rest/http";
+import { getAddedOn } from "@cadl-lang/versioning";
+import { execFileSync } from "child_process";
+import { dump } from "js-yaml";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+
+export async function $onEmit(program: Program) {
+  const yamlMap = createYamlEmitter(program);
+  const yamlPath = resolvePath(program.compilerOptions.outputPath!, "output.yaml");
+  await program.host.writeFile(yamlPath, dump(yamlMap));
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+  const root = resolve(__dirname, "..", "..");
+
+  execFileSync(process.execPath, [
+    `${root}/node_modules/@autorest/python/run-python3.js`,
+    `${root}/node_modules/@autorest/python/run_cadl.py`,
+    `--output-folder=${program.compilerOptions.outputPath!}`,
+    `--cadl-file=${yamlPath}`,
+  ]);
+}
+
+function camelToSnakeCase(name: string): string {
+  const camelToSnakeCaseRe = (str: string) =>
+    str
+      .replace(/\s+/g, "_")
+      .replace(/\$/g, "")
+      .replace(/-/g, "_")
+      .replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+
+  return camelToSnakeCaseRe(name[0].toLowerCase() + name.slice(1));
+}
+
+const typesMap = new Map<Type, Record<string, any>>();
+
+function getDocStr(program: Program, target: Type): string {
+  return getDoc(program, target) ?? "";
+}
+
+function getType(program: Program, type: Type): any {
+  const cached = typesMap.get(type);
+  if (cached) {
+    return cached;
+  }
+  const newValue = emitType(program, type);
+  typesMap.set(type, newValue);
+  return newValue;
+}
+
+// To pass the yaml dump
+function getAddedOnVersion(p: Program, t: Type): string | undefined {
+  return getAddedOn(p as any, t as any)?.value;
+}
+
+function emitParamBase(program: Program, parameter: ModelTypeProperty | Type): Record<string, any> {
+  let optional: boolean;
+  let name: string;
+  let description: string = "";
+  let addedApiVersion: string | undefined;
+
+  if (parameter.kind === "ModelProperty") {
+    optional = parameter.optional;
+    name = parameter.name;
+    description = getDocStr(program, parameter);
+    addedApiVersion = getAddedOnVersion(program, parameter);
+  } else {
+    optional = false;
+    name = "body";
+  }
+
+  return {
+    optional,
+    description,
+    addedApiVersion,
+    clientName: camelToSnakeCase(name),
+    inOverload: false,
+  };
+}
+
+function emitRequestBody(program: Program, bodyType: Type, params: HttpOperationParameters): Record<string, any> {
+  const base = emitParamBase(program, params.bodyParameter ?? bodyType);
+  const contentTypeParam = params.parameters.find((p) => p.type === "header" && p.name === "content-type");
+  const contentTypes = contentTypeParam
+    ? ignoreDiagnostics(getContentTypes(contentTypeParam.param))
+    : ["application/json"];
+  if (contentTypes.length !== 1) {
+    throw Error("Currently only one kind of content-type!");
+  }
+  return {
+    contentTypes,
+    restApiName: params.bodyParameter?.name ?? "body",
+    location: "body",
+    type: getType(program, bodyType),
+    ...base,
+  };
+}
+
+function emitParameter(
+  program: Program,
+  parameter: HttpOperationParameter,
+  implementation: string,
+): Record<string, any> {
+  const base = emitParamBase(program, parameter.param);
+  const paramMap: Record<string, any> = {
+    restApiName: parameter.name,
+    location: parameter.type,
+    type: getType(program, parameter.param.type),
+    implementation: implementation,
+  };
+  let clientDefaultValue = undefined;
+  if (paramMap.type.type === "constant") {
+    clientDefaultValue = paramMap.type.value;
+  }
+  return { clientDefaultValue, ...base, ...paramMap };
+}
+
+function emitResponseHeaders(program: Program, headers?: Record<string, ModelTypeProperty>): Record<string, any>[] {
+  const retval: Record<string, any>[] = [];
+  if (!headers) {
+    return retval;
+  }
+  for (const [key, value] of Object.entries(headers)) {
+    retval.push({
+      type: emitType(program, value.type),
+      restApiName: key,
+    });
+  }
+  return retval;
+}
+
+function emitResponse(
+  program: Program,
+  response: HttpOperationResponse,
+  innerResponse: HttpOperationResponseContent,
+): Record<string, any> {
+  // let type;
+  // if (innerResponse.body?.type) {
+  //   type = getType(program, response.type)
+  // } else {
+  //   type = undefined
+  // }
+  return {
+    headers: emitResponseHeaders(program, innerResponse.headers),
+    statusCodes: [parseInt(response.statusCode)],
+    addedApiVersion: getAddedOnVersion(program, response.type),
+    discriminator: "basic",
+  };
+}
+
+function emitOperation(program: Program, operation: OperationDetails): Record<string, any> {
+  // Set up parameters for operation
+  const parameters: Record<string, any>[] = [];
+  for (const param of operation.parameters.parameters) {
+    parameters.push(emitParameter(program, param, "Method"));
+  }
+
+  // Set up responses for operation
+  const responses: Record<string, any>[] = [];
+  const exceptions: Record<string, any>[] = [];
+  for (const response of operation.responses) {
+    for (const innerResponse of response.responses) {
+      const emittedResponse = emitResponse(program, response, innerResponse);
+      if (isErrorModel(program, response.type)) {
+        // * is valid status code in cadl but invalid for autorest.python
+        if (response.statusCode !== "*") {
+          exceptions.push(emittedResponse);
+        }
+      } else {
+        responses.push(emittedResponse);
+      }
+    }
+  }
+  let requestBody: Record<string, any> | undefined;
+  if (operation.parameters.bodyType === undefined) {
+    requestBody = undefined;
+  } else {
+    requestBody = emitRequestBody(program, operation.parameters.bodyType, operation.parameters);
+  }
+
+  return {
+    name: camelToSnakeCase(operation.operation.name),
+    description: getDocStr(program, operation.operation),
+    url: operation.path,
+    method: operation.verb.toUpperCase(),
+    parameters: parameters,
+    bodyParameter: requestBody,
+    responses: responses,
+    exceptions: exceptions,
+    groupName: capitalize(operation.container.name),
+    addedApiVersion: getAddedOnVersion(program, operation.operation),
+    discriminator: "basic",
+    isOverload: false,
+    overloads: [],
+    apiVersions: [getAddedOnVersion(program, operation.operation)],
+  };
+}
+
+// Return any string literal values for type
+// function getStringValues(type: Type): string[] {
+//   if (type.kind === "String") {
+//     return [type.value];
+//   } else if (type.kind === "Union") {
+//     return type.options.flatMap(getStringValues).filter((v) => v);
+//   }
+//   return [];
+// }
+
+// function getDiscriminatorMapping(
+//   program: Program,
+//   discriminator: any,
+//   childModels: readonly ModelType[]
+// ): Record<string, string> | undefined {
+//   const { propertyName } = discriminator;
+//   const getMapping = (t: ModelType): any => {
+//     const prop = t.properties?.get(propertyName);
+//     if (prop) {
+//       return getStringValues(prop.type).flatMap((v) => [{ [v]: getType(program, t) }]);
+//     }
+//     return undefined;
+//   };
+//   const mappings = childModels.flatMap(getMapping).filter((v) => v); // only defined values
+//   return mappings.length > 0 ? mappings.reduce((a, s) => ({ ...a, ...s }), {}) : undefined;
+// }
+
+function emitProperty(program: Program, property: ModelTypeProperty): Record<string, any> {
+  return {
+    clientName: camelToSnakeCase(property.name),
+    restApiName: property.name,
+    type: getType(program, property.type),
+    optional: property.optional,
+    description: getDocStr(program, property),
+    addedApiVersion: getAddedOnVersion(program, property),
+  };
+}
+
+function emitModel(program: Program, type: ModelType): Record<string, any> {
+  if (type.indexer) {
+    if (isNeverType(type.indexer.key)) {
+    } else {
+      const name = getIntrinsicModelName(program, type.indexer.key);
+      if (name === "string") {
+        return { type: "dict", elementType: getType(program, type.indexer.value!) };
+      } else if (name === "integer") {
+        return { type: "list", elementType: getType(program, type.indexer.value!) };
+      }
+    }
+  }
+
+  for (const decorator of type.decorators) {
+    if (decorator.decorator.name === "$knownValues") {
+      for (const arg of decorator.args) {
+        if (typeof arg.value === "object" && arg.value.kind === "Enum") {
+          const enumResult = emitEnum(program, arg.value);
+          enumResult["name"] = type.name;
+          return enumResult;
+        }
+      }
+    }
+  }
+  const name = getIntrinsicModelName(program, type);
+  switch (name) {
+    case "bytes":
+      return { type: "base64" };
+    case "int8":
+    case "int16":
+    case "int32":
+    case "int64":
+    case "safeint":
+    case "uint8":
+    case "uint16":
+    case "uint32":
+    case "uint64":
+      return { type: "integer" };
+    case "float32":
+    case "float64":
+      return { type: "float" };
+    case "string":
+      return KnownTypes.string;
+    case "boolean":
+      return { type: "boolean" };
+    case "plainDate":
+      return { type: "date" };
+    case "zonedDateTime":
+      return { type: "datetime", format: "date-time" };
+    case "plainTime":
+      return { type: "time" };
+    case "duration":
+      return { type: "duration" };
+    default:
+      // Now we know it's a defined model
+      // const discriminator = getDiscriminator(program, type);
+      // const discriminatorEntry: Record<string, any> | undefined = {};
+      // const childModels: Record<string, any>[] = [];
+      // for (const childModel of type.derivedModels) {
+      //   childModels.push(getType(program, childModel));
+      // }
+      // if (discriminator) {
+      //   const discriminatorMapping = getDiscriminatorMapping(program, discriminator, childModels);
+      //   if (discriminatorMapping) {
+      //     discriminatorEntry.mapping = discriminatorMapping;
+      //   }
+      //   discriminatorEntry.propertyName = discriminator.propertyName;
+      // }
+      const properties: Record<string, any>[] = [];
+      for (const property of type.properties.values()) {
+        properties.push(emitProperty(program, property));
+      }
+      let baseModel = undefined;
+      if (type.baseModel) {
+        baseModel = emitModel(program, type.baseModel);
+      }
+      return {
+        type: "model",
+        name: type.name,
+        description: getDocStr(program, type),
+        baseModels: [baseModel],
+        discriminatedSubtypes: {},
+        properties: properties,
+        addedApiVersion: getAddedOnVersion(program, type),
+        snakeCaseName: type.name,
+      };
+  }
+}
+
+function intOrFloat(value: number): string {
+  return value.toString().indexOf(".") === -1 ? "integer" : "float";
+}
+
+function enumName(name: string): string {
+  if (name.toUpperCase() === name) {
+    return name;
+  }
+  return camelToSnakeCase(name).toUpperCase();
+}
+
+function emitEnum(program: Program, type: EnumType): Record<string, any> {
+  const enumValues = [];
+  for (const m of type.members) {
+    enumValues.push({
+      name: enumName(m.name),
+      value: m.value ?? m.name,
+      description: getDocStr(program, m),
+    });
+  }
+
+  return {
+    type: "enum",
+    name: type.name,
+    description: getDocStr(program, type),
+    valueType: { type: enumMemberType(type.members[0]) },
+    values: enumValues,
+  };
+  function enumMemberType(member: EnumMemberType) {
+    if (typeof member.value === "number") {
+      return intOrFloat(member.value);
+    }
+    return "string";
+  }
+}
+
+function constantType(value: any, valueType: string): Record<string, any> {
+  return { type: "constant", value: value, valueType: { type: valueType } };
+}
+
+function emitType(program: Program, type: Type): Record<string, any> {
+  switch (type.kind) {
+    case "Number":
+      return constantType(type.value, intOrFloat(type.value));
+    case "String":
+      return constantType(type.value, "string");
+    case "Boolean":
+      return constantType(type.value, "boolean");
+    case "Model":
+      return emitModel(program, type);
+    case "Enum":
+      return emitEnum(program, type);
+    default:
+      throw Error(`Not supported ${type.kind}`);
+  }
+}
+
+function capitalize(name: string): string {
+  return name[0].toUpperCase() + name.slice(1);
+}
+
+function emitOperationGroups(program: Program): Record<string, any>[] {
+  const operationGroups: Record<string, any>[] = [];
+  const allOperations = ignoreDiagnostics(getAllRoutes(program));
+  for (const operation of allOperations) {
+    let existingOperationGroup: Record<string, any> | undefined = undefined;
+    for (const operationGroup of operationGroups) {
+      if (operationGroup["className"] === capitalize(operation.container.name)) {
+        existingOperationGroup = operationGroup;
+      }
+    }
+    const emittedOperation = emitOperation(program, operation);
+    if (existingOperationGroup) {
+      existingOperationGroup["operations"].push(emittedOperation);
+    } else {
+      const newOperationGroup = {
+        propertyName: capitalize(operation.container.name),
+        className: capitalize(operation.container.name),
+        operations: [emittedOperation],
+      };
+      operationGroups.push(newOperationGroup);
+    }
+  }
+  return operationGroups;
+}
+
+function createYamlEmitter(program: Program) {
+  const serviceNamespace = getServiceNamespace(program);
+  if (serviceNamespace === undefined) {
+    throw Error("Can not emit yaml for a namespace that doesn't exist.");
+  }
+
+  // let [_, versions] = getVersions(program, serviceNamespace);
+  // if (versions.length === 0 && getServiceVersion(program)) {
+  //   versions = [getServiceVersion(program)];
+  // }
+  const name = getServiceTitle(program).replace(/ /g, "");
+  // Get types
+  const codeModel = {
+    client: {
+      name: name,
+      description: "Service client",
+      moduleName: camelToSnakeCase(name),
+      parameters: [
+        {
+          optional: false,
+          description: "Service host",
+          clientName: "endpoint",
+          clientDefaultValue: "http://localhost:3000",
+          restApiName: "$host",
+          location: "path",
+          type: KnownTypes.string,
+          implementation: "Client",
+          inOverload: false,
+        },
+      ],
+      security: {},
+      namespace: getServiceNamespaceString(program),
+      url: "",
+      apiVersions: [],
+    },
+    operationGroups: emitOperationGroups(program),
+    types: [...typesMap.values(), ...Object.values(KnownTypes)],
+  };
+  return codeModel;
+}
+
+const KnownTypes = {
+  string: { type: "string" },
+};

packages/cadl-python-emitter/src/index.ts

@@ -0,0 +1 @@
+export * from "./emitter.js";

packages/cadl-python-emitter/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": ".",
+    "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo",
+    "types": ["node", "mocha"],
+    "composite": true,
+    "alwaysStrict": true,
+    "forceConsistentCasingInFileNames": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "noImplicitAny": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+    "sourceMap": true,
+    "declarationMap": true,
+    "strict": true,
+    "declaration": true,
+    "stripInternal": true,
+    "noEmitHelpers": false,
+    "target": "es2019",
+    "lib": ["es2019"],
+    "experimentalDecorators": true,
+    "newLine": "LF"
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}