switch to using `npm run regenerate` (#2747)

* add pyodide

* can iterate over subdirectories

* add script for regenerating

* temp

* just missing typetest in package name

* regenerating

* regenerate

* update pipeline to use npm run regenerate for typespec-python

* format and fix deps

* remove tasks.py file, add contributing

* add changeset

* fix for --name and Windows env

* fix pipeline

* fix nightly.yaml

* fix ci.yaml

* fix doc

* update regenerate

* inv

* inv

* regenerate

* fix regenerate for unbranded

* remove outdated folder

* add examples-directory for regenerte

---------

Co-authored-by: iscai-msft <isabellavcai@gmail.com>
Co-authored-by: Yuchao Yan <yuchaoyan@microsoft.com>
Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com>
This commit is contained in:
iscai-msft 2024-08-06 06:04:46 -04:00 коммит произвёл GitHub
Родитель 1ebc2d3be9
Коммит bca7bc43f4
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
283 изменённых файлов: 387 добавлений и 327 удалений

Просмотреть файл

@ -0,0 +1,8 @@
---
changeKind: internal
packages:
- "@autorest/python"
- "@azure-tools/typespec-python"
---
Switch to `npm run regenerate` for typespec package

10
.vscode/launch.json поставляемый
Просмотреть файл

@ -50,6 +50,14 @@
"${workspaceFolder}/core/packages/*/dist/**/*.js",
"${workspaceFolder}/core/packages/*/dist-dev/**/*.js"
]
}
},
{
"name": "npm: regenerate",
"request": "launch",
"type": "node",
"runtimeArgs": ["run", "regenerate", "--", "--name", "type/"],
"runtimeExecutable": "npm",
"cwd": "${workspaceFolder}/packages/typespec-python",
}
]
}

26
CONTRIBUTING.md Normal file
Просмотреть файл

@ -0,0 +1,26 @@
# Contributing
## @azure-tools/typespec-python
### Regenerating
`npm run regenerate`
#### Flags
If you're adding flags, you'll need to add a `--` before including the flags
| Flag | Description |
|------|-------------|
| `--name` | Name of the file you want to generate. Will only generate files that include `--name` |
| `--flavor=azure\|unbranded` | Regenerate either azure or unbranded flavor |
| `--debug` | If you want to debug through the code |
**Example**
`npm run regenerate -- --name api --flavor unbranded`
## @autorest/python
### Regenerating
`inv regenerate`

Просмотреть файл

@ -1,5 +1,4 @@
azure-pylint-guidelines-checker==0.0.8
invoke==2.2.0
colorama==0.4.6
debugpy==1.8.2
pytest==8.3.2

Просмотреть файл

@ -107,8 +107,8 @@ steps:
condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
- script: |
cd test/unittests
tox run -e ci
cd test/unittests
tox run -e ci
displayName: Unit tests
workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/
condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
@ -116,7 +116,12 @@ steps:
- script: inv regenerate
displayName: "Regenerate Code"
workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/
condition: and(succeeded(), ${{ parameters.regenerate }})
condition: and(succeeded(), ${{ parameters.regenerate }}, eq('${{parameters.folderName}}', 'autorest.python'))
- script: npm run regenerate
displayName: "Regenerate Code"
workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/
condition: and(succeeded(), ${{ parameters.regenerate }}, eq('${{parameters.folderName}}', 'typespec-python'))
- script: node ../../../eng/scripts/check-for-changed-files.js
displayName: Fail on regeneration diff in Typespec

Просмотреть файл

@ -4,3 +4,4 @@
-r ../../eng/dev_requirements.txt
ptvsd==4.3.2
types-PyYAML==6.0.12.8
invoke==2.2.0

Просмотреть файл

@ -35,7 +35,8 @@
"lint": "eslint . --ext .ts --max-warnings=0",
"lint:fix": "eslint . --fix --ext .ts",
"install": "node ./scripts/run-python3.cjs ./scripts/install.py",
"prepare": "node ./scripts/run-python3.cjs ./scripts/prepare.py"
"prepare": "node ./scripts/run-python3.cjs ./scripts/prepare.py",
"regenerate": "node ./dist/scripts/regenerate.js"
},
"files": [
"dist/**",
@ -75,6 +76,7 @@
"@types/js-yaml": "~4.0.5",
"@types/mocha": "~10.0.1",
"@types/node": "^18.16.3",
"@types/yargs": "17.0.32",
"@typespec/eslint-config-typespec": "~0.55.0",
"@typespec/openapi": "~0.58.0",
"c8": "~7.13.0",
@ -88,6 +90,7 @@
"@typespec/http": "~0.58.0",
"@typespec/rest": "~0.58.0",
"@typespec/versioning": "~0.58.0",
"@azure-tools/typespec-azure-rulesets": "0.44.0"
"@azure-tools/typespec-azure-rulesets": "0.44.0",
"yargs": "~17.2.1"
}
}

Просмотреть файл

@ -0,0 +1,282 @@
/* eslint-disable no-console */
import { exec as execCallback } from "child_process";
import { promisify } from "util";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { dirname, join, relative, resolve } from "path";
import { promises } from "fs";
import { fileURLToPath } from "url";
// Promisify the exec function
const exec = promisify(execCallback);
// Get the directory of the current file
const PLUGIN_DIR = resolve(fileURLToPath(import.meta.url), "../../../");
const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, string>[]> = {
"resiliency/srv-driven/old.tsp": {
"package-name": "resiliency-srv-driven1",
"package-mode": "azure-dataplane",
"package-pprint-name": "ResiliencySrvDriven1",
},
"resiliency/srv-driven": {
"package-name": "resiliency-srv-driven2",
"package-mode": "azure-dataplane",
"package-pprint-name": "ResiliencySrvDriven2",
},
"authentication/http/custom": {
"package-name": "authentication-http-custom",
},
"authentication/union": {
"package-name": "authentication-union",
},
"type/array": {
"package-name": "typetest-array",
},
"type/dictionary": {
"package-name": "typetest-dictionary",
},
"type/enum/extensible": {
"package-name": "typetest-enum-extensible",
},
"type/enum/fixed": {
"package-name": "typetest-enum-fixed",
},
"type/model/empty": {
"package-name": "typetest-model-empty",
},
"type/model/flatten": {
"package-name": "typetest-model-flatten",
},
"type/model/inheritance/enum-discriminator": {
"package-name": "typetest-model-enumdiscriminator",
},
"type/model/inheritance/nested-discriminator": {
"package-name": "typetest-model-nesteddiscriminator",
},
"type/model/inheritance/not-discriminated": {
"package-name": "typetest-model-notdiscriminated",
},
"type/model/inheritance/single-discriminator": {
"package-name": "typetest-model-singlediscriminator",
},
"type/model/inheritance/recursive": {
"package-name": "typetest-model-recursive",
},
"type/model/usage": {
"package-name": "typetest-model-usage",
},
"type/model/visibility": [
{ "package-name": "typetest-model-visibility" },
{ "package-name": "headasbooleantrue", "head-as-boolean": "true" },
{ "package-name": "headasbooleanfalse", "head-as-boolean": "false" },
],
"type/property/nullable": {
"package-name": "typetest-property-nullable",
},
"type/property/optionality": {
"package-name": "typetest-property-optional",
},
"type/property/additional-properties": {
"package-name": "typetest-property-additionalproperties",
},
"type/scalar": {
"package-name": "typetest-scalar",
},
"type/property/value-types": {
"package-name": "typetest-property-valuetypes",
},
"type/union": {
"package-name": "typetest-union",
},
"azure/core/lro/rpc": {
"package-name": "azurecore-lro-rpc",
},
"client/structure/multi-client": {
"package-name": "client-structure-multiclient",
},
"client/structure/renamed-operation": {
"package-name": "client-structure-renamedoperation",
},
"client/structure/two-operation-group": {
"package-name": "client-structure-twooperationgroup",
},
"mgmt/sphere": [{ "package-name": "azure-mgmt-spheredpg" }],
};
function toPosix(dir: string): string {
return dir.replace(/\\/g, "/");
}
function getEmitterOption(spec: string): Record<string, string>[] {
const result = EMITTER_OPTIONS[toPosix(dirname(relative(CADL_RANCH_DIR, spec)))] || [];
return Array.isArray(result) ? result : [result];
}
// Function to execute CLI commands asynchronously
async function executeCommand(command: string): Promise<void> {
try {
const { stdout, stderr } = await exec(command);
if (stdout) console.log(`stdout: ${stdout}`);
if (stderr) console.error(`stderr: ${stderr}`);
} catch (error) {
console.error(`exec error: ${error}`);
throw error;
}
}
interface RegenerateFlagsInput {
flavor?: "azure" | "unbranded";
debug?: boolean;
name?: string;
}
interface RegenerateFlags {
flavor: "azure" | "unbranded";
debug: boolean;
name?: string;
}
const SpecialFlags: Record<string, Record<string, any>> = {
azure: {
"generate-test": true,
"generate-sample": true,
},
};
async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise<string[]> {
const subdirectories: string[] = [];
async function searchDir(currentDir: string) {
const items = await promises.readdir(currentDir, { withFileTypes: true });
const promisesArray = items.map(async (item) => {
const subDirPath = join(currentDir, item.name);
if (item.isDirectory()) {
const mainTspPath = join(subDirPath, "main.tsp");
const clientTspPath = join(subDirPath, "client.tsp");
const mainTspRelativePath = relative(baseDir, mainTspPath);
if (flags.flavor === "unbranded" && mainTspRelativePath.includes("azure")) return;
const hasMainTsp = await promises
.access(mainTspPath)
.then(() => true)
.catch(() => false);
const hasClientTsp = await promises
.access(clientTspPath)
.then(() => true)
.catch(() => false);
if (
toPosix(mainTspRelativePath)
.toLowerCase()
.includes(flags.name || "")
) {
if (hasClientTsp) {
subdirectories.push(resolve(subDirPath, "client.tsp"));
} else if (hasMainTsp) {
subdirectories.push(resolve(subDirPath, "main.tsp"));
}
}
// Recursively search in the subdirectory
await searchDir(subDirPath);
}
});
await Promise.all(promisesArray);
}
await searchDir(baseDir);
return subdirectories;
}
function defaultPackageName(spec: string): string {
return toPosix(relative(CADL_RANCH_DIR, dirname(spec)))
.replace(/\//g, "-")
.toLowerCase();
}
function addOptions(spec: string, generatedFolder: string, flags: RegenerateFlags): string[] {
let options: Record<string, string> = {};
for (const config of getEmitterOption(spec)) {
options = Object.assign(options, config);
}
options["flavor"] = flags.flavor;
for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) {
options[k] = v;
}
if (options["emitter-output-dir"] === undefined) {
const packageName = options["package-name"] || defaultPackageName(spec);
options["emitter-output-dir"] = `${generatedFolder}/test/${flags.flavor}/generated/${packageName}`;
}
if (flags.debug) {
options["debug"] = "true";
}
if (flags.flavor === "unbranded") {
options["company-name"] = "Unbranded";
}
options["examples-directory"] = join(dirname(spec), "examples");
const emitterConfigs = Object.entries(options).flatMap(([k, v]) => {
return `--option @azure-tools/typespec-python.${k}=${v}`;
});
return emitterConfigs;
}
async function _regenerateSingle(spec: string, flags: RegenerateFlags): Promise<void> {
// Perform some asynchronous operation here
const options = addOptions(spec, PLUGIN_DIR, flags);
const command = `tsp compile ${spec} --emit=${PLUGIN_DIR} ${options.join(" ")}`;
console.log(command);
await executeCommand(command);
}
async function regenerate(flags: RegenerateFlagsInput): Promise<boolean> {
if (flags.flavor === undefined) {
const azureGeneration = await regenerate({ ...flags, flavor: "azure" });
const unbrandedGeneration = await regenerate({ ...flags, flavor: "unbranded" });
return azureGeneration && unbrandedGeneration;
} else {
const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
const CADL_RANCH_DIR = resolve(PLUGIN_DIR, "node_modules/@azure-tools/cadl-ranch-specs/http");
const subdirectories = await getSubdirectories(CADL_RANCH_DIR, flagsResolved);
const promises = subdirectories.map(async (subdirectory) => {
// Perform additional asynchronous operations on each subdirectory here
await _regenerateSingle(subdirectory, flagsResolved);
});
await Promise.all(promises);
return true;
}
}
// try {
// const output = await executeCommand('tsp compile');
// console.log(`Command output: ${output}`);
// } catch (error) {
// console.error(`Command failed: ${error}`);
// }
// PARSE INPUT ARGUMENTS
const argv = yargs(hideBin(process.argv))
.option("flavor", {
type: "string",
choices: ["azure", "unbranded"],
description: "Specify the flavor",
})
.option("debug", {
alias: "d",
type: "boolean",
description: "Debug mode",
})
.option("name", {
alias: "n",
type: "string",
description: "Specify filename if you only want to generate a subset",
}).argv;
regenerate(argv as RegenerateFlags)
.then(() => console.log("Regeneration successful"))
.catch((error) => console.error(`Regeneration failed: ${error.message}`));

Просмотреть файл

@ -1,292 +0,0 @@
# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
import re
import os
from pathlib import Path
from multiprocessing import Pool
from colorama import init, Fore
from invoke import task, run
import shutil
from typing import Dict, List, Any, Optional, Literal
import copy
#######################################################
# Working around for issue https://github.com/pyinvoke/invoke/issues/833 in python3.11
import inspect
if not hasattr(inspect, "getargspec"):
inspect.getargspec = inspect.getfullargspec
#######################################################
init()
PLUGIN_DIR = Path(os.path.dirname(__file__))
PLUGIN = (PLUGIN_DIR / "dist/src/index.js").as_posix()
CADL_RANCH_DIR = PLUGIN_DIR / Path("node_modules/@azure-tools/cadl-ranch-specs/http")
LOCAL_SPECIFICATION_DIR = PLUGIN_DIR / Path("test/azure/specification")
EMITTER_OPTIONS = {
"resiliency/srv-driven/old.tsp": {
"package-name": "resiliency-srv-driven1",
"package-mode": "azure-dataplane",
"package-pprint-name": "ResiliencySrvDriven1",
},
"resiliency/srv-driven": {
"package-name": "resiliency-srv-driven2",
"package-mode": "azure-dataplane",
"package-pprint-name": "ResiliencySrvDriven2",
},
"authentication/http/custom": {
"package-name": "authentication-http-custom",
},
"authentication/union": {
"package-name": "authentication-union",
},
"type/array": {
"package-name": "typetest-array",
},
"type/dictionary": {
"package-name": "typetest-dictionary",
},
"type/enum/extensible": {
"package-name": "typetest-enum-extensible",
},
"type/enum/fixed": {
"package-name": "typetest-enum-fixed",
},
"type/model/empty": {
"package-name": "typetest-model-empty",
},
"type/model/flatten": {
"package-name": "typetest-model-flatten",
},
"type/model/inheritance/enum-discriminator": {
"package-name": "typetest-model-enumdiscriminator",
},
"type/model/inheritance/nested-discriminator": {
"package-name": "typetest-model-nesteddiscriminator",
},
"type/model/inheritance/not-discriminated": {
"package-name": "typetest-model-notdiscriminated",
},
"type/model/inheritance/single-discriminator": {
"package-name": "typetest-model-singlediscriminator",
},
"type/model/inheritance/recursive": {
"package-name": "typetest-model-recursive",
},
"type/model/usage": {
"package-name": "typetest-model-usage",
},
"type/model/visibility": [
{"package-name": "typetest-model-visibility"},
{"package-name": "headasbooleantrue", "head-as-boolean": "true"},
{"package-name": "headasbooleanfalse", "head-as-boolean": "false"},
],
"type/property/nullable": {
"package-name": "typetest-property-nullable",
},
"type/property/optionality": {
"package-name": "typetest-property-optional",
},
"type/property/additional-properties": {
"package-name": "typetest-property-additionalproperties",
},
"type/scalar": {
"package-name": "typetest-scalar",
},
"type/property/value-types": {
"package-name": "typetest-property-valuetypes",
},
"type/union": {
"package-name": "typetest-union",
},
"azure/core/lro/rpc-legacy": {
"package-name": "azurecore-lro-rpclegacy",
},
"azure/core/lro/rpc": {
"package-name": "azurecore-lro-rpc",
},
"client/structure/multi-client": {
"package-name": "client-structure-multiclient",
},
"client/structure/renamed-operation": {
"package-name": "client-structure-renamedoperation",
},
"client/structure/two-operation-group": {
"package-name": "client-structure-twooperationgroup",
},
"mgmt/sphere": [
{"package-name": "azure-mgmt-spheredpg"},
],
}
def _package_name_folder(spec: Path, category: Literal["azure", "unbranded"]) -> str:
for item in _get_specification_dirs(category):
if item.as_posix() in spec.as_posix():
return spec.relative_to(item).as_posix()
raise ValueError(f"Cannot find package name for {spec}")
def _default_package_name(spec: Path, category: Literal["azure", "unbranded"]) -> str:
return _package_name_folder(spec, category).replace("/", "-")
def _get_emitter_option(spec: Path, category: Literal["azure", "unbranded"]) -> List[Dict[str, str]]:
name = _package_name_folder(spec, category)
result = EMITTER_OPTIONS.get(name, [])
if isinstance(result, dict):
return [result]
return result
def _add_options(
spec: Path,
category: Literal["azure", "unbranded"],
generated_foder: Path,
special_flags: Dict[str, Any],
debug=False,
) -> str:
result = {}
for config in _get_emitter_option(spec, category):
config_copy = copy.copy(config)
config_copy["emitter-output-dir"] = f"{generated_foder}/{config['package-name']}"
result.update(config_copy)
if not result:
result.update({"emitter-output-dir": f"{generated_foder}/{_default_package_name(spec, category)}"})
result.update({"examples-directory": str(spec / "examples")})
result.update(special_flags)
if debug:
result.update({"debug": "true"})
return "".join([f" --option @azure-tools/typespec-python.{k}={v}" for k, v in result.items()])
def _entry_file_name(path: Path) -> Path:
if path.is_file():
return path
return (path / "client.tsp") if (path / "client.tsp").exists() else (path / "main.tsp")
def _get_specification_dirs(category: Literal["azure", "unbranded"]) -> List[Path]:
# we should remove the need for this by removing our local definition of mgmt sphere
local_specification_folder = Path(f"test/{category}/specification")
return [CADL_RANCH_DIR, local_specification_folder] if local_specification_folder.exists() else [CADL_RANCH_DIR]
def _all_specification_folders(category: Literal["azure", "unbranded"], filename: str = "main.tsp") -> List[Path]:
return [
s
for item in _get_specification_dirs(category)
for s in item.glob("**/*")
if s.is_dir() and any(f for f in s.iterdir() if f.name == filename)
]
def _regenerate(
c,
specs: List[Path],
category: Literal["azure", "unbranded"],
name: Optional[str] = None,
debug: bool = False,
special_flags={},
):
generated_folder = Path(f"{PLUGIN_DIR}/test/{category}/generated")
if name:
specs = [s for s in specs if name.lower() in str(s)]
if not name or name in "resiliency/srv-driven":
specs.extend(s / "old.tsp" for s in _all_specification_folders(category, filename="old.tsp"))
for spec in specs:
for package_name in _get_package_names(spec, category):
(generated_folder / package_name).mkdir(parents=True, exist_ok=True)
_run_cadl(
[
f"tsp compile {_entry_file_name(spec)} --emit={PLUGIN_DIR} {_add_options(spec, category, generated_folder, special_flags, debug)}"
for spec in specs
]
)
def is_invalid_folder(s: Path, invalid_folders: List[str] = []) -> bool:
if "sphere" in str(s) or len(invalid_folders) == 0:
return False
return any(n in s.relative_to(CADL_RANCH_DIR).as_posix() for n in invalid_folders)
@task
def regenerate_azure(c, name=None, debug=False):
specs = [s for s in _all_specification_folders("azure") if not is_invalid_folder(s)]
special_flags = {"flavor": "azure", "generate-test": "true", "generate-sample": "true"}
_regenerate(
c,
specs,
"azure",
name,
debug,
special_flags=special_flags,
)
@task
def regenerate_unbranded(c, name=None, debug=False):
specs = [
s
for s in _all_specification_folders("unbranded")
if not is_invalid_folder(s, invalid_folders=["azure", "client-request-id"])
]
special_flags = {"company-name": "Unbranded"}
_regenerate(
c,
specs,
"unbranded",
name=name,
debug=debug,
special_flags=special_flags,
)
@task
def regenerate(
c,
name=None,
debug=False,
flavor: Optional[Literal["azure", "unbranded"]] = None,
):
if flavor == "azure":
return regenerate_azure(c, name, debug)
if flavor == "unbranded":
return regenerate_unbranded(c, name, debug)
regenerate_azure(c, name, debug)
regenerate_unbranded(c, name, debug)
def _get_package_names(spec: Path, category: Literal["azure", "unbranded"]) -> List[str]:
result = [config["package-name"] for config in _get_emitter_option(spec, category)]
if not result:
result.append(_default_package_name(spec, category))
return result
def _run_cadl(cmds):
if len(cmds) == 1:
success = _run_single_tsp(cmds[0])
else:
with Pool() as pool:
result = pool.map(_run_single_tsp, cmds)
success = all(result)
if not success:
raise SystemExit("Cadl generation fails")
def _run_single_tsp(cmd):
result = run(cmd, warn=True)
if result.ok:
print(Fore.GREEN + f'Call "{cmd}" done with success')
return True
print(Fore.RED + f'Call "{cmd}" failed with {result.return_code}\n{result.stdout}\n{result.stderr}')
output_folder = re.findall(r"emitter-output-dir=([^\s]+)", cmd)[0]
shutil.rmtree(output_folder, ignore_errors=True)
return False

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше