[SDK generation pipeline] optimize for multi input (#36830)
* optimize for multi input * optimize for multi input * optimize for judge tag is preview or not * fix * fix tsp-client * fix * fix * fix
This commit is contained in:
Yuchao Yan
GitHub
Родитель
d805e21925
Коммит
b7d0d383b2
|
|
@ -7,7 +7,7 @@ from functools import wraps
|
|||
|
||||
from ci_tools.git_tools import get_add_diff_file_list
|
||||
from pathlib import Path
|
||||
from subprocess import check_output, CalledProcessError, check_call
|
||||
from subprocess import check_output, CalledProcessError, check_call, STDOUT
|
||||
from typing import Dict, Any
|
||||
from glob import glob
|
||||
import yaml
|
||||
|
|
@ -27,9 +27,9 @@ DEFAULT_DEST_FOLDER = "./dist"
|
|||
_DPG_README = "README.md"
|
||||
|
||||
|
||||
# input example: "../azure-rest-api-specs/specification/informatica/Informatica.DataManagement"
|
||||
def del_outdated_generated_files(readme: str):
|
||||
tspconfig = Path(readme) / "tspconfig.yaml"
|
||||
# tsp example: "../azure-rest-api-specs/specification/informatica/Informatica.DataManagement"
|
||||
def del_outdated_generated_files(tsp: str):
|
||||
tspconfig = Path(tsp) / "tspconfig.yaml"
|
||||
if not tspconfig.exists():
|
||||
_LOGGER.info(f"do not find tspconfig.yaml: {tspconfig}")
|
||||
return
|
||||
|
|
@ -195,8 +195,11 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name,
|
|||
f.write("".join(includes))
|
||||
|
||||
|
||||
def judge_tag_preview(path: str) -> bool:
|
||||
files = [i for i in Path(path).glob("**/*.py")]
|
||||
@return_origin_path
|
||||
def judge_tag_preview(path: str, package_name: str) -> bool:
|
||||
os.chdir(path)
|
||||
first_level = package_name.split("-")[0]
|
||||
files = [i for i in Path(".").glob(f"{first_level}/**/*.py")]
|
||||
default_api_version = "" # for multi-api
|
||||
api_version = "" # for single-api
|
||||
for file in files:
|
||||
|
|
@ -427,14 +430,22 @@ def gen_typespec(typespec_relative_path: str, spec_folder: str, head_sha: str, r
|
|||
try:
|
||||
tsp_dir = (Path(spec_folder) / typespec_relative_path).resolve()
|
||||
repo_url = rest_repo_url.replace("https://github.com/", "")
|
||||
check_output(
|
||||
f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug",
|
||||
shell=True,
|
||||
)
|
||||
cmd = f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug"
|
||||
_LOGGER.info(f"generation cmd: {cmd}")
|
||||
output = check_output(cmd, stderr=STDOUT, shell=True)
|
||||
except CalledProcessError as e:
|
||||
_LOGGER.error(f"Failed to generate sdk from typespec: {e.output.decode('utf-8')}")
|
||||
raise e
|
||||
|
||||
decode_output = output.decode("utf-8")
|
||||
# before https://github.com/Azure/azure-sdk-tools/issues/8815, have to check output to judge whether sdk generation succeeds
|
||||
if " - error " in decode_output:
|
||||
_LOGGER.error(f"Failed to generate sdk from typespec:")
|
||||
for item in decode_output.split("\n"):
|
||||
if " - error " in item:
|
||||
_LOGGER.error(item)
|
||||
raise Exception(f"Failed to generate sdk from typespec: {decode_output}")
|
||||
|
||||
with open(Path("eng/emitter-package.json"), "r") as file_in:
|
||||
data = json.load(file_in)
|
||||
npm_package_verstion = {
|
||||
|
|
|
|||
|
|
@ -295,85 +295,97 @@ def main(generate_input, generate_output):
|
|||
else:
|
||||
del_outdated_generated_files(str(Path(spec_folder, readme_or_tsp)))
|
||||
config = gen_typespec(readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"])
|
||||
package_names = get_package_names(sdk_folder)
|
||||
except Exception as e:
|
||||
_LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}")
|
||||
for hint_message in [
|
||||
"======================================= Whant Can I do ========================================================================",
|
||||
"If you are from service team, please first check if the failure happens only to Python automation, or for all SDK automations. ",
|
||||
"======================================= Whant Can I do (begin) ========================================================================",
|
||||
f"Fail to generate sdk for {readme_or_tsp}. If you are from service team, please first check if the failure happens only to Python automation, or for all SDK automations. ",
|
||||
"If it happens for all SDK automations, please double check your Swagger / Typespec, and check whether there is error in ModelValidation and LintDiff. ",
|
||||
"If it happens to Python alone, you can open an issue to https://github.com/Azure/autorest.python/issues. Please include the link of this Pull Request in the issue.",
|
||||
"===============================================================================================================================",
|
||||
"======================================= Whant Can I do (end) =========================================================================",
|
||||
]:
|
||||
_LOGGER.error(hint_message)
|
||||
raise e
|
||||
_LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen end. [(packages:{str(package_names)})]")
|
||||
if len(readme_and_tsp) == 1:
|
||||
raise e
|
||||
|
||||
# folder_name: "sdk/containerservice"; package_name: "azure-mgmt-containerservice"
|
||||
package_names = get_package_names(sdk_folder)
|
||||
spec_word = "readmeMd" if "readme.md" in readme_or_tsp else "typespecProject"
|
||||
for folder_name, package_name in package_names:
|
||||
if package_name in package_total:
|
||||
continue
|
||||
|
||||
package_total.add(package_name)
|
||||
sdk_code_path = str(Path(sdk_folder, folder_name, package_name))
|
||||
if package_name not in result:
|
||||
package_entry = {}
|
||||
package_entry["packageName"] = package_name
|
||||
package_entry["path"] = [folder_name]
|
||||
package_entry[spec_word] = [readme_or_tsp]
|
||||
package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path)
|
||||
readme_python_content = get_readme_python_content(str(Path(spec_folder) / readme_or_tsp))
|
||||
package_entry["isMultiapi"] = is_multiapi_package(readme_python_content)
|
||||
result[package_name] = package_entry
|
||||
else:
|
||||
result[package_name]["path"].append(folder_name)
|
||||
result[package_name][spec_word].append(readme_or_tsp)
|
||||
|
||||
# Generate some necessary file for new service
|
||||
init_new_service(package_name, folder_name)
|
||||
format_samples_and_tests(sdk_code_path)
|
||||
|
||||
# Update metadata
|
||||
try:
|
||||
update_servicemetadata(
|
||||
sdk_folder,
|
||||
data,
|
||||
config,
|
||||
folder_name,
|
||||
package_name,
|
||||
spec_folder,
|
||||
readme_or_tsp,
|
||||
)
|
||||
except Exception as e:
|
||||
_LOGGER.error(f"fail to update meta: {str(e)}")
|
||||
|
||||
# Setup package locally
|
||||
check_call(
|
||||
f"pip install --ignore-requires-python -e {sdk_code_path}",
|
||||
shell=True,
|
||||
_LOGGER.info(
|
||||
f"[CODEGEN]({readme_or_tsp})codegen end and new package '{folder_name}/{package_name}' generated"
|
||||
)
|
||||
try:
|
||||
package_total.add(package_name)
|
||||
sdk_code_path = str(Path(sdk_folder, folder_name, package_name))
|
||||
if package_name not in result:
|
||||
package_entry = {}
|
||||
package_entry["packageName"] = package_name
|
||||
package_entry["path"] = [folder_name]
|
||||
package_entry[spec_word] = [readme_or_tsp]
|
||||
package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path, package_name)
|
||||
readme_python_content = get_readme_python_content(str(Path(spec_folder) / readme_or_tsp))
|
||||
package_entry["isMultiapi"] = is_multiapi_package(readme_python_content)
|
||||
result[package_name] = package_entry
|
||||
else:
|
||||
result[package_name]["path"].append(folder_name)
|
||||
result[package_name][spec_word].append(readme_or_tsp)
|
||||
|
||||
# check whether multiapi package has only one api-version in per subfolder
|
||||
# skip check for network for https://github.com/Azure/azure-sdk-for-python/issues/30556#issuecomment-1571341309
|
||||
if "azure-mgmt-network" not in sdk_code_path:
|
||||
check_api_version_in_subfolder(sdk_code_path)
|
||||
# Generate some necessary file for new service
|
||||
init_new_service(package_name, folder_name)
|
||||
format_samples_and_tests(sdk_code_path)
|
||||
|
||||
# use multiapi combiner to combine multiapi package
|
||||
if package_name in ("azure-mgmt-network"):
|
||||
multiapi_combiner(sdk_code_path, package_name)
|
||||
after_multiapi_combiner(sdk_code_path, package_name, folder_name)
|
||||
result[package_name]["afterMultiapiCombiner"] = True
|
||||
else:
|
||||
result[package_name]["afterMultiapiCombiner"] = False
|
||||
# Update metadata
|
||||
try:
|
||||
update_servicemetadata(
|
||||
sdk_folder,
|
||||
data,
|
||||
config,
|
||||
folder_name,
|
||||
package_name,
|
||||
spec_folder,
|
||||
readme_or_tsp,
|
||||
)
|
||||
except Exception as e:
|
||||
_LOGGER.error(f"fail to update meta: {str(e)}")
|
||||
|
||||
# Setup package locally
|
||||
check_call(
|
||||
f"pip install --ignore-requires-python -e {sdk_code_path}",
|
||||
shell=True,
|
||||
)
|
||||
|
||||
# check whether multiapi package has only one api-version in per subfolder
|
||||
# skip check for network for https://github.com/Azure/azure-sdk-for-python/issues/30556#issuecomment-1571341309
|
||||
if "azure-mgmt-network" not in sdk_code_path:
|
||||
check_api_version_in_subfolder(sdk_code_path)
|
||||
|
||||
# use multiapi combiner to combine multiapi package
|
||||
if package_name in ("azure-mgmt-network"):
|
||||
multiapi_combiner(sdk_code_path, package_name)
|
||||
after_multiapi_combiner(sdk_code_path, package_name, folder_name)
|
||||
result[package_name]["afterMultiapiCombiner"] = True
|
||||
else:
|
||||
result[package_name]["afterMultiapiCombiner"] = False
|
||||
except Exception as e:
|
||||
_LOGGER.error(f"fail to setup package: {str(e)}")
|
||||
|
||||
# remove duplicates
|
||||
for value in result.values():
|
||||
value["path"] = list(set(value["path"]))
|
||||
if value.get("typespecProject"):
|
||||
value["typespecProject"] = list(set(value["typespecProject"]))
|
||||
if value.get("readmeMd"):
|
||||
value["readmeMd"] = list(set(value["readmeMd"]))
|
||||
try:
|
||||
for value in result.values():
|
||||
value["path"] = list(set(value["path"]))
|
||||
if value.get("typespecProject"):
|
||||
value["typespecProject"] = list(set(value["typespecProject"]))
|
||||
if value.get("readmeMd"):
|
||||
value["readmeMd"] = list(set(value["readmeMd"]))
|
||||
except Exception as e:
|
||||
_LOGGER.error(f"fail to remove duplicates: {str(e)}")
|
||||
|
||||
if len(result) == 0 and len(readme_and_tsp) > 1:
|
||||
raise Exception("No package is generated, please check the log for details")
|
||||
|
||||
with open(generate_output, "w") as writer:
|
||||
json.dump(result, writer)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче