Improvement for Go (#51)
* Add envs to SwaggerToSdk * Add tools to get swaggertosdk conf link * Download the conf * Download conf from github before the clone * Solve relative path in advanced_conf * Implement clone_dir * Add log * Don't solve clone_dir
This commit is contained in:
Laurent Mazuel
GitHub
Родитель
bc35f094ea
Коммит
25de3e00a8
|
|
@ -7,6 +7,8 @@ import re
|
|||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from github import Github, UnknownObjectException
|
||||
|
||||
from .autorest_tools import (
|
||||
|
|
@ -117,6 +119,10 @@ def read_config(sdk_git_folder, config_file):
|
|||
with open(config_path, 'r') as config_fd:
|
||||
return json.loads(config_fd.read())
|
||||
|
||||
def read_config_from_github(sdk_id, branch="master"):
|
||||
raw_link = str(get_configuration_github_path(sdk_id, branch))
|
||||
content = requests.get(raw_link).text
|
||||
return json.loads(content)
|
||||
|
||||
def extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config):
|
||||
readme_files_in_pr = {readme for readme in swagger_files_in_pr if getattr(readme, "name", readme).lower().endswith("readme.md")}
|
||||
|
|
@ -204,3 +210,7 @@ def solve_relative_path(autorest_options, sdk_root):
|
|||
else:
|
||||
solved_autorest_options[key] = value
|
||||
return solved_autorest_options
|
||||
|
||||
def get_configuration_github_path(sdk_id, branch="master"):
|
||||
gh_token = os.environ.get("GH_TOKEN", None) # Token here is just for private
|
||||
return GithubLink(sdk_id, "raw", branch, CONFIG_FILE, gh_token)
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ import sys
|
|||
from git import Repo
|
||||
|
||||
from .SwaggerToSdkCore import (
|
||||
CONFIG_FILE,
|
||||
read_config,
|
||||
read_config_from_github,
|
||||
extract_conf_from_readmes,
|
||||
get_input_paths,
|
||||
get_readme_files_from_file_list
|
||||
get_readme_files_from_file_list,
|
||||
solve_relative_path
|
||||
)
|
||||
from .github_tools import (
|
||||
manage_git_folder,
|
||||
|
|
@ -24,24 +24,26 @@ from .git_tools import (
|
|||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_sdk(config_path,
|
||||
sdk_git_id, base_branch_name,
|
||||
def generate_sdk(sdk_git_id, base_branch_name,
|
||||
autorest_bin=None):
|
||||
"""Main method of the the file"""
|
||||
|
||||
# On Travis, local folder is restapi git folder
|
||||
restapi_git_folder = '.'
|
||||
|
||||
config = read_config_from_github(sdk_git_id, base_branch_name)
|
||||
global_conf = config["meta"]
|
||||
|
||||
# No token is provided to clone SDK. Do NOT try to clone a private it will fail.
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(None, Path(temp_dir) / Path("sdk"), sdk_git_id+'@'+base_branch_name) as sdk_folder:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
|
||||
clone_dir = Path(temp_dir) / Path(global_conf["advanced_options"].get("clone_dir", "sdk"))
|
||||
_LOGGER.info("Clone dir will be: %s", clone_dir)
|
||||
|
||||
with manage_git_folder(None, clone_dir, sdk_git_id+'@'+base_branch_name) as sdk_folder:
|
||||
|
||||
sdk_repo = Repo(str(sdk_folder))
|
||||
|
||||
config = read_config(sdk_repo.working_tree_dir, config_path)
|
||||
|
||||
global_conf = config["meta"]
|
||||
|
||||
swagger_files_in_pr = get_files_in_commit(restapi_git_folder)
|
||||
_LOGGER.info("Files in PR: %s ", swagger_files_in_pr)
|
||||
swagger_files_in_pr = get_readme_files_from_file_list(swagger_files_in_pr, restapi_git_folder)
|
||||
|
|
@ -94,9 +96,6 @@ def main(argv):
|
|||
parser.add_argument('--base-branch', '-o',
|
||||
dest='base_branch', default='master',
|
||||
help='The base branch to checkout. [default: %(default)s]')
|
||||
parser.add_argument('--config', '-c',
|
||||
dest='config_path', default=CONFIG_FILE,
|
||||
help='The JSON configuration format path [default: %(default)s]')
|
||||
parser.add_argument('--autorest',
|
||||
dest='autorest_bin',
|
||||
help='Force the Autorest to be executed. Must be a executable command.')
|
||||
|
|
@ -117,7 +116,6 @@ def main(argv):
|
|||
logging.basicConfig()
|
||||
main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
|
||||
|
||||
generate_sdk(args.config_path,
|
||||
args.sdk_git_id,
|
||||
generate_sdk(args.sdk_git_id,
|
||||
args.base_branch,
|
||||
args.autorest_bin)
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ import tempfile
|
|||
from git import Repo, GitCommandError
|
||||
|
||||
from .SwaggerToSdkCore import (
|
||||
CONFIG_FILE,
|
||||
read_config,
|
||||
read_config_from_github,
|
||||
DEFAULT_COMMIT_MESSAGE,
|
||||
get_input_paths,
|
||||
extract_conf_from_readmes,
|
||||
|
|
@ -119,9 +118,12 @@ def write_build_file(sdk_root, local_conf):
|
|||
|
||||
def execute_after_script(sdk_root, global_conf, local_conf):
|
||||
after_scripts = merge_options(global_conf, local_conf, "after_scripts", keep_list_order=True) or []
|
||||
local_envs = dict(os.environ)
|
||||
local_envs.update(global_conf.get("envs", {}))
|
||||
|
||||
for script in after_scripts:
|
||||
_LOGGER.info("Execute after script: %s", script)
|
||||
execute_simple_command(script, cwd=sdk_root, shell=True)
|
||||
execute_simple_command(script, cwd=sdk_root, shell=True, env=local_envs)
|
||||
|
||||
|
||||
def get_local_path_dir(root, relative_path):
|
||||
|
|
@ -155,7 +157,8 @@ def build_libraries(config, skip_callback, restapi_git_folder, sdk_repo, temp_di
|
|||
|
||||
global_conf = config["meta"]
|
||||
global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_repo.working_tree_dir)
|
||||
|
||||
global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_repo.working_tree_dir)
|
||||
global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_repo.working_tree_dir)
|
||||
|
||||
for project, local_conf in config.get("projects", {}).items():
|
||||
if skip_callback(project, local_conf):
|
||||
|
|
@ -205,7 +208,6 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
|
|||
This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name"
|
||||
"""
|
||||
gh_token = os.environ["GH_TOKEN"]
|
||||
config_path = CONFIG_FILE
|
||||
message_template = DEFAULT_COMMIT_MESSAGE
|
||||
autorest_bin = None
|
||||
if sdk_tag is None:
|
||||
|
|
@ -221,9 +223,26 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
|
|||
# Always clone SDK from fallback branch that is required to exist
|
||||
branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
|
||||
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder:
|
||||
# I don't know if the destination branch exists, try until it works
|
||||
config = None
|
||||
for branch in base_branch_names + branch_name + fallback_base_branch_name:
|
||||
try:
|
||||
config = read_config_from_github(sdk_git_id, branch)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
break
|
||||
if config is None:
|
||||
raise ValueError("Unable to locate configuration in {}".format(base_branch_names + branch_name + fallback_base_branch_name))
|
||||
global_conf = config["meta"]
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
|
||||
clone_dir = Path(temp_dir) / Path(global_conf["advanced_options"].get("clone_dir", "sdk"))
|
||||
_LOGGER.info("Clone dir will be: %s", clone_dir)
|
||||
|
||||
with manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
|
||||
manage_git_folder(gh_token, clone_dir, branched_sdk_git_id) as sdk_folder:
|
||||
|
||||
swagger_files_in_commit = get_readme_files_from_git_object(git_object, restapi_git_folder)
|
||||
_LOGGER.info("Files in PR: %s ", swagger_files_in_commit)
|
||||
|
|
@ -248,9 +267,6 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
|
|||
|
||||
configure_user(gh_token, sdk_repo)
|
||||
|
||||
config = read_config(sdk_repo.working_tree_dir, config_path)
|
||||
global_conf = config["meta"]
|
||||
|
||||
# Look for configuration in Readme
|
||||
_LOGGER.info('Extract conf from Readmes for target: %s', sdk_git_id)
|
||||
extract_conf_from_readmes(swagger_files_in_commit, restapi_git_folder, sdk_tag, config)
|
||||
|
|
|
|||
|
|
@ -121,14 +121,15 @@ def generate_code(input_file, global_conf, local_conf, output_dir=None, autorest
|
|||
raise ValueError("Autorest call ended with 0, but no files were generated")
|
||||
|
||||
|
||||
def execute_simple_command(cmd_line, cwd=None, shell=False):
|
||||
def execute_simple_command(cmd_line, cwd=None, shell=False, env=None):
|
||||
try:
|
||||
process = subprocess.Popen(cmd_line,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
cwd=cwd,
|
||||
shell=shell)
|
||||
shell=shell,
|
||||
env=env)
|
||||
output_buffer = []
|
||||
for line in process.stdout:
|
||||
output_buffer.append(line.rstrip())
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde
|
|||
global_conf = config["meta"]
|
||||
repotag = get_repo_tag_meta(global_conf)
|
||||
global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder)
|
||||
global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder)
|
||||
global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_folder)
|
||||
if restapi_git_folder:
|
||||
restapi_git_folder = Path(restapi_git_folder).expanduser()
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from git import Repo
|
|||
|
||||
from swaggertosdk.SwaggerToSdkCore import (
|
||||
CONFIG_FILE,
|
||||
read_config,
|
||||
read_config_from_github,
|
||||
build_swaggertosdk_conf_from_json_readme,
|
||||
)
|
||||
from swaggertosdk.git_tools import (
|
||||
|
|
@ -122,6 +122,8 @@ class GithubHandler:
|
|||
path = None # Not such notion of path here, since it's inside SwaggerToSdk conf
|
||||
branched_rest_api_id = rest_api_id + "@" + rest_api_branch
|
||||
|
||||
config = read_config_from_github(pr.head.repo.full_name, branch_name)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \
|
||||
manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder:
|
||||
|
|
@ -129,7 +131,6 @@ class GithubHandler:
|
|||
sdk_repo = Repo(str(sdk_folder))
|
||||
configure_user(self.gh_token, sdk_repo)
|
||||
|
||||
config = read_config(sdk_repo.working_tree_dir, config_path)
|
||||
if path: # Assume this is a Readme path
|
||||
config["projects"] = {} # Wipe out everything
|
||||
build_swaggertosdk_conf_from_json_readme(path, sdkid, config, base_folder=restapi_git_folder)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,11 @@ def test_execute_simple_command():
|
|||
output = execute_simple_command(["python", "--version"])
|
||||
assert "Python" in output
|
||||
|
||||
env = dict(os.environ)
|
||||
env.update({"GOPATH": "something"})
|
||||
output = execute_simple_command(["python", "-c", "import os; print(os.environ['GOPATH'])"], env=env)
|
||||
assert "something" in output
|
||||
|
||||
try:
|
||||
execute_simple_command(["python", "--oiuyertuyerituy"])
|
||||
pytest.fail("This should raise an exception")
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ from swaggertosdk.SwaggerToSdkCore import (
|
|||
extract_conf_from_readmes,
|
||||
get_context_tag_from_git_object,
|
||||
get_readme_files_from_git_object,
|
||||
get_configuration_github_path,
|
||||
read_config_from_github
|
||||
)
|
||||
from swaggertosdk.SwaggerToSdkNewCLI import (
|
||||
solve_relative_path,
|
||||
|
|
@ -281,3 +283,13 @@ def test_extract_conf_from_readmes(mocked_execute_simple_command):
|
|||
extract_conf_from_readmes(swagger_files_in_pr, Path(CWD, "files"), sdk_git_id, config)
|
||||
|
||||
assert len(config["projects"]) == 2
|
||||
|
||||
def test_get_configuration_github_path(github_token):
|
||||
raw_link = str(get_configuration_github_path("Azure/azure-sdk-for-python", "dev"))
|
||||
raw_link = raw_link.replace(github_token, "TOKEN")
|
||||
assert raw_link == "https://TOKEN@raw.githubusercontent.com/Azure/azure-sdk-for-python/dev/swagger_to_sdk_config.json"
|
||||
|
||||
def test_read_config_from_github(github_token):
|
||||
conf = read_config_from_github("Azure/azure-sdk-for-python")
|
||||
# Don't do too much
|
||||
assert "meta" in conf
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче