Improvement for Go (#51)

* Add envs to SwaggerToSdk

* Add tools to get swaggertosdk conf link

* Download the conf

* Download conf from github before the clone

* Solve relative path in advanced_conf

* Implement clone_dir

* Add log

* Don't solve clone_dir

swaggertosdk/SwaggerToSdkCore.py

@@ -7,6 +7,8 @@ import re
 import tempfile
 from pathlib import Path
 
+import requests
+
 from github import Github, UnknownObjectException
 
 from .autorest_tools import (
@@ -117,6 +119,10 @@ def read_config(sdk_git_folder, config_file):
     with open(config_path, 'r') as config_fd:
         return json.loads(config_fd.read())
 
+def read_config_from_github(sdk_id, branch="master"):
+    raw_link = str(get_configuration_github_path(sdk_id, branch))
+    content = requests.get(raw_link).text
+    return json.loads(content)
 
 def extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config):
     readme_files_in_pr = {readme for readme in swagger_files_in_pr if getattr(readme, "name", readme).lower().endswith("readme.md")}
@@ -204,3 +210,7 @@ def solve_relative_path(autorest_options, sdk_root):
         else:
             solved_autorest_options[key] = value
     return solved_autorest_options
+
+def get_configuration_github_path(sdk_id, branch="master"):
+    gh_token = os.environ.get("GH_TOKEN", None)  # Token here is just for private
+    return GithubLink(sdk_id, "raw", branch, CONFIG_FILE, gh_token)

swaggertosdk/SwaggerToSdkMain.py

@@ -8,11 +8,11 @@ import sys
 from git import Repo
 
 from .SwaggerToSdkCore import (
-    CONFIG_FILE,
-    read_config,
+    read_config_from_github,
     extract_conf_from_readmes,
     get_input_paths,
-    get_readme_files_from_file_list
+    get_readme_files_from_file_list,
+    solve_relative_path
 )
 from .github_tools import (
     manage_git_folder,
@@ -24,48 +24,50 @@ from .git_tools import (
 _LOGGER = logging.getLogger(__name__)
 
 
-def generate_sdk(config_path,
-                 sdk_git_id, base_branch_name,
+def generate_sdk(sdk_git_id, base_branch_name,
                  autorest_bin=None):
     """Main method of the the file"""
 
     # On Travis, local folder is restapi git folder
     restapi_git_folder = '.'
 
+    config = read_config_from_github(sdk_git_id, base_branch_name)
+    global_conf = config["meta"]
+
     # No token is provided to clone SDK. Do NOT try to clone a private it will fail.
-    with tempfile.TemporaryDirectory() as temp_dir, \
-            manage_git_folder(None, Path(temp_dir) / Path("sdk"), sdk_git_id+'@'+base_branch_name) as sdk_folder:
+    with tempfile.TemporaryDirectory() as temp_dir:
 
-        sdk_repo = Repo(str(sdk_folder))
+        clone_dir = Path(temp_dir) / Path(global_conf["advanced_options"].get("clone_dir", "sdk"))
+        _LOGGER.info("Clone dir will be: %s", clone_dir)
 
-        config = read_config(sdk_repo.working_tree_dir, config_path)
+        with manage_git_folder(None, clone_dir, sdk_git_id+'@'+base_branch_name) as sdk_folder:
 
-        global_conf = config["meta"]
+            sdk_repo = Repo(str(sdk_folder))
 
-        swagger_files_in_pr = get_files_in_commit(restapi_git_folder)
-        _LOGGER.info("Files in PR: %s ", swagger_files_in_pr)
-        swagger_files_in_pr = get_readme_files_from_file_list(swagger_files_in_pr, restapi_git_folder)
-        _LOGGER.info("Readmes in PR: %s ", swagger_files_in_pr)
+            swagger_files_in_pr = get_files_in_commit(restapi_git_folder)
+            _LOGGER.info("Files in PR: %s ", swagger_files_in_pr)
+            swagger_files_in_pr = get_readme_files_from_file_list(swagger_files_in_pr, restapi_git_folder)
+            _LOGGER.info("Readmes in PR: %s ", swagger_files_in_pr)
 
-        # Look for configuration in Readme
-        extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config)
+            # Look for configuration in Readme
+            extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config)
 
-        def skip_callback(project, local_conf):
-            if not swagger_files_in_pr:
-                return True # Travis with no files found, always skip
+            def skip_callback(project, local_conf):
+                if not swagger_files_in_pr:
+                    return True # Travis with no files found, always skip
 
-            markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf)
+                markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf)
 
-            if swagger_files_in_pr and not (
-                    markdown_relative_path in swagger_files_in_pr or
-                    any(input_file in swagger_files_in_pr for input_file in optional_relative_paths)):
-                _LOGGER.info(f"In project {project} no files involved in this PR")
-                return True
-            return False
+                if swagger_files_in_pr and not (
+                        markdown_relative_path in swagger_files_in_pr or
+                        any(input_file in swagger_files_in_pr for input_file in optional_relative_paths)):
+                    _LOGGER.info(f"In project {project} no files involved in this PR")
+                    return True
+                return False
 
-        from . import SwaggerToSdkNewCLI
-        SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder,
-                                           sdk_repo, temp_dir, autorest_bin)
+            from . import SwaggerToSdkNewCLI
+            SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder,
+                                            sdk_repo, temp_dir, autorest_bin)
 
     _LOGGER.info("Build SDK finished and cleaned")
 
@@ -94,9 +96,6 @@ def main(argv):
     parser.add_argument('--base-branch', '-o',
                         dest='base_branch', default='master',
                         help='The base branch to checkout. [default: %(default)s]')
-    parser.add_argument('--config', '-c',
-                        dest='config_path', default=CONFIG_FILE,
-                        help='The JSON configuration format path [default: %(default)s]')
     parser.add_argument('--autorest',
                         dest='autorest_bin',
                         help='Force the Autorest to be executed. Must be a executable command.')
@@ -117,7 +116,6 @@ def main(argv):
         logging.basicConfig()
         main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
 
-    generate_sdk(args.config_path,
-                 args.sdk_git_id,
+    generate_sdk(args.sdk_git_id,
                  args.base_branch,
                  args.autorest_bin)

swaggertosdk/SwaggerToSdkNewCLI.py

@@ -9,8 +9,7 @@ import tempfile
 from git import Repo, GitCommandError
 
 from .SwaggerToSdkCore import (
-    CONFIG_FILE,
-    read_config,
+    read_config_from_github,
     DEFAULT_COMMIT_MESSAGE,
     get_input_paths,
     extract_conf_from_readmes,
@@ -119,9 +118,12 @@ def write_build_file(sdk_root, local_conf):
 
 def execute_after_script(sdk_root, global_conf, local_conf):
     after_scripts = merge_options(global_conf, local_conf, "after_scripts", keep_list_order=True) or []
+    local_envs = dict(os.environ)
+    local_envs.update(global_conf.get("envs", {}))
+
     for script in after_scripts:
         _LOGGER.info("Execute after script: %s", script)
-        execute_simple_command(script, cwd=sdk_root, shell=True)
+        execute_simple_command(script, cwd=sdk_root, shell=True, env=local_envs)
 
 
 def get_local_path_dir(root, relative_path):
@@ -155,7 +157,8 @@ def build_libraries(config, skip_callback, restapi_git_folder, sdk_repo, temp_di
 
     global_conf = config["meta"]
     global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_repo.working_tree_dir)
-
+    global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_repo.working_tree_dir)
+    global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_repo.working_tree_dir)
 
     for project, local_conf in config.get("projects", {}).items():
         if skip_callback(project, local_conf):
@@ -205,7 +208,6 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
     This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name"
     """
     gh_token = os.environ["GH_TOKEN"]
-    config_path = CONFIG_FILE
     message_template = DEFAULT_COMMIT_MESSAGE
     autorest_bin = None
     if sdk_tag is None:
@@ -221,65 +223,79 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
     # Always clone SDK from fallback branch that is required to exist
     branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name
 
-    with tempfile.TemporaryDirectory() as temp_dir, \
-            manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
-            manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder:
-
-        swagger_files_in_commit = get_readme_files_from_git_object(git_object, restapi_git_folder)
-        _LOGGER.info("Files in PR: %s ", swagger_files_in_commit)
-        if not swagger_files_in_commit:
-            _LOGGER.info("No Readme in PR, quit")
-            return
-
-        # SDK part
-        sdk_repo = Repo(str(sdk_folder))
-
-        for base_branch in base_branch_names:
-            _LOGGER.info('Checkout and create %s', base_branch)
-            checkout_and_create_branch(sdk_repo, base_branch)
-
-        _LOGGER.info('Try to checkout destination branch %s', branch_name)
+    # I don't know if the destination branch exists, try until it works
+    config = None
+    for branch in base_branch_names + branch_name + fallback_base_branch_name:
         try:
-            sdk_repo.git.checkout(branch_name)
-            _LOGGER.info('The branch exists.')
-        except GitCommandError:
-            _LOGGER.info('Destination branch does not exists')
-            # Will be created by do_commit
+            config = read_config_from_github(sdk_git_id, branch)
+        except Exception:
+            pass
+        else:
+            break
+    if config is None:
+        raise ValueError("Unable to locate configuration in {}".format(base_branch_names + branch_name + fallback_base_branch_name))
+    global_conf = config["meta"]
 
-        configure_user(gh_token, sdk_repo)
+    with tempfile.TemporaryDirectory() as temp_dir:
 
-        config = read_config(sdk_repo.working_tree_dir, config_path)
-        global_conf = config["meta"]
+        clone_dir = Path(temp_dir) / Path(global_conf["advanced_options"].get("clone_dir", "sdk"))
+        _LOGGER.info("Clone dir will be: %s", clone_dir)
 
-        # Look for configuration in Readme
-        _LOGGER.info('Extract conf from Readmes for target: %s', sdk_git_id)
-        extract_conf_from_readmes(swagger_files_in_commit, restapi_git_folder, sdk_tag, config)
-        _LOGGER.info('End of extraction')
+        with manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
+            manage_git_folder(gh_token, clone_dir, branched_sdk_git_id) as sdk_folder:
 
-        def skip_callback(project, local_conf):
-            # We know "project" is based on Path in "swagger_files_in_commit"
-            if Path(project) in swagger_files_in_commit:
-                return False
-            # Might be a regular project
-            markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf)
-            if not (
-                    markdown_relative_path in swagger_files_in_commit or
-                    any(input_file in swagger_files_in_commit for input_file in optional_relative_paths)):
-                _LOGGER.info(f"In project {project} no files involved in this commit")
-                return True
-            return False
+            swagger_files_in_commit = get_readme_files_from_git_object(git_object, restapi_git_folder)
+            _LOGGER.info("Files in PR: %s ", swagger_files_in_commit)
+            if not swagger_files_in_commit:
+                _LOGGER.info("No Readme in PR, quit")
+                return
 
-        build_libraries(config, skip_callback, restapi_git_folder,
-                        sdk_repo, temp_dir, autorest_bin)
+            # SDK part
+            sdk_repo = Repo(str(sdk_folder))
 
-        try:
-            commit_for_sha = git_object.commit   # Commit
-        except AttributeError:
-            commit_for_sha = list(git_object.get_commits())[-1].commit  # PR
-        message = message_template + "\n\n" + commit_for_sha.message
-        commit_sha = do_commit(sdk_repo, message, branch_name, commit_for_sha.sha)
-        if commit_sha:
             for base_branch in base_branch_names:
-                sdk_repo.git.push('origin', base_branch, set_upstream=True)
-            sdk_repo.git.push('origin', branch_name, set_upstream=True)
-            return "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha)
+                _LOGGER.info('Checkout and create %s', base_branch)
+                checkout_and_create_branch(sdk_repo, base_branch)
+
+            _LOGGER.info('Try to checkout destination branch %s', branch_name)
+            try:
+                sdk_repo.git.checkout(branch_name)
+                _LOGGER.info('The branch exists.')
+            except GitCommandError:
+                _LOGGER.info('Destination branch does not exists')
+                # Will be created by do_commit
+
+            configure_user(gh_token, sdk_repo)
+
+            # Look for configuration in Readme
+            _LOGGER.info('Extract conf from Readmes for target: %s', sdk_git_id)
+            extract_conf_from_readmes(swagger_files_in_commit, restapi_git_folder, sdk_tag, config)
+            _LOGGER.info('End of extraction')
+
+            def skip_callback(project, local_conf):
+                # We know "project" is based on Path in "swagger_files_in_commit"
+                if Path(project) in swagger_files_in_commit:
+                    return False
+                # Might be a regular project
+                markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf)
+                if not (
+                        markdown_relative_path in swagger_files_in_commit or
+                        any(input_file in swagger_files_in_commit for input_file in optional_relative_paths)):
+                    _LOGGER.info(f"In project {project} no files involved in this commit")
+                    return True
+                return False
+
+            build_libraries(config, skip_callback, restapi_git_folder,
+                            sdk_repo, temp_dir, autorest_bin)
+
+            try:
+                commit_for_sha = git_object.commit   # Commit
+            except AttributeError:
+                commit_for_sha = list(git_object.get_commits())[-1].commit  # PR
+            message = message_template + "\n\n" + commit_for_sha.message
+            commit_sha = do_commit(sdk_repo, message, branch_name, commit_for_sha.sha)
+            if commit_sha:
+                for base_branch in base_branch_names:
+                    sdk_repo.git.push('origin', base_branch, set_upstream=True)
+                sdk_repo.git.push('origin', branch_name, set_upstream=True)
+                return "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha)

swaggertosdk/autorest_tools.py

@@ -121,14 +121,15 @@ def generate_code(input_file, global_conf, local_conf, output_dir=None, autorest
         raise ValueError("Autorest call ended with 0, but no files were generated")
 
 
-def execute_simple_command(cmd_line, cwd=None, shell=False):
+def execute_simple_command(cmd_line, cwd=None, shell=False, env=None):
     try:
         process = subprocess.Popen(cmd_line,
                                    stderr=subprocess.STDOUT,
                                    stdout=subprocess.PIPE,
                                    universal_newlines=True,
                                    cwd=cwd,
-                                   shell=shell)
+                                   shell=shell,
+                                   env=env)
         output_buffer = []
         for line in process.stdout:
             output_buffer.append(line.rstrip())

swaggertosdk/generate_sdk.py

@@ -26,6 +26,8 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde
     global_conf = config["meta"]
     repotag = get_repo_tag_meta(global_conf)
     global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder)
+    global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder)
+    global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_folder)
     if restapi_git_folder:
         restapi_git_folder = Path(restapi_git_folder).expanduser()
 

swaggertosdk/restapi/sdkbot.py

@@ -7,7 +7,7 @@ from git import Repo
 
 from swaggertosdk.SwaggerToSdkCore import (
     CONFIG_FILE,
-    read_config,
+    read_config_from_github,
     build_swaggertosdk_conf_from_json_readme,
 )
 from swaggertosdk.git_tools import (
@@ -122,6 +122,8 @@ class GithubHandler:
             path = None  # Not such notion of path here, since it's inside SwaggerToSdk conf
         branched_rest_api_id = rest_api_id + "@" + rest_api_branch
 
+        config = read_config_from_github(pr.head.repo.full_name, branch_name)
+
         with tempfile.TemporaryDirectory() as temp_dir, \
                 manage_git_folder(token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \
                 manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder:
@@ -129,7 +131,6 @@ class GithubHandler:
             sdk_repo = Repo(str(sdk_folder))
             configure_user(self.gh_token, sdk_repo)
 
-            config = read_config(sdk_repo.working_tree_dir, config_path)
             if path: # Assume this is a Readme path
                 config["projects"] = {} # Wipe out everything
                 build_swaggertosdk_conf_from_json_readme(path, sdkid, config, base_folder=restapi_git_folder)

tests/test_autorest_tools.py

@@ -23,6 +23,11 @@ def test_execute_simple_command():
     output = execute_simple_command(["python", "--version"])
     assert "Python" in output
 
+    env = dict(os.environ)
+    env.update({"GOPATH": "something"})
+    output = execute_simple_command(["python", "-c", "import os; print(os.environ['GOPATH'])"], env=env)
+    assert "something" in output
+
     try:
         execute_simple_command(["python", "--oiuyertuyerituy"])
         pytest.fail("This should raise an exception")

tests/test_swaggertosdk.py

@@ -10,6 +10,8 @@ from swaggertosdk.SwaggerToSdkCore import (
     extract_conf_from_readmes,
     get_context_tag_from_git_object,
     get_readme_files_from_git_object,
+    get_configuration_github_path,
+    read_config_from_github
 )
 from swaggertosdk.SwaggerToSdkNewCLI import (
     solve_relative_path,
@@ -281,3 +283,13 @@ def test_extract_conf_from_readmes(mocked_execute_simple_command):
     extract_conf_from_readmes(swagger_files_in_pr, Path(CWD, "files"), sdk_git_id, config)
 
     assert len(config["projects"]) == 2
+
+def test_get_configuration_github_path(github_token):
+    raw_link = str(get_configuration_github_path("Azure/azure-sdk-for-python", "dev"))
+    raw_link = raw_link.replace(github_token, "TOKEN")
+    assert raw_link == "https://TOKEN@raw.githubusercontent.com/Azure/azure-sdk-for-python/dev/swagger_to_sdk_config.json"
+
+def test_read_config_from_github(github_token):
+    conf = read_config_from_github("Azure/azure-sdk-for-python")
+    # Don't do too much
+    assert "meta" in conf