RestAPI (#21)
* Add Github http link to SDK/Rest parameters * Cast folder as str (can be Path) * Basic JSON-RPC server * Allow all hosts * Gitattributes for EOL * Add Github handler basic * Fix getting token * Plug generate from README * Fix indentation * Fix input path * Add exception handler * Add exists clause * Fix base branch * Error does not have a required message * Add SDK subfolder * Conf user + fix reponse * Handle creation of issue * Improve comments * Improve error message * Add help * Improve message * Fix return type of build_sdk * Add rest endpoint * Take SDKID as query parameter * Add PR hook * Handle PR from a fork * Improve already existing PR * Improve do pr to return already existing one if there is * Fix conflict PR code * Add rebuild endpoint * Fix endpoint * Fix rebuild command * Add local log * Fix build sdk for Windows * Switch current to master * Fix build_libraries API after rebase against master * Adapt webhook code to skip_callback * Enabling SwaggerToSdk logs * Add PR close action * Add synchronize/pr event * Add Consume thread * Fix thread loop * Fix comment on close * Add Delivery in log * Plug auto-conf detection * Add labels to SDK PRs * Add sdkbase and make branch creation more robust
This commit is contained in:
Laurent Mazuel
GitHub
Родитель
bcd65625d4
Коммит
784a5d3d70
|
|
@ -0,0 +1 @@
|
|||
*.py text
|
||||
|
|
@ -41,7 +41,7 @@ ENV LC_ALL en_US.UTF-8
|
|||
COPY setup.py /tmp
|
||||
COPY swaggertosdk /tmp/swaggertosdk/
|
||||
WORKDIR /tmp
|
||||
RUN pip3.6 install .
|
||||
RUN pip3.6 install .[rest]
|
||||
|
||||
WORKDIR /git-restapi
|
||||
ENTRYPOINT ["python3.6", "-m", "swaggertosdk"]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
-e .
|
||||
-e .[rest]
|
||||
pytest-cov
|
||||
pytest>=3.2.0
|
||||
pylint
|
||||
|
|
|
|||
10
setup.py
10
setup.py
|
|
@ -24,5 +24,13 @@ setup(
|
|||
"requests",
|
||||
"mistune",
|
||||
"pyyaml",
|
||||
]
|
||||
"cookiecutter",
|
||||
"wheel"
|
||||
],
|
||||
extras_require={
|
||||
'rest': [
|
||||
'flask',
|
||||
'json-rpc'
|
||||
]
|
||||
}
|
||||
)
|
||||
|
|
@ -1,13 +1,14 @@
|
|||
import os
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from git import Repo
|
||||
from github import Github, GithubException, UnknownObjectException
|
||||
|
|
@ -116,9 +117,9 @@ def do_commit(repo, message_template, branch_name, hexsha):
|
|||
|
||||
checkout_and_create_branch(repo, branch_name)
|
||||
msg = message_template.format(hexsha=hexsha)
|
||||
repo.index.commit(msg)
|
||||
commit = repo.index.commit(msg)
|
||||
_LOGGER.info("Commit done: %s", msg)
|
||||
return True
|
||||
return commit.hexsha
|
||||
|
||||
|
||||
def sync_fork(gh_token, github_repo_id, repo):
|
||||
|
|
@ -242,7 +243,7 @@ def configure_user(gh_token, repo):
|
|||
git config --global user.name "Your Name"
|
||||
"""
|
||||
user = user_from_token(gh_token)
|
||||
repo.git.config('user.email', user.email or 'autorestci@microsoft.com')
|
||||
repo.git.config('user.email', user.email or 'aspysdk2@microsoft.com')
|
||||
repo.git.config('user.name', user.name or 'SwaggerToSDK Automation')
|
||||
|
||||
|
||||
|
|
@ -258,7 +259,7 @@ def compute_branch_name(branch_name, gh_token=None):
|
|||
return DEFAULT_TRAVIS_BRANCH_NAME.format(branch=os.environ['TRAVIS_BRANCH'])
|
||||
return DEFAULT_TRAVIS_PR_BRANCH_NAME.format(number=pr_object.number)
|
||||
|
||||
def do_pr(gh_token, sdk_git_id, sdk_pr_target_repo_id, branch_name, base_branch):
|
||||
def do_pr(gh_token, sdk_git_id, sdk_pr_target_repo_id, branch_name, base_branch, pr_body=""):
|
||||
"Do the PR"
|
||||
if not gh_token:
|
||||
_LOGGER.info('Skipping the PR, no token found')
|
||||
|
|
@ -276,31 +277,30 @@ def do_pr(gh_token, sdk_git_id, sdk_pr_target_repo_id, branch_name, base_branch)
|
|||
head_name = "{}:{}".format(sdk_git_owner, branch_name)
|
||||
else:
|
||||
head_name = branch_name
|
||||
sdk_git_repo = github_con.get_repo(sdk_git_id)
|
||||
sdk_git_owner = sdk_git_repo.owner.login
|
||||
|
||||
body = ''
|
||||
rest_api_pr = get_initial_pr(gh_token)
|
||||
if rest_api_pr:
|
||||
body += "Generated from RestAPI PR: {}".format(rest_api_pr.html_url)
|
||||
try:
|
||||
github_pr = sdk_pr_target_repo.create_pull(
|
||||
title='Automatic PR from {}'.format(branch_name),
|
||||
body=body,
|
||||
body=pr_body,
|
||||
head=head_name,
|
||||
base=base_branch
|
||||
)
|
||||
except GithubException as err:
|
||||
if err.status == 422 and err.data['errors'][0]['message'].startswith('A pull request already exists'):
|
||||
_LOGGER.info('PR already exists, it was a commit on an open PR')
|
||||
return
|
||||
if err.status == 422 and err.data['errors'][0].get('message', '').startswith('A pull request already exists'):
|
||||
matching_pulls = sdk_pr_target_repo.get_pulls(base=base_branch, head=sdk_git_owner+":"+head_name)
|
||||
matching_pull = matching_pulls[0]
|
||||
_LOGGER.info('PR already exists: '+matching_pull.html_url)
|
||||
return matching_pull
|
||||
raise
|
||||
_LOGGER.info("Made PR %s", github_pr.html_url)
|
||||
comment = compute_pr_comment_with_sdk_pr(github_pr.html_url, sdk_git_id, branch_name)
|
||||
add_comment_to_initial_pr(gh_token, comment)
|
||||
return github_pr
|
||||
|
||||
|
||||
def get_swagger_hexsha(restapi_git_folder):
|
||||
"""Get the SHA1 of the current repo"""
|
||||
repo = Repo(restapi_git_folder)
|
||||
repo = Repo(str(restapi_git_folder))
|
||||
if repo.bare:
|
||||
not_git_hexsha = "notgitrepo"
|
||||
_LOGGER.warning("Not a git repo, SHA1 used will be: %s", not_git_hexsha)
|
||||
|
|
@ -330,9 +330,16 @@ def add_comment_to_initial_pr(gh_token, comment):
|
|||
return True
|
||||
|
||||
|
||||
def clone_to_path(gh_token, temp_dir, sdk_git_id):
|
||||
"""Clone the given repo_id to the 'sdk' folder in given temp_dir"""
|
||||
def clone_to_path(gh_token, temp_dir, sdk_git_id, branch=None):
|
||||
"""Clone the given repo_id to the temp_dir folder.
|
||||
|
||||
:param str branch: If specified, switch to this branch. Branch must exist.
|
||||
"""
|
||||
_LOGGER.info("Clone SDK repository %s", sdk_git_id)
|
||||
url_parsing = urlparse(sdk_git_id)
|
||||
sdk_git_id = url_parsing.path
|
||||
if sdk_git_id.startswith("/"):
|
||||
sdk_git_id = sdk_git_id[1:]
|
||||
|
||||
credentials_part = ''
|
||||
if gh_token:
|
||||
|
|
@ -348,11 +355,14 @@ def clone_to_path(gh_token, temp_dir, sdk_git_id):
|
|||
credentials=credentials_part,
|
||||
sdk_git_id=sdk_git_id
|
||||
)
|
||||
sdk_path = os.path.join(temp_dir, 'sdk')
|
||||
Repo.clone_from(https_authenticated_url, sdk_path)
|
||||
_LOGGER.info("Clone success")
|
||||
repo = Repo.clone_from(https_authenticated_url, str(temp_dir))
|
||||
# Do NOT clone and set branch at the same time, since we allow branch to be a SHA1
|
||||
# And you can't clone a SHA1
|
||||
if branch:
|
||||
_LOGGER.info("Checkout branch %s", branch)
|
||||
repo.git.checkout(branch)
|
||||
|
||||
return sdk_path
|
||||
_LOGGER.info("Clone success")
|
||||
|
||||
def remove_readonly(func, path, _):
|
||||
"Clear the readonly bit and reattempt the removal"
|
||||
|
|
@ -360,17 +370,23 @@ def remove_readonly(func, path, _):
|
|||
func(path)
|
||||
|
||||
@contextmanager
|
||||
def manage_sdk_folder(gh_token, temp_dir, sdk_git_id):
|
||||
def manage_git_folder(gh_token, temp_dir, git_id):
|
||||
"""Context manager to avoid readonly problem while cleanup the temp dir"""
|
||||
sdk_path = clone_to_path(gh_token, temp_dir, sdk_git_id)
|
||||
_LOGGER.debug("SDK path %s", sdk_path)
|
||||
_LOGGER.debug("Git ID %s", git_id)
|
||||
if Path(git_id).exists():
|
||||
yield git_id
|
||||
return None # Do not erase a local folder, just skip here
|
||||
|
||||
# Clone the specific branch
|
||||
split_git_id = git_id.split("@")
|
||||
branch = split_git_id[1] if len(split_git_id) > 1 else None
|
||||
clone_to_path(gh_token, temp_dir, split_git_id[0], branch=branch)
|
||||
try:
|
||||
yield sdk_path
|
||||
yield temp_dir
|
||||
# Pre-cleanup for Windows http://bugs.python.org/issue26660
|
||||
finally:
|
||||
_LOGGER.debug("Preclean SDK folder")
|
||||
shutil.rmtree(sdk_path, onerror=remove_readonly)
|
||||
|
||||
_LOGGER.debug("Preclean Rest folder")
|
||||
shutil.rmtree(temp_dir, onerror=remove_readonly)
|
||||
|
||||
def get_full_sdk_id(gh_token, sdk_git_id):
|
||||
"""If the SDK git id is incomplete, try to complete it with user login"""
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import os
|
|||
import logging
|
||||
import tempfile
|
||||
from git import Repo, GitCommandError
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from .SwaggerToSdkCore import (
|
||||
IS_TRAVIS,
|
||||
|
|
@ -12,7 +14,7 @@ from .SwaggerToSdkCore import (
|
|||
DEFAULT_TRAVIS_BRANCH_NAME,
|
||||
DEFAULT_TRAVIS_PR_BRANCH_NAME,
|
||||
get_full_sdk_id,
|
||||
manage_sdk_folder,
|
||||
manage_git_folder,
|
||||
compute_branch_name,
|
||||
configure_user,
|
||||
sync_fork,
|
||||
|
|
@ -22,6 +24,7 @@ from .SwaggerToSdkCore import (
|
|||
do_commit,
|
||||
do_pr,
|
||||
add_comment_to_initial_pr,
|
||||
compute_pr_comment_with_sdk_pr,
|
||||
get_swagger_project_files_in_pr,
|
||||
get_commit_object_from_travis,
|
||||
extract_conf_from_readmes,
|
||||
|
|
@ -34,16 +37,17 @@ from .autorest_tools import (
|
|||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_sdk(gh_token, config_path, project_pattern, restapi_git_folder,
|
||||
def generate_sdk(gh_token, config_path, project_pattern, restapi_git_id,
|
||||
sdk_git_id, pr_repo_id, message_template, base_branch_name, branch_name,
|
||||
autorest_bin=None):
|
||||
"""Main method of the the file"""
|
||||
sdk_git_id = get_full_sdk_id(gh_token, sdk_git_id)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_sdk_folder(gh_token, temp_dir, sdk_git_id) as sdk_folder:
|
||||
manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), restapi_git_id) as restapi_git_folder, \
|
||||
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), sdk_git_id) as sdk_folder:
|
||||
|
||||
sdk_repo = Repo(sdk_folder)
|
||||
sdk_repo = Repo(str(sdk_folder))
|
||||
if gh_token:
|
||||
branch_name = compute_branch_name(branch_name, gh_token)
|
||||
|
||||
|
|
@ -104,7 +108,10 @@ def generate_sdk(gh_token, config_path, project_pattern, restapi_git_folder,
|
|||
if do_commit(sdk_repo, message_template, branch_name, hexsha):
|
||||
sdk_repo.git.push('origin', branch_name, set_upstream=True)
|
||||
if pr_repo_id:
|
||||
do_pr(gh_token, sdk_git_id, pr_repo_id, branch_name, base_branch_name)
|
||||
pr_body = "Generated from PR: {}".format(initial_pr.html_url)
|
||||
github_pr = do_pr(gh_token, sdk_git_id, pr_repo_id, branch_name, base_branch_name, pr_body)
|
||||
comment = compute_pr_comment_with_sdk_pr(github_pr.html_url, sdk_git_id, branch_name)
|
||||
add_comment_to_initial_pr(gh_token, comment)
|
||||
else:
|
||||
add_comment_to_initial_pr(gh_token, "No modification for {}".format(sdk_git_id))
|
||||
else:
|
||||
|
|
@ -113,8 +120,29 @@ def generate_sdk(gh_token, config_path, project_pattern, restapi_git_folder,
|
|||
_LOGGER.info("Build SDK finished and cleaned")
|
||||
|
||||
|
||||
def main():
|
||||
def main(argv):
|
||||
"""Main method"""
|
||||
|
||||
if 'GH_TOKEN' not in os.environ:
|
||||
gh_token = None
|
||||
else:
|
||||
gh_token = os.environ['GH_TOKEN']
|
||||
|
||||
if "--rest-server" in argv:
|
||||
from .restapi import app
|
||||
log_level = logging.WARNING
|
||||
if "-v" in argv or "--verbose" in argv:
|
||||
log_level = logging.INFO
|
||||
if "--debug" in argv:
|
||||
log_level = logging.DEBUG
|
||||
|
||||
main_logger = logging.getLogger()
|
||||
logging.basicConfig()
|
||||
main_logger.setLevel(log_level)
|
||||
|
||||
app.run(debug=log_level == logging.DEBUG, host='0.0.0.0')
|
||||
sys.exit(0)
|
||||
|
||||
epilog = "\n".join([
|
||||
'The script activates this additional behaviour if Travis is detected:',
|
||||
' --branch is setted by default to "{}" if triggered by a PR, "{}" otherwise'.format(
|
||||
|
|
@ -165,12 +193,7 @@ def main():
|
|||
'Otherwise, you can use the syntax username/repoid')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if 'GH_TOKEN' not in os.environ:
|
||||
gh_token = None
|
||||
else:
|
||||
gh_token = os.environ['GH_TOKEN']
|
||||
|
||||
|
||||
main_logger = logging.getLogger()
|
||||
if args.verbose or args.debug:
|
||||
logging.basicConfig()
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
from .SwaggerToSdkMain import main
|
||||
main()
|
||||
import sys
|
||||
from swaggertosdk.SwaggerToSdkMain import main
|
||||
main(sys.argv)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,173 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
# --------------------------------------------------------------------------------------------
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from cookiecutter.main import cookiecutter
|
||||
from swaggertosdk.SwaggerToSdkNewCLI import generate_code
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
def guess_service_info_from_path(spec_path):
|
||||
"""Guess Python Autorest options based on the spec path.
|
||||
|
||||
Expected path:
|
||||
specification/compute/resource-manager/readme.md
|
||||
"""
|
||||
spec_path = spec_path.lower()
|
||||
spec_path = spec_path[spec_path.index("specification"):] # Might raise and it's ok
|
||||
split_spec_path = spec_path.split("/") if "/" in spec_path else spec_path.split("\\")
|
||||
|
||||
rp_name = split_spec_path[1]
|
||||
is_arm = split_spec_path[2] == "resource-manager"
|
||||
|
||||
return {
|
||||
"rp_name": rp_name,
|
||||
"is_arm": is_arm
|
||||
}
|
||||
|
||||
def get_data(spec_path):
|
||||
if spec_path.startswith("http"):
|
||||
response = requests.get(spec_path)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
else:
|
||||
with open(spec_path, "r") as fd:
|
||||
return fd.read()
|
||||
|
||||
def guess_service_info_from_content(spec_path):
|
||||
data = get_data(spec_path)
|
||||
|
||||
lines = data.splitlines()
|
||||
while lines:
|
||||
line = lines.pop(0)
|
||||
if line.startswith("# "):
|
||||
pretty_name = line[2:]
|
||||
break
|
||||
else:
|
||||
raise ValueError("Unable to find main title in this README")
|
||||
return {
|
||||
'pretty_name': pretty_name
|
||||
}
|
||||
|
||||
def guess_service_info(spec_path):
|
||||
result = guess_service_info_from_content(spec_path)
|
||||
result.update(guess_service_info_from_path(spec_path))
|
||||
return result
|
||||
|
||||
def create_swagger_to_sdk_conf(spec_path, service_info, autorest_options):
|
||||
type_str = ".mgmt" if service_info["is_arm"] else ".data"
|
||||
return {
|
||||
"{}{}".format(service_info["rp_name"], type_str): {
|
||||
"markdown": spec_path[spec_path.index("specification"):],
|
||||
"autorest_options": {
|
||||
"namespace": autorest_options["namespace"],
|
||||
"package-version": autorest_options["package-version"],
|
||||
"package-name": autorest_options["package-name"]
|
||||
},
|
||||
"output_dir": "{}/{}".format(autorest_options["package-name"], autorest_options["package-name"].replace("-","/")),
|
||||
"build_dir": autorest_options["package-name"]
|
||||
}
|
||||
}
|
||||
|
||||
def create_package_service_mapping(spec_path, service_info, autorest_options):
|
||||
type_str = "Management" if service_info["is_arm"] else "Client"
|
||||
return {
|
||||
autorest_options["package-name"]: {
|
||||
"service_name": service_info["pretty_name"],
|
||||
"category": type_str,
|
||||
"namespaces": [
|
||||
autorest_options["namespace"]
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
def generate(spec_path, cwd='.'):
|
||||
service_info = guess_service_info(spec_path)
|
||||
|
||||
autorest_options = {}
|
||||
if service_info["is_arm"]:
|
||||
autorest_options["azure-arm"] = True
|
||||
namespace = "azure.mgmt." + service_info["rp_name"]
|
||||
package_name = "azure-mgmt-" + service_info["rp_name"]
|
||||
else:
|
||||
autorest_options["add-credentials"] = True
|
||||
namespace = "azure." + service_info["rp_name"]
|
||||
package_name = "azure-" + service_info["rp_name"]
|
||||
|
||||
# Create output folder
|
||||
output_dir = Path(cwd).resolve() / Path(package_name)
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
_LOGGER.info("Output folder: %s", output_dir)
|
||||
|
||||
_LOGGER.info("Calling Autorest")
|
||||
autorest_options.update({
|
||||
"use": "@microsoft.azure/autorest.python@preview",
|
||||
"license-header": "MICROSOFT_MIT_NO_VERSION",
|
||||
"payload-flattening-threshold": 2,
|
||||
"python": "",
|
||||
"package-version": "0.1.0",
|
||||
"python.output-folder": output_dir
|
||||
})
|
||||
|
||||
autorest_options["namespace"] = namespace
|
||||
autorest_options["package-name"] = package_name
|
||||
|
||||
generate_code(
|
||||
input_file=Path(spec_path) if not spec_path.startswith("http") else spec_path,
|
||||
output_dir=output_dir,
|
||||
global_conf={"autorest_options": autorest_options},
|
||||
local_conf={}
|
||||
)
|
||||
|
||||
_LOGGER.info("Rebuilding packaging with Cookiecutter")
|
||||
pretty_name = service_info["pretty_name"]
|
||||
cookiecutter('gh:Azure/cookiecutter-azuresdk-pypackage',
|
||||
no_input=True,
|
||||
extra_context={
|
||||
'package_name': package_name,
|
||||
'package_pprint_name': pretty_name
|
||||
},
|
||||
overwrite_if_exists=True,
|
||||
output_dir=cwd
|
||||
)
|
||||
|
||||
entry = create_swagger_to_sdk_conf(spec_path, service_info, autorest_options)
|
||||
swagger_to_sdk = Path(cwd) / Path("swagger_to_sdk_config.json")
|
||||
if swagger_to_sdk.exists():
|
||||
_LOGGER.info("Updating swagger_to_sdk_config.json")
|
||||
with swagger_to_sdk.open() as fd:
|
||||
data_conf = json.load(fd)
|
||||
data_conf["projects"].update(entry)
|
||||
with swagger_to_sdk.open("w") as fd:
|
||||
json.dump(data_conf, fd, indent=2, sort_keys=True)
|
||||
|
||||
package_service_mapping = Path(cwd) / Path("package_service_mapping.json")
|
||||
if package_service_mapping.exists():
|
||||
_LOGGER.info("Updating package_service_mapping.json")
|
||||
package_service_mapping_entry = create_package_service_mapping(spec_path, service_info, autorest_options)
|
||||
with package_service_mapping.open() as fd:
|
||||
data_conf = json.load(fd)
|
||||
data_conf.update(package_service_mapping_entry)
|
||||
with package_service_mapping.open("w") as fd:
|
||||
json.dump(data_conf, fd, indent=2, sort_keys=True)
|
||||
|
||||
_LOGGER.info("Done! Enjoy your Python SDK!!")
|
||||
return entry
|
||||
|
||||
def main(spec_path):
|
||||
generate(spec_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig()
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
main(sys.argv[1])
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
from flask import Flask
|
||||
from jsonrpc.backend.flask import api
|
||||
|
||||
from ..SwaggerToSdkMain import generate_sdk
|
||||
from ..SwaggerToSdkCore import CONFIG_FILE, DEFAULT_COMMIT_MESSAGE
|
||||
|
||||
app = Flask(__name__)
|
||||
app.add_url_rule('/', 'api', api.as_view(), methods=['POST'])
|
||||
|
||||
@api.dispatcher.add_method
|
||||
def ping(*args, **kwargs):
|
||||
return "Pong!"
|
||||
|
||||
@api.dispatcher.add_method
|
||||
def generate_project(*args, **kwargs):
|
||||
# Get required parameter
|
||||
rest_api_id = kwargs['rest_api_id']
|
||||
sdk_id = kwargs['sdk_id']
|
||||
project = kwargs['project']
|
||||
|
||||
generate_sdk(
|
||||
os.environ['GH_TOKEN'],
|
||||
CONFIG_FILE,
|
||||
project,
|
||||
rest_api_id,
|
||||
sdk_id,
|
||||
None, # No PR repo id
|
||||
DEFAULT_COMMIT_MESSAGE,
|
||||
'master',
|
||||
None # Destination branch
|
||||
)
|
||||
|
||||
from .github import *
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
from swaggertosdk.restapi import app
|
||||
app.run(debug=True)
|
||||
|
|
@ -0,0 +1,350 @@
|
|||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from functools import wraps, lru_cache
|
||||
import logging
|
||||
from queue import Queue
|
||||
import re
|
||||
import traceback
|
||||
from threading import Thread
|
||||
|
||||
from flask import request, jsonify
|
||||
|
||||
from github import Github, GithubException, UnknownObjectException
|
||||
|
||||
import hmac, hashlib
|
||||
import os
|
||||
|
||||
from .github_handler import (
|
||||
build_from_issue_comment,
|
||||
build_from_issues,
|
||||
GithubHandler as LocalHandler,
|
||||
generate_sdk_from_commit
|
||||
)
|
||||
from . import app
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
_QUEUE = Queue(64)
|
||||
|
||||
|
||||
# Webhook secreet to authenticate message (bytes)
|
||||
SECRET = b'mydeepsecret'
|
||||
|
||||
_HMAC_CHECK = False
|
||||
|
||||
def check_hmac(request, secret):
|
||||
data = request.get_data()
|
||||
hmac_tester = hmac.HMAC(b'mydeepsecret', data, hashlib.sha1)
|
||||
if not 'X-Hub-Signature' in request.headers:
|
||||
raise ValueError('X-Hub-Signature is mandatory on this WebService')
|
||||
if request.headers['X-Hub-Signature'] == 'sha1='+hmac_tester.hexdigest():
|
||||
return True
|
||||
raise ValueError('Bad X-Hub-Signature signature')
|
||||
|
||||
@app.route('/github', methods=['POST'])
|
||||
def notify():
|
||||
"""Github main endpoint."""
|
||||
github_index = {
|
||||
'ping': ping,
|
||||
'issue_comment': issue_comment,
|
||||
'issues': issues
|
||||
}
|
||||
return handle_github_webhook(
|
||||
github_index,
|
||||
request.headers['X-GitHub-Event'],
|
||||
request.get_json()
|
||||
)
|
||||
|
||||
@app.route('/github/rest', methods=['POST'])
|
||||
def rest_notify():
|
||||
"""Github rest endpoint."""
|
||||
github_index = {
|
||||
'ping': ping,
|
||||
'push': push,
|
||||
'pull_request': rest_pull_request
|
||||
}
|
||||
return handle_github_webhook(
|
||||
github_index,
|
||||
request.headers['X-GitHub-Event'],
|
||||
request.get_json()
|
||||
)
|
||||
|
||||
def handle_github_webhook(github_index, gh_event_type, json_body):
|
||||
if _HMAC_CHECK:
|
||||
check_hmac(request, SECRET)
|
||||
_LOGGER.info("Received Webhook %s", request.headers.get("X-GitHub-Delivery"))
|
||||
|
||||
json_answer = notify_github(github_index, gh_event_type, json_body)
|
||||
return jsonify(json_answer)
|
||||
|
||||
@lru_cache()
|
||||
def robot_name():
|
||||
github_con = Github(os.environ["GH_TOKEN"])
|
||||
return github_con.get_user().login
|
||||
|
||||
def notify_github(github_index, event_type, json_body):
|
||||
if json_body['sender']['login'].lower() == robot_name().lower():
|
||||
return {'message': 'I don\'t talk to myself, I\'m not schizo'}
|
||||
if event_type in github_index:
|
||||
return github_index[event_type](json_body)
|
||||
return {'message': 'Not handled currently'}
|
||||
|
||||
def ping(body):
|
||||
return {'message': 'Moi aussi zen beaucoup'}
|
||||
|
||||
def issue_comment(body):
|
||||
if body["action"] in ["created", "edited"]:
|
||||
webhook_data = build_from_issue_comment(body)
|
||||
response = manage_comment(webhook_data)
|
||||
if response:
|
||||
return response
|
||||
return {'message': 'Nothing for me'}
|
||||
|
||||
def issues(body):
|
||||
if body["action"] in ["opened"]:
|
||||
webhook_data = build_from_issues(body)
|
||||
response = manage_comment(webhook_data)
|
||||
if response:
|
||||
return response
|
||||
return {'message': 'Nothing for me'}
|
||||
|
||||
def manage_comment(webhook_data):
|
||||
handler = LocalHandler()
|
||||
|
||||
# Is someone talking to me:
|
||||
message = re.search("@{} (.*)".format(robot_name()), webhook_data.text, re.I)
|
||||
if message:
|
||||
command = message.group(1)
|
||||
try:
|
||||
response = handler.act_and_response(webhook_data, command)
|
||||
except Exception as err:
|
||||
response = traceback.format_exc()
|
||||
if response:
|
||||
return {'message': response}
|
||||
|
||||
def push(body):
|
||||
sdkid = request.args.get("sdkid")
|
||||
if not sdkid:
|
||||
return {'message': 'sdkid is a required query parameter'}
|
||||
sdkbase = request.args.get("sdkbase", "master")
|
||||
|
||||
rest_api_branch_name = body["ref"][len("refs/heads/"):]
|
||||
if rest_api_branch_name == "master":
|
||||
return {'message': 'Webhook disabled for RestAPI master'}
|
||||
|
||||
gh_token = os.environ["GH_TOKEN"]
|
||||
github_con = Github(gh_token)
|
||||
restapi_git_id = body['repository']['full_name']
|
||||
repo = github_con.get_repo(restapi_git_id)
|
||||
|
||||
commit_obj = repo.get_commit(body["after"])
|
||||
generate_sdk_from_commit(
|
||||
commit_obj,
|
||||
"restapi_auto_"+rest_api_branch_name,
|
||||
restapi_git_id,
|
||||
sdkid,
|
||||
None, # I don't know if the origin branch comes from "master", assume it.
|
||||
sdkbase
|
||||
)
|
||||
return {'message': 'No return for this endpoint'}
|
||||
|
||||
def rest_pull_request(body):
|
||||
sdkid = request.args.get("sdkid")
|
||||
if not sdkid:
|
||||
return {'message': 'sdkid is a required query parameter'}
|
||||
sdkbase = request.args.get("sdkbase", "master")
|
||||
|
||||
_LOGGER.info("Received PR action %s", body["action"])
|
||||
_QUEUE.put((body, sdkid, sdkbase))
|
||||
_LOGGER.info("Received action has been queued. Queue size: %d", _QUEUE.qsize())
|
||||
|
||||
return {'message': 'Current queue size: {}'.format(_QUEUE.qsize())}
|
||||
|
||||
def rest_handle_action(body, sdkid, sdkbase):
|
||||
"""First method in the thread.
|
||||
"""
|
||||
_LOGGER.info("Rest handle action")
|
||||
gh_token = os.environ["GH_TOKEN"]
|
||||
github_con = Github(gh_token)
|
||||
|
||||
sdk_pr_target_repo = github_con.get_repo(sdkid)
|
||||
|
||||
restapi_git_id = body['repository']['full_name']
|
||||
restapi_repo = github_con.get_repo(restapi_git_id)
|
||||
|
||||
_LOGGER.info("Received PR action %s", body["action"])
|
||||
if body["action"] in ["opened", "reopened"]:
|
||||
return rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase)
|
||||
if body["action"] == "closed":
|
||||
return rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase)
|
||||
if body["action"] == "synchronize": # push to a PR from a fork
|
||||
return rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase)
|
||||
|
||||
def rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master"):
|
||||
|
||||
rest_basebranch = body["pull_request"]["base"]["ref"]
|
||||
dest_branch = body["pull_request"]["head"]["ref"]
|
||||
origin_repo = body["pull_request"]["head"]["repo"]["full_name"]
|
||||
|
||||
if rest_basebranch == "master":
|
||||
sdk_base = sdk_default_base
|
||||
sdk_checkout_base = None
|
||||
else:
|
||||
sdk_base = "restapi_auto_" + rest_basebranch
|
||||
sdk_checkout_base = sdk_base
|
||||
|
||||
rest_pr = restapi_repo.get_pull(body["number"])
|
||||
|
||||
if origin_repo != restapi_repo.full_name:
|
||||
_LOGGER.info("This comes from a fork, I need generation first, since targetted branch does not exist")
|
||||
fork_repo = github_con.get_repo(origin_repo)
|
||||
fork_owner = fork_repo.owner.login
|
||||
commit_obj = fork_repo.get_commit(body["pull_request"]["head"]["sha"])
|
||||
subbranch_name_part = fork_owner+"_"+dest_branch
|
||||
sdk_dest_branch = "restapi_auto_" + subbranch_name_part
|
||||
generate_sdk_from_commit(
|
||||
commit_obj,
|
||||
sdk_dest_branch,
|
||||
origin_repo,
|
||||
sdk_pr_target_repo.full_name,
|
||||
sdk_checkout_base,
|
||||
sdk_default_base
|
||||
)
|
||||
else:
|
||||
sdk_dest_branch = "restapi_auto_" + dest_branch
|
||||
|
||||
try:
|
||||
github_pr = sdk_pr_target_repo.create_pull(
|
||||
title='Automatic PR of {} into {}'.format(sdk_dest_branch, sdk_base),
|
||||
body="Created to sync {}".format(rest_pr.html_url),
|
||||
head=sdk_dest_branch,
|
||||
base=sdk_base
|
||||
)
|
||||
sdk_pr_as_issue = sdk_pr_target_repo.get_issue(github_pr.number)
|
||||
sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress))
|
||||
except GithubException as err:
|
||||
if err.status == 422 and err.data['errors'][0].get('message', '').startswith('A pull request already exists'):
|
||||
_LOGGER.info('PR already exists, it was a commit on an open PR')
|
||||
sdk_pr = list(sdk_pr_target_repo.get_pulls(
|
||||
head=sdk_pr_target_repo.owner.login+":"+sdk_dest_branch,
|
||||
base=sdk_base
|
||||
))[0]
|
||||
sdk_pr_as_issue = sdk_pr_target_repo.get_issue(sdk_pr.number)
|
||||
sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress))
|
||||
safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.refused))
|
||||
safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.merged))
|
||||
return {'message': 'PR already exists'}
|
||||
else:
|
||||
return {'message': err.data}
|
||||
except Exception as err:
|
||||
response = traceback.format_exc()
|
||||
return {'message': response}
|
||||
|
||||
|
||||
class SwaggerToSdkLabels(Enum):
|
||||
merged = "RestPRMerged", "0e8a16"
|
||||
refused = "RestPRRefused", "b60205"
|
||||
in_progress = "RestPRInProgress", "fbca04"
|
||||
|
||||
def get_or_create_label(sdk_pr_target_repo, label_enum):
|
||||
try:
|
||||
return sdk_pr_target_repo.get_label(label_enum.value[0])
|
||||
except UnknownObjectException:
|
||||
return sdk_pr_target_repo.create_label(*label_enum.value)
|
||||
|
||||
def safe_remove_label(issue, label):
|
||||
"""Remove a label, does not fail if label was not there.
|
||||
"""
|
||||
try:
|
||||
issue.remove_from_labels(label)
|
||||
except GithubException:
|
||||
pass
|
||||
|
||||
def rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master"):
|
||||
_LOGGER.info("Received a PR closed event")
|
||||
sdkid = sdk_pr_target_repo.full_name
|
||||
rest_pr = restapi_repo.get_pull(body["number"])
|
||||
|
||||
# What was "head" name
|
||||
origin_repo = body["pull_request"]["head"]["repo"]["full_name"]
|
||||
dest_branch = body["pull_request"]["head"]["ref"]
|
||||
if origin_repo != restapi_repo.full_name: # This PR comes from a fork
|
||||
fork_repo = github_con.get_repo(origin_repo)
|
||||
fork_owner = fork_repo.owner.login
|
||||
subbranch_name_part = fork_owner+"_"+dest_branch
|
||||
sdk_dest_branch = "restapi_auto_" + subbranch_name_part
|
||||
else:
|
||||
sdk_dest_branch = "restapi_auto_" + dest_branch
|
||||
_LOGGER.info("SDK head branch should be %s", sdk_dest_branch)
|
||||
full_head = sdk_pr_target_repo.owner.login+":"+sdk_dest_branch
|
||||
_LOGGER.info("Will filter with %s", full_head)
|
||||
|
||||
# What was "base"
|
||||
rest_basebranch = body["pull_request"]["base"]["ref"]
|
||||
sdk_base = sdk_default_base if rest_basebranch == "master" else "restapi_auto_" + rest_basebranch
|
||||
_LOGGER.info("SDK base branch should be %s", sdk_base)
|
||||
|
||||
sdk_prs = list(sdk_pr_target_repo.get_pulls(
|
||||
head=full_head,
|
||||
base=sdk_base
|
||||
))
|
||||
if not sdk_prs:
|
||||
rest_pr.create_issue_comment("Was unable to find SDK {} PR for this closed PR.".format(sdkid))
|
||||
elif len(sdk_prs) == 1:
|
||||
sdk_pr = sdk_prs[0]
|
||||
sdk_pr_as_issue = sdk_pr_target_repo.get_issue(sdk_pr.number)
|
||||
safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress))
|
||||
try:
|
||||
if body["pull_request"]["merged"]:
|
||||
sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.merged))
|
||||
else:
|
||||
sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.refused))
|
||||
except GithubException:
|
||||
sdk_pr.create_issue_comment("Cannot set labels. Initial PR has been closed with merged status: {}".format(body["pull_request"]["merged"]))
|
||||
else:
|
||||
# Should be impossible, create_pull would have sent a 422
|
||||
pr_list = "\n".join(["- {}".format(pr.html_url) for pr in sdk_prs])
|
||||
rest_pr.create_issue_comment("We found several SDK {} PRs and didn't notify closing event.\n{}".format(sdkid, pr_list))
|
||||
|
||||
def rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master"):
|
||||
|
||||
if body["before"] == body["after"]:
|
||||
return {'message': 'No commit id change'}
|
||||
|
||||
# What was "head" name
|
||||
origin_repo = body["pull_request"]["head"]["repo"]["full_name"]
|
||||
|
||||
if origin_repo == restapi_repo.full_name:
|
||||
_LOGGER.info("This will be handled by 'push' event on the branch")
|
||||
|
||||
dest_branch = body["pull_request"]["head"]["ref"]
|
||||
fork_repo = github_con.get_repo(origin_repo)
|
||||
fork_owner = fork_repo.owner.login
|
||||
commit_obj = fork_repo.get_commit(body["pull_request"]["head"]["sha"])
|
||||
subbranch_name_part = fork_owner+"_"+dest_branch
|
||||
generate_sdk_from_commit(
|
||||
commit_obj,
|
||||
"restapi_auto_"+subbranch_name_part,
|
||||
origin_repo,
|
||||
sdk_pr_target_repo.full_name,
|
||||
None, # I don't know if the origin branch comes from "master", assume it.
|
||||
sdk_default_base
|
||||
)
|
||||
|
||||
return {'message': 'No return for this endpoint'}
|
||||
|
||||
def consume():
|
||||
"""Consume action and block if there is not.
|
||||
"""
|
||||
while True:
|
||||
body, sdkid, sdkbase = _QUEUE.get()
|
||||
try:
|
||||
rest_handle_action(body, sdkid, sdkbase)
|
||||
except Exception as err:
|
||||
_LOGGER.critical("Worked thread issue:\n%s", traceback.format_exc())
|
||||
_LOGGER.info("End of WorkerThread")
|
||||
|
||||
_WORKER_THREAD = Thread(
|
||||
target=consume,
|
||||
name="WorkerThread"
|
||||
)
|
||||
_WORKER_THREAD.start()
|
||||
|
|
@ -0,0 +1,276 @@
|
|||
from collections import namedtuple
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import traceback
|
||||
|
||||
from github import Github
|
||||
from git import Repo, GitCommandError
|
||||
|
||||
from swaggertosdk.build_sdk import generate as build_sdk
|
||||
from swaggertosdk.SwaggerToSdkCore import (
|
||||
manage_git_folder,
|
||||
checkout_and_create_branch,
|
||||
do_commit,
|
||||
do_pr,
|
||||
configure_user,
|
||||
CONFIG_FILE,
|
||||
read_config,
|
||||
DEFAULT_COMMIT_MESSAGE,
|
||||
get_input_paths,
|
||||
extract_conf_from_readmes,
|
||||
checkout_and_create_branch
|
||||
)
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
WebhookMetadata = namedtuple(
|
||||
'WebhookMetadata',
|
||||
['repo', 'issue', 'text']
|
||||
)
|
||||
|
||||
def build_from_issue_comment(body):
|
||||
gh_token = os.environ["GH_TOKEN"]
|
||||
github_con = Github(gh_token)
|
||||
repo = github_con.get_repo(body['repository']['full_name'])
|
||||
issue = repo.get_issue(body['issue']['number'])
|
||||
text = body['comment']['body']
|
||||
return WebhookMetadata(repo, issue, text)
|
||||
|
||||
def build_from_issues(body):
|
||||
gh_token = os.environ["GH_TOKEN"]
|
||||
github_con = Github(gh_token)
|
||||
repo = github_con.get_repo(body['repository']['full_name'])
|
||||
issue = repo.get_issue(body['issue']['number'])
|
||||
text = body['issue']['body']
|
||||
return WebhookMetadata(repo, issue, text)
|
||||
|
||||
class GithubHandler:
|
||||
def __init__(self):
|
||||
self.gh_token = os.environ["GH_TOKEN"]
|
||||
|
||||
def act_and_response(self, webhook_data, command):
|
||||
issue = webhook_data.issue
|
||||
|
||||
try:
|
||||
response = self.comment_command(issue, command)
|
||||
except Exception as err:
|
||||
response = "Something's wrong:\n```python\n{}\n```\n".format(traceback.format_exc())
|
||||
|
||||
new_comment = issue.create_comment(response)
|
||||
return 'Posted: {}'.format(new_comment.html_url)
|
||||
|
||||
def comment_command(self, issue, text):
|
||||
split_text = text.lower().split()
|
||||
if split_text[0] == "generate":
|
||||
return self.generate(issue, split_text[1])
|
||||
elif split_text[0] == "rebuild":
|
||||
return self.rebuild(issue, split_text[1])
|
||||
elif split_text[0] == "help":
|
||||
return self.help(issue)
|
||||
else:
|
||||
return "I didn't understand your command:\n```bash\n{}\n```\nin this context, sorry :(".format(text)
|
||||
|
||||
def help(self, issue):
|
||||
message = """This is what I can do:
|
||||
- `help` : this help message
|
||||
- `generate <raw github path to a readme>` : create a PR for this README
|
||||
"""
|
||||
new_comment = issue.create_comment(message)
|
||||
|
||||
def generate(self, issue, readme_parameter):
|
||||
# Do a start comment
|
||||
new_comment = issue.create_comment("Working on generating this for you!!!")
|
||||
|
||||
# Clone SDK repo
|
||||
sdk_git_id = issue.repository.full_name
|
||||
pr_repo_id = sdk_git_id
|
||||
base_branch_name = "master"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(self.gh_token, temp_dir + "/sdk", sdk_git_id) as sdk_folder:
|
||||
|
||||
sdk_conf = build_sdk(readme_parameter, sdk_folder)
|
||||
branch_name = list(sdk_conf.keys()).pop()
|
||||
|
||||
new_comment.edit("Generated! Let's see if there is something to PR.")
|
||||
|
||||
sdk_repo = Repo(str(sdk_folder))
|
||||
configure_user(self.gh_token, sdk_repo)
|
||||
modification = do_commit(
|
||||
sdk_repo,
|
||||
"Generated from {}".format(issue.html_url),
|
||||
branch_name,
|
||||
""
|
||||
)
|
||||
new_comment.delete()
|
||||
if modification:
|
||||
sdk_repo.git.push('origin', branch_name, set_upstream=True)
|
||||
pip_command = 'pip install "git+{}@{}#egg={}&subdirectory={}"'.format(
|
||||
issue.repository.html_url,
|
||||
branch_name,
|
||||
sdk_conf[branch_name]["autorest_options"]["package-name"],
|
||||
sdk_conf[branch_name]["autorest_options"]["package-name"]
|
||||
)
|
||||
local_command = 'pip install -e ./{}'.format(sdk_conf[branch_name]["autorest_options"]["package-name"])
|
||||
|
||||
pr_body = """Generated from Issue: {}
|
||||
|
||||
You can install the new package of this PR for testing using the following command:
|
||||
`{}`
|
||||
|
||||
If you have a local clone of this repo in the folder /home/git/repo, please checkout this branch and do:
|
||||
`{}`
|
||||
""".format(issue.html_url, pip_command, local_command)
|
||||
|
||||
pr = do_pr(self.gh_token, sdk_git_id, pr_repo_id, branch_name, base_branch_name, pr_body)
|
||||
|
||||
answer = """
|
||||
Done! I created this branch and this PR:
|
||||
- {}
|
||||
- {}
|
||||
""".format(branch_name, pr.html_url, pip_command)
|
||||
return answer
|
||||
else:
|
||||
return "Sorry, there is nothing to PR"
|
||||
|
||||
def rebuild(self, issue, project_pattern):
|
||||
if not issue.pull_request:
|
||||
return "Rebuild is just supported in PR for now"
|
||||
pr = issue.repository.get_pull(issue.number)
|
||||
|
||||
new_comment = issue.create_comment("Working on generating {} for you!!!".format(project_pattern))
|
||||
|
||||
config_path = CONFIG_FILE
|
||||
message = "Rebuild by "+issue.html_url
|
||||
initial_pr = None # There is no initial PR to test for files
|
||||
autorest_bin = None
|
||||
|
||||
branch_name = pr.head.ref
|
||||
|
||||
rest_api_id = "Azure/azure-rest-api-specs" # current
|
||||
branched_sdk_id = pr.head.repo.full_name+'@'+branch_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(self.gh_token, Path(temp_dir) / Path("rest"), rest_api_id) as restapi_git_folder, \
|
||||
manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder:
|
||||
|
||||
sdk_repo = Repo(str(sdk_folder))
|
||||
configure_user(self.gh_token, sdk_repo)
|
||||
|
||||
config = read_config(sdk_repo.working_tree_dir, config_path)
|
||||
|
||||
def skip_callback(project, local_conf):
|
||||
if not project.startswith(project_pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
from swaggertosdk import SwaggerToSdkNewCLI
|
||||
SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder,
|
||||
sdk_repo, temp_dir, autorest_bin)
|
||||
new_comment.edit("End of generation, doing commit")
|
||||
commit_sha = do_commit(sdk_repo, message, branch_name, "")
|
||||
if commit_sha:
|
||||
new_comment.edit("Pushing")
|
||||
sdk_repo.git.push('origin', branch_name, set_upstream=True)
|
||||
new_comment.delete()
|
||||
else:
|
||||
new_comment.delete()
|
||||
return "Nothing to rebuild, this PR is up to date"
|
||||
|
||||
_LOGGER.info("Build SDK finished and cleaned")
|
||||
return "Build SDK finished and cleaned"
|
||||
|
||||
|
||||
def generate_sdk_from_commit_safe(commit_obj, branch_name, restapi_git_id, sdkid, base_branch_name, fallback_base_branch_name="master"):
|
||||
try:
|
||||
response = generate_sdk_from_commit(commit_obj, branch_name, restapi_git_id, sdkid, base_branch_name, fallback_base_branch_name)
|
||||
except Exception as err:
|
||||
response = "Something's wrong:\n```python\n{}\n```\n".format(traceback.format_exc())
|
||||
|
||||
new_comment = commit_obj.create_comment(response)
|
||||
return 'Posted: {}'.format(new_comment.html_url)
|
||||
|
||||
def generate_sdk_from_commit(commit_obj, branch_name, restapi_git_id, sdk_git_id, base_branch_name, fallback_base_branch_name="master"):
|
||||
"""Generate SDK from a commit.
|
||||
|
||||
commit_obj is the initial commit_obj from the RestAPI repo. restapi_git_id explains where to clone the repo.
|
||||
sdk_git_id explains where to push the commit.
|
||||
branch_name is the expected branch name in the SDK repo.
|
||||
- If this branch exists, use it.
|
||||
- If not, use the base branch to create that branch (base branch is where I intend to do my PR)
|
||||
- If base_branch is not provided, use fallback_base_branch_name as base
|
||||
- If this base branch is provided and does not exists, create this base branch first using fallback_base_branch_name (this one is required to exist)
|
||||
|
||||
WARNING:
|
||||
This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name"
|
||||
"""
|
||||
gh_token = os.environ["GH_TOKEN"]
|
||||
config_path = CONFIG_FILE
|
||||
message_template = DEFAULT_COMMIT_MESSAGE
|
||||
autorest_bin = None
|
||||
|
||||
branched_rest_api_id = restapi_git_id+'@'+commit_obj.sha
|
||||
branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \
|
||||
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder:
|
||||
|
||||
sdk_repo = Repo(str(sdk_folder))
|
||||
_LOGGER.info('Destination branch for generated code is %s', branch_name)
|
||||
try:
|
||||
_LOGGER.info('Try to checkout the destination branch if it already exists')
|
||||
sdk_repo.git.checkout(branch_name)
|
||||
except GitCommandError:
|
||||
_LOGGER.info('Destination branch does not exists')
|
||||
if base_branch_name is not None:
|
||||
_LOGGER.info('Try to checkout base branch {} '.format(base_branch_name))
|
||||
try:
|
||||
sdk_repo.git.checkout(base_branch_name)
|
||||
except GitCommandError:
|
||||
_LOGGER.info('Base branch does not exists, create it from {}'.format(fallback_base_branch_name))
|
||||
checkout_and_create_branch(sdk_repo, base_branch_name)
|
||||
sdk_repo.git.push('origin', base_branch_name, set_upstream=True)
|
||||
|
||||
configure_user(gh_token, sdk_repo)
|
||||
|
||||
config = read_config(sdk_repo.working_tree_dir, config_path)
|
||||
global_conf = config["meta"]
|
||||
|
||||
from swaggertosdk import SwaggerToSdkNewCLI
|
||||
from swaggertosdk import SwaggerToSdkCore
|
||||
swagger_files_in_commit = SwaggerToSdkCore.get_swagger_project_files_in_pr(commit_obj, restapi_git_folder)
|
||||
_LOGGER.info("Files in PR: %s ", swagger_files_in_commit)
|
||||
|
||||
# Look for configuration in Readme
|
||||
extract_conf_from_readmes(gh_token, swagger_files_in_commit, restapi_git_folder, sdk_git_id, config)
|
||||
|
||||
def skip_callback(project, local_conf):
|
||||
if not swagger_files_in_commit:
|
||||
return True
|
||||
markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf)
|
||||
if not (
|
||||
markdown_relative_path in swagger_files_in_commit or
|
||||
any(input_file in swagger_files_in_commit for input_file in optional_relative_paths)):
|
||||
_LOGGER.info(f"In project {project} no files involved in this commit")
|
||||
return True
|
||||
return False
|
||||
|
||||
SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder,
|
||||
sdk_repo, temp_dir, autorest_bin)
|
||||
|
||||
message = message_template + "\n\n" + commit_obj.commit.message
|
||||
commit_sha = do_commit(sdk_repo, message, branch_name, commit_obj.sha)
|
||||
if commit_sha:
|
||||
sdk_repo.git.push('origin', branch_name, set_upstream=True)
|
||||
commit_url = "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha)
|
||||
commit_obj.create_comment("Did a commit to SDK for Python:\n{}".format(commit_url))
|
||||
else:
|
||||
commit_obj.create_comment("This commit was treated and no generation was made for Python")
|
||||
|
||||
_LOGGER.info("Build SDK finished and cleaned")
|
||||
return "Build SDK finished and cleaned"
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
from . import app
|
||||
from jsonrpc.backend.flask import api
|
||||
|
||||
@app.route("/")
|
||||
def hello():
|
||||
return "Hello World!"
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
from swaggertosdk.build_sdk import *
|
||||
|
||||
def test_guess_autorest_options():
|
||||
assert guess_service_info_from_path("specification/compute/resource-manager/readme.md") == {"rp_name": "compute", "is_arm": True}
|
||||
assert guess_service_info_from_path("specification/servicefabric/data-plane/readme.md") == {"rp_name": "servicefabric", "is_arm": False}
|
||||
|
|
@ -5,6 +5,8 @@ import tempfile
|
|||
from pathlib import Path
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
from git import GitCommandError
|
||||
|
||||
# Fake Travis before importing the Script
|
||||
os.environ['TRAVIS'] = 'true'
|
||||
|
||||
|
|
@ -113,6 +115,77 @@ class TestSwaggerToSDK(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
def test_manage_git_folder(self):
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(GH_TOKEN, temp_dir, "lmazuel/TestingRepo") as rest_repo:
|
||||
|
||||
self.assertTrue((Path(rest_repo) / Path("README.md")).exists())
|
||||
|
||||
finished = True
|
||||
except (PermissionError, FileNotFoundError):
|
||||
if not finished:
|
||||
raise
|
||||
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||
manage_git_folder(GH_TOKEN, temp_dir, "lmazuel/TestingRepo@lmazuel-patch-1") as rest_repo:
|
||||
|
||||
self.assertTrue((Path(rest_repo) / Path("README.md")).exists())
|
||||
self.assertTrue(Repo(rest_repo).active_branch, "lmazuel-patch-1")
|
||||
|
||||
finished = True
|
||||
except (PermissionError, FileNotFoundError):
|
||||
if not finished:
|
||||
raise
|
||||
|
||||
def test_clone_path(self):
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
clone_to_path(GH_TOKEN, temp_dir, "lmazuel/TestingRepo")
|
||||
self.assertTrue((Path(temp_dir) / Path("README.md")).exists())
|
||||
|
||||
finished = True
|
||||
except PermissionError:
|
||||
if not finished:
|
||||
raise
|
||||
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
clone_to_path(GH_TOKEN, temp_dir, "https://github.com/lmazuel/TestingRepo")
|
||||
self.assertTrue((Path(temp_dir) / Path("README.md")).exists())
|
||||
|
||||
finished = True
|
||||
except PermissionError:
|
||||
if not finished:
|
||||
raise
|
||||
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
clone_to_path(GH_TOKEN, temp_dir, "lmazuel/TestingRepo", "lmazuel-patch-1")
|
||||
self.assertTrue((Path(temp_dir) / Path("README.md")).exists())
|
||||
|
||||
finished = True
|
||||
except PermissionError:
|
||||
if not finished:
|
||||
raise
|
||||
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with self.assertRaises(GitCommandError):
|
||||
clone_to_path(GH_TOKEN, temp_dir, "lmazuel/TestingRepo", "fakebranch")
|
||||
|
||||
finished = True
|
||||
except (PermissionError, FileNotFoundError):
|
||||
if not finished:
|
||||
raise
|
||||
|
||||
def test_do_commit(self):
|
||||
finished = False # Authorize PermissionError on cleanup
|
||||
try:
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче