From 784a5d3d7072175718f394aefe0aeeb4abd44a6f Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 23 Jan 2018 11:42:09 -0800 Subject: [PATCH] RestAPI (#21) * Add Github http link to SDK/Rest parameters * Cast folder as str (can be Path) * Basic JSON-RPC server * Allow all hosts * Gitattributes for EOL * Add Github handler basic * Fix getting token * Plug generate from README * Fix indentation * Fix input path * Add exception handler * Add exists clause * Fix base branch * Error does not have a required message * Add SDK subfolder * Conf user + fix reponse * Handle creation of issue * Improve comments * Improve error message * Add help * Improve message * Fix return type of build_sdk * Add rest endpoint * Take SDKID as query parameter * Add PR hook * Handle PR from a fork * Improve already existing PR * Improve do pr to return already existing one if there is * Fix conflict PR code * Add rebuild endpoint * Fix endpoint * Fix rebuild command * Add local log * Fix build sdk for Windows * Switch current to master * Fix build_libraries API after rebase against master * Adapt webhook code to skip_callback * Enabling SwaggerToSdk logs * Add PR close action * Add synchronize/pr event * Add Consume thread * Fix thread loop * Fix comment on close * Add Delivery in log * Plug auto-conf detection * Add labels to SDK PRs * Add sdkbase and make branch creation more robust --- .gitattributes | 1 + Dockerfile | 2 +- dev_requirements.txt | 2 +- setup.py | 10 +- swaggertosdk/SwaggerToSdkCore.py | 78 +++--- swaggertosdk/SwaggerToSdkMain.py | 47 +++- swaggertosdk/__main__.py | 5 +- swaggertosdk/build_sdk.py | 173 ++++++++++++ swaggertosdk/restapi/__init__.py | 33 +++ swaggertosdk/restapi/__main__.py | 2 + swaggertosdk/restapi/github.py | 350 +++++++++++++++++++++++++ swaggertosdk/restapi/github_handler.py | 276 +++++++++++++++++++ swaggertosdk/restapi/views.py | 6 + tests/test_build_sdk.py | 5 + tests/test_swaggertosdk.py | 73 ++++++ 15 files changed, 1015 insertions(+), 48 deletions(-) create mode 100644 .gitattributes create mode 100644 swaggertosdk/build_sdk.py create mode 100644 swaggertosdk/restapi/__init__.py create mode 100644 swaggertosdk/restapi/__main__.py create mode 100644 swaggertosdk/restapi/github.py create mode 100644 swaggertosdk/restapi/github_handler.py create mode 100644 swaggertosdk/restapi/views.py create mode 100644 tests/test_build_sdk.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b9d5dc8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.py text \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a4b8422..546911b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ ENV LC_ALL en_US.UTF-8 COPY setup.py /tmp COPY swaggertosdk /tmp/swaggertosdk/ WORKDIR /tmp -RUN pip3.6 install . +RUN pip3.6 install .[rest] WORKDIR /git-restapi ENTRYPOINT ["python3.6", "-m", "swaggertosdk"] diff --git a/dev_requirements.txt b/dev_requirements.txt index 8380c2f..6cd6da5 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,4 @@ --e . +-e .[rest] pytest-cov pytest>=3.2.0 pylint diff --git a/setup.py b/setup.py index 34fa690..c96713e 100644 --- a/setup.py +++ b/setup.py @@ -24,5 +24,13 @@ setup( "requests", "mistune", "pyyaml", - ] + "cookiecutter", + "wheel" + ], + extras_require={ + 'rest': [ + 'flask', + 'json-rpc' + ] + } ) \ No newline at end of file diff --git a/swaggertosdk/SwaggerToSdkCore.py b/swaggertosdk/SwaggerToSdkCore.py index 8425d8d..7f193d2 100644 --- a/swaggertosdk/SwaggerToSdkCore.py +++ b/swaggertosdk/SwaggerToSdkCore.py @@ -1,13 +1,14 @@ -import os -import logging +from contextlib import contextmanager import json +import logging +import os import re import shutil import stat import subprocess import tempfile from pathlib import Path -from contextlib import contextmanager +from urllib.parse import urlparse from git import Repo from github import Github, GithubException, UnknownObjectException @@ -116,9 +117,9 @@ def do_commit(repo, message_template, branch_name, hexsha): checkout_and_create_branch(repo, branch_name) msg = message_template.format(hexsha=hexsha) - repo.index.commit(msg) + commit = repo.index.commit(msg) _LOGGER.info("Commit done: %s", msg) - return True + return commit.hexsha def sync_fork(gh_token, github_repo_id, repo): @@ -242,7 +243,7 @@ def configure_user(gh_token, repo): git config --global user.name "Your Name" """ user = user_from_token(gh_token) - repo.git.config('user.email', user.email or 'autorestci@microsoft.com') + repo.git.config('user.email', user.email or 'aspysdk2@microsoft.com') repo.git.config('user.name', user.name or 'SwaggerToSDK Automation') @@ -258,7 +259,7 @@ def compute_branch_name(branch_name, gh_token=None): return DEFAULT_TRAVIS_BRANCH_NAME.format(branch=os.environ['TRAVIS_BRANCH']) return DEFAULT_TRAVIS_PR_BRANCH_NAME.format(number=pr_object.number) -def do_pr(gh_token, sdk_git_id, sdk_pr_target_repo_id, branch_name, base_branch): +def do_pr(gh_token, sdk_git_id, sdk_pr_target_repo_id, branch_name, base_branch, pr_body=""): "Do the PR" if not gh_token: _LOGGER.info('Skipping the PR, no token found') @@ -276,31 +277,30 @@ def do_pr(gh_token, sdk_git_id, sdk_pr_target_repo_id, branch_name, base_branch) head_name = "{}:{}".format(sdk_git_owner, branch_name) else: head_name = branch_name + sdk_git_repo = github_con.get_repo(sdk_git_id) + sdk_git_owner = sdk_git_repo.owner.login - body = '' - rest_api_pr = get_initial_pr(gh_token) - if rest_api_pr: - body += "Generated from RestAPI PR: {}".format(rest_api_pr.html_url) try: github_pr = sdk_pr_target_repo.create_pull( title='Automatic PR from {}'.format(branch_name), - body=body, + body=pr_body, head=head_name, base=base_branch ) except GithubException as err: - if err.status == 422 and err.data['errors'][0]['message'].startswith('A pull request already exists'): - _LOGGER.info('PR already exists, it was a commit on an open PR') - return + if err.status == 422 and err.data['errors'][0].get('message', '').startswith('A pull request already exists'): + matching_pulls = sdk_pr_target_repo.get_pulls(base=base_branch, head=sdk_git_owner+":"+head_name) + matching_pull = matching_pulls[0] + _LOGGER.info('PR already exists: '+matching_pull.html_url) + return matching_pull raise _LOGGER.info("Made PR %s", github_pr.html_url) - comment = compute_pr_comment_with_sdk_pr(github_pr.html_url, sdk_git_id, branch_name) - add_comment_to_initial_pr(gh_token, comment) + return github_pr def get_swagger_hexsha(restapi_git_folder): """Get the SHA1 of the current repo""" - repo = Repo(restapi_git_folder) + repo = Repo(str(restapi_git_folder)) if repo.bare: not_git_hexsha = "notgitrepo" _LOGGER.warning("Not a git repo, SHA1 used will be: %s", not_git_hexsha) @@ -330,9 +330,16 @@ def add_comment_to_initial_pr(gh_token, comment): return True -def clone_to_path(gh_token, temp_dir, sdk_git_id): - """Clone the given repo_id to the 'sdk' folder in given temp_dir""" +def clone_to_path(gh_token, temp_dir, sdk_git_id, branch=None): + """Clone the given repo_id to the temp_dir folder. + + :param str branch: If specified, switch to this branch. Branch must exist. + """ _LOGGER.info("Clone SDK repository %s", sdk_git_id) + url_parsing = urlparse(sdk_git_id) + sdk_git_id = url_parsing.path + if sdk_git_id.startswith("/"): + sdk_git_id = sdk_git_id[1:] credentials_part = '' if gh_token: @@ -348,11 +355,14 @@ def clone_to_path(gh_token, temp_dir, sdk_git_id): credentials=credentials_part, sdk_git_id=sdk_git_id ) - sdk_path = os.path.join(temp_dir, 'sdk') - Repo.clone_from(https_authenticated_url, sdk_path) - _LOGGER.info("Clone success") + repo = Repo.clone_from(https_authenticated_url, str(temp_dir)) + # Do NOT clone and set branch at the same time, since we allow branch to be a SHA1 + # And you can't clone a SHA1 + if branch: + _LOGGER.info("Checkout branch %s", branch) + repo.git.checkout(branch) - return sdk_path + _LOGGER.info("Clone success") def remove_readonly(func, path, _): "Clear the readonly bit and reattempt the removal" @@ -360,17 +370,23 @@ def remove_readonly(func, path, _): func(path) @contextmanager -def manage_sdk_folder(gh_token, temp_dir, sdk_git_id): +def manage_git_folder(gh_token, temp_dir, git_id): """Context manager to avoid readonly problem while cleanup the temp dir""" - sdk_path = clone_to_path(gh_token, temp_dir, sdk_git_id) - _LOGGER.debug("SDK path %s", sdk_path) + _LOGGER.debug("Git ID %s", git_id) + if Path(git_id).exists(): + yield git_id + return None # Do not erase a local folder, just skip here + + # Clone the specific branch + split_git_id = git_id.split("@") + branch = split_git_id[1] if len(split_git_id) > 1 else None + clone_to_path(gh_token, temp_dir, split_git_id[0], branch=branch) try: - yield sdk_path + yield temp_dir # Pre-cleanup for Windows http://bugs.python.org/issue26660 finally: - _LOGGER.debug("Preclean SDK folder") - shutil.rmtree(sdk_path, onerror=remove_readonly) - + _LOGGER.debug("Preclean Rest folder") + shutil.rmtree(temp_dir, onerror=remove_readonly) def get_full_sdk_id(gh_token, sdk_git_id): """If the SDK git id is incomplete, try to complete it with user login""" diff --git a/swaggertosdk/SwaggerToSdkMain.py b/swaggertosdk/SwaggerToSdkMain.py index eb5b3be..d7b2824 100644 --- a/swaggertosdk/SwaggerToSdkMain.py +++ b/swaggertosdk/SwaggerToSdkMain.py @@ -3,6 +3,8 @@ import os import logging import tempfile from git import Repo, GitCommandError +from pathlib import Path +import sys from .SwaggerToSdkCore import ( IS_TRAVIS, @@ -12,7 +14,7 @@ from .SwaggerToSdkCore import ( DEFAULT_TRAVIS_BRANCH_NAME, DEFAULT_TRAVIS_PR_BRANCH_NAME, get_full_sdk_id, - manage_sdk_folder, + manage_git_folder, compute_branch_name, configure_user, sync_fork, @@ -22,6 +24,7 @@ from .SwaggerToSdkCore import ( do_commit, do_pr, add_comment_to_initial_pr, + compute_pr_comment_with_sdk_pr, get_swagger_project_files_in_pr, get_commit_object_from_travis, extract_conf_from_readmes, @@ -34,16 +37,17 @@ from .autorest_tools import ( _LOGGER = logging.getLogger(__name__) -def generate_sdk(gh_token, config_path, project_pattern, restapi_git_folder, +def generate_sdk(gh_token, config_path, project_pattern, restapi_git_id, sdk_git_id, pr_repo_id, message_template, base_branch_name, branch_name, autorest_bin=None): """Main method of the the file""" sdk_git_id = get_full_sdk_id(gh_token, sdk_git_id) with tempfile.TemporaryDirectory() as temp_dir, \ - manage_sdk_folder(gh_token, temp_dir, sdk_git_id) as sdk_folder: + manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), restapi_git_id) as restapi_git_folder, \ + manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), sdk_git_id) as sdk_folder: - sdk_repo = Repo(sdk_folder) + sdk_repo = Repo(str(sdk_folder)) if gh_token: branch_name = compute_branch_name(branch_name, gh_token) @@ -104,7 +108,10 @@ def generate_sdk(gh_token, config_path, project_pattern, restapi_git_folder, if do_commit(sdk_repo, message_template, branch_name, hexsha): sdk_repo.git.push('origin', branch_name, set_upstream=True) if pr_repo_id: - do_pr(gh_token, sdk_git_id, pr_repo_id, branch_name, base_branch_name) + pr_body = "Generated from PR: {}".format(initial_pr.html_url) + github_pr = do_pr(gh_token, sdk_git_id, pr_repo_id, branch_name, base_branch_name, pr_body) + comment = compute_pr_comment_with_sdk_pr(github_pr.html_url, sdk_git_id, branch_name) + add_comment_to_initial_pr(gh_token, comment) else: add_comment_to_initial_pr(gh_token, "No modification for {}".format(sdk_git_id)) else: @@ -113,8 +120,29 @@ def generate_sdk(gh_token, config_path, project_pattern, restapi_git_folder, _LOGGER.info("Build SDK finished and cleaned") -def main(): +def main(argv): """Main method""" + + if 'GH_TOKEN' not in os.environ: + gh_token = None + else: + gh_token = os.environ['GH_TOKEN'] + + if "--rest-server" in argv: + from .restapi import app + log_level = logging.WARNING + if "-v" in argv or "--verbose" in argv: + log_level = logging.INFO + if "--debug" in argv: + log_level = logging.DEBUG + + main_logger = logging.getLogger() + logging.basicConfig() + main_logger.setLevel(log_level) + + app.run(debug=log_level == logging.DEBUG, host='0.0.0.0') + sys.exit(0) + epilog = "\n".join([ 'The script activates this additional behaviour if Travis is detected:', ' --branch is setted by default to "{}" if triggered by a PR, "{}" otherwise'.format( @@ -165,12 +193,7 @@ def main(): 'Otherwise, you can use the syntax username/repoid') args = parser.parse_args() - - if 'GH_TOKEN' not in os.environ: - gh_token = None - else: - gh_token = os.environ['GH_TOKEN'] - + main_logger = logging.getLogger() if args.verbose or args.debug: logging.basicConfig() diff --git a/swaggertosdk/__main__.py b/swaggertosdk/__main__.py index ac19ce5..caa871f 100644 --- a/swaggertosdk/__main__.py +++ b/swaggertosdk/__main__.py @@ -1,2 +1,3 @@ -from .SwaggerToSdkMain import main -main() +import sys +from swaggertosdk.SwaggerToSdkMain import main +main(sys.argv) diff --git a/swaggertosdk/build_sdk.py b/swaggertosdk/build_sdk.py new file mode 100644 index 0000000..e021bd3 --- /dev/null +++ b/swaggertosdk/build_sdk.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import json +import logging +import os.path +from pathlib import Path +import sys + +import requests +from cookiecutter.main import cookiecutter +from swaggertosdk.SwaggerToSdkNewCLI import generate_code + +_LOGGER = logging.getLogger(__name__) + +def guess_service_info_from_path(spec_path): + """Guess Python Autorest options based on the spec path. + + Expected path: + specification/compute/resource-manager/readme.md + """ + spec_path = spec_path.lower() + spec_path = spec_path[spec_path.index("specification"):] # Might raise and it's ok + split_spec_path = spec_path.split("/") if "/" in spec_path else spec_path.split("\\") + + rp_name = split_spec_path[1] + is_arm = split_spec_path[2] == "resource-manager" + + return { + "rp_name": rp_name, + "is_arm": is_arm + } + +def get_data(spec_path): + if spec_path.startswith("http"): + response = requests.get(spec_path) + response.raise_for_status() + return response.text + else: + with open(spec_path, "r") as fd: + return fd.read() + +def guess_service_info_from_content(spec_path): + data = get_data(spec_path) + + lines = data.splitlines() + while lines: + line = lines.pop(0) + if line.startswith("# "): + pretty_name = line[2:] + break + else: + raise ValueError("Unable to find main title in this README") + return { + 'pretty_name': pretty_name + } + +def guess_service_info(spec_path): + result = guess_service_info_from_content(spec_path) + result.update(guess_service_info_from_path(spec_path)) + return result + +def create_swagger_to_sdk_conf(spec_path, service_info, autorest_options): + type_str = ".mgmt" if service_info["is_arm"] else ".data" + return { + "{}{}".format(service_info["rp_name"], type_str): { + "markdown": spec_path[spec_path.index("specification"):], + "autorest_options": { + "namespace": autorest_options["namespace"], + "package-version": autorest_options["package-version"], + "package-name": autorest_options["package-name"] + }, + "output_dir": "{}/{}".format(autorest_options["package-name"], autorest_options["package-name"].replace("-","/")), + "build_dir": autorest_options["package-name"] + } + } + +def create_package_service_mapping(spec_path, service_info, autorest_options): + type_str = "Management" if service_info["is_arm"] else "Client" + return { + autorest_options["package-name"]: { + "service_name": service_info["pretty_name"], + "category": type_str, + "namespaces": [ + autorest_options["namespace"] + ] + } + } + +def generate(spec_path, cwd='.'): + service_info = guess_service_info(spec_path) + + autorest_options = {} + if service_info["is_arm"]: + autorest_options["azure-arm"] = True + namespace = "azure.mgmt." + service_info["rp_name"] + package_name = "azure-mgmt-" + service_info["rp_name"] + else: + autorest_options["add-credentials"] = True + namespace = "azure." + service_info["rp_name"] + package_name = "azure-" + service_info["rp_name"] + + # Create output folder + output_dir = Path(cwd).resolve() / Path(package_name) + output_dir.mkdir(exist_ok=True) + _LOGGER.info("Output folder: %s", output_dir) + + _LOGGER.info("Calling Autorest") + autorest_options.update({ + "use": "@microsoft.azure/autorest.python@preview", + "license-header": "MICROSOFT_MIT_NO_VERSION", + "payload-flattening-threshold": 2, + "python": "", + "package-version": "0.1.0", + "python.output-folder": output_dir + }) + + autorest_options["namespace"] = namespace + autorest_options["package-name"] = package_name + + generate_code( + input_file=Path(spec_path) if not spec_path.startswith("http") else spec_path, + output_dir=output_dir, + global_conf={"autorest_options": autorest_options}, + local_conf={} + ) + + _LOGGER.info("Rebuilding packaging with Cookiecutter") + pretty_name = service_info["pretty_name"] + cookiecutter('gh:Azure/cookiecutter-azuresdk-pypackage', + no_input=True, + extra_context={ + 'package_name': package_name, + 'package_pprint_name': pretty_name + }, + overwrite_if_exists=True, + output_dir=cwd + ) + + entry = create_swagger_to_sdk_conf(spec_path, service_info, autorest_options) + swagger_to_sdk = Path(cwd) / Path("swagger_to_sdk_config.json") + if swagger_to_sdk.exists(): + _LOGGER.info("Updating swagger_to_sdk_config.json") + with swagger_to_sdk.open() as fd: + data_conf = json.load(fd) + data_conf["projects"].update(entry) + with swagger_to_sdk.open("w") as fd: + json.dump(data_conf, fd, indent=2, sort_keys=True) + + package_service_mapping = Path(cwd) / Path("package_service_mapping.json") + if package_service_mapping.exists(): + _LOGGER.info("Updating package_service_mapping.json") + package_service_mapping_entry = create_package_service_mapping(spec_path, service_info, autorest_options) + with package_service_mapping.open() as fd: + data_conf = json.load(fd) + data_conf.update(package_service_mapping_entry) + with package_service_mapping.open("w") as fd: + json.dump(data_conf, fd, indent=2, sort_keys=True) + + _LOGGER.info("Done! Enjoy your Python SDK!!") + return entry + +def main(spec_path): + generate(spec_path) + +if __name__ == "__main__": + logging.basicConfig() + logging.getLogger().setLevel(logging.INFO) + + main(sys.argv[1]) diff --git a/swaggertosdk/restapi/__init__.py b/swaggertosdk/restapi/__init__.py new file mode 100644 index 0000000..25e350f --- /dev/null +++ b/swaggertosdk/restapi/__init__.py @@ -0,0 +1,33 @@ +from flask import Flask +from jsonrpc.backend.flask import api + +from ..SwaggerToSdkMain import generate_sdk +from ..SwaggerToSdkCore import CONFIG_FILE, DEFAULT_COMMIT_MESSAGE + +app = Flask(__name__) +app.add_url_rule('/', 'api', api.as_view(), methods=['POST']) + +@api.dispatcher.add_method +def ping(*args, **kwargs): + return "Pong!" + +@api.dispatcher.add_method +def generate_project(*args, **kwargs): + # Get required parameter + rest_api_id = kwargs['rest_api_id'] + sdk_id = kwargs['sdk_id'] + project = kwargs['project'] + + generate_sdk( + os.environ['GH_TOKEN'], + CONFIG_FILE, + project, + rest_api_id, + sdk_id, + None, # No PR repo id + DEFAULT_COMMIT_MESSAGE, + 'master', + None # Destination branch + ) + +from .github import * diff --git a/swaggertosdk/restapi/__main__.py b/swaggertosdk/restapi/__main__.py new file mode 100644 index 0000000..b7e9c55 --- /dev/null +++ b/swaggertosdk/restapi/__main__.py @@ -0,0 +1,2 @@ +from swaggertosdk.restapi import app +app.run(debug=True) \ No newline at end of file diff --git a/swaggertosdk/restapi/github.py b/swaggertosdk/restapi/github.py new file mode 100644 index 0000000..c044408 --- /dev/null +++ b/swaggertosdk/restapi/github.py @@ -0,0 +1,350 @@ +from datetime import datetime +from enum import Enum +from functools import wraps, lru_cache +import logging +from queue import Queue +import re +import traceback +from threading import Thread + +from flask import request, jsonify + +from github import Github, GithubException, UnknownObjectException + +import hmac, hashlib +import os + +from .github_handler import ( + build_from_issue_comment, + build_from_issues, + GithubHandler as LocalHandler, + generate_sdk_from_commit +) +from . import app + +_LOGGER = logging.getLogger(__name__) +_QUEUE = Queue(64) + + +# Webhook secreet to authenticate message (bytes) +SECRET = b'mydeepsecret' + +_HMAC_CHECK = False + +def check_hmac(request, secret): + data = request.get_data() + hmac_tester = hmac.HMAC(b'mydeepsecret', data, hashlib.sha1) + if not 'X-Hub-Signature' in request.headers: + raise ValueError('X-Hub-Signature is mandatory on this WebService') + if request.headers['X-Hub-Signature'] == 'sha1='+hmac_tester.hexdigest(): + return True + raise ValueError('Bad X-Hub-Signature signature') + +@app.route('/github', methods=['POST']) +def notify(): + """Github main endpoint.""" + github_index = { + 'ping': ping, + 'issue_comment': issue_comment, + 'issues': issues + } + return handle_github_webhook( + github_index, + request.headers['X-GitHub-Event'], + request.get_json() + ) + +@app.route('/github/rest', methods=['POST']) +def rest_notify(): + """Github rest endpoint.""" + github_index = { + 'ping': ping, + 'push': push, + 'pull_request': rest_pull_request + } + return handle_github_webhook( + github_index, + request.headers['X-GitHub-Event'], + request.get_json() + ) + +def handle_github_webhook(github_index, gh_event_type, json_body): + if _HMAC_CHECK: + check_hmac(request, SECRET) + _LOGGER.info("Received Webhook %s", request.headers.get("X-GitHub-Delivery")) + + json_answer = notify_github(github_index, gh_event_type, json_body) + return jsonify(json_answer) + +@lru_cache() +def robot_name(): + github_con = Github(os.environ["GH_TOKEN"]) + return github_con.get_user().login + +def notify_github(github_index, event_type, json_body): + if json_body['sender']['login'].lower() == robot_name().lower(): + return {'message': 'I don\'t talk to myself, I\'m not schizo'} + if event_type in github_index: + return github_index[event_type](json_body) + return {'message': 'Not handled currently'} + +def ping(body): + return {'message': 'Moi aussi zen beaucoup'} + +def issue_comment(body): + if body["action"] in ["created", "edited"]: + webhook_data = build_from_issue_comment(body) + response = manage_comment(webhook_data) + if response: + return response + return {'message': 'Nothing for me'} + +def issues(body): + if body["action"] in ["opened"]: + webhook_data = build_from_issues(body) + response = manage_comment(webhook_data) + if response: + return response + return {'message': 'Nothing for me'} + +def manage_comment(webhook_data): + handler = LocalHandler() + + # Is someone talking to me: + message = re.search("@{} (.*)".format(robot_name()), webhook_data.text, re.I) + if message: + command = message.group(1) + try: + response = handler.act_and_response(webhook_data, command) + except Exception as err: + response = traceback.format_exc() + if response: + return {'message': response} + +def push(body): + sdkid = request.args.get("sdkid") + if not sdkid: + return {'message': 'sdkid is a required query parameter'} + sdkbase = request.args.get("sdkbase", "master") + + rest_api_branch_name = body["ref"][len("refs/heads/"):] + if rest_api_branch_name == "master": + return {'message': 'Webhook disabled for RestAPI master'} + + gh_token = os.environ["GH_TOKEN"] + github_con = Github(gh_token) + restapi_git_id = body['repository']['full_name'] + repo = github_con.get_repo(restapi_git_id) + + commit_obj = repo.get_commit(body["after"]) + generate_sdk_from_commit( + commit_obj, + "restapi_auto_"+rest_api_branch_name, + restapi_git_id, + sdkid, + None, # I don't know if the origin branch comes from "master", assume it. + sdkbase + ) + return {'message': 'No return for this endpoint'} + +def rest_pull_request(body): + sdkid = request.args.get("sdkid") + if not sdkid: + return {'message': 'sdkid is a required query parameter'} + sdkbase = request.args.get("sdkbase", "master") + + _LOGGER.info("Received PR action %s", body["action"]) + _QUEUE.put((body, sdkid, sdkbase)) + _LOGGER.info("Received action has been queued. Queue size: %d", _QUEUE.qsize()) + + return {'message': 'Current queue size: {}'.format(_QUEUE.qsize())} + +def rest_handle_action(body, sdkid, sdkbase): + """First method in the thread. + """ + _LOGGER.info("Rest handle action") + gh_token = os.environ["GH_TOKEN"] + github_con = Github(gh_token) + + sdk_pr_target_repo = github_con.get_repo(sdkid) + + restapi_git_id = body['repository']['full_name'] + restapi_repo = github_con.get_repo(restapi_git_id) + + _LOGGER.info("Received PR action %s", body["action"]) + if body["action"] in ["opened", "reopened"]: + return rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase) + if body["action"] == "closed": + return rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase) + if body["action"] == "synchronize": # push to a PR from a fork + return rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase) + +def rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master"): + + rest_basebranch = body["pull_request"]["base"]["ref"] + dest_branch = body["pull_request"]["head"]["ref"] + origin_repo = body["pull_request"]["head"]["repo"]["full_name"] + + if rest_basebranch == "master": + sdk_base = sdk_default_base + sdk_checkout_base = None + else: + sdk_base = "restapi_auto_" + rest_basebranch + sdk_checkout_base = sdk_base + + rest_pr = restapi_repo.get_pull(body["number"]) + + if origin_repo != restapi_repo.full_name: + _LOGGER.info("This comes from a fork, I need generation first, since targetted branch does not exist") + fork_repo = github_con.get_repo(origin_repo) + fork_owner = fork_repo.owner.login + commit_obj = fork_repo.get_commit(body["pull_request"]["head"]["sha"]) + subbranch_name_part = fork_owner+"_"+dest_branch + sdk_dest_branch = "restapi_auto_" + subbranch_name_part + generate_sdk_from_commit( + commit_obj, + sdk_dest_branch, + origin_repo, + sdk_pr_target_repo.full_name, + sdk_checkout_base, + sdk_default_base + ) + else: + sdk_dest_branch = "restapi_auto_" + dest_branch + + try: + github_pr = sdk_pr_target_repo.create_pull( + title='Automatic PR of {} into {}'.format(sdk_dest_branch, sdk_base), + body="Created to sync {}".format(rest_pr.html_url), + head=sdk_dest_branch, + base=sdk_base + ) + sdk_pr_as_issue = sdk_pr_target_repo.get_issue(github_pr.number) + sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress)) + except GithubException as err: + if err.status == 422 and err.data['errors'][0].get('message', '').startswith('A pull request already exists'): + _LOGGER.info('PR already exists, it was a commit on an open PR') + sdk_pr = list(sdk_pr_target_repo.get_pulls( + head=sdk_pr_target_repo.owner.login+":"+sdk_dest_branch, + base=sdk_base + ))[0] + sdk_pr_as_issue = sdk_pr_target_repo.get_issue(sdk_pr.number) + sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress)) + safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.refused)) + safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.merged)) + return {'message': 'PR already exists'} + else: + return {'message': err.data} + except Exception as err: + response = traceback.format_exc() + return {'message': response} + + +class SwaggerToSdkLabels(Enum): + merged = "RestPRMerged", "0e8a16" + refused = "RestPRRefused", "b60205" + in_progress = "RestPRInProgress", "fbca04" + +def get_or_create_label(sdk_pr_target_repo, label_enum): + try: + return sdk_pr_target_repo.get_label(label_enum.value[0]) + except UnknownObjectException: + return sdk_pr_target_repo.create_label(*label_enum.value) + +def safe_remove_label(issue, label): + """Remove a label, does not fail if label was not there. + """ + try: + issue.remove_from_labels(label) + except GithubException: + pass + +def rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master"): + _LOGGER.info("Received a PR closed event") + sdkid = sdk_pr_target_repo.full_name + rest_pr = restapi_repo.get_pull(body["number"]) + + # What was "head" name + origin_repo = body["pull_request"]["head"]["repo"]["full_name"] + dest_branch = body["pull_request"]["head"]["ref"] + if origin_repo != restapi_repo.full_name: # This PR comes from a fork + fork_repo = github_con.get_repo(origin_repo) + fork_owner = fork_repo.owner.login + subbranch_name_part = fork_owner+"_"+dest_branch + sdk_dest_branch = "restapi_auto_" + subbranch_name_part + else: + sdk_dest_branch = "restapi_auto_" + dest_branch + _LOGGER.info("SDK head branch should be %s", sdk_dest_branch) + full_head = sdk_pr_target_repo.owner.login+":"+sdk_dest_branch + _LOGGER.info("Will filter with %s", full_head) + + # What was "base" + rest_basebranch = body["pull_request"]["base"]["ref"] + sdk_base = sdk_default_base if rest_basebranch == "master" else "restapi_auto_" + rest_basebranch + _LOGGER.info("SDK base branch should be %s", sdk_base) + + sdk_prs = list(sdk_pr_target_repo.get_pulls( + head=full_head, + base=sdk_base + )) + if not sdk_prs: + rest_pr.create_issue_comment("Was unable to find SDK {} PR for this closed PR.".format(sdkid)) + elif len(sdk_prs) == 1: + sdk_pr = sdk_prs[0] + sdk_pr_as_issue = sdk_pr_target_repo.get_issue(sdk_pr.number) + safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress)) + try: + if body["pull_request"]["merged"]: + sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.merged)) + else: + sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.refused)) + except GithubException: + sdk_pr.create_issue_comment("Cannot set labels. Initial PR has been closed with merged status: {}".format(body["pull_request"]["merged"])) + else: + # Should be impossible, create_pull would have sent a 422 + pr_list = "\n".join(["- {}".format(pr.html_url) for pr in sdk_prs]) + rest_pr.create_issue_comment("We found several SDK {} PRs and didn't notify closing event.\n{}".format(sdkid, pr_list)) + +def rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master"): + + if body["before"] == body["after"]: + return {'message': 'No commit id change'} + + # What was "head" name + origin_repo = body["pull_request"]["head"]["repo"]["full_name"] + + if origin_repo == restapi_repo.full_name: + _LOGGER.info("This will be handled by 'push' event on the branch") + + dest_branch = body["pull_request"]["head"]["ref"] + fork_repo = github_con.get_repo(origin_repo) + fork_owner = fork_repo.owner.login + commit_obj = fork_repo.get_commit(body["pull_request"]["head"]["sha"]) + subbranch_name_part = fork_owner+"_"+dest_branch + generate_sdk_from_commit( + commit_obj, + "restapi_auto_"+subbranch_name_part, + origin_repo, + sdk_pr_target_repo.full_name, + None, # I don't know if the origin branch comes from "master", assume it. + sdk_default_base + ) + + return {'message': 'No return for this endpoint'} + +def consume(): + """Consume action and block if there is not. + """ + while True: + body, sdkid, sdkbase = _QUEUE.get() + try: + rest_handle_action(body, sdkid, sdkbase) + except Exception as err: + _LOGGER.critical("Worked thread issue:\n%s", traceback.format_exc()) + _LOGGER.info("End of WorkerThread") + +_WORKER_THREAD = Thread( + target=consume, + name="WorkerThread" +) +_WORKER_THREAD.start() diff --git a/swaggertosdk/restapi/github_handler.py b/swaggertosdk/restapi/github_handler.py new file mode 100644 index 0000000..c9e8f96 --- /dev/null +++ b/swaggertosdk/restapi/github_handler.py @@ -0,0 +1,276 @@ +from collections import namedtuple +import logging +import os +import re +from pathlib import Path +import tempfile +import traceback + +from github import Github +from git import Repo, GitCommandError + +from swaggertosdk.build_sdk import generate as build_sdk +from swaggertosdk.SwaggerToSdkCore import ( + manage_git_folder, + checkout_and_create_branch, + do_commit, + do_pr, + configure_user, + CONFIG_FILE, + read_config, + DEFAULT_COMMIT_MESSAGE, + get_input_paths, + extract_conf_from_readmes, + checkout_and_create_branch +) + +_LOGGER = logging.getLogger(__name__) + +WebhookMetadata = namedtuple( + 'WebhookMetadata', + ['repo', 'issue', 'text'] +) + +def build_from_issue_comment(body): + gh_token = os.environ["GH_TOKEN"] + github_con = Github(gh_token) + repo = github_con.get_repo(body['repository']['full_name']) + issue = repo.get_issue(body['issue']['number']) + text = body['comment']['body'] + return WebhookMetadata(repo, issue, text) + +def build_from_issues(body): + gh_token = os.environ["GH_TOKEN"] + github_con = Github(gh_token) + repo = github_con.get_repo(body['repository']['full_name']) + issue = repo.get_issue(body['issue']['number']) + text = body['issue']['body'] + return WebhookMetadata(repo, issue, text) + +class GithubHandler: + def __init__(self): + self.gh_token = os.environ["GH_TOKEN"] + + def act_and_response(self, webhook_data, command): + issue = webhook_data.issue + + try: + response = self.comment_command(issue, command) + except Exception as err: + response = "Something's wrong:\n```python\n{}\n```\n".format(traceback.format_exc()) + + new_comment = issue.create_comment(response) + return 'Posted: {}'.format(new_comment.html_url) + + def comment_command(self, issue, text): + split_text = text.lower().split() + if split_text[0] == "generate": + return self.generate(issue, split_text[1]) + elif split_text[0] == "rebuild": + return self.rebuild(issue, split_text[1]) + elif split_text[0] == "help": + return self.help(issue) + else: + return "I didn't understand your command:\n```bash\n{}\n```\nin this context, sorry :(".format(text) + + def help(self, issue): + message = """This is what I can do: +- `help` : this help message +- `generate ` : create a PR for this README +""" + new_comment = issue.create_comment(message) + + def generate(self, issue, readme_parameter): + # Do a start comment + new_comment = issue.create_comment("Working on generating this for you!!!") + + # Clone SDK repo + sdk_git_id = issue.repository.full_name + pr_repo_id = sdk_git_id + base_branch_name = "master" + + with tempfile.TemporaryDirectory() as temp_dir, \ + manage_git_folder(self.gh_token, temp_dir + "/sdk", sdk_git_id) as sdk_folder: + + sdk_conf = build_sdk(readme_parameter, sdk_folder) + branch_name = list(sdk_conf.keys()).pop() + + new_comment.edit("Generated! Let's see if there is something to PR.") + + sdk_repo = Repo(str(sdk_folder)) + configure_user(self.gh_token, sdk_repo) + modification = do_commit( + sdk_repo, + "Generated from {}".format(issue.html_url), + branch_name, + "" + ) + new_comment.delete() + if modification: + sdk_repo.git.push('origin', branch_name, set_upstream=True) + pip_command = 'pip install "git+{}@{}#egg={}&subdirectory={}"'.format( + issue.repository.html_url, + branch_name, + sdk_conf[branch_name]["autorest_options"]["package-name"], + sdk_conf[branch_name]["autorest_options"]["package-name"] + ) + local_command = 'pip install -e ./{}'.format(sdk_conf[branch_name]["autorest_options"]["package-name"]) + + pr_body = """Generated from Issue: {} + +You can install the new package of this PR for testing using the following command: +`{}` + +If you have a local clone of this repo in the folder /home/git/repo, please checkout this branch and do: +`{}` +""".format(issue.html_url, pip_command, local_command) + + pr = do_pr(self.gh_token, sdk_git_id, pr_repo_id, branch_name, base_branch_name, pr_body) + + answer = """ +Done! I created this branch and this PR: +- {} +- {} +""".format(branch_name, pr.html_url, pip_command) + return answer + else: + return "Sorry, there is nothing to PR" + + def rebuild(self, issue, project_pattern): + if not issue.pull_request: + return "Rebuild is just supported in PR for now" + pr = issue.repository.get_pull(issue.number) + + new_comment = issue.create_comment("Working on generating {} for you!!!".format(project_pattern)) + + config_path = CONFIG_FILE + message = "Rebuild by "+issue.html_url + initial_pr = None # There is no initial PR to test for files + autorest_bin = None + + branch_name = pr.head.ref + + rest_api_id = "Azure/azure-rest-api-specs" # current + branched_sdk_id = pr.head.repo.full_name+'@'+branch_name + + with tempfile.TemporaryDirectory() as temp_dir, \ + manage_git_folder(self.gh_token, Path(temp_dir) / Path("rest"), rest_api_id) as restapi_git_folder, \ + manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder: + + sdk_repo = Repo(str(sdk_folder)) + configure_user(self.gh_token, sdk_repo) + + config = read_config(sdk_repo.working_tree_dir, config_path) + + def skip_callback(project, local_conf): + if not project.startswith(project_pattern): + return True + return False + + from swaggertosdk import SwaggerToSdkNewCLI + SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder, + sdk_repo, temp_dir, autorest_bin) + new_comment.edit("End of generation, doing commit") + commit_sha = do_commit(sdk_repo, message, branch_name, "") + if commit_sha: + new_comment.edit("Pushing") + sdk_repo.git.push('origin', branch_name, set_upstream=True) + new_comment.delete() + else: + new_comment.delete() + return "Nothing to rebuild, this PR is up to date" + + _LOGGER.info("Build SDK finished and cleaned") + return "Build SDK finished and cleaned" + + +def generate_sdk_from_commit_safe(commit_obj, branch_name, restapi_git_id, sdkid, base_branch_name, fallback_base_branch_name="master"): + try: + response = generate_sdk_from_commit(commit_obj, branch_name, restapi_git_id, sdkid, base_branch_name, fallback_base_branch_name) + except Exception as err: + response = "Something's wrong:\n```python\n{}\n```\n".format(traceback.format_exc()) + + new_comment = commit_obj.create_comment(response) + return 'Posted: {}'.format(new_comment.html_url) + +def generate_sdk_from_commit(commit_obj, branch_name, restapi_git_id, sdk_git_id, base_branch_name, fallback_base_branch_name="master"): + """Generate SDK from a commit. + + commit_obj is the initial commit_obj from the RestAPI repo. restapi_git_id explains where to clone the repo. + sdk_git_id explains where to push the commit. + branch_name is the expected branch name in the SDK repo. + - If this branch exists, use it. + - If not, use the base branch to create that branch (base branch is where I intend to do my PR) + - If base_branch is not provided, use fallback_base_branch_name as base + - If this base branch is provided and does not exists, create this base branch first using fallback_base_branch_name (this one is required to exist) + + WARNING: + This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name" + """ + gh_token = os.environ["GH_TOKEN"] + config_path = CONFIG_FILE + message_template = DEFAULT_COMMIT_MESSAGE + autorest_bin = None + + branched_rest_api_id = restapi_git_id+'@'+commit_obj.sha + branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name + + with tempfile.TemporaryDirectory() as temp_dir, \ + manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \ + manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder: + + sdk_repo = Repo(str(sdk_folder)) + _LOGGER.info('Destination branch for generated code is %s', branch_name) + try: + _LOGGER.info('Try to checkout the destination branch if it already exists') + sdk_repo.git.checkout(branch_name) + except GitCommandError: + _LOGGER.info('Destination branch does not exists') + if base_branch_name is not None: + _LOGGER.info('Try to checkout base branch {} '.format(base_branch_name)) + try: + sdk_repo.git.checkout(base_branch_name) + except GitCommandError: + _LOGGER.info('Base branch does not exists, create it from {}'.format(fallback_base_branch_name)) + checkout_and_create_branch(sdk_repo, base_branch_name) + sdk_repo.git.push('origin', base_branch_name, set_upstream=True) + + configure_user(gh_token, sdk_repo) + + config = read_config(sdk_repo.working_tree_dir, config_path) + global_conf = config["meta"] + + from swaggertosdk import SwaggerToSdkNewCLI + from swaggertosdk import SwaggerToSdkCore + swagger_files_in_commit = SwaggerToSdkCore.get_swagger_project_files_in_pr(commit_obj, restapi_git_folder) + _LOGGER.info("Files in PR: %s ", swagger_files_in_commit) + + # Look for configuration in Readme + extract_conf_from_readmes(gh_token, swagger_files_in_commit, restapi_git_folder, sdk_git_id, config) + + def skip_callback(project, local_conf): + if not swagger_files_in_commit: + return True + markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf) + if not ( + markdown_relative_path in swagger_files_in_commit or + any(input_file in swagger_files_in_commit for input_file in optional_relative_paths)): + _LOGGER.info(f"In project {project} no files involved in this commit") + return True + return False + + SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder, + sdk_repo, temp_dir, autorest_bin) + + message = message_template + "\n\n" + commit_obj.commit.message + commit_sha = do_commit(sdk_repo, message, branch_name, commit_obj.sha) + if commit_sha: + sdk_repo.git.push('origin', branch_name, set_upstream=True) + commit_url = "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha) + commit_obj.create_comment("Did a commit to SDK for Python:\n{}".format(commit_url)) + else: + commit_obj.create_comment("This commit was treated and no generation was made for Python") + + _LOGGER.info("Build SDK finished and cleaned") + return "Build SDK finished and cleaned" + \ No newline at end of file diff --git a/swaggertosdk/restapi/views.py b/swaggertosdk/restapi/views.py new file mode 100644 index 0000000..bef0b0b --- /dev/null +++ b/swaggertosdk/restapi/views.py @@ -0,0 +1,6 @@ +from . import app +from jsonrpc.backend.flask import api + +@app.route("/") +def hello(): + return "Hello World!" \ No newline at end of file diff --git a/tests/test_build_sdk.py b/tests/test_build_sdk.py new file mode 100644 index 0000000..618ef9c --- /dev/null +++ b/tests/test_build_sdk.py @@ -0,0 +1,5 @@ +from swaggertosdk.build_sdk import * + +def test_guess_autorest_options(): + assert guess_service_info_from_path("specification/compute/resource-manager/readme.md") == {"rp_name": "compute", "is_arm": True} + assert guess_service_info_from_path("specification/servicefabric/data-plane/readme.md") == {"rp_name": "servicefabric", "is_arm": False} \ No newline at end of file diff --git a/tests/test_swaggertosdk.py b/tests/test_swaggertosdk.py index 0f29206..3b0e23c 100644 --- a/tests/test_swaggertosdk.py +++ b/tests/test_swaggertosdk.py @@ -5,6 +5,8 @@ import tempfile from pathlib import Path logging.basicConfig(level=logging.INFO) +from git import GitCommandError + # Fake Travis before importing the Script os.environ['TRAVIS'] = 'true' @@ -113,6 +115,77 @@ class TestSwaggerToSDK(unittest.TestCase): ) + def test_manage_git_folder(self): + finished = False # Authorize PermissionError on cleanup + try: + with tempfile.TemporaryDirectory() as temp_dir, \ + manage_git_folder(GH_TOKEN, temp_dir, "lmazuel/TestingRepo") as rest_repo: + + self.assertTrue((Path(rest_repo) / Path("README.md")).exists()) + + finished = True + except (PermissionError, FileNotFoundError): + if not finished: + raise + + finished = False # Authorize PermissionError on cleanup + try: + with tempfile.TemporaryDirectory() as temp_dir, \ + manage_git_folder(GH_TOKEN, temp_dir, "lmazuel/TestingRepo@lmazuel-patch-1") as rest_repo: + + self.assertTrue((Path(rest_repo) / Path("README.md")).exists()) + self.assertTrue(Repo(rest_repo).active_branch, "lmazuel-patch-1") + + finished = True + except (PermissionError, FileNotFoundError): + if not finished: + raise + + def test_clone_path(self): + finished = False # Authorize PermissionError on cleanup + try: + with tempfile.TemporaryDirectory() as temp_dir: + clone_to_path(GH_TOKEN, temp_dir, "lmazuel/TestingRepo") + self.assertTrue((Path(temp_dir) / Path("README.md")).exists()) + + finished = True + except PermissionError: + if not finished: + raise + + finished = False # Authorize PermissionError on cleanup + try: + with tempfile.TemporaryDirectory() as temp_dir: + clone_to_path(GH_TOKEN, temp_dir, "https://github.com/lmazuel/TestingRepo") + self.assertTrue((Path(temp_dir) / Path("README.md")).exists()) + + finished = True + except PermissionError: + if not finished: + raise + + finished = False # Authorize PermissionError on cleanup + try: + with tempfile.TemporaryDirectory() as temp_dir: + clone_to_path(GH_TOKEN, temp_dir, "lmazuel/TestingRepo", "lmazuel-patch-1") + self.assertTrue((Path(temp_dir) / Path("README.md")).exists()) + + finished = True + except PermissionError: + if not finished: + raise + + finished = False # Authorize PermissionError on cleanup + try: + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(GitCommandError): + clone_to_path(GH_TOKEN, temp_dir, "lmazuel/TestingRepo", "fakebranch") + + finished = True + except (PermissionError, FileNotFoundError): + if not finished: + raise + def test_do_commit(self): finished = False # Authorize PermissionError on cleanup try: