diff --git a/doc/releasing_components.md b/doc/releasing_components.md deleted file mode 100644 index e4a2e17a4..000000000 --- a/doc/releasing_components.md +++ /dev/null @@ -1,17 +0,0 @@ -Releasing Components -==================== - -To create a release for a component, create a PR with title `Release ` - - e.g. 'Release azure-cli-vm 0.1.1' - -The 'Release' label should be added to the PR. - -PR checklist: - -- [ ] The PR title (commit) has format `Release `. -- [ ] `setup.py` has been modified with the same version as in the PR title. -- [ ] If required, `__version__` defined in any `__init__.py` should also be modified to match. -- [ ] `HISTORY.rst` has been modified with appropriate release notes. - -When the PR is approved and merged, the component will be released and available on PyPI. diff --git a/scripts/github_bot/HISTORY.rst b/scripts/github_bot/HISTORY.rst deleted file mode 100644 index 7b586d8ee..000000000 --- a/scripts/github_bot/HISTORY.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. :changelog: - -Release History -=============== - -0.1.2 (2017-05-31) -++++++++++++++++++ - -* Add link to docs.microsoft.com release notes. - -0.1.1 (2017-03-13) -++++++++++++++++++ - -* Return 200 if webhook is skipped by the bot instead of 400. -* Create release on async thread to prevent GitHub webhook timeouts. -* Clone and tag the correct commit to prevent race conditions with multiple merges to the repo at once. - -0.1.0 (2017-01-13) -++++++++++++++++++ - -* First release. -* Uploads source code, .tar.gz and .whl assets to the GitHub release. -* Bot comments on opened Release PR with checklist -* Bot comments if release successful or not -* Bot applies 'Release' label automatically -* Bot creates a GitHub release diff --git a/scripts/github_bot/README.md b/scripts/github_bot/README.md deleted file mode 100644 index 98854c3a7..000000000 --- a/scripts/github_bot/README.md +++ /dev/null @@ -1,93 +0,0 @@ -GitHub bot for Azure CLI -======================== - -Description ------------ -This is a GitHub bot/webhook for automated component/package releases. - -The server can be extended to handle more GitHub webhook events in the future as needed. - - -Run locally with `export FLASK_APP=app.py; flask run -h '0.0.0.0'` - -How to Build ------------- -``` -$ sudo docker build --no-cache -t azuresdk/azure-cli-bot:0.1.0 . -``` - -How to Run with Docker ----------------------- -The following environment variables are required for the server to run: - -`REPO_NAME` - The name of the GitHub repo (e.g. azure/azure-cli) -`GITHUB_SECRET_TOKEN` - The Secret that is configured in GitHub Webhook settings. -`GITHUB_USER` - User id of the bot that will post comments and create releases. -`GITHUB_USER_TOKEN` - Access token for this user. -`ALLOWED_USERS` - Space separated list of GitHub usernames that can create releases. -`PYPI_REPO` - URL to PyPI (e.g. https://testpypi.python.org/pypi or https://pypi.python.org/pypi). -`TWINE_USERNAME` - Username to authenticate with PyPI. -`TWINE_PASSWORD` - Password to authenticate with PyPI. - -The `GITHUB_USER` should have the following GitHub OAuth scopes: -- repo_deployment (to create GitHub releases) -- public_repo (to post comments on the repo) - -For example: - -``` -$ sudo docker run -d -e "REPO_NAME=azure/azure-cli" -e "GITHUB_SECRET_TOKEN=" -e "GITHUB_USER=user1" \ --e "GITHUB_USER_TOKEN=" -e "ALLOWED_USERS=user1 user2 user3" \ --e "PYPI_REPO=https://testpypi.python.org/pypi" -e "TWINE_USERNAME=" -e "TWINE_PASSWORD=" \ --p 80:80 azuresdk/azure-cli-bot:0.1.0 -``` - - -Verify server running ---------------------- -``` -$ curl -X GET -i 'http://:/' -HTTP/1.1 200 OK -Server: gunicorn/19.6.0 -Connection: close -Content-Type: application/json -Content-Length: 58 - -{ - "message": "API is running!", - "version": "0.1.0" -} -``` - - -Deploy on Azure Linux Web App (with Docker container) ------------------------------------------------------ -The docker image should be available on a Docker registry. - -``` -$ sudo docker push azuresdk/azure-cli-bot:0.1.0 -``` - - -For example: - -``` -$ az appservice plan create -g -n --sku s1 --is-linux -$ az appservice web create -g --plan -n -$ az appservice web config container update -g -n \ ---docker-custom-image-name -$ az appservice web config appsettings update -g -n \ ---settings 'REPO_NAME=azure/azure-cli' 'GITHUB_SECRET_TOKEN=' 'GITHUB_USER=user1' 'GITHUB_USER_TOKEN=' \ -'ALLOWED_USERS=user1 user2 user3' 'PYPI_REPO=https://testpypi.python.org/pypi' \ -'TWINE_USERNAME=' 'TWINE_PASSWORD=' -$ az appservice web browse -g -n -``` - - -Registering the Webhook in GitHub ---------------------------------- - -- Enter the appropriate payload URL (i.e. https://:/github-webhook). -- The payload URL *should* be over SSL (i.e. https://). -- Set the Secret to be a value defined in the running service. -- Set the webhook to trigger for 'Pull request' events. diff --git a/scripts/github_bot/api/Dockerfile b/scripts/github_bot/api/Dockerfile deleted file mode 100644 index 44cfdf347..000000000 --- a/scripts/github_bot/api/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -FROM python:3.5.2-alpine - -RUN apk upgrade --no-cache && \ - apk add --no-cache bash git openssh - -RUN pip install --no-cache-dir --upgrade pip gunicorn Flask wheel twine requests uritemplate.py - -ADD app.py / - -ENV FLASK_APP app.py - -CMD gunicorn --log-level DEBUG -w 10 -b 0.0.0.0:80 app:app diff --git a/scripts/github_bot/api/app.py b/scripts/github_bot/api/app.py deleted file mode 100644 index 77d7c98a0..000000000 --- a/scripts/github_bot/api/app.py +++ /dev/null @@ -1,221 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# pylint: disable=line-too-long - -import os -import hmac -import tempfile -import shutil -import requests -import threading -from hashlib import sha1 -from flask import Flask, jsonify, request, Response -from subprocess import check_call, CalledProcessError -from uritemplate import URITemplate, expand - -VERSION = '0.1.2' - -# GitHub API constants -GITHUB_UA_PREFIX = 'GitHub-Hookshot/' -GITHUB_EVENT_NAME_PING = 'ping' -GITHUB_EVENT_NAME_PR = 'pull_request' -GITHUB_ALLOWED_EVENTS = [GITHUB_EVENT_NAME_PING, GITHUB_EVENT_NAME_PR] - -# Environment variables -ENV_REPO_NAME = os.environ.get('REPO_NAME') -ENV_GITHUB_SECRET_TOKEN = os.environ.get('GITHUB_SECRET_TOKEN') -ENV_GITHUB_API_USER = os.environ.get('GITHUB_USER') -ENV_GITHUB_API_USER_TOKEN = os.environ.get('GITHUB_USER_TOKEN') -ENV_ALLOWED_USERS = os.environ.get('ALLOWED_USERS') -ENV_PYPI_REPO = os.environ.get('PYPI_REPO') -# although not used directly here, twine env vars are needed for releasing -ENV_PYPI_USERNAME = os.environ.get('TWINE_USERNAME') -ENV_PYPI_PASSWORD = os.environ.get('TWINE_PASSWORD') - -assert (ENV_REPO_NAME and ENV_GITHUB_SECRET_TOKEN and ENV_ALLOWED_USERS and ENV_PYPI_REPO and ENV_PYPI_USERNAME and ENV_PYPI_PASSWORD and ENV_GITHUB_API_USER and ENV_GITHUB_API_USER_TOKEN),\ - "Not all required environment variables have been set. "\ - "Set REPO_NAME, GITHUB_SECRET_TOKEN, GITHUB_USER, GITHUB_USER_TOKEN, ALLOWED_USERS, PYPI_REPO, TWINE_USERNAME, TWINE_PASSWORD" - -GITHUB_API_AUTH = (ENV_GITHUB_API_USER, ENV_GITHUB_API_USER_TOKEN) -GITHUB_API_HEADERS = {'Accept': 'application/vnd.github.v3+json', 'user-agent': 'azure-cli-bot/{}'.format(VERSION)} - -RELEASE_LABEL = 'Release' -SOURCE_ARCHIVE_NAME = 'source.tar.gz' - -OPENED_RELEASE_PR_COMMENT = """ -Reviewers, please verify the following for this PR: - -- [ ] The PR title has format `Release `. -- [ ] `setup.py` has been modified with the same version as in the PR title. -- [ ] `HISTORY.rst` has been modified with appropriate release notes. -- [ ] If releasing azure-cli or azure-cli-core, `__version__` defined in `__init__.py` should also be modified to match. -""" - -GITHUB_RELEASE_BODY_TMPL = """ -The module has been published to PyPI. - -View HISTORY.rst of the module for a changelog. - -{} - -Full release notes at https://docs.microsoft.com/en-us/cli/azure/release-notes-azure-cli - -""" - -def _verify_parse_request(req): - headers = req.headers - raw_payload = req.data - payload = req.get_json() - # Verify User Agent - ua = headers.get('User-Agent') - assert ua.startswith(GITHUB_UA_PREFIX), "Invalid User-Agent '{}'".format(ua) - # Verify Signature - gh_sig = headers.get('X-Hub-Signature') - computed_sig = 'sha1=' + hmac.new(bytearray(ENV_GITHUB_SECRET_TOKEN, 'utf-8'), msg=raw_payload, digestmod=sha1).hexdigest() - assert hmac.compare_digest(gh_sig, computed_sig), "Signatures didn't match" - # Verify GitHub event - gh_event = headers.get('X-GitHub-Event') - assert gh_event in GITHUB_ALLOWED_EVENTS, "Webhook does not support event '{}'".format(gh_event) - # Verify the repository - event_repo = payload['repository']['full_name'] - assert event_repo == ENV_REPO_NAME, "Not listening for events from repo '{}'".format(event_repo) - # Verify the sender (user who performed/triggered the event) - event_sender = payload['sender']['login'] - assert event_sender in ENV_ALLOWED_USERS.split(), "Not listening for events from sender '{}'".format(event_sender) - return {'event': gh_event, 'payload': payload} - -def _handle_ping_event(_): - return jsonify(ok=True) - -def _parse_pr_title(title): - assert title - part1, component_name, version = title.split() - assert part1.lower() == 'release' - assert component_name.lower().startswith('azure-cli') - return component_name, version - -def create_release(component_name, release_assets_dir, merge_commit_sha): - working_dir = tempfile.mkdtemp() - check_call(['git', 'clone', 'https://github.com/{}'.format(ENV_REPO_NAME), working_dir]) - check_call(['git', 'checkout', merge_commit_sha], cwd=working_dir) - check_call(['pip', 'install', '-e', 'scripts'], cwd=working_dir) - check_call(['python', '-m', 'scripts.automation.release.run', '-c', component_name, - '-r', ENV_PYPI_REPO, '--dest', release_assets_dir], cwd=working_dir) - shutil.rmtree(working_dir, ignore_errors=True) - -def apply_release_label(issue_url): - payload = [RELEASE_LABEL] - r = requests.post('{}/labels'.format(issue_url), json=payload, auth=GITHUB_API_AUTH, headers=GITHUB_API_HEADERS) - return True if r.status_code in [200, 201] else False - -def comment_on_pr(comments_url, comment_body): - payload = {'body': comment_body} - r = requests.post(comments_url, json=payload, auth=GITHUB_API_AUTH, headers=GITHUB_API_HEADERS) - return True if r.status_code == 201 else False - -def upload_asset(upload_uri_tmpl, filepath, label): - filename = os.path.basename(filepath) - upload_url = URITemplate(upload_uri_tmpl).expand(name=filename, label=label) - headers = GITHUB_API_HEADERS - headers['Content-Type'] = 'application/octet-stream' - with open(filepath, 'rb') as payload: - requests.post(upload_url, data=payload, auth=GITHUB_API_AUTH, headers=headers) - -def upload_assets_for_github_release(upload_uri_tmpl, component_name, component_version, release_assets_dir): - for filename in os.listdir(release_assets_dir): - fullpath = os.path.join(release_assets_dir, filename) - if filename == SOURCE_ARCHIVE_NAME: - upload_asset(upload_uri_tmpl, fullpath, '{} {} source code (.tar.gz)'.format(component_name, component_version)) - elif filename.endswith('.tar.gz'): - upload_asset(upload_uri_tmpl, fullpath, '{} {} Source Distribution (.tar.gz)'.format(component_name, component_version)) - elif filename.endswith('.whl'): - upload_asset(upload_uri_tmpl, fullpath, '{} {} Python Wheel (.whl)'.format(component_name, component_version)) - -def create_github_release(component_name, component_version, released_pypi_url, release_assets_dir, merge_commit_sha): - tag_name = '{}-{}'.format(component_name, component_version) - release_name = "{} {}".format(component_name, component_version) - payload = {'tag_name': tag_name, "target_commitish": merge_commit_sha, "name": release_name, "body": GITHUB_RELEASE_BODY_TMPL.format(released_pypi_url), "prerelease": False} - r = requests.post('https://api.github.com/repos/{}/releases'.format(ENV_REPO_NAME), json=payload, auth=GITHUB_API_AUTH, headers=GITHUB_API_HEADERS) - if r.status_code == 201: - upload_url = r.json()['upload_url'] - upload_assets_for_github_release(upload_url, component_name, component_version, release_assets_dir) - return True - return False - -def _task_async_release(component_name=None, component_version=None, comment_url=None, merge_commit_sha=None): - try: - comment_on_pr(comment_url, "Started releasing {} {}...".format(component_name, component_version)) - release_assets_dir = tempfile.mkdtemp() - create_release(component_name, release_assets_dir, merge_commit_sha) - released_pypi_url = '{}/{}/{}'.format(ENV_PYPI_REPO, component_name, component_version) - msg = "Release of '{}' with version '{}' successful. View at {}.".format(component_name, component_version, released_pypi_url) - comment_on_pr(comment_url, msg) - success = create_github_release(component_name, component_version, released_pypi_url, release_assets_dir, merge_commit_sha) - if success: - comment_on_pr(comment_url, 'GitHub release created. https://github.com/{}/releases.'.format(ENV_REPO_NAME)) - else: - comment_on_pr(comment_url, 'GitHub release creation unsuccessful. Please create a release at https://github.com/{}/releases.'.format(ENV_REPO_NAME)) - except CalledProcessError: - err_msg = "Release of '{}' with version '{}' unsuccessful. Admins, please release manually.".format(component_name, component_version) - comment_on_pr(comment_url, err_msg) - - -def _handle_pr_event(payload): - pr_action = payload['action'] - pr = payload['pull_request'] - pr_merged = pr_action == 'closed' and pr['merged'] - try: - component_name, component_version = _parse_pr_title(pr['title']) - issue_url = pr['_links']['issue']['href'] - comment_url = pr['_links']['comments']['href'] - if pr_action == 'opened': - apply_release_label(issue_url) - comment_created = comment_on_pr(comment_url, OPENED_RELEASE_PR_COMMENT) - if comment_created: - return jsonify(msg="Commented on PR with release PR checklist.") - else: - return jsonify(msg="Attempted to comment on PR but API request did not succeed.") - elif not pr_merged: - return jsonify(msg="Ignoring event. PR not merged.") - else: - # Merged PR - merge_commit_sha = pr['merge_commit_sha'] - msg = "Received merge PR. Asynchronously creating release." - task_args = {'component_name': component_name, - 'component_version': component_version, - 'merge_commit_sha': merge_commit_sha, - 'comment_url': comment_url} - async_task = threading.Thread(target=_task_async_release, kwargs=task_args) - async_task.start() - return jsonify(msg=msg) - except (AssertionError, ValueError): - return jsonify(msg="Ignoring merged PR as not a Release PR. Expecting title to have format 'Release '") - return jsonify(error='Unable to handle PR event.'), 500 - -app = Flask(__name__) - -@app.route('/') -def hello(): - return jsonify(version=VERSION, message='API is running!') - -@app.route('/github-webhook', methods=['POST']) -def handle_github_webhook(): - try: - parsed_req = _verify_parse_request(request) - event = parsed_req['event'] - payload = parsed_req['payload'] - if event == GITHUB_EVENT_NAME_PING: - return _handle_ping_event(payload) - elif event == GITHUB_EVENT_NAME_PR: - return _handle_pr_event(payload) - else: - return jsonify(error="Event '{}' not supported.".format(event)) - except AssertionError as e: - return jsonify(error=str(e)) - return jsonify(error='Unable to handle request.'), 500 - -if __name__ == "__main__": - app.run()