RPM Package scripts (#4408)

packaged_releases/rpm/README.md

@@ -0,0 +1,65 @@
+RPM Packaging
+================
+
+Building the RPM package
+------------------------
+
+On a build machine (e.g. new CentOS 7 VM) run the following.
+
+Install dependencies required to build:
+```
+# Required for rpm build tools.
+sudo yum install -y gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools
+
+# Required to build the CLI.
+sudo yum install -y gcc python libffi-devel python-devel openssl-devel
+```
+
+Set up directory structure for build:
+```
+mkdir -p ~/rpmbuild/
+cd ~/rpmbuild/
+mkdir -p BUILD RPMS SOURCES SPECS SRPMS
+> SPECS/azure-cli.spec; vi SPECS/azure-cli.spec
+```
+
+Set the CLI version and SHA256 for the archive:
+```
+export CLI_VERSION=2.0.16
+export CLI_DOWNLOAD_SHA256=22c048d2911c13738c6b901a741ea655f277e0d9eb756c4fb9aee6bb6c2b0109
+```
+
+RPM Build:
+```
+rpmbuild -v -bb --clean SPECS/azure-cli.spec
+```
+
+Verification
+------------
+
+```
+sudo rpm -i RPMS/*/azure-cli-2.0.16-1.noarch.rpm
+az --version
+```
+
+Check the file permissions of the package:  
+```
+rpmlint RPMS/*/azure-cli-2.0.16-1.x86_64.rpm
+```
+
+Check the file permissions of the package:  
+```
+rpm -qlvp RPMS/*/azure-cli-2.0.16-1.x86_64.rpm
+```
+
+To remove:  
+```
+sudo rpm -e azure-cli
+```
+
+Links
+-----
+
+https://fedoraproject.org/wiki/How_to_create_an_RPM_package
+
+https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros

packaged_releases/rpm/azure-cli.spec

@@ -0,0 +1,81 @@
+# RPM spec file for Azure CLI 2.0
+# Definition of macros used - https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros
+
+# .el7.centos -> .el7
+%if 0%{?rhel} == 7
+  %define dist .el7
+%endif
+
+%define name           azure-cli
+%define release        1%{?dist}
+%define version        %{getenv:CLI_VERSION}
+%define source_sha256  %{getenv:CLI_DOWNLOAD_SHA256}
+%define source_url     https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_%{version}.tar.gz
+%define venv_url       https://pypi.python.org/packages/source/v/virtualenv/virtualenv-15.0.0.tar.gz
+%define venv_sha256    70d63fb7e949d07aeb37f6ecc94e8b60671edb15b890aa86dba5dfaf2225dc19
+%define cli_lib_dir    %{_libdir}/az
+
+Summary:        Azure CLI 2.0
+License:        MIT
+Name:           %{name}
+Version:        %{version}
+Release:        %{release}
+Source0:        %{source_url}
+Url:            https://docs.microsoft.com/en-us/cli/azure/install-azure-cli
+BuildArch:      x86_64
+Requires:       python
+
+BuildRequires:  gcc
+BuildRequires:  python
+BuildRequires:  libffi-devel
+BuildRequires:  python-devel
+BuildRequires:  openssl-devel
+
+%description
+A great cloud needs great tools; we're excited to introduce Azure CLI 2.0,
+ our next generation multi-platform command line experience for Azure.
+
+%prep
+# Create some tmp files
+tmp_venv_archive=$(mktemp)
+tmp_source_archive=$(mktemp)
+
+# Download, Extract Source
+wget %{source_url} -qO $tmp_source_archive
+echo "%{source_sha256}  $tmp_source_archive" | sha256sum -c -
+tar -xvzf $tmp_source_archive -C %{_builddir}
+
+# Download, Extract Virtualenv
+wget %{venv_url} -qO $tmp_venv_archive
+echo "%{venv_sha256}  $tmp_venv_archive" | sha256sum -c -
+tar -xvzf $tmp_venv_archive -C %{_builddir}
+
+%install
+# Create the venv
+python %{_builddir}/virtualenv-15.0.0/virtualenv.py --python python %{buildroot}%{cli_lib_dir}
+
+# Build the wheels from the source
+source_dir=%{_builddir}/azure-cli_packaged_%{version}
+dist_dir=$(mktemp -d)
+for d in $source_dir/src/azure-cli $source_dir/src/azure-cli-core $source_dir/src/azure-cli-nspkg $source_dir/src/azure-cli-command_modules-nspkg $source_dir/src/command_modules/azure-cli-*/; \
+do cd $d; %{buildroot}%{cli_lib_dir}/bin/python setup.py bdist_wheel -d $dist_dir; cd -; done;
+
+# Install the CLI
+%{buildroot}%{cli_lib_dir}/bin/pip install azure-cli --find-links $dist_dir
+%{buildroot}%{cli_lib_dir}/bin/pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
+
+# Fix up %{buildroot} appearing in some files...
+for d in %{buildroot}%{cli_lib_dir}/bin/*; do perl -p -i -e "s#%{buildroot}##g" $d; done;
+
+# Create executable
+mkdir -p %{buildroot}%{_bindir}
+printf '#!/usr/bin/env bash\n%{cli_lib_dir}/bin/python -Esm azure.cli "$@"' > %{buildroot}%{_bindir}/az
+
+# Set up tab completion
+mkdir -p %{buildroot}%{_sysconfdir}/bash_completion.d/
+cat $source_dir/az.completion > %{buildroot}%{_sysconfdir}/bash_completion.d/azure-cli
+
+%files
+%attr(-,root,root) %{cli_lib_dir}
+%config(noreplace) %{_sysconfdir}/bash_completion.d/azure-cli
+%attr(0755,root,root) %{_bindir}/az

scripts/releaser/HISTORY.rst

@@ -3,6 +3,11 @@
 Release History
 ===============
 
+0.1.4 (2017-09-07)
+++++++++++++++++++
+
+* Add release script for RPM releases.
+
 0.1.3 (2017-08-18)
 ++++++++++++++++++
 

scripts/releaser/README.md

@@ -29,6 +29,7 @@ These can be set in the initial `docker run` command above or by using `export E
 python release.py
 python release-docker.py
 python release-debian.py
+python release-rpm.py
 ```
 
 Environment Variables
@@ -51,7 +52,9 @@ The `GITHUB_USER` should have the following GitHub OAuth scopes:
 `DOCKER_USERNAME` - The Docker username that has push permissions to the above Docker repo.
 `DOCKER_PASSWORD` - The Docker password for the user.
 
+`MS_REPO_URL` - The repository URL to publish .deb/.rpm packages.
+`MS_REPO_USERNAME` - The repository username to publish .deb/.rpm packages.
+`MS_REPO_PASSWORD` - The user password to publish the .deb/.rpm package.
+
 `DEBIAN_REPO_ID` - The repository ID to publish the .deb package.
-`DEBIAN_REPO_URL` - The repository URL to publish the .deb package.
+`YUM_REPO_ID` - The repository ID to publish the .rpm package.
-`DEBIAN_REPO_USERNAME` - The repository username to publish the .deb package.
-`DEBIAN_REPO_PASSWORD` - The user password to publish the .deb package.

scripts/releaser/release-debian.py

@@ -26,9 +26,9 @@ add_script_env('CLI_VERSION')
 add_script_env('CLI_DOWNLOAD_SHA256')
 add_script_env('AZURE_STORAGE_CONNECTION_STRING')
 add_script_env('DEBIAN_REPO_ID')
-add_script_env('DEBIAN_REPO_URL')
+add_script_env('MS_REPO_URL')
-add_script_env('DEBIAN_REPO_USERNAME')
+add_script_env('MS_REPO_USERNAME')
-add_script_env('DEBIAN_REPO_PASSWORD')
+add_script_env('MS_REPO_PASSWORD')
 
 assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set.  {}".format(script_env)
 
@@ -133,9 +133,9 @@ def main():
     upload_script = REPO_UPLOAD_SCRIPT_TMPL.format(cli_version=script_env.get('CLI_VERSION'),
                                                    repo_id=script_env.get('DEBIAN_REPO_ID'),
                                                    source_url=deb_url,
-                                                   repo_package_url=script_env.get('DEBIAN_REPO_URL'),
+                                                   repo_package_url=script_env.get('MS_REPO_URL'),
-                                                   repo_user=script_env.get('DEBIAN_REPO_USERNAME'),
+                                                   repo_user=script_env.get('MS_REPO_USERNAME'),
-                                                   repo_pass=script_env.get('DEBIAN_REPO_PASSWORD'))
+                                                   repo_pass=script_env.get('MS_REPO_PASSWORD'))
     my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
     my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
     print_status('Done. :)')

scripts/releaser/release-rpm.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+# This script is interactive as you need to log in to 'az'.
+
+from __future__ import print_function
+
+import os
+import sys
+import time
+from datetime import datetime
+from six import StringIO
+from sh import az, ssh
+
+
+script_env = {}
+
+def add_script_env(name):
+    script_env[name] = os.environ.get(name)
+
+add_script_env('REPO_NAME')
+add_script_env('CLI_VERSION')
+add_script_env('CLI_DOWNLOAD_SHA256')
+add_script_env('AZURE_STORAGE_CONNECTION_STRING')
+add_script_env('YUM_REPO_ID')
+add_script_env('MS_REPO_URL')
+add_script_env('MS_REPO_USERNAME')
+add_script_env('MS_REPO_PASSWORD')
+
+assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set.  {}".format(script_env)
+
+REPO_UPLOAD_SCRIPT_TMPL = """
+import os, requests
+payload = {{'name': 'azure-cli', 'version': '{cli_version}', 'repositoryId': '{repo_id}', 'sourceUrl': '{source_url}'}}
+r = requests.post('{repo_package_url}', verify=False, auth=('{repo_user}', '{repo_pass}'), json=payload)
+print('Status Code')
+print(r.status_code)
+print('Query with a GET to the following:')
+print(r.headers['Location'])
+"""
+
+def print_env_vars():
+    for n in script_env:
+        print('{} = {}'.format(n, script_env[n]))
+
+
+def print_status(msg=''):
+    print('-- '+msg)
+
+def print_heading(heading):
+    print('{0}\n{1}\n{0}'.format('=' * len(heading), heading))
+
+def give_chance_to_cancel(msg_prefix=''):
+    cancel_time_secs = 10
+    msg_tmpl = '{}: Starting in {} seconds.'
+    for i in range(cancel_time_secs, 0, -1):
+        print_status(msg_tmpl.format(msg_prefix, i))
+        time.sleep(1)
+
+def main():
+    print_env_vars()
+    time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S')
+    az(["login"], _out=sys.stdout, _err=sys.stdout)
+    resource_group = 'azurecli-release-rpm-' + time_str
+    vm_name = 'vm-rpm-' + time_str
+    print_status('Creating resource group.')
+    az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout)
+    print_status('Creating VM.')
+    az(['vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh',
+        '--image', 'OpenLogic:CentOS:7.3:latest', '--admin-username', 'myuser'],
+       _out=sys.stdout, _err=sys.stdout)
+    io = StringIO()
+    print_status('Getting VM IP address.')
+    az(['vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name,
+        '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress'], _out=io)
+    ip_address = io.getvalue().strip().replace('"', '')
+    print_status('VM IP address is {}'.format(ip_address))
+    io.close()
+    vm_connect_str = "myuser@{}".format(ip_address)
+    my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str])
+    print_status('Installing git.')
+    build_prereqs = "sudo yum update -y && sudo yum install -y git gcc rpm-build rpm-devel rpmlint make python bash coreutils " \
+                    "diffutils patch rpmdevtools python libffi-devel python-devel openssl-devel"
+    my_vm(build_prereqs.split(),
+          _out=sys.stdout, _err=sys.stdout)
+    my_vm("mkdir -p ~/rpmbuild/BUILD ~/rpmbuild/RPMS ~/rpmbuild/SOURCES ~/rpmbuild/SPECS ~/rpmbuild/SRPMS".split(), _out=io)
+    io = StringIO()
+    my_vm(['mktemp', '-d'], _out=io)
+    repo_dir = io.getvalue().strip()
+    io.close()
+    print_status('Cloning repo.')
+    my_vm(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_dir], _out=sys.stdout, _err=sys.stdout)
+    path_to_spec_file = os.path.join(repo_dir, 'packaged_releases', 'rpm', 'azure-cli.spec')
+    print_status('Running build script.')
+    my_vm(['export', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), '&&',
+           'export', 'CLI_DOWNLOAD_SHA256={}'.format(script_env.get('CLI_DOWNLOAD_SHA256')), '&&',
+           'rpmbuild', '-v', '-bb', '--clean', path_to_spec_file],
+          _out=sys.stdout, _err=sys.stdout)
+    print_status('Build complete.')
+    io = StringIO()
+    my_vm(['ls', '~/rpmbuild/RPMS/*/*'], _out=io)
+    rpm_file_path = io.getvalue().strip()
+    io.close()
+    artifact_name = rpm_file_path.split('/')[-1]
+    print_status('Installing the .rpm on the build machine')
+    my_vm(['sudo', 'rpm', '-i', rpm_file_path], _out=sys.stdout, _err=sys.stdout)
+    # Upload to Azure Storage
+    print_status('Uploading .rpm to Azure storage.')
+    my_vm(['az', 'storage', 'container', 'create', '--name', 'rpms', '--public-access', 'blob',
+           '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))],
+          _out=sys.stdout, _err=sys.stdout)
+    my_vm(['az', 'storage', 'blob', 'upload', '-f', rpm_file_path,
+           '-n', artifact_name, '-c', 'rpms', '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))],
+          _out=sys.stdout, _err=sys.stdout)
+    io = StringIO()
+    my_vm(['az', 'storage', 'blob', 'url', '-n', artifact_name, '-c', 'rpms', '--output', 'tsv',
+           '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], _out=io)
+    rpm_url = io.getvalue().strip()
+    io.close()
+    print_status('RPM file uploaded to the following URL.')
+    print_status(rpm_url)
+    # Publish to service
+    my_vm(['wget', '-q', 'https://bootstrap.pypa.io/get-pip.py'], _out=sys.stdout, _err=sys.stdout)
+    my_vm(['sudo', 'python', 'get-pip.py'], _out=sys.stdout, _err=sys.stdout)
+    my_vm(['sudo', 'pip', 'install', '--upgrade', 'requests'], _out=sys.stdout, _err=sys.stdout)
+    upload_script = REPO_UPLOAD_SCRIPT_TMPL.format(cli_version=script_env.get('CLI_VERSION'),
+                                                   repo_id=script_env.get('YUM_REPO_ID'),
+                                                   source_url=rpm_url,
+                                                   repo_package_url=script_env.get('MS_REPO_URL'),
+                                                   repo_user=script_env.get('MS_REPO_USERNAME'),
+                                                   repo_pass=script_env.get('MS_REPO_PASSWORD'))
+    # Keeping this code commented for when we can automate the signing of RPM packages.
+    # my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
+    # my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
+    print_status('PRINTING OUT REPO UPLOAD SCRIPT AS THE UNSIGNED RPM NEEDS TO BE FIRST SIGNED BEFORE UPLOADING...')
+    my_vm(['cat', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
+    print_status('Done. :)')
+    give_chance_to_cancel('Delete resource group (in background)')
+    az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout)
+    print_status('Finished. :)')
+
+if __name__ == '__main__':
+    main()