From 1d67790540035adaa4f361c90c6aa541d0d66419 Mon Sep 17 00:00:00 2001 From: Derek Bekoe Date: Fri, 11 Aug 2017 09:45:47 -0700 Subject: [PATCH] Update docs for example module (#4196) --- .../example_module_template/HISTORY.rst | 4 +- .../azure_bdist_wheel.py | 533 ++++++++++++++++++ .../example_module_template/setup.cfg | 3 + .../example_module_template/setup.py | 15 +- 4 files changed, 549 insertions(+), 6 deletions(-) create mode 100644 doc/authoring_command_modules/example_module_template/azure_bdist_wheel.py create mode 100644 doc/authoring_command_modules/example_module_template/setup.cfg diff --git a/doc/authoring_command_modules/example_module_template/HISTORY.rst b/doc/authoring_command_modules/example_module_template/HISTORY.rst index 06a33ea39..09e6486ca 100644 --- a/doc/authoring_command_modules/example_module_template/HISTORY.rst +++ b/doc/authoring_command_modules/example_module_template/HISTORY.rst @@ -3,7 +3,7 @@ Release History =============== -0.0.1b1 (YYYY-MM-DD) -^^^^^^^^^^^^^^^^^^^^ +unreleased +++++++++++ * Initial release of module. diff --git a/doc/authoring_command_modules/example_module_template/azure_bdist_wheel.py b/doc/authoring_command_modules/example_module_template/azure_bdist_wheel.py new file mode 100644 index 000000000..3ffa5ea50 --- /dev/null +++ b/doc/authoring_command_modules/example_module_template/azure_bdist_wheel.py @@ -0,0 +1,533 @@ +""" +"wheel" copyright (c) 2012-2017 Daniel Holth and +contributors. + +The MIT License + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +Create a Azure wheel (.whl) distribution (a wheel is a built archive format). + +This file is a copy of the official bdist_wheel file from wheel 0.30.0a0, enhanced +of the bottom with some Microsoft extension for Azure SDK for Python + +""" + +import csv +import hashlib +import os +import subprocess +import warnings +import shutil +import json +import sys + +try: + import sysconfig +except ImportError: # pragma nocover + # Python < 2.7 + import distutils.sysconfig as sysconfig + +import pkg_resources + +safe_name = pkg_resources.safe_name +safe_version = pkg_resources.safe_version + +from shutil import rmtree +from email.generator import Generator + +from distutils.core import Command +from distutils.sysconfig import get_python_version + +from distutils import log as logger + +from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag, get_platform +from wheel.util import native, open_for_csv +from wheel.archive import archive_wheelfile +from wheel.pkginfo import read_pkg_info, write_pkg_info +from wheel.metadata import pkginfo_to_dict +from wheel import pep425tags, metadata +from wheel import __version__ as wheel_version + +def safer_name(name): + return safe_name(name).replace('-', '_') + +def safer_version(version): + return safe_version(version).replace('-', '_') + +class bdist_wheel(Command): + + description = 'create a wheel distribution' + + user_options = [('bdist-dir=', 'b', + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('relative', None, + "build the archive using relative paths" + "(default: false)"), + ('owner=', 'u', + "Owner name used when creating a tar file" + " [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file" + " [default: current group]"), + ('universal', None, + "make a universal wheel" + " (default: false)"), + ('python-tag=', None, + "Python implementation compatibility tag" + " (default: py%s)" % get_impl_ver()[0]), + ] + + boolean_options = ['keep-temp', 'skip-build', 'relative', 'universal'] + + def initialize_options(self): + self.bdist_dir = None + self.data_dir = None + self.plat_name = None + self.plat_tag = None + self.format = 'zip' + self.keep_temp = False + self.dist_dir = None + self.distinfo_dir = None + self.egginfo_dir = None + self.root_is_pure = None + self.skip_build = None + self.relative = False + self.owner = None + self.group = None + self.universal = False + self.python_tag = 'py' + get_impl_ver()[0] + self.plat_name_supplied = False + + def finalize_options(self): + if self.bdist_dir is None: + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'wheel') + + self.data_dir = self.wheel_dist_name + '.data' + self.plat_name_supplied = self.plat_name is not None + + need_options = ('dist_dir', 'plat_name', 'skip_build') + + self.set_undefined_options('bdist', + *zip(need_options, need_options)) + + self.root_is_pure = not (self.distribution.has_ext_modules() + or self.distribution.has_c_libraries()) + + # Support legacy [wheel] section for setting universal + wheel = self.distribution.get_option_dict('wheel') + if 'universal' in wheel: + # please don't define this in your global configs + val = wheel['universal'][1].strip() + if val.lower() in ('1', 'true', 'yes'): + self.universal = True + + @property + def wheel_dist_name(self): + """Return distribution full name with - replaced with _""" + return '-'.join((safer_name(self.distribution.get_name()), + safer_version(self.distribution.get_version()))) + + def get_tag(self): + # bdist sets self.plat_name if unset, we should only use it for purepy + # wheels if the user supplied it. + if self.plat_name_supplied: + plat_name = self.plat_name + elif self.root_is_pure: + plat_name = 'any' + else: + plat_name = self.plat_name or get_platform() + if plat_name in ('linux-x86_64', 'linux_x86_64') and sys.maxsize == 2147483647: + plat_name = 'linux_i686' + plat_name = plat_name.replace('-', '_').replace('.', '_') + + + if self.root_is_pure: + if self.universal: + impl = 'py2.py3' + else: + impl = self.python_tag + tag = (impl, 'none', plat_name) + else: + impl_name = get_abbr_impl() + impl_ver = get_impl_ver() + # PEP 3149 + abi_tag = str(get_abi_tag()).lower() + tag = (impl_name + impl_ver, abi_tag, plat_name) + supported_tags = pep425tags.get_supported( + supplied_platform=plat_name if self.plat_name_supplied else None) + # XXX switch to this alternate implementation for non-pure: + assert tag == supported_tags[0], "%s != %s" % (tag, supported_tags[0]) + return tag + + def get_archive_basename(self): + """Return archive name without extension""" + + impl_tag, abi_tag, plat_tag = self.get_tag() + + archive_basename = "%s-%s-%s-%s" % ( + self.wheel_dist_name, + impl_tag, + abi_tag, + plat_tag) + return archive_basename + + def run(self): + build_scripts = self.reinitialize_command('build_scripts') + build_scripts.executable = 'python' + + if not self.skip_build: + self.run_command('build') + + install = self.reinitialize_command('install', + reinit_subcommands=True) + install.root = self.bdist_dir + install.compile = False + install.skip_build = self.skip_build + install.warn_dir = False + + # A wheel without setuptools scripts is more cross-platform. + # Use the (undocumented) `no_ep` option to setuptools' + # install_scripts command to avoid creating entry point scripts. + install_scripts = self.reinitialize_command('install_scripts') + install_scripts.no_ep = True + + # Use a custom scheme for the archive, because we have to decide + # at installation time which scheme to use. + for key in ('headers', 'scripts', 'data', 'purelib', 'platlib'): + setattr(install, + 'install_' + key, + os.path.join(self.data_dir, key)) + + basedir_observed = '' + + if os.name == 'nt': + # win32 barfs if any of these are ''; could be '.'? + # (distutils.command.install:change_roots bug) + basedir_observed = os.path.normpath(os.path.join(self.data_dir, '..')) + self.install_libbase = self.install_lib = basedir_observed + + setattr(install, + 'install_purelib' if self.root_is_pure else 'install_platlib', + basedir_observed) + + logger.info("installing to %s", self.bdist_dir) + + self.run_command('install') + + archive_basename = self.get_archive_basename() + + pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) + if not self.relative: + archive_root = self.bdist_dir + else: + archive_root = os.path.join( + self.bdist_dir, + self._ensure_relative(install.install_base)) + + self.set_undefined_options( + 'install_egg_info', ('target', 'egginfo_dir')) + self.distinfo_dir = os.path.join(self.bdist_dir, + '%s.dist-info' % self.wheel_dist_name) + self.egg2dist(self.egginfo_dir, + self.distinfo_dir) + + self.write_wheelfile(self.distinfo_dir) + + self.write_record(self.bdist_dir, self.distinfo_dir) + + # Make the archive + if not os.path.exists(self.dist_dir): + os.makedirs(self.dist_dir) + wheel_name = archive_wheelfile(pseudoinstall_root, archive_root) + + # Sign the archive + if 'WHEEL_TOOL' in os.environ: + subprocess.call([os.environ['WHEEL_TOOL'], 'sign', wheel_name]) + + # Add to 'Distribution.dist_files' so that the "upload" command works + getattr(self.distribution, 'dist_files', []).append( + ('bdist_wheel', get_python_version(), wheel_name)) + + if not self.keep_temp: + if self.dry_run: + logger.info('removing %s', self.bdist_dir) + else: + rmtree(self.bdist_dir) + + def write_wheelfile(self, wheelfile_base, generator='bdist_wheel (' + wheel_version + ')'): + from email.message import Message + msg = Message() + msg['Wheel-Version'] = '1.0' # of the spec + msg['Generator'] = generator + msg['Root-Is-Purelib'] = str(self.root_is_pure).lower() + + # Doesn't work for bdist_wininst + impl_tag, abi_tag, plat_tag = self.get_tag() + for impl in impl_tag.split('.'): + for abi in abi_tag.split('.'): + for plat in plat_tag.split('.'): + msg['Tag'] = '-'.join((impl, abi, plat)) + + wheelfile_path = os.path.join(wheelfile_base, 'WHEEL') + logger.info('creating %s', wheelfile_path) + with open(wheelfile_path, 'w') as f: + Generator(f, maxheaderlen=0).flatten(msg) + + def _ensure_relative(self, path): + # copied from dir_util, deleted + drive, path = os.path.splitdrive(path) + if path[0:1] == os.sep: + path = drive + path[1:] + return path + + def _pkginfo_to_metadata(self, egg_info_path, pkginfo_path): + return metadata.pkginfo_to_metadata(egg_info_path, pkginfo_path) + + def license_file(self): + """Return license filename from a license-file key in setup.cfg, or None.""" + metadata = self.distribution.get_option_dict('metadata') + if not 'license_file' in metadata: + return None + return metadata['license_file'][1] + + def setupcfg_requirements(self): + """Generate requirements from setup.cfg as + ('Requires-Dist', 'requirement; qualifier') tuples. From a metadata + section in setup.cfg: + + [metadata] + provides-extra = extra1 + extra2 + requires-dist = requirement; qualifier + another; qualifier2 + unqualified + + Yields + + ('Provides-Extra', 'extra1'), + ('Provides-Extra', 'extra2'), + ('Requires-Dist', 'requirement; qualifier'), + ('Requires-Dist', 'another; qualifier2'), + ('Requires-Dist', 'unqualified') + """ + metadata = self.distribution.get_option_dict('metadata') + + # our .ini parser folds - to _ in key names: + for key, title in (('provides_extra', 'Provides-Extra'), + ('requires_dist', 'Requires-Dist')): + if not key in metadata: + continue + field = metadata[key] + for line in field[1].splitlines(): + line = line.strip() + if not line: + continue + yield (title, line) + + def add_requirements(self, metadata_path): + """Add additional requirements from setup.cfg to file metadata_path""" + additional = list(self.setupcfg_requirements()) + if not additional: return + pkg_info = read_pkg_info(metadata_path) + if 'Provides-Extra' in pkg_info or 'Requires-Dist' in pkg_info: + warnings.warn('setup.cfg requirements overwrite values from setup.py') + del pkg_info['Provides-Extra'] + del pkg_info['Requires-Dist'] + for k, v in additional: + pkg_info[k] = v + write_pkg_info(metadata_path, pkg_info) + + def egg2dist(self, egginfo_path, distinfo_path): + """Convert an .egg-info directory into a .dist-info directory""" + def adios(p): + """Appropriately delete directory, file or link.""" + if os.path.exists(p) and not os.path.islink(p) and os.path.isdir(p): + shutil.rmtree(p) + elif os.path.exists(p): + os.unlink(p) + + adios(distinfo_path) + + if not os.path.exists(egginfo_path): + # There is no egg-info. This is probably because the egg-info + # file/directory is not named matching the distribution name used + # to name the archive file. Check for this case and report + # accordingly. + import glob + pat = os.path.join(os.path.dirname(egginfo_path), '*.egg-info') + possible = glob.glob(pat) + err = "Egg metadata expected at %s but not found" % (egginfo_path,) + if possible: + alt = os.path.basename(possible[0]) + err += " (%s found - possible misnamed archive file?)" % (alt,) + + raise ValueError(err) + + if os.path.isfile(egginfo_path): + # .egg-info is a single file + pkginfo_path = egginfo_path + pkg_info = self._pkginfo_to_metadata(egginfo_path, egginfo_path) + os.mkdir(distinfo_path) + else: + # .egg-info is a directory + pkginfo_path = os.path.join(egginfo_path, 'PKG-INFO') + pkg_info = self._pkginfo_to_metadata(egginfo_path, pkginfo_path) + + # ignore common egg metadata that is useless to wheel + shutil.copytree(egginfo_path, distinfo_path, + ignore=lambda x, y: set(('PKG-INFO', + 'requires.txt', + 'SOURCES.txt', + 'not-zip-safe',))) + + # delete dependency_links if it is only whitespace + dependency_links_path = os.path.join(distinfo_path, 'dependency_links.txt') + with open(dependency_links_path, 'r') as dependency_links_file: + dependency_links = dependency_links_file.read().strip() + if not dependency_links: + adios(dependency_links_path) + + write_pkg_info(os.path.join(distinfo_path, 'METADATA'), pkg_info) + + # XXX deprecated. Still useful for current distribute/setuptools. + metadata_path = os.path.join(distinfo_path, 'METADATA') + self.add_requirements(metadata_path) + + # XXX intentionally a different path than the PEP. + metadata_json_path = os.path.join(distinfo_path, 'metadata.json') + pymeta = pkginfo_to_dict(metadata_path, + distribution=self.distribution) + + if 'description' in pymeta: + description_filename = 'DESCRIPTION.rst' + description_text = pymeta.pop('description') + description_path = os.path.join(distinfo_path, + description_filename) + with open(description_path, "wb") as description_file: + description_file.write(description_text.encode('utf-8')) + pymeta['extensions']['python.details']['document_names']['description'] = description_filename + + # XXX heuristically copy any LICENSE/LICENSE.txt? + license = self.license_file() + if license: + license_filename = 'LICENSE.txt' + shutil.copy(license, os.path.join(self.distinfo_dir, license_filename)) + pymeta['extensions']['python.details']['document_names']['license'] = license_filename + + with open(metadata_json_path, "w") as metadata_json: + json.dump(pymeta, metadata_json, sort_keys=True) + + adios(egginfo_path) + + def write_record(self, bdist_dir, distinfo_dir): + from wheel.util import urlsafe_b64encode + + record_path = os.path.join(distinfo_dir, 'RECORD') + record_relpath = os.path.relpath(record_path, bdist_dir) + + def walk(): + for dir, dirs, files in os.walk(bdist_dir): + dirs.sort() + for f in sorted(files): + yield os.path.join(dir, f) + + def skip(path): + """Wheel hashes every possible file.""" + return (path == record_relpath) + + with open_for_csv(record_path, 'w+') as record_file: + writer = csv.writer(record_file) + for path in walk(): + relpath = os.path.relpath(path, bdist_dir) + if skip(relpath): + hash = '' + size = '' + else: + with open(path, 'rb') as f: + data = f.read() + digest = hashlib.sha256(data).digest() + hash = 'sha256=' + native(urlsafe_b64encode(digest)) + size = len(data) + record_path = os.path.relpath( + path, bdist_dir).replace(os.path.sep, '/') + writer.writerow((record_path, hash, size)) + + +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +from distutils import log as logger +import os.path + +#from wheel.bdist_wheel import bdist_wheel +class azure_bdist_wheel(bdist_wheel): + + description = "Create an Azure wheel distribution" + + user_options = bdist_wheel.user_options + \ + [('azure-namespace-package=', None, + "Name of the deepest nspkg used")] + + def initialize_options(self): + bdist_wheel.initialize_options(self) + self.azure_namespace_package = None + + def finalize_options(self): + bdist_wheel.finalize_options(self) + if self.azure_namespace_package and not self.azure_namespace_package.endswith("-nspkg"): + raise ValueError("azure_namespace_package must finish by -nspkg") + + def run(self): + if not self.distribution.install_requires: + self.distribution.install_requires = [] + self.distribution.install_requires.append( + "{}>=2.0.0".format(self.azure_namespace_package.replace('_', '-'))) + bdist_wheel.run(self) + + def write_record(self, bdist_dir, distinfo_dir): + if self.azure_namespace_package: + # Split and remove last part, assuming it's "nspkg" + subparts = self.azure_namespace_package.split('-')[0:-1] + folder_with_init = [os.path.join(*subparts[0:i+1]) for i in range(len(subparts))] + for azure_sub_package in folder_with_init: + init_file = os.path.join(bdist_dir, azure_sub_package, '__init__.py') + if os.path.isfile(init_file): + logger.info("manually remove {} while building the wheel".format(init_file)) + os.remove(init_file) + else: + raise ValueError("Unable to find {}. Are you sure of your namespace package?".format(init_file)) + bdist_wheel.write_record(self, bdist_dir, distinfo_dir) +cmdclass = { + 'bdist_wheel': azure_bdist_wheel, +} diff --git a/doc/authoring_command_modules/example_module_template/setup.cfg b/doc/authoring_command_modules/example_module_template/setup.cfg new file mode 100644 index 000000000..3326c62a7 --- /dev/null +++ b/doc/authoring_command_modules/example_module_template/setup.cfg @@ -0,0 +1,3 @@ +[bdist_wheel] +universal=1 +azure-namespace-package=azure-cli-command_modules-nspkg diff --git a/doc/authoring_command_modules/example_module_template/setup.py b/doc/authoring_command_modules/example_module_template/setup.py index 8cb2e9eee..311b50d7f 100644 --- a/doc/authoring_command_modules/example_module_template/setup.py +++ b/doc/authoring_command_modules/example_module_template/setup.py @@ -7,7 +7,14 @@ from codecs import open from setuptools import setup -VERSION = "0.0.1b1+dev" +try: + from azure_bdist_wheel import cmdclass +except ImportError: + from distutils import log as logger + logger.warn("Wheel is not available, disabling bdist_wheel hook") + cmdclass = {} + +VERSION = "0.0.1+dev" # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers @@ -25,6 +32,7 @@ CLASSIFIERS = [ ] DEPENDENCIES = [ + 'azure-cli-core', ] with open('README.rst', 'r', encoding='utf-8') as f: @@ -42,13 +50,12 @@ setup( author_email='azpycli@microsoft.com', url='https://github.com/Azure/azure-cli', classifiers=CLASSIFIERS, - namespace_packages = [ + packages=[ 'azure', 'azure.cli', 'azure.cli.command_modules', - ], - packages=[ 'azure.cli.command_modules.example', ], install_requires=DEPENDENCIES, + cmdclass=cmdclass )