diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7d42484..10d5c65 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,18 +2,21 @@ jobs: - job: 'Test' - pool: 'Hosted macOS' + pool: + vmImage: 'macOS-latest' strategy: matrix: - Python36: - python.version: '3.6' Python37: python.version: '3.7' Python38: python.version: '3.8' Python39: python.version: '3.9' - maxParallel: 4 + Python310: + python.version: '3.10' + Python311: + python.version: '3.11' + maxParallel: 8 steps: - task: UsePythonVersion@0 @@ -21,7 +24,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python + - script: curl -sSL https://install.python-poetry.org | python displayName: Install Poetry - script: python -m venv $(System.DefaultWorkingDirectory) @@ -29,7 +32,7 @@ jobs: - script: | source bin/activate - $HOME/.poetry/bin/poetry install + $HOME/.local/bin/poetry install displayName: 'Install dependencies' - script: | diff --git a/poetry.lock b/poetry.lock index 705e8c0..2d83476 100644 --- a/poetry.lock +++ b/poetry.lock @@ -45,7 +45,6 @@ python-versions = ">=3.6.2" [package.dependencies] click = ">=7.1.2" -dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0,<1" platformdirs = ">=2" @@ -117,14 +116,6 @@ tomli = {version = "*", optional = true, markers = "extra == \"toml\""} [package.extras] toml = ["tomli"] -[[package]] -name = "dataclasses" -version = "0.8" -description = "A backport of the dataclasses module for Python 3.6" -category = "dev" -optional = false -python-versions = ">=3.6, <3.7" - [[package]] name = "deserialize" version = "1.8.3" @@ -377,15 +368,16 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "setuptools" -version = "59.6.0" +version = "65.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] -docs = ["furo", "jaraco.packaging (>=8.2)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-inline-tabs", "sphinxcontrib-towncrier"] -testing = ["flake8-2020", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "paver", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "pytest-virtualenv (>=1.2.7)", "pytest-xdist", "sphinx", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -511,8 +503,8 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black ( [metadata] lock-version = "1.1" -python-versions = "^3.6.2" -content-hash = "cc0653fcf2407acf507376a400656ddb8a74dfcfa000f0388898804dd92ddea5" +python-versions = "^3.7.0" +content-hash = "ca8f6af06cbb59c7ea49f12740baf0df6f39c5fee3f54881338a8eff3e2cfde6" [metadata.files] astroid = [ @@ -596,10 +588,6 @@ coverage = [ {file = "coverage-6.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:5829192582c0ec8ca4a2532407bc14c2f338d9878a10442f5d03804a95fac9de"}, {file = "coverage-6.2.tar.gz", hash = "sha256:e2cad8093172b7d1595b4ad66f24270808658e11acf43a8f95b41276162eb5b8"}, ] -dataclasses = [ - {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, - {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, -] deserialize = [ {file = "deserialize-1.8.3-py3-none-any.whl", hash = "sha256:48844d7bdebfe4b440cb3548d2a32a89d429a657e6c1947c703d32105763bde1"}, {file = "deserialize-1.8.3.tar.gz", hash = "sha256:d1aa33990e046f9ccbe2bb4d1d21631329df39b52462589dd4f80c0d59919306"}, @@ -851,8 +839,8 @@ requests = [ {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] setuptools = [ - {file = "setuptools-59.6.0-py3-none-any.whl", hash = "sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"}, - {file = "setuptools-59.6.0.tar.gz", hash = "sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373"}, + {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, + {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, diff --git a/pyproject.toml b/pyproject.toml index b3cb383..b429900 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] [tool.poetry.dependencies] -python = "^3.6.2" +python = "^3.7.0" deserialize = "^1.5.1" requests = "^2.21" tenacity = "^6.2.0" diff --git a/simple_ado/git.py b/simple_ado/git.py index 4d4b174..9a4ff47 100755 --- a/simple_ado/git.py +++ b/simple_ado/git.py @@ -247,6 +247,7 @@ class ADOGitClient(ADOBaseClient): output_path: str, project_id: str, repository_id: str, + callback: Optional[Callable[[int, int], None]] = None ) -> None: """Download the zip of the branch specified. @@ -254,6 +255,9 @@ class ADOGitClient(ADOBaseClient): :param str output_path: The path to write the output to. :param str project_id: The ID of the project :param str repository_id: The ID for the repository + :param callback: The callback for download progress updates. First + parameter is bytes downloaded, second is total bytes. + The latter will be 0 if the content length is unknown. :raises ADOException: If the output path already exists :raises ADOHTTPException: If we fail to fetch the zip for any reason @@ -278,7 +282,12 @@ class ADOGitClient(ADOBaseClient): raise ADOException("The output path already exists") with self.http_client.get(request_url, stream=True) as response: - download_from_response_stream(response=response, output_path=output_path, log=self.log) + download_from_response_stream( + response=response, + output_path=output_path, + log=self.log, + callback=callback + ) # pylint: disable=too-many-locals def get_refs( diff --git a/simple_ado/utilities.py b/simple_ado/utilities.py index 228fc49..d739d7f 100644 --- a/simple_ado/utilities.py +++ b/simple_ado/utilities.py @@ -1,6 +1,7 @@ """Utilities for dealing with the ADO REST API.""" import logging +from typing import Callable, Optional import requests @@ -18,13 +19,18 @@ def boolstr(value: bool) -> str: def download_from_response_stream( - *, response: requests.Response, output_path: str, log: logging.Logger + *, + response: requests.Response, + output_path: str, + log: logging.Logger, + callback: Optional[Callable[[int, int], None]] = None ) -> None: """Downloads a file from an already open response stream. :param requests.Response response: The response to download from :param str output_path: The path to write the file out to :param logging.Logger log: The log to use for progress updates + :param callback: If supplied, this will be called on every new chunk to update progress to the caller :raises ADOHTTPException: If we fail to fetch the file for any reason """ @@ -46,6 +52,9 @@ def download_from_response_stream( total_downloaded += len(data) output_file.write(data) + if callback is not None: + callback(total_downloaded, total_size) + if total_size != 0: progress = int((total_downloaded * 100.0) / total_size) log.info(f"Download progress: {progress}%")