Merge branch 'microsoft:main' into device_flow

azure-pipelines.yml

@@ -2,18 +2,21 @@
 jobs:
 
 - job: 'Test'
-  pool: 'Hosted macOS'
+  pool:
+    vmImage: 'macOS-latest'
   strategy:
     matrix:
-      Python36:
-        python.version: '3.6'
       Python37:
         python.version: '3.7'
       Python38:
         python.version: '3.8'
       Python39:
         python.version: '3.9'
-    maxParallel: 4
+      Python310:
+        python.version: '3.10'
+      Python311:
+        python.version: '3.11'
+    maxParallel: 8
 
   steps:
   - task: UsePythonVersion@0
@@ -21,7 +24,7 @@ jobs:
       versionSpec: '$(python.version)'
       architecture: 'x64'
 
-  - script: curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+  - script: curl -sSL https://install.python-poetry.org | python
     displayName: Install Poetry
 
   - script: python -m venv $(System.DefaultWorkingDirectory)
@@ -29,7 +32,7 @@ jobs:
 
   - script: |
       source bin/activate
-      $HOME/.poetry/bin/poetry install
+      $HOME/.local/bin/poetry install
     displayName: 'Install dependencies'
 
   - script: |

poetry.lock

@@ -45,7 +45,6 @@ python-versions = ">=3.6.2"
 
 [package.dependencies]
 click = ">=7.1.2"
-dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""}
 mypy-extensions = ">=0.4.3"
 pathspec = ">=0.9.0,<1"
 platformdirs = ">=2"
@@ -117,14 +116,6 @@ tomli = {version = "*", optional = true, markers = "extra == \"toml\""}
 [package.extras]
 toml = ["tomli"]
 
-[[package]]
-name = "dataclasses"
-version = "0.8"
-description = "A backport of the dataclasses module for Python 3.6"
-category = "dev"
-optional = false
-python-versions = ">=3.6, <3.7"
-
 [[package]]
 name = "deserialize"
 version = "1.8.3"
@@ -377,15 +368,16 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<5)"]
 
 [[package]]
 name = "setuptools"
-version = "59.6.0"
+version = "65.5.1"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
 category = "dev"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"
 
 [package.extras]
-docs = ["furo", "jaraco.packaging (>=8.2)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-inline-tabs", "sphinxcontrib-towncrier"]
+docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
-testing = ["flake8-2020", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "paver", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "pytest-virtualenv (>=1.2.7)", "pytest-xdist", "sphinx", "virtualenv (>=13.0.0)", "wheel"]
+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
 
 [[package]]
 name = "six"
@@ -511,8 +503,8 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black (
 
 [metadata]
 lock-version = "1.1"
-python-versions = "^3.6.2"
+python-versions = "^3.7.0"
-content-hash = "cc0653fcf2407acf507376a400656ddb8a74dfcfa000f0388898804dd92ddea5"
+content-hash = "ca8f6af06cbb59c7ea49f12740baf0df6f39c5fee3f54881338a8eff3e2cfde6"
 
 [metadata.files]
 astroid = [
@@ -596,10 +588,6 @@ coverage = [
     {file = "coverage-6.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:5829192582c0ec8ca4a2532407bc14c2f338d9878a10442f5d03804a95fac9de"},
     {file = "coverage-6.2.tar.gz", hash = "sha256:e2cad8093172b7d1595b4ad66f24270808658e11acf43a8f95b41276162eb5b8"},
 ]
-dataclasses = [
-    {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"},
-    {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"},
-]
 deserialize = [
     {file = "deserialize-1.8.3-py3-none-any.whl", hash = "sha256:48844d7bdebfe4b440cb3548d2a32a89d429a657e6c1947c703d32105763bde1"},
     {file = "deserialize-1.8.3.tar.gz", hash = "sha256:d1aa33990e046f9ccbe2bb4d1d21631329df39b52462589dd4f80c0d59919306"},
@@ -851,8 +839,8 @@ requests = [
     {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
 ]
 setuptools = [
-    {file = "setuptools-59.6.0-py3-none-any.whl", hash = "sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"},
+    {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"},
-    {file = "setuptools-59.6.0.tar.gz", hash = "sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373"},
+    {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"},
 ]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},

pyproject.toml

@@ -31,7 +31,7 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.6.2"
+python = "^3.7.0"
 deserialize = "^1.5.1"
 requests = "^2.21"
 tenacity = "^6.2.0"

simple_ado/git.py

@@ -247,6 +247,7 @@ class ADOGitClient(ADOBaseClient):
         output_path: str,
         project_id: str,
         repository_id: str,
+        callback: Optional[Callable[[int, int], None]] = None
     ) -> None:
         """Download the zip of the branch specified.
 
@@ -254,6 +255,9 @@ class ADOGitClient(ADOBaseClient):
         :param str output_path: The path to write the output to.
         :param str project_id: The ID of the project
         :param str repository_id: The ID for the repository
+        :param callback: The callback for download progress updates. First
+                         parameter is bytes downloaded, second is total bytes.
+                         The latter will be 0 if the content length is unknown.
 
         :raises ADOException: If the output path already exists
         :raises ADOHTTPException: If we fail to fetch the zip for any reason
@@ -278,7 +282,12 @@ class ADOGitClient(ADOBaseClient):
             raise ADOException("The output path already exists")
 
         with self.http_client.get(request_url, stream=True) as response:
-            download_from_response_stream(response=response, output_path=output_path, log=self.log)
+            download_from_response_stream(
+                response=response,
+                output_path=output_path,
+                log=self.log,
+                callback=callback
+            )
 
     # pylint: disable=too-many-locals
     def get_refs(

simple_ado/utilities.py

@@ -1,6 +1,7 @@
 """Utilities for dealing with the ADO REST API."""
 
 import logging
+from typing import Callable, Optional
 
 import requests
 
@@ -18,13 +19,18 @@ def boolstr(value: bool) -> str:
 
 
 def download_from_response_stream(
-    *, response: requests.Response, output_path: str, log: logging.Logger
+    *,
+    response: requests.Response,
+    output_path: str,
+    log: logging.Logger,
+    callback: Optional[Callable[[int, int], None]] = None
 ) -> None:
     """Downloads a file from an already open response stream.
 
     :param requests.Response response: The response to download from
     :param str output_path: The path to write the file out to
     :param logging.Logger log: The log to use for progress updates
+    :param callback: If supplied, this will be called on every new chunk to update progress to the caller
 
     :raises ADOHTTPException: If we fail to fetch the file for any reason
     """
@@ -46,6 +52,9 @@ def download_from_response_stream(
             total_downloaded += len(data)
             output_file.write(data)
 
+            if callback is not None:
+                callback(total_downloaded, total_size)
+
             if total_size != 0:
                 progress = int((total_downloaded * 100.0) / total_size)
                 log.info(f"Download progress: {progress}%")